Merge pull request #13165 from surajshetty3416/python-distributed-testing

.github/workflows/ci-tests.yml → .github/workflows/server-mariadb-tests.yml

@@ -1,10 +1,8 @@
-name: CI
+name: Server
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
-  push:
 
 jobs:
   test:
@@ -13,23 +11,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
+       container: [1, 2]
 
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
+    name: Python Unit Tests (MariaDB)
 
     services:
       mysql:
@@ -40,18 +24,6 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Clone
         uses: actions/checkout@v2
@@ -63,7 +35,7 @@ jobs:
 
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
           check-latest: true
 
       - name: Add to Hosts
@@ -104,68 +76,54 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: ${{ runner.os }}-cypress-
-          restore-keys: |
-            ${{ runner.os }}-cypress-
-            ${{ runner.os }}-
-
       - name: Install Dependencies
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
           AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
+          TYPE: server
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          DB: mariadb
+          TYPE: server
 
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
-        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
 
       - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
-      - name: Coverage - Pull Request
-        if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
+      - name: Upload Coverage Data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
-          
-      - name: Coverage - Push
-        if: matrix.TYPE == 'server' && github.event_name == 'push'
+          COVERALLS_FLAG_NAME: run-${{ matrix.container }}
+          COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
+          COVERALLS_PARALLEL: true
+
+  coveralls:
+    name: Coverage Wrap Up
+    needs: test
+    container: python:3-slim
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Coveralls Finished
         run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github-actions
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,100 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (Postgres)
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

.github/workflows/ui-tests.yml

@@ -0,0 +1,105 @@
+name: UI
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: UI Tests (Cypress)
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ui
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: ui
+
+      - name: Site Setup
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+
+      - name: UI Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

.mergify.yml

@@ -3,9 +3,12 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -16,9 +19,12 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

frappe/commands/utils.py

@@ -585,12 +585,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 	if os.environ.get('CI'):
 		sys.exit(ret)
 
+@click.command('run-parallel-tests')
+@click.option('--app', help="For App", default='frappe')
+@click.option('--build-number', help="Build number", default=1)
+@click.option('--total-builds', help="Total number of builds", default=1)
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
+@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
+@pass_context
+def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
+	site = get_site(context)
+	if use_orchestrator:
+		from frappe.parallel_test_runner import ParallelTestWithOrchestrator
+		ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
+	else:
+		from frappe.parallel_test_runner import ParallelTestRunner
+		ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
+@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False):
+def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -622,6 +639,12 @@ def run_ui_tests(context, app, headless=False):
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 
+	if parallel:
+		formatted_command += ' --parallel'
+
+	if ci_build_id:
+		formatted_command += ' --ci-build-id {}'.format(ci_build_id)
+
 	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
@@ -797,5 +820,6 @@ commands = [
 	watch,
 	bulk_rename,
 	add_to_email_queue,
-	rebuild_global_search
+	rebuild_global_search,
+	run_parallel_tests
 ]

frappe/contacts/doctype/contact/test_contact.py

@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 
 import frappe
 import unittest
-from frappe.exceptions import ValidationError
+
+test_dependencies = ['Contact', 'Salutation']
 
 class TestContact(unittest.TestCase):
 
@@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
 	if save:
 		doc.insert()
 
-	return doc
+	return doc

frappe/core/doctype/activity_log/test_activity_log.py

@@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase):
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()

frappe/core/doctype/data_export/exporter.py

@@ -282,7 +282,7 @@ class DataExporter:
 				try:
 					sflags = self.docs_to_export.get("flags", "I,U").upper()
 					flags = 0
-					for a in re.split('\W+',sflags):
+					for a in re.split(r'\W+', sflags):
 						flags = flags | reflags.get(a,0)
 
 					c = re.compile(names, flags)

frappe/core/doctype/docshare/test_docshare.py

@@ -7,6 +7,8 @@ import frappe.share
 import unittest
 from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype
 
+test_dependencies = ['User']
+
 class TestDocShare(unittest.TestCase):
 	def setUp(self):
 		self.user = "test@example.com"
@@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase):
 		self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
 		self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
 
-		frappe.share.remove(doctype, submittable_doc.name, self.user)
+		frappe.share.remove(doctype, submittable_doc.name, self.user)

frappe/core/doctype/doctype/doctype.py

@@ -964,7 +964,7 @@ def validate_fields(meta):
 		for field in depends_on_fields:
 			depends_on = docfield.get(field, None)
 			if depends_on and ("=" in depends_on) and \
-				re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on):
+				re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on):
 				frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError)
 
 	def check_table_multiselect_option(docfield):

frappe/core/doctype/doctype/test_doctype.py

@@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase):
 			fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\
 				"read_only_depends_on", "fieldname", "fieldtype"])
 
-		pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+"""
+		pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+'
 		for field in docfields:
 			for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]:
 				condition = field.get(depends_on)
@@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None):
 		for f in fields:
 			doc.append('fields', f)
 
-	return doc
+	return doc

frappe/core/doctype/file/test_file.py

@@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase):
 
 class TestFile(unittest.TestCase):
 	def setUp(self):
+		frappe.set_user('Administrator')
 		self.delete_test_data()
 		self.upload_file()
 

frappe/core/doctype/role_profile/test_role_profile.py

@@ -5,6 +5,8 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
+test_dependencies = ['Role']
+
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
@@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase):
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
-		self.assertEqual(new_role_profile.roles, [])
+		self.assertEqual(new_role_profile.roles, [])

frappe/core/doctype/system_settings/system_settings.py

@@ -42,7 +42,7 @@ class SystemSettings(Document):
 
 	def on_update(self):
 		for df in self.meta.get("fields"):
-			if df.fieldtype not in no_value_fields:
+			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
 				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
 
 		if self.language:

frappe/desk/doctype/todo/test_todo.py

@@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery
 from frappe.permissions import add_permission, reset_perms
 from frappe.core.doctype.doctype.doctype import clear_permissions_cache
 
-# test_records = frappe.get_test_records('ToDo')
-test_user_records = frappe.get_test_records('User')
+test_dependencies = ['User']
 
 class TestToDo(unittest.TestCase):
 	def test_delete(self):
@@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase):
 		frappe.set_user('test4@example.com')
 		#owner and assigned_by is test4
 		todo3 = create_new_todo('Test3', 'test4@example.com')
-		
+
 		# user without any role to read or write todo document
 		self.assertFalse(todo1.has_permission("read"))
 		self.assertFalse(todo1.has_permission("write"))

frappe/email/doctype/notification/test_notification.py

@@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler
 from frappe.desk.form import assign_to
 import unittest
 
-test_records = frappe.get_test_records('Notification')
-
-test_dependencies = ["User"]
+test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):

frappe/parallel_test_runner.py

@@ -0,0 +1,282 @@
+import json
+import os
+import re
+import sys
+import time
+import unittest
+import click
+import frappe
+import requests
+
+from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config)
+
+click_ctx = click.get_current_context(True)
+if click_ctx:
+	click_ctx.color = True
+
+class ParallelTestRunner():
+	def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False):
+		self.app = app
+		self.site = site
+		self.with_coverage = with_coverage
+		self.build_number = frappe.utils.cint(build_number) or 1
+		self.total_builds = frappe.utils.cint(total_builds)
+		self.setup_test_site()
+		self.run_tests()
+
+	def setup_test_site(self):
+		frappe.init(site=self.site)
+		if not frappe.db:
+			frappe.connect()
+
+		frappe.flags.in_test = True
+		frappe.clear_cache()
+		frappe.utils.scheduler.disable_scheduler()
+		set_test_email_config()
+		self.before_test_setup()
+
+	def before_test_setup(self):
+		start_time = time.time()
+		for fn in frappe.get_hooks("before_tests", app_name=self.app):
+			frappe.get_attr(fn)()
+
+		test_module = frappe.get_module(f'{self.app}.tests')
+
+		if hasattr(test_module, "global_test_dependencies"):
+			for doctype in test_module.global_test_dependencies:
+				make_test_records(doctype)
+
+		elapsed = time.time() - start_time
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red')
+		click.echo(f'Before Test {elapsed}')
+
+	def run_tests(self):
+		self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2)
+
+		self.start_coverage()
+
+		for test_file_info in self.get_test_file_list():
+			self.run_tests_for_file(test_file_info)
+
+		self.save_coverage()
+		self.print_result()
+
+	def run_tests_for_file(self, file_info):
+		if not file_info: return
+
+		frappe.set_user('Administrator')
+		path, filename = file_info
+		module = self.get_module(path, filename)
+		self.create_test_dependency_records(module, path, filename)
+		test_suite = unittest.TestSuite()
+		module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
+		test_suite.addTest(module_test_cases)
+		test_suite(self.test_result)
+
+	def create_test_dependency_records(self, module, path, filename):
+		if hasattr(module, "test_dependencies"):
+			for doctype in module.test_dependencies:
+				make_test_records(doctype)
+
+		if os.path.basename(os.path.dirname(path)) == "doctype":
+			# test_data_migration_connector.py > data_migration_connector.json
+			test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json")
+			test_record_file_path = os.path.join(path, test_record_filename)
+			if os.path.exists(test_record_file_path):
+				with open(test_record_file_path, 'r') as f:
+					doc = json.loads(f.read())
+					doctype = doc["name"]
+					make_test_records(doctype)
+
+	def get_module(self, path, filename):
+		app_path = frappe.get_pymodule_path(self.app)
+		relative_path = os.path.relpath(path, app_path)
+		if relative_path == '.':
+			module_name = self.app
+		else:
+			relative_path = relative_path.replace('/', '.')
+			module_name = os.path.splitext(filename)[0]
+			module_name = f'{self.app}.{relative_path}.{module_name}'
+
+		return frappe.get_module(module_name)
+
+	def print_result(self):
+		self.test_result.printErrors()
+		click.echo(self.test_result)
+		if self.test_result.failures or self.test_result.errors:
+			if os.environ.get('CI'):
+				sys.exit(1)
+
+	def start_coverage(self):
+		if self.with_coverage:
+			from coverage import Coverage
+			from frappe.utils import get_bench_path
+
+			# Generate coverage report only for app that is being tested
+			source_path = os.path.join(get_bench_path(), 'apps', self.app)
+			omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss',
+				'*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*']
+
+			if self.app == 'frappe':
+				omit.append('*/commands/*')
+
+			self.coverage = Coverage(source=[source_path], omit=omit)
+			self.coverage.start()
+
+	def save_coverage(self):
+		if not self.with_coverage:
+			return
+		self.coverage.stop()
+		self.coverage.save()
+
+	def get_test_file_list(self):
+		test_list = get_all_tests(self.app)
+		split_size = frappe.utils.ceil(len(test_list) / self.total_builds)
+		# [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2
+		test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)]
+		return test_chunks[self.build_number - 1]
+
+
+class ParallelTestResult(unittest.TextTestResult):
+	def startTest(self, test):
+		self._started_at = time.time()
+		super(unittest.TextTestResult, self).startTest(test)
+		test_class = unittest.util.strclass(test.__class__)
+		if not hasattr(self, 'current_test_class') or self.current_test_class != test_class:
+			click.echo(f"\n{unittest.util.strclass(test.__class__)}")
+			self.current_test_class = test_class
+
+	def getTestMethodName(self, test):
+		return test._testMethodName if hasattr(test, '_testMethodName') else str(test)
+
+	def addSuccess(self, test):
+		super(unittest.TextTestResult, self).addSuccess(test)
+		elapsed = time.time() - self._started_at
+		threshold_passed = elapsed >= SLOW_TEST_THRESHOLD
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else ''
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}")
+
+	def addError(self, test, err):
+		super(unittest.TextTestResult, self).addError(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addFailure(self, test, err):
+		super(unittest.TextTestResult, self).addFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addSkip(self, test, reason):
+		super(unittest.TextTestResult, self).addSkip(test, reason)
+		click.echo(f"  {click.style(' = ', fg='white')} {self.getTestMethodName(test)}")
+
+	def addExpectedFailure(self, test, err):
+		super(unittest.TextTestResult, self).addExpectedFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addUnexpectedSuccess(self, test):
+		super(unittest.TextTestResult, self).addUnexpectedSuccess(test)
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}")
+
+	def printErrors(self):
+		click.echo('\n')
+		self.printErrorList(' ERROR ', self.errors, 'red')
+		self.printErrorList(' FAIL ', self.failures, 'red')
+
+	def printErrorList(self, flavour, errors, color):
+		for test, err in errors:
+			click.echo(self.separator1)
+			click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}")
+			click.echo(self.separator2)
+			click.echo(err)
+
+	def __str__(self):
+		return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}"
+
+def get_all_tests(app):
+	test_file_list = []
+	for path, folders, files in os.walk(frappe.get_pymodule_path(app)):
+		for dontwalk in ('locals', '.git', 'public', '__pycache__'):
+			if dontwalk in folders:
+				folders.remove(dontwalk)
+
+		# for predictability
+		folders.sort()
+		files.sort()
+
+		if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path:
+			# in /doctype/doctype/boilerplate/
+			continue
+
+		for filename in files:
+			if filename.startswith("test_") and filename.endswith(".py") \
+				and filename != 'test_runner.py':
+				test_file_list.append([path, filename])
+
+	return test_file_list
+
+
+class ParallelTestWithOrchestrator(ParallelTestRunner):
+	'''
+		This can be used to balance-out test time across multiple instances
+		This is dependent on external orchestrator which returns next test to run
+
+		orchestrator endpoints
+		- register-instance (<build_id>, <instance_id>, test_spec_list)
+		- get-next-test-spec (<build_id>, <instance_id>)
+		- test-completed (<build_id>, <instance_id>)
+	'''
+	def __init__(self, app, site, with_coverage=False):
+		self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL')
+		if not self.orchestrator_url:
+			click.echo('ORCHESTRATOR_URL environment variable not found!')
+			click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator')
+			sys.exit(1)
+
+		self.ci_build_id = os.environ.get('CI_BUILD_ID')
+		self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10)
+		if not self.ci_build_id:
+			click.echo('CI_BUILD_ID environment variable not found!')
+			sys.exit(1)
+
+		ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage)
+
+	def run_tests(self):
+		self.test_status = 'ongoing'
+		self.register_instance()
+		super().run_tests()
+
+	def get_test_file_list(self):
+		while self.test_status == 'ongoing':
+			yield self.get_next_test()
+
+	def register_instance(self):
+		test_spec_list = get_all_tests(self.app)
+		response_data = self.call_orchestrator('register-instance', data={
+			'test_spec_list': test_spec_list
+		})
+		self.is_master = response_data.get('is_master')
+
+	def get_next_test(self):
+		response_data = self.call_orchestrator('get-next-test-spec')
+		self.test_status = response_data.get('status')
+		return response_data.get('next_test')
+
+	def print_result(self):
+		self.call_orchestrator('test-completed')
+		return super().print_result()
+
+	def call_orchestrator(self, endpoint, data={}):
+		# add repo token header
+		# build id in header
+		headers = {
+			'CI-BUILD-ID': self.ci_build_id,
+			'CI-INSTANCE-ID': self.ci_instance_id,
+			'REPO-TOKEN': '2948288382838DE'
+		}
+		url = f'{self.orchestrator_url}/{endpoint}'
+		res = requests.get(url, json=data, headers=headers)
+		res.raise_for_status()
+		response_data = {}
+		if 'application/json' in res.headers.get('content-type'):
+			response_data = res.json()
+
+		return response_data

frappe/patches/v5_0/fix_text_editor_file_urls.py

@@ -33,8 +33,7 @@ def execute():
 def scrub_relative_urls(html):
 	"""prepend a slash before a relative url"""
 	try:
-		return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html)
-		# return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html)
+		return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html)
 	except:
 		print("Error", html)
 		raise

frappe/printing/doctype/print_format/test_print_format.py

@@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase):
 	def test_print_user(self, style=None):
 		print_html = frappe.get_print("User", "Administrator", style=style)
 		self.assertTrue("<label>First Name: </label>" in print_html)
-		self.assertTrue(re.findall('<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
+		self.assertTrue(re.findall(r'<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
 		return print_html
 
 	def test_print_user_standard(self):
 		print_html = self.test_print_user("Standard")
-		self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html))
-		self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html))
+		self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html))
+		self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html))
 		self.assertFalse("font-family: serif;" in print_html)
 
 	def test_print_user_modern(self):

frappe/test_runner.py

@@ -9,7 +9,6 @@ import time
 import xmlrunner
 import importlib
 from frappe.modules import load_doctype_module, get_module_name
-from frappe.utils import cstr
 import frappe.utils.scheduler
 import cProfile, pstats
 from six import StringIO
@@ -308,6 +307,8 @@ def get_dependencies(doctype):
 			if doctype_name in options_list:
 				options_list.remove(doctype_name)
 
+	options_list.sort()
+
 	return options_list
 
 def make_test_records_for_doctype(doctype, verbose=0, force=False):

frappe/tests/__init__.py

@@ -3,7 +3,10 @@ import frappe
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()
 
 def get_system_setting(key):
 	return frappe.db.get_single_value("System Settings", key)
+
+global_test_dependencies = ['User']

frappe/tests/test_auth.py

@@ -110,7 +110,7 @@ class TestLoginAttemptTracker(unittest.TestCase):
 	def test_account_unlock(self):
 		"""Make sure that locked account gets unlocked after lock_interval of time.
 		"""
-		lock_interval = 10 # In sec
+		lock_interval = 2 # In sec
 		tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=1, lock_interval=lock_interval)
 		# Clear the cache by setting attempt as success
 		tracker.add_success_attempt()

frappe/tests/test_document.py

@@ -6,9 +6,8 @@ import os
 import unittest
 
 import frappe
-from frappe.utils import cint, add_to_date, now
+from frappe.utils import cint
 from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series
-from frappe.exceptions import DoesNotExistError
 
 
 class TestDocument(unittest.TestCase):
@@ -87,13 +86,13 @@ class TestDocument(unittest.TestCase):
 		d.insert()
 		self.assertEqual(frappe.db.get_value("User", d.name), d.name)
 
-	def test_confict_validation(self):
+	def test_conflict_validation(self):
 		d1 = self.test_insert()
 		d2 = frappe.get_doc(d1.doctype, d1.name)
 		d1.save()
 		self.assertRaises(frappe.TimestampMismatchError, d2.save)
 
-	def test_confict_validation_single(self):
+	def test_conflict_validation_single(self):
 		d1 = frappe.get_doc("Website Settings", "Website Settings")
 		d1.home_page = "test-web-page-1"
 
@@ -110,7 +109,7 @@ class TestDocument(unittest.TestCase):
 
 	def test_permission_single(self):
 		frappe.set_user("Guest")
-		d = frappe.get_doc("Website Settings", "Website Settigns")
+		d = frappe.get_doc("Website Settings", "Website Settings")
 		self.assertRaises(frappe.PermissionError, d.save)
 		frappe.set_user("Administrator")
 
@@ -196,41 +195,6 @@ class TestDocument(unittest.TestCase):
 		self.assertTrue(xss not in d.subject)
 		self.assertTrue(escaped_xss in d.subject)
 
-	def test_link_count(self):
-		if os.environ.get('CI'):
-			# cannot run this test reliably in travis due to its handling
-			# of parallelism
-			return
-
-		from frappe.model.utils.link_count import update_link_count
-
-		update_link_count()
-
-		doctype, name = 'User', 'test@example.com'
-
-		d = self.test_insert()
-		d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name})
-
-		d.save()
-
-		link_count = frappe.cache().get_value('_link_count') or {}
-		old_count = link_count.get((doctype, name)) or 0
-
-		frappe.db.commit()
-
-		link_count = frappe.cache().get_value('_link_count') or {}
-		new_count = link_count.get((doctype, name)) or 0
-
-		self.assertEqual(old_count + 1, new_count)
-
-		before_update = frappe.db.get_value(doctype, name, 'idx')
-
-		update_link_count()
-
-		after_update = frappe.db.get_value(doctype, name, 'idx')
-
-		self.assertEqual(before_update + new_count, after_update)
-
 	def test_naming_series(self):
 		data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"]
 

frappe/tests/test_email.py

@@ -6,11 +6,7 @@ from __future__ import unicode_literals
 import unittest, frappe, re, email
 from six import PY3
 
-from frappe.test_runner import make_test_records
-
-make_test_records("User")
-make_test_records("Email Account")
-
+test_dependencies = ['Email Account']
 
 class TestEmail(unittest.TestCase):
 	def setUp(self):

frappe/tests/test_fmt_datetime.py

@@ -45,6 +45,7 @@ class TestFmtDatetime(unittest.TestCase):
 		frappe.db.set_default("time_format", self.pre_test_time_format)
 		frappe.local.user_date_format = None
 		frappe.local.user_time_format = None
+		frappe.db.rollback()
 
 	# Test utility functions
 
@@ -97,28 +98,12 @@ class TestFmtDatetime(unittest.TestCase):
 			self.assertEqual(formatdate(test_date), valid_fmt)
 
 	# Test time formatters
-
 	def test_format_time_forced(self):
 		# Test with forced time formats
 		self.assertEqual(
 			format_time(test_time, 'ss:mm:HH'),
 			test_date_obj.strftime('%S:%M:%H'))
 
-	@unittest.expectedFailure
-	def test_format_time_forced_broken_locale(self):
-		# Test with forced time formats
-		# Currently format_time defaults to HH:mm:ss if the locale is
-		# broken, so this is an expected failure.
-		lang = frappe.local.lang
-		try:
-			# Force fallback from Babel
-			frappe.local.lang = 'FAKE'
-			self.assertEqual(
-				format_time(test_time, 'ss:mm:HH'),
-				test_date_obj.strftime('%S:%M:%H'))
-		finally:
-			frappe.local.lang = lang
-
 	def test_format_time(self):
 		# Test format_time with various default time formats set
 		for fmt, valid_fmt in test_time_formats.items():
@@ -135,21 +120,6 @@ class TestFmtDatetime(unittest.TestCase):
 			format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
 			test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
 
-	@unittest.expectedFailure
-	def test_format_datetime_forced_broken_locale(self):
-		# Test with forced datetime formats
-		# Currently format_datetime defaults to yyyy-MM-dd HH:mm:ss
-		# if the locale is broken, so this is an expected failure.
-		lang = frappe.local.lang
-		# Force fallback from Babel
-		try:
-			frappe.local.lang = 'FAKE'
-			self.assertEqual(
-				format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
-				test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
-		finally:
-			frappe.local.lang = lang
-
 	def test_format_datetime(self):
 		# Test formatdate with various default date formats set
 		for date_fmt, valid_date in test_date_formats.items():

frappe/tests/test_seen.py

@@ -41,6 +41,7 @@ class TestSeen(unittest.TestCase):
 		self.assertTrue('test1@example.com' in json.loads(ev._seen))
 
 		ev.save()
+		ev = frappe.get_doc('Event', ev.name)
 
 		self.assertFalse('test@example.com' in json.loads(ev._seen))
 		self.assertTrue('test1@example.com' in json.loads(ev._seen))

frappe/tests/test_twofactor.py

@@ -8,7 +8,7 @@ from frappe.utils import cint
 from frappe.utils import set_request
 from frappe.auth import validate_ip_address, get_login_attempt_tracker
 from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass,
-	two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj)
+	two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, ExpiredLoginException)
 from . import update_system_settings, get_system_setting
 
 import time
@@ -111,6 +111,7 @@ class TestTwoFactor(unittest.TestCase):
 
 	def test_confirm_otp_token(self):
 		'''Ensure otp is confirmed'''
+		frappe.flags.otp_expiry = 2
 		authenticate_for_2factor(self.user)
 		tmp_id = frappe.local.response['tmp_id']
 		otp = 'wrongotp'
@@ -118,10 +119,11 @@ class TestTwoFactor(unittest.TestCase):
 			confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
 		otp = get_otp(self.user)
 		self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id))
+		frappe.flags.otp_expiry = None
 		if frappe.flags.tests_verbose:
-			print('Sleeping for 30secs to confirm token expires..')
-		time.sleep(30)
-		with self.assertRaises(frappe.AuthenticationError):
+			print('Sleeping for 2 secs to confirm token expires..')
+		time.sleep(2)
+		with self.assertRaises(ExpiredLoginException):
 			confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
 
 	def test_get_verification_obj(self):
@@ -208,12 +210,14 @@ def enable_2fa(bypass_two_factor_auth=0, bypass_restrict_ip_check=0):
 	system_settings.bypass_2fa_for_retricted_ip_users = cint(bypass_two_factor_auth)
 	system_settings.bypass_restrict_ip_check_if_2fa_enabled = cint(bypass_restrict_ip_check)
 	system_settings.two_factor_method = 'OTP App'
+	system_settings.flags.ignore_mandatory = True
 	system_settings.save(ignore_permissions=True)
 	frappe.db.commit()
 
 def disable_2fa():
 	system_settings = frappe.get_doc('System Settings')
 	system_settings.enable_two_factor_auth = 0
+	system_settings.flags.ignore_mandatory = True
 	system_settings.save(ignore_permissions=True)
 	frappe.db.commit()
 

frappe/twofactor.py

@@ -73,11 +73,11 @@ def cache_2fa_data(user, token, otp_secret, tmp_id):
 
 	# set increased expiry time for SMS and Email
 	if verification_method in ['SMS', 'Email']:
-		expiry_time = 300
+		expiry_time = frappe.flags.token_expiry or 300
 		frappe.cache().set(tmp_id + '_token', token)
 		frappe.cache().expire(tmp_id + '_token', expiry_time)
 	else:
-		expiry_time = 180
+		expiry_time = frappe.flags.otp_expiry or 180
 	for k, v in iteritems({'_usr': user, '_pwd': pwd, '_otp_secret': otp_secret}):
 		frappe.cache().set("{0}{1}".format(tmp_id, k), v)
 		frappe.cache().expire("{0}{1}".format(tmp_id, k), expiry_time)

frappe/utils/formatters.py

@@ -78,7 +78,7 @@ def format_value(value, df=None, doc=None, currency=None, translated=False):
 		return "{}%".format(flt(value, 2))
 
 	elif df.get("fieldtype") in ("Text", "Small Text"):
-		if not re.search("(\<br|\<div|\<p)", value):
+		if not re.search(r"(<br|<div|<p)", value):
 			return frappe.safe_decode(value).replace("\n", "<br>")
 
 	elif df.get("fieldtype") == "Markdown Editor":

frappe/utils/global_search.py

@@ -348,7 +348,7 @@ def get_formatted_value(value, field):
 
 	if getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]:
 		value = unescape_html(frappe.safe_decode(value))
-		value = (re.subn(r'<[\s]*(script|style).*?</\1>(?s)', '', text_type(value))[0])
+		value = (re.subn(r'(?s)<[\s]*(script|style).*?</\1>', '', text_type(value))[0])
 		value = ' '.join(value.split())
 	return field.label + " : " + strip_html_tags(text_type(value))
 

frappe/website/doctype/blog_post/test_blog_post.py

@@ -13,6 +13,8 @@ from frappe.website.doctype.blog_post.blog_post import get_blog_list
 from frappe.website.website_generator import WebsiteGenerator
 from frappe.custom.doctype.customize_form.customize_form import reset_customization
 
+test_dependencies = ['Blog Post']
+
 class TestBlogPost(unittest.TestCase):
 	def setUp(self):
 		reset_customization('Blog Post')

frappe/website/doctype/web_form/test_web_form.py

@@ -8,7 +8,7 @@ import unittest, json
 from frappe.website.render import build_page
 from frappe.website.doctype.web_form.web_form import accept
 
-test_records = frappe.get_test_records('Web Form')
+test_dependencies = ['Web Form']
 
 class TestWebForm(unittest.TestCase):
 	def setUp(self):

frappe/website/doctype/web_page/test_web_page.py

@@ -39,7 +39,7 @@ class TestWebPage(unittest.TestCase):
 			published = 1,
 			content_type = 'Rich Text',
 			main_section = 'rich text',
-			main_section_md = '# h1\n\markdown content',
+			main_section_md = '# h1\nmarkdown content',
 			main_section_html = '<div>html content</div>'
 		)).insert()
 

frappe/website/router.py

@@ -158,7 +158,7 @@ def evaluate_dynamic_routes(rules, path):
 	route_map = Map(rules)
 	endpoint = None
 
-	if frappe.local.request:
+	if hasattr(frappe.local, 'request') and frappe.local.request.environ:
 		urls = route_map.bind_to_environ(frappe.local.request.environ)
 		try:
 			endpoint, args = urls.match("/" + path)

requirements.txt

@@ -6,7 +6,8 @@ boto3~=1.17.53
 braintree~=4.8.0
 chardet~=4.0.0
 Click~=7.1.2
-coverage~=4.5.4
+colorama~=0.4.4
+coverage==5.5
 croniter~=1.0.11
 cryptography~=3.4.7
 dropbox~=11.7.0