Merge remote-tracking branch 'upstream/develop' into esbuild

.flake8

@@ -29,4 +29,5 @@ ignore =
     B950,
     W191,
 
-max-line-length = 200
+max-line-length = 200
+exclude=.github/helper/semgrep_rules

.github/helper/semgrep_rules/frappe_correctness.py

@@ -4,25 +4,61 @@ from frappe import _, flt
 from frappe.model.document import Document
 
 
+# ruleid: frappe-modifying-but-not-comitting
 def on_submit(self):
 	if self.value_of_goods == 0:
 		frappe.throw(_('Value of goods cannot be 0'))
-	# ruleid: frappe-modifying-after-submit
 	self.status = 'Submitted'
 
-def on_submit(self):   # noqa
-	if flt(self.per_billed) < 100:
-		self.update_billing_status()
-	else:
-		# todook: frappe-modifying-after-submit
-		self.status = "Completed"
-		self.db_set("status", "Completed")
-
-class TestDoc(Document):
-	pass
-
-	def validate(self):
-		#ruleid: frappe-modifying-child-tables-while-iterating
-		for item in self.child_table:
-			if item.value < 0:
-				self.remove(item)
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	self.status = 'Submitted'
+	self.db_set('status', 'Submitted')
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	x = "y"
+	self.status = x
+	self.db_set('status', x)
+
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	x = "y"
+	self.status = x
+	self.save()
+
+# ruleid: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "uptate"
+
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "update"
+		self.db_set("status", "update")
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+		self.save()
+
+	def tainted_method(self):
+		self.status = "uptate"

.github/helper/semgrep_rules/translate.js

@@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.')
 // ruleid: frappe-translation-js-splitting
 __('You have {0} subscribers' +
     'in your mailing list', [subscribers.length])
+
+// ok: frappe-translation-js-splitting
+__("Ctrl+Enter to add comment")
+
+// ruleid: frappe-translation-js-splitting
+__('You have {0} subscribers \
+    in your mailing list', [subscribers.length])

.github/helper/semgrep_rules/translate.py

@@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool")
 _("")
 # ruleid: frappe-translation-empty-string
 _('')
+
+
+class Test:
+	# ok: frappe-translation-python-splitting
+	def __init__(
+			args
+			):
+		pass

.github/helper/semgrep_rules/translate.yml

@@ -44,8 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
-  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
+  - pattern-regex: '[\s\.]_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '[\s\.]_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations
@@ -54,8 +54,8 @@ rules:
 
 - id: frappe-translation-js-splitting
   pattern-either:
-  - pattern-regex: '__\([^\)]*[\+\\]\s*'
-  - pattern: __('...' + '...')
+  - pattern-regex: '__\([^\)]*[\\]\s+'
+  - pattern: __('...' + '...', ...)
   - pattern: __('...') + __('...')
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.

.github/workflows/semgrep.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - develop
+      - version-13-hotfix
+      - version-13-pre-release
 jobs:
   semgrep:
     name: Frappe Linter

.github/workflows/ci-tests.yml → .github/workflows/server-mariadb-tests.yml

@@ -1,10 +1,8 @@
-name: CI
+name: Server
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
-  push:
 
 jobs:
   test:
@@ -13,23 +11,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
+       container: [1, 2]
 
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
+    name: Python Unit Tests (MariaDB)
 
     services:
       mysql:
@@ -40,18 +24,6 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Clone
         uses: actions/checkout@v2
@@ -104,68 +76,54 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: ${{ runner.os }}-cypress-
-          restore-keys: |
-            ${{ runner.os }}-cypress-
-            ${{ runner.os }}-
-
       - name: Install Dependencies
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
           AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
+          TYPE: server
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
-        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          DB: mariadb
+          TYPE: server
 
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
 
       - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
-      - name: Coverage - Pull Request
-        if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
+      - name: Upload Coverage Data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
+          COVERALLS_FLAG_NAME: run-${{ matrix.container }}
+          COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
+          COVERALLS_PARALLEL: true
+
+  coveralls:
+    name: Coverage Wrap Up
+    needs: test
+    container: python:3-slim
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
 
-      - name: Coverage - Push
-        if: matrix.TYPE == 'server' && github.event_name == 'push'
+      - name: Coveralls Finished
         run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github-actions
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,100 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (Postgres)
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

.github/workflows/ui-tests.yml

@@ -0,0 +1,105 @@
+name: UI
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: UI Tests (Cypress)
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ui
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: ui
+
+      - name: Site Setup
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+
+      - name: UI Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

.mergify.yml

@@ -3,9 +3,12 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -16,9 +19,12 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

frappe/app.py

@@ -99,17 +99,7 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
-			frappe.logger("frappe.web", allow_site=frappe.local.site).info({
-				"site": get_site_name(request.host),
-				"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
-				"base_url": getattr(request, "base_url", "NOTFOUND"),
-				"full_path": getattr(request, "full_path", "NOTFOUND"),
-				"method": getattr(request, "method", "NOTFOUND"),
-				"scheme": getattr(request, "scheme", "NOTFOUND"),
-				"http_status_code": getattr(response, "status_code", "NOTFOUND")
-			})
-
+		log_request(request, response)
 		process_response(response)
 		frappe.destroy()
 
@@ -137,6 +127,19 @@ def init_request(request):
 	if request.method != "OPTIONS":
 		frappe.local.http_request = frappe.auth.HTTPRequest()
 
+def log_request(request, response):
+	if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
+		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+			"site": get_site_name(request.host),
+			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
+			"base_url": getattr(request, "base_url", "NOTFOUND"),
+			"full_path": getattr(request, "full_path", "NOTFOUND"),
+			"method": getattr(request, "method", "NOTFOUND"),
+			"scheme": getattr(request, "scheme", "NOTFOUND"),
+			"http_status_code": getattr(response, "status_code", "NOTFOUND")
+		})
+
+
 def process_response(response):
 	if not response:
 		return

frappe/build.py

@@ -1,14 +1,11 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import json
 import shutil
-import warnings
-import tempfile
+from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
 
 import frappe
@@ -16,8 +13,8 @@ from frappe.utils.minify import JavascriptMinify
 
 import click
 import psutil
-from six import iteritems, text_type
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+from simple_chalk import green
 
 
 timestamps = {}
@@ -76,8 +73,8 @@ def get_assets_link(frappe_head):
 	from requests import head
 
 	tag = getoutput(
-			"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			" refs/tags/,,' -e 's/\^{}//'"
+			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+			r" refs/tags/,,' -e 's/\^{}//'"
 			% frappe_head
 		)
 
@@ -98,9 +95,7 @@ def download_frappe_assets(verbose=True):
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from simple_chalk import green
 	from subprocess import getoutput
-	from tempfile import mkdtemp
 
 	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
@@ -167,7 +162,7 @@ def symlink(target, link_name, overwrite=False):
 
 	# Create link to target with temporary filename
 	while True:
-		temp_link_name = tempfile.mktemp(dir=link_dir)
+		temp_link_name = mktemp(dir=link_dir)
 
 		# os.* functions mimic as closely as possible system functions
 		# The POSIX symlink() returns EEXIST if link_name already exists
@@ -194,7 +189,8 @@ def symlink(target, link_name, overwrite=False):
 
 
 def setup():
-	global app_paths
+	global app_paths, assets_path
+
 	pymodules = []
 	for app in frappe.get_all_apps(True):
 		try:
@@ -202,12 +198,13 @@ def setup():
 		except ImportError:
 			pass
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
+	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
 
-def bundle(mode, apps=None, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
+def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
 	"""concat / minify js files"""
 	setup()
-	make_asset_dirs(make_copy=make_copy, restore=restore)
+	make_asset_dirs(hard_link=hard_link)
 
 	mode = "production" if mode == "production" else "build"
 	command = "yarn run {mode}".format(mode=mode)
@@ -264,75 +261,101 @@ def get_safe_max_old_space_size():
 
 	return safe_max_old_space_size
 
-def make_asset_dirs(make_copy=False, restore=False):
-	# don't even think of making assets_path absolute - rm -rf ahead.
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
+def generate_assets_map():
+	symlinks = {}
 
-	for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
+	for app_name in frappe.get_all_apps():
+		app_doc_path = None
 
-	for app_name in frappe.get_all_apps(True):
 		pymodule = frappe.get_module(app_name)
 		app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
-
-		symlinks = []
 		app_public_path = os.path.join(app_base_path, "public")
-		# app/public > assets/app
-		symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
-		# app/node_modules > assets/app/node_modules
-		if os.path.exists(os.path.abspath(app_public_path)):
-			symlinks.append(
-				[
-					os.path.join(app_base_path, "..", "node_modules"),
-					os.path.join(assets_path, app_name, "node_modules"),
-				]
-			)
+		app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
+		app_docs_path = os.path.join(app_base_path, "docs")
+		app_www_docs_path = os.path.join(app_base_path, "www", "docs")
 
-		app_doc_path = None
-		if os.path.isdir(os.path.join(app_base_path, "docs")):
-			app_doc_path = os.path.join(app_base_path, "docs")
+		app_assets = os.path.abspath(app_public_path)
+		app_node_modules = os.path.abspath(app_node_modules_path)
 
-		elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
-			app_doc_path = os.path.join(app_base_path, "www", "docs")
+		# {app}/public > assets/{app}
+		if os.path.isdir(app_assets):
+			symlinks[app_assets] = os.path.join(assets_path, app_name)
+
+		# {app}/node_modules > assets/{app}/node_modules
+		if os.path.isdir(app_node_modules):
+			symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
 
+		# {app}/docs > assets/{app}_docs
+		if os.path.isdir(app_docs_path):
+			app_doc_path = os.path.join(app_base_path, "docs")
+		elif os.path.isdir(app_www_docs_path):
+			app_doc_path = os.path.join(app_base_path, "www", "docs")
 		if app_doc_path:
-			symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
-
-		for source, target in symlinks:
-			source = os.path.abspath(source)
-			if os.path.exists(source):
-				if restore:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-						shutil.copytree(source, target)
-				elif make_copy:
-					if os.path.exists(target):
-						warnings.warn("Target {target} already exists.".format(target=target))
-					else:
-						shutil.copytree(source, target)
-				else:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-					try:
-						symlink(source, target, overwrite=True)
-					except OSError:
-						print("Cannot link {} to {}".format(source, target))
-			else:
-				warnings.warn('Source {source} does not exist.'.format(source = source))
-				pass
+			app_docs = os.path.abspath(app_doc_path)
+			symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
 
+	return symlinks
+
+
+def setup_assets_dirs():
+	for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
+		os.makedirs(dir_path, exist_ok=True)
+
+
+def clear_broken_symlinks():
+	for path in os.listdir(assets_path):
+		path = os.path.join(assets_path, path)
+		if os.path.islink(path) and not os.path.exists(path):
+			os.remove(path)
 
-def build(no_compress=False, verbose=False):
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
-	for target, sources in iteritems(get_build_maps()):
+
+def unstrip(message):
+	try:
+		max_str = os.get_terminal_size().columns
+	except Exception:
+		max_str = 80
+	_len = len(message)
+	_rem = max_str - _len
+	return f"{message}{' ' * _rem}"
+
+
+def make_asset_dirs(hard_link=False):
+	setup_assets_dirs()
+	clear_broken_symlinks()
+	symlinks = generate_assets_map()
+
+	for source, target in symlinks.items():
+		start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
+		fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
+
+		try:
+			print(start_message, end="\r")
+			link_assets_dir(source, target, hard_link=hard_link)
+		except Exception:
+			print(fail_message, end="\r")
+
+	print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
+
+
+def link_assets_dir(source, target, hard_link=False):
+	if not os.path.exists(source):
+		return
+
+	if os.path.exists(target):
+		if os.path.islink(target):
+			os.unlink(target)
+		else:
+			shutil.rmtree(target)
+
+	if hard_link:
+		shutil.copytree(source, target, dirs_exist_ok=True)
+	else:
+		symlink(source, target, overwrite=True)
+
+
+def build(no_compress=False, verbose=False):
+	for target, sources in get_build_maps().items():
 		pack(os.path.join(assets_path, target), sources, no_compress, verbose)
 
 
@@ -346,7 +369,7 @@ def get_build_maps():
 		if os.path.exists(path):
 			with open(path) as f:
 				try:
-					for target, sources in iteritems(json.loads(f.read())):
+					for target, sources in (json.loads(f.read() or "{}")).items():
 						# update app path
 						source_paths = []
 						for source in sources:
@@ -379,7 +402,7 @@ def pack(target, sources, no_compress, verbose):
 		timestamps[f] = os.path.getmtime(f)
 		try:
 			with open(f, "r") as sourcefile:
-				data = text_type(sourcefile.read(), "utf-8", errors="ignore")
+				data = str(sourcefile.read(), "utf-8", errors="ignore")
 
 			extn = f.rsplit(".", 1)[1]
 
@@ -394,7 +417,7 @@ def pack(target, sources, no_compress, verbose):
 				jsm.minify(tmpin, tmpout)
 				minified = tmpout.getvalue()
 				if minified:
-					outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
+					outtxt += str(minified or "", "utf-8").strip("\n") + ";"
 
 				if verbose:
 					print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@@ -424,16 +447,16 @@ def html_to_js_template(path, content):
 def scrub_html_template(content):
 	"""Returns HTML content with removed whitespace and comments"""
 	# remove whitespace to a single space
-	content = re.sub("\s+", " ", content)
+	content = re.sub(r"\s+", " ", content)
 
 	# strip comments
-	content = re.sub("(<!--.*?-->)", "", content)
+	content = re.sub(r"(<!--.*?-->)", "", content)
 
 	return content.replace("'", "\'")
 
 
 def files_dirty():
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		for f in sources:
 			if ":" in f:
 				f, suffix = f.split(":")

frappe/commands/__init__.py

@@ -28,6 +28,10 @@ def pass_context(f):
 		except frappe.exceptions.SiteNotSpecifiedError as e:
 			click.secho(str(e), fg='yellow')
 			sys.exit(1)
+		except frappe.exceptions.IncorrectSitePath:
+			site = ctx.obj.get("sites", "")[0]
+			click.secho(f'Site {site} does not exist!', fg='yellow')
+			sys.exit(1)
 
 		if profile:
 			pr.disable()

frappe/commands/utils.py

@@ -15,17 +15,17 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
 
 
 @click.command('build')
-@click.option('--app', help='Build assets for specific app')
+@click.option('--app', help='Build assets for app')
 @click.option('--apps', help='Build assets for specific apps')
-@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
-@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
+@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
+@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
+@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
 @click.option('--production', is_flag=True, default=False, help='Build assets in production mode')
-@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
-def build(app=None, apps=None, make_copy=False, restore=False, production=False, verbose=False, force=False):
+@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
+def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False):
 	"Compile JS and CSS source files"
 	from frappe.build import bundle, download_frappe_assets
-
 	frappe.init('')
 
 	if not apps and app:
@@ -44,7 +44,15 @@ def build(app=None, apps=None, make_copy=False, restore=False, production=False,
 	if production:
 		mode = "production"
 
-	bundle(mode, apps=apps, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
+	if make_copy or restore:
+		hard_link = make_copy or restore
+		click.secho(
+			"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
+			fg="yellow",
+		)
+
+	bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe)
+
 
 
 @click.command('watch')
@@ -499,6 +507,8 @@ frappe.db.connect()
 @pass_context
 def console(context):
 	"Start ipython console for a site"
+	import warnings
+
 	site = get_site(context)
 	frappe.init(site=site)
 	frappe.connect()
@@ -519,6 +529,7 @@ def console(context):
 	if failed_to_import:
 		print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
 
+	warnings.simplefilter('ignore')
 	IPython.embed(display_banner="", header="", colors="neutral")
 
 
@@ -596,12 +607,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 	if os.environ.get('CI'):
 		sys.exit(ret)
 
+@click.command('run-parallel-tests')
+@click.option('--app', help="For App", default='frappe')
+@click.option('--build-number', help="Build number", default=1)
+@click.option('--total-builds', help="Total number of builds", default=1)
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
+@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
+@pass_context
+def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
+	site = get_site(context)
+	if use_orchestrator:
+		from frappe.parallel_test_runner import ParallelTestWithOrchestrator
+		ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
+	else:
+		from frappe.parallel_test_runner import ParallelTestRunner
+		ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
+@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False):
+def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -633,6 +661,12 @@ def run_ui_tests(context, app, headless=False):
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 
+	if parallel:
+		formatted_command += ' --parallel'
+
+	if ci_build_id:
+		formatted_command += ' --ci-build-id {}'.format(ci_build_id)
+
 	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
@@ -808,5 +842,6 @@ commands = [
 	watch,
 	bulk_rename,
 	add_to_email_queue,
-	rebuild_global_search
+	rebuild_global_search,
+	run_parallel_tests
 ]

frappe/contacts/doctype/contact/test_contact.py

@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 
 import frappe
 import unittest
-from frappe.exceptions import ValidationError
+
+test_dependencies = ['Contact', 'Salutation']
 
 class TestContact(unittest.TestCase):
 
@@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
 	if save:
 		doc.insert()
 
-	return doc
+	return doc

frappe/core/doctype/activity_log/test_activity_log.py

@@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase):
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()

frappe/core/doctype/data_export/exporter.py

@@ -282,7 +282,7 @@ class DataExporter:
 				try:
 					sflags = self.docs_to_export.get("flags", "I,U").upper()
 					flags = 0
-					for a in re.split('\W+',sflags):
+					for a in re.split(r'\W+', sflags):
 						flags = flags | reflags.get(a,0)
 
 					c = re.compile(names, flags)

frappe/core/doctype/data_import/importer.py

@@ -641,7 +641,7 @@ class Row:
 				return
 		elif df.fieldtype == "Duration":
 			import re
-			is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
+			is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
 			if not is_valid_duration:
 				self.warnings.append(
 					{
@@ -929,10 +929,7 @@ class Column:
 				self.warnings.append(
 					{
 						"col": self.column_number,
-						"message": _(
-							"Date format could not be determined from the values in"
-							" this column. Defaulting to yyyy-mm-dd."
-						),
+						"message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."),
 						"type": "info",
 					}
 				)

frappe/core/doctype/docshare/test_docshare.py

@@ -7,6 +7,8 @@ import frappe.share
 import unittest
 from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype
 
+test_dependencies = ['User']
+
 class TestDocShare(unittest.TestCase):
 	def setUp(self):
 		self.user = "test@example.com"
@@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase):
 		self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
 		self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
 
-		frappe.share.remove(doctype, submittable_doc.name, self.user)
+		frappe.share.remove(doctype, submittable_doc.name, self.user)

frappe/core/doctype/doctype/doctype.py

@@ -671,12 +671,12 @@ class DocType(Document):
 		flags = {"flags": re.ASCII} if six.PY3 else {}
 
 		# a DocType name should not start or end with an empty space
-		if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
+		if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
 			frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
 
 		# a DocType's name should not start with a number or underscore
 		# and should only contain letters, numbers and underscore
-		if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
+		if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags):
 			frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
 
 		validate_route_conflict(self.doctype, self.name)
@@ -964,7 +964,7 @@ def validate_fields(meta):
 		for field in depends_on_fields:
 			depends_on = docfield.get(field, None)
 			if depends_on and ("=" in depends_on) and \
-				re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on):
+				re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on):
 				frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError)
 
 	def check_table_multiselect_option(docfield):

frappe/core/doctype/doctype/test_doctype.py

@@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase):
 			fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\
 				"read_only_depends_on", "fieldname", "fieldtype"])
 
-		pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+"""
+		pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+'
 		for field in docfields:
 			for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]:
 				condition = field.get(depends_on)
@@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None):
 		for f in fields:
 			doc.append('fields', f)
 
-	return doc
+	return doc

frappe/core/doctype/file/file.py

@@ -498,7 +498,7 @@ class File(Document):
 		self.file_size = self.check_max_file_size()
 
 		if (
-			self.content_type and "image" in self.content_type
+			self.content_type and self.content_type == "image/jpeg"
 			and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
 		):
 			self.content = strip_exif_data(self.content, self.content_type)
@@ -912,7 +912,7 @@ def extract_images_from_html(doc, content):
 		return '<img src="{file_url}"'.format(file_url=file_url)
 
 	if content and isinstance(content, string_types):
-		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+		content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
 
 	return content
 

frappe/core/doctype/file/test_file.py

@@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase):
 
 class TestFile(unittest.TestCase):
 	def setUp(self):
+		frappe.set_user('Administrator')
 		self.delete_test_data()
 		self.upload_file()
 

frappe/core/doctype/role_profile/test_role_profile.py

@@ -5,6 +5,8 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
+test_dependencies = ['Role']
+
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
@@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase):
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
-		self.assertEqual(new_role_profile.roles, [])
+		self.assertEqual(new_role_profile.roles, [])

frappe/core/doctype/system_settings/system_settings.py

@@ -42,7 +42,7 @@ class SystemSettings(Document):
 
 	def on_update(self):
 		for df in self.meta.get("fields"):
-			if df.fieldtype not in no_value_fields:
+			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
 				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
 
 		if self.language:

frappe/desk/doctype/todo/test_todo.py

@@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery
 from frappe.permissions import add_permission, reset_perms
 from frappe.core.doctype.doctype.doctype import clear_permissions_cache
 
-# test_records = frappe.get_test_records('ToDo')
-test_user_records = frappe.get_test_records('User')
+test_dependencies = ['User']
 
 class TestToDo(unittest.TestCase):
 	def test_delete(self):
@@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase):
 		frappe.set_user('test4@example.com')
 		#owner and assigned_by is test4
 		todo3 = create_new_todo('Test3', 'test4@example.com')
-		
+
 		# user without any role to read or write todo document
 		self.assertFalse(todo1.has_permission("read"))
 		self.assertFalse(todo1.has_permission("write"))

frappe/desk/doctype/workspace_link/workspace_link.json

@@ -8,13 +8,13 @@
   "type",
   "label",
   "icon",
+  "only_for",
   "hidden",
   "link_details_section",
   "link_type",
   "link_to",
   "column_break_7",
   "dependencies",
-  "only_for",
   "onboard",
   "is_query_report"
  ],
@@ -84,7 +84,7 @@
   {
    "fieldname": "only_for",
    "fieldtype": "Link",
-   "label": "Only for ",
+   "label": "Only for",
    "options": "Country"
   },
   {
@@ -104,7 +104,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-01-12 13:13:12.379443",
+ "modified": "2021-05-13 13:10:18.128512",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Link",

frappe/email/doctype/notification/test_notification.py

@@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler
 from frappe.desk.form import assign_to
 import unittest
 
-test_records = frappe.get_test_records('Notification')
-
-test_dependencies = ["User"]
+test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):

frappe/email/receive.py

@@ -284,7 +284,7 @@ class EmailServer:
 
 		flags = []
 		for flag in imaplib.ParseFlags(flag_string) or []:
-			pattern = re.compile("\w+")
+			pattern = re.compile(r"\w+")
 			match = re.search(pattern, frappe.as_unicode(flag))
 			flags.append(match.group(0))
 
@@ -555,7 +555,7 @@ class Email:
 
 	def get_thread_id(self):
 		"""Extract thread ID from `[]`"""
-		l = re.findall('(?<=\[)[\w/-]+', self.subject)
+		l = re.findall(r'(?<=\[)[\w/-]+', self.subject)
 		return l and l[0] or None
 
 

frappe/hooks.py

@@ -228,7 +228,6 @@ scheduler_events = {
 		"frappe.desk.doctype.event.event.send_event_digest",
 		"frappe.sessions.clear_expired_sessions",
 		"frappe.email.doctype.notification.notification.trigger_daily_alerts",
-		"frappe.realtime.remove_old_task_logs",
 		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",

frappe/installer.py

@@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None):
 
 
 def make_site_dirs():
-	site_public_path = os.path.join(frappe.local.site_path, 'public')
-	site_private_path = os.path.join(frappe.local.site_path, 'private')
-	for dir_path in (
-			os.path.join(site_private_path, 'backups'),
-			os.path.join(site_public_path, 'files'),
-			os.path.join(site_private_path, 'files'),
-			os.path.join(frappe.local.site_path, 'logs'),
-			os.path.join(frappe.local.site_path, 'task-logs')):
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
-	locks_dir = frappe.get_site_path('locks')
-	if not os.path.exists(locks_dir):
-			os.makedirs(locks_dir)
+	for dir_path in [
+		os.path.join("public", "files"),
+		os.path.join("private", "backups"),
+		os.path.join("private", "files"),
+		"error-snapshots",
+		"locks",
+		"logs",
+	]:
+		path = frappe.get_site_path(dir_path)
+		os.makedirs(path, exist_ok=True)
 
 
 def add_module_defs(app):

frappe/model/base_document.py

@@ -870,7 +870,7 @@ class BaseDocument(object):
 			from frappe.model.meta import get_default_df
 			df = get_default_df(fieldname)
 
-		if not currency:
+		if not currency and df:
 			currency = self.get(df.get("options"))
 			if not frappe.db.exists('Currency', currency, cache=True):
 				currency = None

frappe/model/naming.py

@@ -283,7 +283,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
 	filters.update({fieldname: value})
 	exists = frappe.db.exists(doctype, filters)
 
-	regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator)
+	regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator)
 
 	if exists:
 		last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}`

frappe/parallel_test_runner.py

@@ -0,0 +1,282 @@
+import json
+import os
+import re
+import sys
+import time
+import unittest
+import click
+import frappe
+import requests
+
+from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config)
+
+click_ctx = click.get_current_context(True)
+if click_ctx:
+	click_ctx.color = True
+
+class ParallelTestRunner():
+	def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False):
+		self.app = app
+		self.site = site
+		self.with_coverage = with_coverage
+		self.build_number = frappe.utils.cint(build_number) or 1
+		self.total_builds = frappe.utils.cint(total_builds)
+		self.setup_test_site()
+		self.run_tests()
+
+	def setup_test_site(self):
+		frappe.init(site=self.site)
+		if not frappe.db:
+			frappe.connect()
+
+		frappe.flags.in_test = True
+		frappe.clear_cache()
+		frappe.utils.scheduler.disable_scheduler()
+		set_test_email_config()
+		self.before_test_setup()
+
+	def before_test_setup(self):
+		start_time = time.time()
+		for fn in frappe.get_hooks("before_tests", app_name=self.app):
+			frappe.get_attr(fn)()
+
+		test_module = frappe.get_module(f'{self.app}.tests')
+
+		if hasattr(test_module, "global_test_dependencies"):
+			for doctype in test_module.global_test_dependencies:
+				make_test_records(doctype)
+
+		elapsed = time.time() - start_time
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red')
+		click.echo(f'Before Test {elapsed}')
+
+	def run_tests(self):
+		self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2)
+
+		self.start_coverage()
+
+		for test_file_info in self.get_test_file_list():
+			self.run_tests_for_file(test_file_info)
+
+		self.save_coverage()
+		self.print_result()
+
+	def run_tests_for_file(self, file_info):
+		if not file_info: return
+
+		frappe.set_user('Administrator')
+		path, filename = file_info
+		module = self.get_module(path, filename)
+		self.create_test_dependency_records(module, path, filename)
+		test_suite = unittest.TestSuite()
+		module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
+		test_suite.addTest(module_test_cases)
+		test_suite(self.test_result)
+
+	def create_test_dependency_records(self, module, path, filename):
+		if hasattr(module, "test_dependencies"):
+			for doctype in module.test_dependencies:
+				make_test_records(doctype)
+
+		if os.path.basename(os.path.dirname(path)) == "doctype":
+			# test_data_migration_connector.py > data_migration_connector.json
+			test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json")
+			test_record_file_path = os.path.join(path, test_record_filename)
+			if os.path.exists(test_record_file_path):
+				with open(test_record_file_path, 'r') as f:
+					doc = json.loads(f.read())
+					doctype = doc["name"]
+					make_test_records(doctype)
+
+	def get_module(self, path, filename):
+		app_path = frappe.get_pymodule_path(self.app)
+		relative_path = os.path.relpath(path, app_path)
+		if relative_path == '.':
+			module_name = self.app
+		else:
+			relative_path = relative_path.replace('/', '.')
+			module_name = os.path.splitext(filename)[0]
+			module_name = f'{self.app}.{relative_path}.{module_name}'
+
+		return frappe.get_module(module_name)
+
+	def print_result(self):
+		self.test_result.printErrors()
+		click.echo(self.test_result)
+		if self.test_result.failures or self.test_result.errors:
+			if os.environ.get('CI'):
+				sys.exit(1)
+
+	def start_coverage(self):
+		if self.with_coverage:
+			from coverage import Coverage
+			from frappe.utils import get_bench_path
+
+			# Generate coverage report only for app that is being tested
+			source_path = os.path.join(get_bench_path(), 'apps', self.app)
+			omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss',
+				'*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*']
+
+			if self.app == 'frappe':
+				omit.append('*/commands/*')
+
+			self.coverage = Coverage(source=[source_path], omit=omit)
+			self.coverage.start()
+
+	def save_coverage(self):
+		if not self.with_coverage:
+			return
+		self.coverage.stop()
+		self.coverage.save()
+
+	def get_test_file_list(self):
+		test_list = get_all_tests(self.app)
+		split_size = frappe.utils.ceil(len(test_list) / self.total_builds)
+		# [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2
+		test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)]
+		return test_chunks[self.build_number - 1]
+
+
+class ParallelTestResult(unittest.TextTestResult):
+	def startTest(self, test):
+		self._started_at = time.time()
+		super(unittest.TextTestResult, self).startTest(test)
+		test_class = unittest.util.strclass(test.__class__)
+		if not hasattr(self, 'current_test_class') or self.current_test_class != test_class:
+			click.echo(f"\n{unittest.util.strclass(test.__class__)}")
+			self.current_test_class = test_class
+
+	def getTestMethodName(self, test):
+		return test._testMethodName if hasattr(test, '_testMethodName') else str(test)
+
+	def addSuccess(self, test):
+		super(unittest.TextTestResult, self).addSuccess(test)
+		elapsed = time.time() - self._started_at
+		threshold_passed = elapsed >= SLOW_TEST_THRESHOLD
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else ''
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}")
+
+	def addError(self, test, err):
+		super(unittest.TextTestResult, self).addError(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addFailure(self, test, err):
+		super(unittest.TextTestResult, self).addFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addSkip(self, test, reason):
+		super(unittest.TextTestResult, self).addSkip(test, reason)
+		click.echo(f"  {click.style(' = ', fg='white')} {self.getTestMethodName(test)}")
+
+	def addExpectedFailure(self, test, err):
+		super(unittest.TextTestResult, self).addExpectedFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addUnexpectedSuccess(self, test):
+		super(unittest.TextTestResult, self).addUnexpectedSuccess(test)
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}")
+
+	def printErrors(self):
+		click.echo('\n')
+		self.printErrorList(' ERROR ', self.errors, 'red')
+		self.printErrorList(' FAIL ', self.failures, 'red')
+
+	def printErrorList(self, flavour, errors, color):
+		for test, err in errors:
+			click.echo(self.separator1)
+			click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}")
+			click.echo(self.separator2)
+			click.echo(err)
+
+	def __str__(self):
+		return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}"
+
+def get_all_tests(app):
+	test_file_list = []
+	for path, folders, files in os.walk(frappe.get_pymodule_path(app)):
+		for dontwalk in ('locals', '.git', 'public', '__pycache__'):
+			if dontwalk in folders:
+				folders.remove(dontwalk)
+
+		# for predictability
+		folders.sort()
+		files.sort()
+
+		if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path:
+			# in /doctype/doctype/boilerplate/
+			continue
+
+		for filename in files:
+			if filename.startswith("test_") and filename.endswith(".py") \
+				and filename != 'test_runner.py':
+				test_file_list.append([path, filename])
+
+	return test_file_list
+
+
+class ParallelTestWithOrchestrator(ParallelTestRunner):
+	'''
+		This can be used to balance-out test time across multiple instances
+		This is dependent on external orchestrator which returns next test to run
+
+		orchestrator endpoints
+		- register-instance (<build_id>, <instance_id>, test_spec_list)
+		- get-next-test-spec (<build_id>, <instance_id>)
+		- test-completed (<build_id>, <instance_id>)
+	'''
+	def __init__(self, app, site, with_coverage=False):
+		self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL')
+		if not self.orchestrator_url:
+			click.echo('ORCHESTRATOR_URL environment variable not found!')
+			click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator')
+			sys.exit(1)
+
+		self.ci_build_id = os.environ.get('CI_BUILD_ID')
+		self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10)
+		if not self.ci_build_id:
+			click.echo('CI_BUILD_ID environment variable not found!')
+			sys.exit(1)
+
+		ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage)
+
+	def run_tests(self):
+		self.test_status = 'ongoing'
+		self.register_instance()
+		super().run_tests()
+
+	def get_test_file_list(self):
+		while self.test_status == 'ongoing':
+			yield self.get_next_test()
+
+	def register_instance(self):
+		test_spec_list = get_all_tests(self.app)
+		response_data = self.call_orchestrator('register-instance', data={
+			'test_spec_list': test_spec_list
+		})
+		self.is_master = response_data.get('is_master')
+
+	def get_next_test(self):
+		response_data = self.call_orchestrator('get-next-test-spec')
+		self.test_status = response_data.get('status')
+		return response_data.get('next_test')
+
+	def print_result(self):
+		self.call_orchestrator('test-completed')
+		return super().print_result()
+
+	def call_orchestrator(self, endpoint, data={}):
+		# add repo token header
+		# build id in header
+		headers = {
+			'CI-BUILD-ID': self.ci_build_id,
+			'CI-INSTANCE-ID': self.ci_instance_id,
+			'REPO-TOKEN': '2948288382838DE'
+		}
+		url = f'{self.orchestrator_url}/{endpoint}'
+		res = requests.get(url, json=data, headers=headers)
+		res.raise_for_status()
+		response_data = {}
+		if 'application/json' in res.headers.get('content-type'):
+			response_data = res.json()
+
+		return response_data

frappe/patches/v5_0/fix_text_editor_file_urls.py

@@ -33,8 +33,7 @@ def execute():
 def scrub_relative_urls(html):
 	"""prepend a slash before a relative url"""
 	try:
-		return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html)
-		# return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html)
+		return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html)
 	except:
 		print("Error", html)
 		raise

frappe/printing/doctype/print_format/test_print_format.py

@@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase):
 	def test_print_user(self, style=None):
 		print_html = frappe.get_print("User", "Administrator", style=style)
 		self.assertTrue("<label>First Name: </label>" in print_html)
-		self.assertTrue(re.findall('<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
+		self.assertTrue(re.findall(r'<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
 		return print_html
 
 	def test_print_user_standard(self):
 		print_html = self.test_print_user("Standard")
-		self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html))
-		self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html))
+		self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html))
+		self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html))
 		self.assertFalse("font-family: serif;" in print_html)
 
 	def test_print_user_modern(self):

frappe/public/js/frappe/form/controls/date.js

@@ -13,9 +13,11 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat
 		super.set_formatted_input(value);
 		if (this.timepicker_only) return;
 		if (!this.datepicker) return;
-		if(!value) {
+		if (!value) {
 			this.datepicker.clear();
 			return;
+		} else if (value === "Today") {
+			value = this.get_now_date();
 		}
 
 		let should_refresh = this.last_value && this.last_value !== value;

frappe/public/js/frappe/form/grid.js

@@ -910,6 +910,10 @@ export default class Grid {
 
 	update_docfield_property(fieldname, property, value) {
 		// update the docfield of each row
+		if (!this.grid_rows) {
+			return;
+		}
+
 		for (let row of this.grid_rows) {
 			let docfield = row.docfields.find(d => d.fieldname === fieldname);
 			if (docfield) {

frappe/public/js/frappe/form/layout.js

@@ -510,7 +510,7 @@ frappe.ui.form.Layout = class Layout {
 			form_obj = this;
 		}
 		if (form_obj) {
-			if (this.doc && this.doc.parent) {
+			if (this.doc && this.doc.parent && this.doc.parentfield) {
 				form_obj.setting_dependency = true;
 				form_obj.set_df_property(this.doc.parentfield, property, value, this.doc.parent, fieldname, this.doc.name);
 				form_obj.setting_dependency = false;

frappe/realtime.py

@@ -1,56 +1,23 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-
-
 import frappe
 import os
-import time
 import redis
-from io import FileIO
-from frappe.utils import get_site_path
-from frappe import conf
 
-END_LINE = '<!-- frappe: end-file -->'
-TASK_LOG_MAX_AGE = 86400  # 1 day in seconds
 redis_server = None
 
+
 @frappe.whitelist()
 def get_pending_tasks_for_doc(doctype, docname):
 	return frappe.db.sql_list("select name from `tabAsync Task` where status in ('Queued', 'Running') and reference_doctype=%s and reference_name=%s", (doctype, docname))
 
 
-def set_task_status(task_id, status, response=None):
-	if not response:
-		response = {}
-	response.update({
-		"status": status,
-		"task_id": task_id
-	})
-	emit_via_redis("task_status_change", response, room="task:" + task_id)
-
-
-def remove_old_task_logs():
-	logs_path = get_site_path('task-logs')
-
-	def full_path(_file):
-		return os.path.join(logs_path, _file)
-
-	files_to_remove = [full_path(_file) for _file in os.listdir(logs_path)]
-	files_to_remove = [_file for _file in files_to_remove if is_file_old(_file) and os.path.isfile(_file)]
-	for _file in files_to_remove:
-		os.remove(_file)
-
-
-def is_file_old(file_path):
-	return ((time.time() - os.stat(file_path).st_mtime) > TASK_LOG_MAX_AGE)
-
 def publish_progress(percent, title=None, doctype=None, docname=None, description=None):
 	publish_realtime('progress', {'percent': percent, 'title': title, 'description': description},
 		user=frappe.session.user, doctype=doctype, docname=docname)
 
+
 def publish_realtime(event=None, message=None, room=None,
 	user=None, doctype=None, docname=None, task_id=None,
 	after_commit=False):
@@ -103,6 +70,7 @@ def publish_realtime(event=None, message=None, room=None,
 	else:
 		emit_via_redis(event, message, room)
 
+
 def emit_via_redis(event, message, room):
 	"""Publish real-time updates via redis
 
@@ -117,57 +85,17 @@ def emit_via_redis(event, message, room):
 		# print(frappe.get_traceback())
 		pass
 
-def put_log(line_no, line, task_id=None):
-	r = get_redis_server()
-	if not task_id:
-		task_id = frappe.local.task_id
-	task_progress_room = get_task_progress_room(task_id)
-	task_log_key = "task_log:" + task_id
-	publish_realtime('task_progress', {
-		"message": {
-			"lines": {line_no: line}
-		},
-		"task_id": task_id
-	}, room=task_progress_room)
-	r.hset(task_log_key, line_no, line)
-	r.expire(task_log_key, 3600)
-
 
 def get_redis_server():
 	"""returns redis_socketio connection."""
 	global redis_server
 	if not redis_server:
 		from redis import Redis
-		redis_server = Redis.from_url(conf.get("redis_socketio")
+		redis_server = Redis.from_url(frappe.conf.redis_socketio
 			or "redis://localhost:12311")
 	return redis_server
 
 
-class FileAndRedisStream(FileIO):
-	def __init__(self, *args, **kwargs):
-		ret = super(FileAndRedisStream, self).__init__(*args, **kwargs)
-		self.count = 0
-		return ret
-
-	def write(self, data):
-		ret = super(FileAndRedisStream, self).write(data)
-		if frappe.local.task_id:
-			put_log(self.count, data, task_id=frappe.local.task_id)
-			self.count += 1
-		return ret
-
-
-def get_std_streams(task_id):
-	stdout = FileAndRedisStream(get_task_log_file_path(task_id, 'stdout'), 'w')
-	# stderr = FileAndRedisStream(get_task_log_file_path(task_id, 'stderr'), 'w')
-	return stdout, stdout
-
-
-def get_task_log_file_path(task_id, stream_type):
-	logs_dir = frappe.utils.get_site_path('task-logs')
-	return os.path.join(logs_dir, task_id + '.' + stream_type)
-
-
 @frappe.whitelist(allow_guest=True)
 def can_subscribe_doc(doctype, docname):
 	if os.environ.get('CI'):
@@ -201,9 +129,7 @@ def get_site_room():
 def get_task_progress_room(task_id):
 	return "".join([frappe.local.site, ":task_progress:", task_id])
 
-# frappe.chat
 def get_chat_room(room):
 	room = ''.join([frappe.local.site, ":room:", room])
 
 	return room
-# end frappe.chat room

frappe/search/full_text_search.py

@@ -1,8 +1,8 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
 import frappe
+from frappe.utils import update_progress_bar
 
 from whoosh.index import create_in, open_dir, EmptyIndexError
 from whoosh.fields import TEXT, ID, Schema
@@ -95,9 +95,10 @@ class FullTextSearch:
 		ix = self.create_index()
 		writer = ix.writer()
 
-		for document in self.documents:
+		for i, document in enumerate(self.documents):
 			if document:
 				writer.add_document(**document)
+			update_progress_bar("Building Index", i, len(self.documents))
 
 		writer.commit(optimize=True)
 

frappe/search/website_search.py

@@ -1,14 +1,15 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
-import frappe
+import os
+
 from bs4 import BeautifulSoup
-from whoosh.fields import TEXT, ID, Schema
+from whoosh.fields import ID, TEXT, Schema
+
+import frappe
 from frappe.search.full_text_search import FullTextSearch
+from frappe.utils import set_request, update_progress_bar
 from frappe.website.render import render_page
-from frappe.utils import set_request
-import os
 
 INDEX_NAME = "web_routes"
 
@@ -30,11 +31,21 @@ class WebsiteSearch(FullTextSearch):
 		Returns:
 			self (object): FullTextSearch Instance
 		"""
-		routes = get_static_pages_from_all_apps()
-		routes += slugs_with_web_view()
 
-		documents = [self.get_document_to_index(route) for route in routes]
-		return documents
+		if getattr(self, "_items_to_index", False):
+			return self._items_to_index
+
+		routes = get_static_pages_from_all_apps() + slugs_with_web_view()
+
+		self._items_to_index = []
+
+		for i, route in enumerate(routes):
+			update_progress_bar("Retrieving Routes", i, len(routes))
+			self._items_to_index += [self.get_document_to_index(route)]
+
+		print()
+
+		return self.get_items_to_index()
 
 	def get_document_to_index(self, route):
 		"""Render a page and parse it using BeautifulSoup
@@ -114,4 +125,4 @@ def remove_document_from_index(path):
 
 def build_index_for_all_routes():
 	ws = WebsiteSearch(INDEX_NAME)
-	return ws.build()
+	return ws.build()

frappe/test_runner.py

@@ -9,7 +9,6 @@ import time
 import xmlrunner
 import importlib
 from frappe.modules import load_doctype_module, get_module_name
-from frappe.utils import cstr
 import frappe.utils.scheduler
 import cProfile, pstats
 from six import StringIO
@@ -308,6 +307,8 @@ def get_dependencies(doctype):
 			if doctype_name in options_list:
 				options_list.remove(doctype_name)
 
+	options_list.sort()
+
 	return options_list
 
 def make_test_records_for_doctype(doctype, verbose=0, force=False):

frappe/tests/__init__.py

@@ -3,7 +3,10 @@ import frappe
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()
 
 def get_system_setting(key):
 	return frappe.db.get_single_value("System Settings", key)
+
+global_test_dependencies = ['User']

frappe/tests/test_auth.py

@@ -110,7 +110,7 @@ class TestLoginAttemptTracker(unittest.TestCase):
 	def test_account_unlock(self):
 		"""Make sure that locked account gets unlocked after lock_interval of time.
 		"""
-		lock_interval = 10 # In sec
+		lock_interval = 2 # In sec
 		tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=1, lock_interval=lock_interval)
 		# Clear the cache by setting attempt as success
 		tracker.add_success_attempt()

frappe/tests/test_document.py

@@ -6,9 +6,8 @@ import os
 import unittest
 
 import frappe
-from frappe.utils import cint, add_to_date, now
+from frappe.utils import cint
 from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series
-from frappe.exceptions import DoesNotExistError
 
 
 class TestDocument(unittest.TestCase):
@@ -87,13 +86,13 @@ class TestDocument(unittest.TestCase):
 		d.insert()
 		self.assertEqual(frappe.db.get_value("User", d.name), d.name)
 
-	def test_confict_validation(self):
+	def test_conflict_validation(self):
 		d1 = self.test_insert()
 		d2 = frappe.get_doc(d1.doctype, d1.name)
 		d1.save()
 		self.assertRaises(frappe.TimestampMismatchError, d2.save)
 
-	def test_confict_validation_single(self):
+	def test_conflict_validation_single(self):
 		d1 = frappe.get_doc("Website Settings", "Website Settings")
 		d1.home_page = "test-web-page-1"
 
@@ -110,7 +109,7 @@ class TestDocument(unittest.TestCase):
 
 	def test_permission_single(self):
 		frappe.set_user("Guest")
-		d = frappe.get_doc("Website Settings", "Website Settigns")
+		d = frappe.get_doc("Website Settings", "Website Settings")
 		self.assertRaises(frappe.PermissionError, d.save)
 		frappe.set_user("Administrator")
 
@@ -196,41 +195,6 @@ class TestDocument(unittest.TestCase):
 		self.assertTrue(xss not in d.subject)
 		self.assertTrue(escaped_xss in d.subject)
 
-	def test_link_count(self):
-		if os.environ.get('CI'):
-			# cannot run this test reliably in travis due to its handling
-			# of parallelism
-			return
-
-		from frappe.model.utils.link_count import update_link_count
-
-		update_link_count()
-
-		doctype, name = 'User', 'test@example.com'
-
-		d = self.test_insert()
-		d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name})
-
-		d.save()
-
-		link_count = frappe.cache().get_value('_link_count') or {}
-		old_count = link_count.get((doctype, name)) or 0
-
-		frappe.db.commit()
-
-		link_count = frappe.cache().get_value('_link_count') or {}
-		new_count = link_count.get((doctype, name)) or 0
-
-		self.assertEqual(old_count + 1, new_count)
-
-		before_update = frappe.db.get_value(doctype, name, 'idx')
-
-		update_link_count()
-
-		after_update = frappe.db.get_value(doctype, name, 'idx')
-
-		self.assertEqual(before_update + new_count, after_update)
-
 	def test_naming_series(self):
 		data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"]
 

frappe/tests/test_email.py

@@ -6,11 +6,7 @@ from __future__ import unicode_literals
 import unittest, frappe, re, email
 from six import PY3
 
-from frappe.test_runner import make_test_records
-
-make_test_records("User")
-make_test_records("Email Account")
-
+test_dependencies = ['Email Account']
 
 class TestEmail(unittest.TestCase):
 	def setUp(self):

frappe/tests/test_fmt_datetime.py

@@ -45,6 +45,7 @@ class TestFmtDatetime(unittest.TestCase):
 		frappe.db.set_default("time_format", self.pre_test_time_format)
 		frappe.local.user_date_format = None
 		frappe.local.user_time_format = None
+		frappe.db.rollback()
 
 	# Test utility functions
 
@@ -97,28 +98,12 @@ class TestFmtDatetime(unittest.TestCase):
 			self.assertEqual(formatdate(test_date), valid_fmt)
 
 	# Test time formatters
-
 	def test_format_time_forced(self):
 		# Test with forced time formats
 		self.assertEqual(
 			format_time(test_time, 'ss:mm:HH'),
 			test_date_obj.strftime('%S:%M:%H'))
 
-	@unittest.expectedFailure
-	def test_format_time_forced_broken_locale(self):
-		# Test with forced time formats
-		# Currently format_time defaults to HH:mm:ss if the locale is
-		# broken, so this is an expected failure.
-		lang = frappe.local.lang
-		try:
-			# Force fallback from Babel
-			frappe.local.lang = 'FAKE'
-			self.assertEqual(
-				format_time(test_time, 'ss:mm:HH'),
-				test_date_obj.strftime('%S:%M:%H'))
-		finally:
-			frappe.local.lang = lang
-
 	def test_format_time(self):
 		# Test format_time with various default time formats set
 		for fmt, valid_fmt in test_time_formats.items():
@@ -135,21 +120,6 @@ class TestFmtDatetime(unittest.TestCase):
 			format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
 			test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
 
-	@unittest.expectedFailure
-	def test_format_datetime_forced_broken_locale(self):
-		# Test with forced datetime formats
-		# Currently format_datetime defaults to yyyy-MM-dd HH:mm:ss
-		# if the locale is broken, so this is an expected failure.
-		lang = frappe.local.lang
-		# Force fallback from Babel
-		try:
-			frappe.local.lang = 'FAKE'
-			self.assertEqual(
-				format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
-				test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
-		finally:
-			frappe.local.lang = lang
-
 	def test_format_datetime(self):
 		# Test formatdate with various default date formats set
 		for date_fmt, valid_date in test_date_formats.items():

frappe/tests/test_seen.py

@@ -41,6 +41,7 @@ class TestSeen(unittest.TestCase):
 		self.assertTrue('test1@example.com' in json.loads(ev._seen))
 
 		ev.save()
+		ev = frappe.get_doc('Event', ev.name)
 
 		self.assertFalse('test@example.com' in json.loads(ev._seen))
 		self.assertTrue('test1@example.com' in json.loads(ev._seen))

frappe/tests/test_twofactor.py

@@ -8,7 +8,7 @@ from frappe.utils import cint
 from frappe.utils import set_request
 from frappe.auth import validate_ip_address, get_login_attempt_tracker
 from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass,
-	two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj)
+	two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, ExpiredLoginException)
 from . import update_system_settings, get_system_setting
 
 import time
@@ -111,6 +111,7 @@ class TestTwoFactor(unittest.TestCase):
 
 	def test_confirm_otp_token(self):
 		'''Ensure otp is confirmed'''
+		frappe.flags.otp_expiry = 2
 		authenticate_for_2factor(self.user)
 		tmp_id = frappe.local.response['tmp_id']
 		otp = 'wrongotp'
@@ -118,10 +119,11 @@ class TestTwoFactor(unittest.TestCase):
 			confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
 		otp = get_otp(self.user)
 		self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id))
+		frappe.flags.otp_expiry = None
 		if frappe.flags.tests_verbose:
-			print('Sleeping for 30secs to confirm token expires..')
-		time.sleep(30)
-		with self.assertRaises(frappe.AuthenticationError):
+			print('Sleeping for 2 secs to confirm token expires..')
+		time.sleep(2)
+		with self.assertRaises(ExpiredLoginException):
 			confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
 
 	def test_get_verification_obj(self):
@@ -208,12 +210,14 @@ def enable_2fa(bypass_two_factor_auth=0, bypass_restrict_ip_check=0):
 	system_settings.bypass_2fa_for_retricted_ip_users = cint(bypass_two_factor_auth)
 	system_settings.bypass_restrict_ip_check_if_2fa_enabled = cint(bypass_restrict_ip_check)
 	system_settings.two_factor_method = 'OTP App'
+	system_settings.flags.ignore_mandatory = True
 	system_settings.save(ignore_permissions=True)
 	frappe.db.commit()
 
 def disable_2fa():
 	system_settings = frappe.get_doc('System Settings')
 	system_settings.enable_two_factor_auth = 0
+	system_settings.flags.ignore_mandatory = True
 	system_settings.save(ignore_permissions=True)
 	frappe.db.commit()
 

frappe/twofactor.py

@@ -73,11 +73,11 @@ def cache_2fa_data(user, token, otp_secret, tmp_id):
 
 	# set increased expiry time for SMS and Email
 	if verification_method in ['SMS', 'Email']:
-		expiry_time = 300
+		expiry_time = frappe.flags.token_expiry or 300
 		frappe.cache().set(tmp_id + '_token', token)
 		frappe.cache().expire(tmp_id + '_token', expiry_time)
 	else:
-		expiry_time = 180
+		expiry_time = frappe.flags.otp_expiry or 180
 	for k, v in iteritems({'_usr': user, '_pwd': pwd, '_otp_secret': otp_secret}):
 		frappe.cache().set("{0}{1}".format(tmp_id, k), v)
 		frappe.cache().expire("{0}{1}".format(tmp_id, k), expiry_time)

frappe/utils/__init__.py

@@ -161,7 +161,7 @@ def validate_url(txt, throw=False, valid_schemes=None):
 
 		Parameters:
 			throw (`bool`): throws a validationError if URL is not valid
-			valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this 
+			valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this
 
 		Returns:
 			bool: if `txt` represents a valid URL
@@ -225,14 +225,17 @@ def get_gravatar(email):
 
 	return gravatar_url
 
-def get_traceback():
+def get_traceback() -> str:
 	"""
 		 Returns the traceback of the Exception
 	"""
 	exc_type, exc_value, exc_tb = sys.exc_info()
+
+	if not any([exc_type, exc_value, exc_tb]):
+		return ""
+
 	trace_list = traceback.format_exception(exc_type, exc_value, exc_tb)
-	body = "".join(cstr(t) for t in trace_list)
-	return body
+	return "".join(cstr(t) for t in trace_list)
 
 def log(event, details):
 	frappe.logger().info(details)
@@ -425,7 +428,7 @@ def get_test_client():
 	return Client(application)
 
 def get_hook_method(hook_name, fallback=None):
-	method = (frappe.get_hooks().get(hook_name))
+	method = frappe.get_hooks().get(hook_name)
 	if method:
 		method = frappe.get_attr(method[0])
 		return method
@@ -439,6 +442,16 @@ def call_hook_method(hook, *args, **kwargs):
 
 	return out
 
+def is_cli() -> bool:
+	"""Returns True if current instance is being run via a terminal
+	"""
+	invoked_from_terminal = False
+	try:
+		invoked_from_terminal = bool(os.get_terminal_size())
+	except Exception:
+		invoked_from_terminal = sys.stdin.isatty()
+	return invoked_from_terminal
+
 def update_progress_bar(txt, i, l):
 	if os.environ.get("CI"):
 		if i == 0:
@@ -448,7 +461,7 @@ def update_progress_bar(txt, i, l):
 		sys.stdout.flush()
 		return
 
-	if not getattr(frappe.local, 'request', None):
+	if not getattr(frappe.local, 'request', None) or is_cli():
 		lt = len(txt)
 		try:
 			col = 40 if os.get_terminal_size().columns > 80 else 20

frappe/utils/boilerplate.py

@@ -40,7 +40,7 @@ def make_boilerplate(dest, app_name):
 			if hook_key=="app_name" and hook_val.lower().replace(" ", "_") != hook_val:
 				print("App Name must be all lowercase and without spaces")
 				hook_val = ""
-			elif hook_key=="app_title" and not re.match("^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
+			elif hook_key=="app_title" and not re.match(r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
 				print("App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores")
 				hook_val = ""
 

frappe/utils/data.py

@@ -1278,7 +1278,9 @@ def make_filter_dict(filters):
 
 def sanitize_column(column_name):
 	from frappe import _
+	import sqlparse
 	regex = re.compile("^.*[,'();].*")
+	column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower")
 	blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or']
 
 	def _raise_exception():

frappe/utils/file_manager.py

@@ -418,7 +418,7 @@ def extract_images_from_html(doc, content):
 		return '<img src="{file_url}"'.format(file_url=file_url)
 
 	if content:
-		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+		content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
 
 	return content
 

frappe/utils/formatters.py

@@ -78,7 +78,7 @@ def format_value(value, df=None, doc=None, currency=None, translated=False):
 		return "{}%".format(flt(value, 2))
 
 	elif df.get("fieldtype") in ("Text", "Small Text"):
-		if not re.search("(\<br|\<div|\<p)", value):
+		if not re.search(r"(<br|<div|<p)", value):
 			return frappe.safe_decode(value).replace("\n", "<br>")
 
 	elif df.get("fieldtype") == "Markdown Editor":

frappe/utils/global_search.py

@@ -348,7 +348,7 @@ def get_formatted_value(value, field):
 
 	if getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]:
 		value = unescape_html(frappe.safe_decode(value))
-		value = (re.subn(r'<[\s]*(script|style).*?</\1>(?s)', '', text_type(value))[0])
+		value = (re.subn(r'(?s)<[\s]*(script|style).*?</\1>', '', text_type(value))[0])
 		value = ' '.join(value.split())
 	return field.label + " : " + strip_html_tags(text_type(value))
 

frappe/utils/redis_wrapper.py

@@ -98,7 +98,7 @@ class RedisWrapper(redis.Redis):
 			return self.keys(key)
 
 		except redis.exceptions.ConnectionError:
-			regex = re.compile(cstr(key).replace("|", "\|").replace("*", "[\w]*"))
+			regex = re.compile(cstr(key).replace("|", r"\|").replace("*", r"[\w]*"))
 			return [k for k in list(frappe.local.cache) if regex.match(cstr(k))]
 
 	def delete_keys(self, key):

frappe/website/context.py

@@ -61,7 +61,7 @@ def update_controller_context(context, controller):
 			except (frappe.PermissionError, frappe.PageDoesNotExistError, frappe.Redirect):
 				raise
 			except:
-				if not frappe.flags.in_migrate:
+				if not any([frappe.flags.in_migrate, frappe.flags.in_website_search_build]):
 					frappe.errprint(frappe.utils.get_traceback())
 
 		if hasattr(module, "get_children"):

frappe/website/doctype/blog_post/test_blog_post.py

@@ -13,6 +13,8 @@ from frappe.website.doctype.blog_post.blog_post import get_blog_list
 from frappe.website.website_generator import WebsiteGenerator
 from frappe.custom.doctype.customize_form.customize_form import reset_customization
 
+test_dependencies = ['Blog Post']
+
 class TestBlogPost(unittest.TestCase):
 	def setUp(self):
 		reset_customization('Blog Post')

frappe/website/doctype/web_form/test_web_form.py

@@ -8,7 +8,7 @@ import unittest, json
 from frappe.website.render import build_page
 from frappe.website.doctype.web_form.web_form import accept
 
-test_records = frappe.get_test_records('Web Form')
+test_dependencies = ['Web Form']
 
 class TestWebForm(unittest.TestCase):
 	def setUp(self):

frappe/website/doctype/web_page/test_web_page.py

@@ -39,7 +39,7 @@ class TestWebPage(unittest.TestCase):
 			published = 1,
 			content_type = 'Rich Text',
 			main_section = 'rich text',
-			main_section_md = '# h1\n\markdown content',
+			main_section_md = '# h1\nmarkdown content',
 			main_section_html = '<div>html content</div>'
 		)).insert()
 

frappe/website/router.py

@@ -158,7 +158,7 @@ def evaluate_dynamic_routes(rules, path):
 	route_map = Map(rules)
 	endpoint = None
 
-	if frappe.local.request:
+	if hasattr(frappe.local, 'request') and frappe.local.request.environ:
 		urls = route_map.bind_to_environ(frappe.local.request.environ)
 		try:
 			endpoint, args = urls.match("/" + path)

frappe/www/app.py

@@ -34,10 +34,10 @@ def get_context(context):
 	boot_json = frappe.as_json(boot)
 
 	# remove script tags from boot
-	boot_json = re.sub("\<script[^<]*\</script\>", "", boot_json)
+	boot_json = re.sub(r"\<script[^<]*\</script\>", "", boot_json)
 
 	# TODO: Find better fix
-	boot_json = re.sub("</script\>", "", boot_json)
+	boot_json = re.sub(r"</script\>", "", boot_json)
 
 	context.update({
 		"no_cache": 1,

frappe/www/printview.py

@@ -409,7 +409,7 @@ def get_print_style(style=None, print_format=None, for_legacy=False):
 		css = css + '\n' + frappe.db.get_value('Print Style', style, 'css')
 
 	# move @import to top
-	for at_import in list(set(re.findall("(@import url\([^\)]+\)[;]?)", css))):
+	for at_import in list(set(re.findall(r"(@import url\([^\)]+\)[;]?)", css))):
 		css = css.replace(at_import, "")
 
 		# prepend css with at_import

requirements.txt

@@ -6,7 +6,8 @@ boto3~=1.17.53
 braintree~=4.8.0
 chardet~=4.0.0
 Click~=7.1.2
-coverage~=4.5.4
+colorama~=0.4.4
+coverage==5.5
 croniter~=1.0.11
 cryptography~=3.4.7
 dropbox~=11.7.0