Merge branch 'develop' into workspace-deleted-issue

.flake8

@@ -29,4 +29,5 @@ ignore =
     B950,
     W191,
 
-max-line-length = 200
+max-line-length = 200
+exclude=.github/helper/semgrep_rules

.github/helper/semgrep_rules/frappe_correctness.py

@@ -4,25 +4,61 @@ from frappe import _, flt
 from frappe.model.document import Document
 
 
+# ruleid: frappe-modifying-but-not-comitting
 def on_submit(self):
 	if self.value_of_goods == 0:
 		frappe.throw(_('Value of goods cannot be 0'))
-	# ruleid: frappe-modifying-after-submit
 	self.status = 'Submitted'
 
-def on_submit(self):   # noqa
-	if flt(self.per_billed) < 100:
-		self.update_billing_status()
-	else:
-		# todook: frappe-modifying-after-submit
-		self.status = "Completed"
-		self.db_set("status", "Completed")
-
-class TestDoc(Document):
-	pass
-
-	def validate(self):
-		#ruleid: frappe-modifying-child-tables-while-iterating
-		for item in self.child_table:
-			if item.value < 0:
-				self.remove(item)
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	self.status = 'Submitted'
+	self.db_set('status', 'Submitted')
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	x = "y"
+	self.status = x
+	self.db_set('status', x)
+
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	x = "y"
+	self.status = x
+	self.save()
+
+# ruleid: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "uptate"
+
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "update"
+		self.db_set("status", "update")
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+		self.save()
+
+	def tainted_method(self):
+		self.status = "uptate"

.github/helper/semgrep_rules/translate.js

@@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.')
 // ruleid: frappe-translation-js-splitting
 __('You have {0} subscribers' +
     'in your mailing list', [subscribers.length])
+
+// ok: frappe-translation-js-splitting
+__("Ctrl+Enter to add comment")
+
+// ruleid: frappe-translation-js-splitting
+__('You have {0} subscribers \
+    in your mailing list', [subscribers.length])

.github/helper/semgrep_rules/translate.py

@@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool")
 _("")
 # ruleid: frappe-translation-empty-string
 _('')
+
+
+class Test:
+	# ok: frappe-translation-python-splitting
+	def __init__(
+			args
+			):
+		pass

.github/helper/semgrep_rules/translate.yml

@@ -44,8 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
-  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
+  - pattern-regex: '[\s\.]_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '[\s\.]_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations
@@ -54,8 +54,8 @@ rules:
 
 - id: frappe-translation-js-splitting
   pattern-either:
-  - pattern-regex: '__\([^\)]*[\+\\]\s*'
-  - pattern: __('...' + '...')
+  - pattern-regex: '__\([^\)]*[\\]\s+'
+  - pattern: __('...' + '...', ...)
   - pattern: __('...') + __('...')
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.

.github/workflows/semgrep.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - develop
+      - version-13-hotfix
+      - version-13-pre-release
 jobs:
   semgrep:
     name: Frappe Linter

.github/workflows/ci-tests.yml → .github/workflows/server-mariadb-tests.yml

@@ -1,10 +1,8 @@
-name: CI
+name: Server
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
-  push:
 
 jobs:
   test:
@@ -13,23 +11,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
+       container: [1, 2]
 
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
+    name: Python Unit Tests (MariaDB)
 
     services:
       mysql:
@@ -40,18 +24,6 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Clone
         uses: actions/checkout@v2
@@ -63,7 +35,7 @@ jobs:
 
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: '14'
           check-latest: true
 
       - name: Add to Hosts
@@ -104,68 +76,54 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: ${{ runner.os }}-cypress-
-          restore-keys: |
-            ${{ runner.os }}-cypress-
-            ${{ runner.os }}-
-
       - name: Install Dependencies
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
           AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
+          TYPE: server
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          DB: mariadb
+          TYPE: server
 
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
-        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
 
       - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
-      - name: Coverage - Pull Request
-        if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
+      - name: Upload Coverage Data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
-          
-      - name: Coverage - Push
-        if: matrix.TYPE == 'server' && github.event_name == 'push'
+          COVERALLS_FLAG_NAME: run-${{ matrix.container }}
+          COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
+          COVERALLS_PARALLEL: true
+
+  coveralls:
+    name: Coverage Wrap Up
+    needs: test
+    container: python:3-slim
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Coveralls Finished
         run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github-actions
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,100 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (Postgres)
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

.github/workflows/ui-tests.yml

@@ -0,0 +1,105 @@
+name: UI
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: UI Tests (Cypress)
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '12'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ui
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: ui
+
+      - name: Site Setup
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+
+      - name: UI Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

.mergify.yml

@@ -3,9 +3,12 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -16,9 +19,12 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

frappe/__init__.py

@@ -11,7 +11,6 @@ be used to build database driven apps.
 Read the documentation: https://frappeframework.com/docs
 """
 
-from six import iteritems, binary_type, text_type, string_types
 from werkzeug.local import Local, release_local
 import os, sys, importlib, inspect, json, warnings
 import typing
@@ -91,14 +90,14 @@ def _(msg, lang=None, context=None):
 
 def as_unicode(text, encoding='utf-8'):
 	'''Convert to unicode if required'''
-	if isinstance(text, text_type):
+	if isinstance(text, str):
 		return text
 	elif text==None:
 		return ''
-	elif isinstance(text, binary_type):
-		return text_type(text, encoding)
+	elif isinstance(text, bytes):
+		return str(text, encoding)
 	else:
-		return text_type(text)
+		return str(text)
 
 def get_lang_dict(fortype, name=None):
 	"""Returns the translated language dict for the given type and name.
@@ -591,7 +590,7 @@ def is_whitelisted(method):
 		# strictly sanitize form_dict
 		# escapes html characters like <> except for predefined tags like a, b, ul etc.
 		for key, value in form_dict.items():
-			if isinstance(value, string_types):
+			if isinstance(value, str):
 				form_dict[key] = sanitize_html(value)
 
 def read_only():
@@ -715,7 +714,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc
 		user = session.user
 
 	if doc:
-		if isinstance(doc, string_types):
+		if isinstance(doc, str):
 			doc = get_doc(doctype, doc)
 
 		doctype = doc.doctype
@@ -784,7 +783,7 @@ def set_value(doctype, docname, fieldname, value=None):
 	return frappe.client.set_value(doctype, docname, fieldname, value)
 
 def get_cached_doc(*args, **kwargs):
-	if args and len(args) > 1 and isinstance(args[1], text_type):
+	if args and len(args) > 1 and isinstance(args[1], str):
 		key = get_document_cache_key(args[0], args[1])
 		# local cache
 		doc = local.document_cache.get(key)
@@ -815,7 +814,7 @@ def clear_document_cache(doctype, name):
 
 def get_cached_value(doctype, name, fieldname, as_dict=False):
 	doc = get_cached_doc(doctype, name)
-	if isinstance(fieldname, string_types):
+	if isinstance(fieldname, str):
 		if as_dict:
 			throw('Cannot make dict for single fieldname')
 		return doc.get(fieldname)
@@ -1021,7 +1020,7 @@ def get_doc_hooks():
 	if not hasattr(local, 'doc_events_hooks'):
 		hooks = get_hooks('doc_events', {})
 		out = {}
-		for key, value in iteritems(hooks):
+		for key, value in hooks.items():
 			if isinstance(key, tuple):
 				for doctype in key:
 					append_hook(out, doctype, value)
@@ -1138,7 +1137,7 @@ def get_file_json(path):
 
 def read_file(path, raise_not_found=False):
 	"""Open a file and return its content as Unicode."""
-	if isinstance(path, text_type):
+	if isinstance(path, str):
 		path = path.encode("utf-8")
 
 	if os.path.exists(path):
@@ -1161,7 +1160,7 @@ def get_attr(method_string):
 
 def call(fn, *args, **kwargs):
 	"""Call a function and match arguments."""
-	if isinstance(fn, string_types):
+	if isinstance(fn, str):
 		fn = get_attr(fn)
 
 	newargs = get_newargs(fn, kwargs)
@@ -1172,13 +1171,9 @@ def get_newargs(fn, kwargs):
 	if hasattr(fn, 'fnargs'):
 		fnargs = fn.fnargs
 	else:
-		try:
-			fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
-		except ValueError:
-			fnargs = inspect.getfullargspec(fn).args
-			varargs = inspect.getfullargspec(fn).varargs
-			varkw = inspect.getfullargspec(fn).varkw
-			defaults = inspect.getfullargspec(fn).defaults
+		fnargs = inspect.getfullargspec(fn).args
+		fnargs.extend(inspect.getfullargspec(fn).kwonlyargs)
+		varkw = inspect.getfullargspec(fn).varkw
 
 	newargs = {}
 	for a in kwargs:
@@ -1620,6 +1615,12 @@ def enqueue(*args, **kwargs):
 	import frappe.utils.background_jobs
 	return frappe.utils.background_jobs.enqueue(*args, **kwargs)
 
+def task(**task_kwargs):
+	def decorator_task(f):
+		f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs)
+		return f
+	return decorator_task
+
 def enqueue_doc(*args, **kwargs):
 	'''
 		Enqueue method to be executed using a background worker

frappe/app.py

@@ -99,17 +99,7 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
-			frappe.logger("frappe.web", allow_site=frappe.local.site).info({
-				"site": get_site_name(request.host),
-				"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
-				"base_url": getattr(request, "base_url", "NOTFOUND"),
-				"full_path": getattr(request, "full_path", "NOTFOUND"),
-				"method": getattr(request, "method", "NOTFOUND"),
-				"scheme": getattr(request, "scheme", "NOTFOUND"),
-				"http_status_code": getattr(response, "status_code", "NOTFOUND")
-			})
-
+		log_request(request, response)
 		process_response(response)
 		frappe.destroy()
 
@@ -137,6 +127,19 @@ def init_request(request):
 	if request.method != "OPTIONS":
 		frappe.local.http_request = frappe.auth.HTTPRequest()
 
+def log_request(request, response):
+	if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
+		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+			"site": get_site_name(request.host),
+			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
+			"base_url": getattr(request, "base_url", "NOTFOUND"),
+			"full_path": getattr(request, "full_path", "NOTFOUND"),
+			"method": getattr(request, "method", "NOTFOUND"),
+			"scheme": getattr(request, "scheme", "NOTFOUND"),
+			"http_status_code": getattr(response, "status_code", "NOTFOUND")
+		})
+
+
 def process_response(response):
 	if not response:
 		return

frappe/automation/doctype/auto_repeat/auto_repeat.js

@@ -103,7 +103,7 @@ frappe.ui.form.on('Auto Repeat', {
 frappe.auto_repeat.render_schedule = function(frm) {
 	if (!frm.is_dirty() && frm.doc.status !== 'Disabled') {
 		frm.call("get_auto_repeat_schedule").then(r => {
-			frm.dashboard.wrapper.empty();
+			frm.dashboard.reset();
 			frm.dashboard.add_section(
 				frappe.render_template("auto_repeat_schedule", {
 					schedule_details: r.message || []

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -173,7 +173,7 @@ class TestAutoRepeat(unittest.TestCase):
 			fields=['docstatus'],
 			limit=1
 		)
-		self.assertEquals(docnames[0].docstatus, 1)
+		self.assertEqual(docnames[0].docstatus, 1)
 
 
 def make_auto_repeat(**args):

frappe/build.py

@@ -1,14 +1,11 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import json
 import shutil
-import warnings
-import tempfile
+from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
 
 import frappe
@@ -16,8 +13,8 @@ from frappe.utils.minify import JavascriptMinify
 
 import click
 import psutil
-from six import iteritems, text_type
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+from simple_chalk import green
 
 
 timestamps = {}
@@ -75,8 +72,8 @@ def get_assets_link(frappe_head):
 	from requests import head
 
 	tag = getoutput(
-			"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			" refs/tags/,,' -e 's/\^{}//'"
+			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+			r" refs/tags/,,' -e 's/\^{}//'"
 			% frappe_head
 		)
 
@@ -97,9 +94,7 @@ def download_frappe_assets(verbose=True):
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from simple_chalk import green
 	from subprocess import getoutput
-	from tempfile import mkdtemp
 
 	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
@@ -166,7 +161,7 @@ def symlink(target, link_name, overwrite=False):
 
 	# Create link to target with temporary filename
 	while True:
-		temp_link_name = tempfile.mktemp(dir=link_dir)
+		temp_link_name = mktemp(dir=link_dir)
 
 		# os.* functions mimic as closely as possible system functions
 		# The POSIX symlink() returns EEXIST if link_name already exists
@@ -193,7 +188,8 @@ def symlink(target, link_name, overwrite=False):
 
 
 def setup():
-	global app_paths
+	global app_paths, assets_path
+
 	pymodules = []
 	for app in frappe.get_all_apps(True):
 		try:
@@ -201,6 +197,7 @@ def setup():
 		except ImportError:
 			pass
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
+	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
 
 def get_node_pacman():
@@ -210,10 +207,10 @@ def get_node_pacman():
 	raise ValueError("Yarn not found")
 
 
-def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
+def bundle(no_compress, app=None, hard_link=False, verbose=False, skip_frappe=False):
 	"""concat / minify js files"""
 	setup()
-	make_asset_dirs(make_copy=make_copy, restore=restore)
+	make_asset_dirs(hard_link=hard_link)
 
 	pacman = get_node_pacman()
 	mode = "build" if no_compress else "production"
@@ -266,75 +263,101 @@ def get_safe_max_old_space_size():
 
 	return safe_max_old_space_size
 
-def make_asset_dirs(make_copy=False, restore=False):
-	# don't even think of making assets_path absolute - rm -rf ahead.
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
+def generate_assets_map():
+	symlinks = {}
 
-	for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
+	for app_name in frappe.get_all_apps():
+		app_doc_path = None
 
-	for app_name in frappe.get_all_apps(True):
 		pymodule = frappe.get_module(app_name)
 		app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
-
-		symlinks = []
 		app_public_path = os.path.join(app_base_path, "public")
-		# app/public > assets/app
-		symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
-		# app/node_modules > assets/app/node_modules
-		if os.path.exists(os.path.abspath(app_public_path)):
-			symlinks.append(
-				[
-					os.path.join(app_base_path, "..", "node_modules"),
-					os.path.join(assets_path, app_name, "node_modules"),
-				]
-			)
+		app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
+		app_docs_path = os.path.join(app_base_path, "docs")
+		app_www_docs_path = os.path.join(app_base_path, "www", "docs")
 
-		app_doc_path = None
-		if os.path.isdir(os.path.join(app_base_path, "docs")):
-			app_doc_path = os.path.join(app_base_path, "docs")
+		app_assets = os.path.abspath(app_public_path)
+		app_node_modules = os.path.abspath(app_node_modules_path)
 
-		elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
-			app_doc_path = os.path.join(app_base_path, "www", "docs")
+		# {app}/public > assets/{app}
+		if os.path.isdir(app_assets):
+			symlinks[app_assets] = os.path.join(assets_path, app_name)
+
+		# {app}/node_modules > assets/{app}/node_modules
+		if os.path.isdir(app_node_modules):
+			symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
 
+		# {app}/docs > assets/{app}_docs
+		if os.path.isdir(app_docs_path):
+			app_doc_path = os.path.join(app_base_path, "docs")
+		elif os.path.isdir(app_www_docs_path):
+			app_doc_path = os.path.join(app_base_path, "www", "docs")
 		if app_doc_path:
-			symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
-
-		for source, target in symlinks:
-			source = os.path.abspath(source)
-			if os.path.exists(source):
-				if restore:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-						shutil.copytree(source, target)
-				elif make_copy:
-					if os.path.exists(target):
-						warnings.warn("Target {target} already exists.".format(target=target))
-					else:
-						shutil.copytree(source, target)
-				else:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-					try:
-						symlink(source, target, overwrite=True)
-					except OSError:
-						print("Cannot link {} to {}".format(source, target))
-			else:
-				# warnings.warn('Source {source} does not exist.'.format(source = source))
-				pass
+			app_docs = os.path.abspath(app_doc_path)
+			symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
 
+	return symlinks
+
+
+def setup_assets_dirs():
+	for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
+		os.makedirs(dir_path, exist_ok=True)
+
+
+def clear_broken_symlinks():
+	for path in os.listdir(assets_path):
+		path = os.path.join(assets_path, path)
+		if os.path.islink(path) and not os.path.exists(path):
+			os.remove(path)
 
-def build(no_compress=False, verbose=False):
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
-	for target, sources in iteritems(get_build_maps()):
+
+def unstrip(message):
+	try:
+		max_str = os.get_terminal_size().columns
+	except Exception:
+		max_str = 80
+	_len = len(message)
+	_rem = max_str - _len
+	return f"{message}{' ' * _rem}"
+
+
+def make_asset_dirs(hard_link=False):
+	setup_assets_dirs()
+	clear_broken_symlinks()
+	symlinks = generate_assets_map()
+
+	for source, target in symlinks.items():
+		start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
+		fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
+
+		try:
+			print(start_message, end="\r")
+			link_assets_dir(source, target, hard_link=hard_link)
+		except Exception:
+			print(fail_message, end="\r")
+
+	print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
+
+
+def link_assets_dir(source, target, hard_link=False):
+	if not os.path.exists(source):
+		return
+
+	if os.path.exists(target):
+		if os.path.islink(target):
+			os.unlink(target)
+		else:
+			shutil.rmtree(target)
+
+	if hard_link:
+		shutil.copytree(source, target, dirs_exist_ok=True)
+	else:
+		symlink(source, target, overwrite=True)
+
+
+def build(no_compress=False, verbose=False):
+	for target, sources in get_build_maps().items():
 		pack(os.path.join(assets_path, target), sources, no_compress, verbose)
 
 
@@ -348,7 +371,7 @@ def get_build_maps():
 		if os.path.exists(path):
 			with open(path) as f:
 				try:
-					for target, sources in iteritems(json.loads(f.read())):
+					for target, sources in (json.loads(f.read() or "{}")).items():
 						# update app path
 						source_paths = []
 						for source in sources:
@@ -381,7 +404,7 @@ def pack(target, sources, no_compress, verbose):
 		timestamps[f] = os.path.getmtime(f)
 		try:
 			with open(f, "r") as sourcefile:
-				data = text_type(sourcefile.read(), "utf-8", errors="ignore")
+				data = str(sourcefile.read(), "utf-8", errors="ignore")
 
 			extn = f.rsplit(".", 1)[1]
 
@@ -396,7 +419,7 @@ def pack(target, sources, no_compress, verbose):
 				jsm.minify(tmpin, tmpout)
 				minified = tmpout.getvalue()
 				if minified:
-					outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
+					outtxt += str(minified or "", "utf-8").strip("\n") + ";"
 
 				if verbose:
 					print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@@ -426,16 +449,16 @@ def html_to_js_template(path, content):
 def scrub_html_template(content):
 	"""Returns HTML content with removed whitespace and comments"""
 	# remove whitespace to a single space
-	content = re.sub("\s+", " ", content)
+	content = re.sub(r"\s+", " ", content)
 
 	# strip comments
-	content = re.sub("(<!--.*?-->)", "", content)
+	content = re.sub(r"(<!--.*?-->)", "", content)
 
 	return content.replace("'", "\'")
 
 
 def files_dirty():
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		for f in sources:
 			if ":" in f:
 				f, suffix = f.split(":")

frappe/change_log/v13/v13_3_0.md

@@ -0,0 +1,49 @@
+# Version 13.3.0 Release Notes
+
+### Features & Enhancements
+
+- Deletion Steps in Data Deletion Tool ([#13124](https://github.com/frappe/frappe/pull/13124))
+- Format Option for list-apps in bench CLI ([#13125](https://github.com/frappe/frappe/pull/13125))
+- Add password fieldtype option for Web Form ([#13093](https://github.com/frappe/frappe/pull/13093))
+- Add simple __repr__ for DocTypes ([#13151](https://github.com/frappe/frappe/pull/13151))
+- Switch theme with left/right keys ([#13077](https://github.com/frappe/frappe/pull/13077))
+- sourceURL for injected javascript ([#13022](https://github.com/frappe/frappe/pull/13022))
+
+### Fixes
+
+- Decode uri before importing file via weblink ([#13026](https://github.com/frappe/frappe/pull/13026))
+- Respond to /api requests as JSON by default ([#13053](https://github.com/frappe/frappe/pull/13053))
+- Disabled checkbox should be disabled ([#13021](https://github.com/frappe/frappe/pull/13021))
+- Moving Site folder across different FileSystems failed ([#13038](https://github.com/frappe/frappe/pull/13038))
+- Freeze screen till the background request is complete ([#13078](https://github.com/frappe/frappe/pull/13078))
+- Added conditional rendering for content field in split section w… ([#13075](https://github.com/frappe/frappe/pull/13075))
+- Show delete button on portal if user has permission to delete document ([#13149](https://github.com/frappe/frappe/pull/13149))
+- Dont disable dialog scroll on focusing a Link/Autocomplete field ([#13119](https://github.com/frappe/frappe/pull/13119))
+- Typo in RecorderDetail.vue ([#13086](https://github.com/frappe/frappe/pull/13086))
+- Error for bench drop-site. Added missing import. ([#13064](https://github.com/frappe/frappe/pull/13064))
+- Report column context ([#13090](https://github.com/frappe/frappe/pull/13090))
+- Different service name for push and pull request events ([#13094](https://github.com/frappe/frappe/pull/13094))
+- Moving Site folder across different FileSystems failed ([#13033](https://github.com/frappe/frappe/pull/13033))
+- Consistent checkboxes on all browsers ([#13042](https://github.com/frappe/frappe/pull/13042))
+- Changed shorcut widgets color picker to dropdown ([#13073](https://github.com/frappe/frappe/pull/13073))
+- Error while exporting reports with duration field ([#13118](https://github.com/frappe/frappe/pull/13118))
+- Add margin to download backup card ([#13079](https://github.com/frappe/frappe/pull/13079))
+- Move mention list generation logic to server-side ([#13074](https://github.com/frappe/frappe/pull/13074))
+- Make strings translatable ([#13046](https://github.com/frappe/frappe/pull/13046))
+- Don't evaluate dynamic properties to check if conflicts exist ([#13186](https://github.com/frappe/frappe/pull/13186))
+- Add __ function in vue global for translation in recorder ([#13089](https://github.com/frappe/frappe/pull/13089))
+- Make strings translatable ([#13076](https://github.com/frappe/frappe/pull/13076))
+- Show config in bench CLI ([#13128](https://github.com/frappe/frappe/pull/13128))
+- Add breadcrumbs for list view ([#13091](https://github.com/frappe/frappe/pull/13091))
+- Do not skip data in save while using shortcut ([#13182](https://github.com/frappe/frappe/pull/13182))
+- Use docfields from options if no docfields are returned from meta ([#13188](https://github.com/frappe/frappe/pull/13188))
+- Disable reloading files in `__pycache__` directory ([#13109](https://github.com/frappe/frappe/pull/13109))
+- RTL stylesheet route to load RTL style on demand. ([#13007](https://github.com/frappe/frappe/pull/13007))
+- Do not show messsage when exception is handled ([#13111](https://github.com/frappe/frappe/pull/13111))
+- Replace parseFloat by Number ([#13082](https://github.com/frappe/frappe/pull/13082))
+- Add margin to download backup card ([#13050](https://github.com/frappe/frappe/pull/13050))
+- Translate report column labels ([#13083](https://github.com/frappe/frappe/pull/13083))
+- Grid row color picker field not working ([#13040](https://github.com/frappe/frappe/pull/13040))
+- Improve oauthlib implementation ([#13045](https://github.com/frappe/frappe/pull/13045))
+- Replace filter_by like with full text filter ([#13126](https://github.com/frappe/frappe/pull/13126))
+- Focus jumps to first field ([#13067](https://github.com/frappe/frappe/pull/13067))

frappe/commands/__init__.py

@@ -28,6 +28,10 @@ def pass_context(f):
 		except frappe.exceptions.SiteNotSpecifiedError as e:
 			click.secho(str(e), fg='yellow')
 			sys.exit(1)
+		except frappe.exceptions.IncorrectSitePath:
+			site = ctx.obj.get("sites", "")[0]
+			click.secho(f'Site {site} does not exist!', fg='yellow')
+			sys.exit(1)
 
 		if profile:
 			pr.disable()

frappe/commands/utils.py

@@ -16,13 +16,13 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
 
 @click.command('build')
 @click.option('--app', help='Build assets for app')
-@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
-@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
+@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
+@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
+@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
 @click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
-def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
+def build(app=None, hard_link=False, make_copy=False, restore=False, verbose=False, force=False):
 	"Minify + concatenate JS and CSS files, build translations"
-	import frappe.build
 	frappe.init('')
 	# don't minify in developer_mode for faster builds
 	no_compress = frappe.local.conf.developer_mode or False
@@ -34,7 +34,20 @@ def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
 	else:
 		skip_frappe = False
 
-	frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
+	if make_copy or restore:
+		hard_link = make_copy or restore
+		click.secho(
+			"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
+			fg="yellow",
+		)
+
+	frappe.build.bundle(
+		skip_frappe=skip_frappe,
+		no_compress=no_compress,
+		hard_link=hard_link,
+		verbose=verbose,
+		app=app,
+	)
 
 
 @click.command('watch')
@@ -488,6 +501,8 @@ frappe.db.connect()
 @pass_context
 def console(context):
 	"Start ipython console for a site"
+	import warnings
+
 	site = get_site(context)
 	frappe.init(site=site)
 	frappe.connect()
@@ -508,6 +523,7 @@ def console(context):
 	if failed_to_import:
 		print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
 
+	warnings.simplefilter('ignore')
 	IPython.embed(display_banner="", header="", colors="neutral")
 
 
@@ -585,12 +601,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 	if os.environ.get('CI'):
 		sys.exit(ret)
 
+@click.command('run-parallel-tests')
+@click.option('--app', help="For App", default='frappe')
+@click.option('--build-number', help="Build number", default=1)
+@click.option('--total-builds', help="Total number of builds", default=1)
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
+@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
+@pass_context
+def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
+	site = get_site(context)
+	if use_orchestrator:
+		from frappe.parallel_test_runner import ParallelTestWithOrchestrator
+		ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
+	else:
+		from frappe.parallel_test_runner import ParallelTestRunner
+		ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
+@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False):
+def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -622,6 +655,12 @@ def run_ui_tests(context, app, headless=False):
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 
+	if parallel:
+		formatted_command += ' --parallel'
+
+	if ci_build_id:
+		formatted_command += ' --ci-build-id {}'.format(ci_build_id)
+
 	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
@@ -797,5 +836,6 @@ commands = [
 	watch,
 	bulk_rename,
 	add_to_email_queue,
-	rebuild_global_search
+	rebuild_global_search,
+	run_parallel_tests
 ]

frappe/contacts/doctype/contact/test_contact.py

@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 
 import frappe
 import unittest
-from frappe.exceptions import ValidationError
+
+test_dependencies = ['Contact', 'Salutation']
 
 class TestContact(unittest.TestCase):
 
@@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
 	if save:
 		doc.insert()
 
-	return doc
+	return doc

frappe/core/doctype/activity_log/test_activity_log.py

@@ -65,12 +65,12 @@ class TestActivityLog(unittest.TestCase):
 		frappe.local.login_manager = LoginManager()
 
 		auth_log = self.get_auth_log()
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test user logout log
 		frappe.local.login_manager.logout()
 		auth_log = self.get_auth_log(operation='Logout')
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test invalid login
 		frappe.form_dict.update({ 'pwd': 'password' })
@@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase):
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()

frappe/core/doctype/data_export/exporter.py

@@ -282,7 +282,7 @@ class DataExporter:
 				try:
 					sflags = self.docs_to_export.get("flags", "I,U").upper()
 					flags = 0
-					for a in re.split('\W+',sflags):
+					for a in re.split(r'\W+', sflags):
 						flags = flags | reflags.get(a,0)
 
 					c = re.compile(names, flags)

frappe/core/doctype/data_import/importer.py

@@ -233,7 +233,7 @@ class Importer:
 			return updated_doc
 		else:
 			# throw if no changes
-			frappe.throw("No changes to update")
+			frappe.throw(_("No changes to update"))
 
 	def get_eta(self, current, total, processing_time):
 		self.last_eta = getattr(self, "last_eta", 0)
@@ -319,7 +319,7 @@ class ImportFile:
 		self.warnings = []
 
 		self.file_doc = self.file_path = self.google_sheets_url = None
-		if isinstance(file, frappe.string_types):
+		if isinstance(file, str):
 			if frappe.db.exists("File", {"file_url": file}):
 				self.file_doc = frappe.get_doc("File", {"file_url": file})
 			elif "docs.google.com/spreadsheets" in file:
@@ -626,7 +626,7 @@ class Row:
 				return
 		elif df.fieldtype in ["Date", "Datetime"]:
 			value = self.get_date(value, col)
-			if isinstance(value, frappe.string_types):
+			if isinstance(value, str):
 				# value was not parsed as datetime object
 				self.warnings.append(
 					{
@@ -641,7 +641,7 @@ class Row:
 				return
 		elif df.fieldtype == "Duration":
 			import re
-			is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
+			is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
 			if not is_valid_duration:
 				self.warnings.append(
 					{
@@ -929,10 +929,7 @@ class Column:
 				self.warnings.append(
 					{
 						"col": self.column_number,
-						"message": _(
-							"Date format could not be determined from the values in"
-							" this column. Defaulting to yyyy-mm-dd."
-						),
+						"message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."),
 						"type": "info",
 					}
 				)

frappe/core/doctype/docshare/test_docshare.py

@@ -7,6 +7,8 @@ import frappe.share
 import unittest
 from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype
 
+test_dependencies = ['User']
+
 class TestDocShare(unittest.TestCase):
 	def setUp(self):
 		self.user = "test@example.com"
@@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase):
 		self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
 		self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
 
-		frappe.share.remove(doctype, submittable_doc.name, self.user)
+		frappe.share.remove(doctype, submittable_doc.name, self.user)

frappe/core/doctype/doctype/doctype.py

@@ -83,12 +83,61 @@ class DocType(Document):
 		if not self.is_new():
 			self.before_update = frappe.get_doc('DocType', self.name)
 			self.setup_fields_to_fetch()
+			self.validate_field_name_conflicts()
 
 		check_email_append_to(self)
 
 		if self.default_print_format and not self.custom:
 			frappe.throw(_('Standard DocType cannot have default print format, use Customize Form'))
 
+		if frappe.conf.get('developer_mode'):
+			self.owner = 'Administrator'
+			self.modified_by = 'Administrator'
+
+	def validate_field_name_conflicts(self):
+		"""Check if field names dont conflict with controller properties and methods"""
+		core_doctypes = [
+			"Custom DocPerm",
+			"DocPerm",
+			"Custom Field",
+			"Customize Form Field",
+			"DocField",
+		]
+
+		if self.name in core_doctypes:
+			return
+
+		from frappe.model.base_document import get_controller
+
+		try:
+			controller = get_controller(self.name)
+		except ImportError:
+			controller = Document
+
+		available_objects = {x for x in dir(controller) if isinstance(x, str)}
+		property_set = {
+			x for x in available_objects if isinstance(getattr(controller, x, None), property)
+		}
+		method_set = {
+			x for x in available_objects if x not in property_set and callable(getattr(controller, x, None))
+		}
+
+		for docfield in self.get("fields") or []:
+			conflict_type = None
+			field = docfield.fieldname
+			field_label = docfield.label or docfield.fieldname
+
+			if docfield.fieldname in method_set:
+				conflict_type = "controller method"
+			if docfield.fieldname in property_set:
+				conflict_type = "class property"
+
+			if conflict_type:
+				frappe.throw(
+					_("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}")
+						.format(field_label, conflict_type, field, self.name)
+				)
+
 	def after_insert(self):
 		# clear user cache so that on the next reload this doctype is included in boot
 		clear_user_cache(frappe.session.user)
@@ -622,12 +671,12 @@ class DocType(Document):
 		flags = {"flags": re.ASCII} if six.PY3 else {}
 
 		# a DocType name should not start or end with an empty space
-		if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
+		if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
 			frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
 
 		# a DocType's name should not start with a number or underscore
 		# and should only contain letters, numbers and underscore
-		if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
+		if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags):
 			frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
 
 		validate_route_conflict(self.doctype, self.name)
@@ -915,7 +964,7 @@ def validate_fields(meta):
 		for field in depends_on_fields:
 			depends_on = docfield.get(field, None)
 			if depends_on and ("=" in depends_on) and \
-				re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on):
+				re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on):
 				frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError)
 
 	def check_table_multiselect_option(docfield):
@@ -1174,11 +1223,19 @@ def make_module_and_roles(doc, perm_fieldname="permissions"):
 		else:
 			raise
 
-def check_if_fieldname_conflicts_with_methods(doctype, fieldname):
-	doc = frappe.get_doc({"doctype": doctype})
-	method_list = [method for method in dir(doc) if isinstance(method, str) and callable(getattr(doc, method))]
+def check_fieldname_conflicts(doctype, fieldname):
+	"""Checks if fieldname conflicts with methods or properties"""
 
-	if fieldname in method_list:
+	doc = frappe.get_doc({"doctype": doctype})
+	available_objects = [x for x in dir(doc) if isinstance(x, str)]
+	property_list = [
+		x for x in available_objects if isinstance(getattr(type(doc), x, None), property)
+	]
+	method_list = [
+		x for x in available_objects if x not in property_list and callable(getattr(doc, x))
+	]
+
+	if fieldname in method_list + property_list:
 		frappe.throw(_("Fieldname {0} conflicting with meta object").format(fieldname))
 
 def clear_linked_doctype_cache():

frappe/core/doctype/doctype/test_doctype.py

@@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase):
 			fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\
 				"read_only_depends_on", "fieldname", "fieldtype"])
 
-		pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+"""
+		pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+'
 		for field in docfields:
 			for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]:
 				condition = field.get(depends_on)
@@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None):
 		for f in fields:
 			doc.append('fields', f)
 
-	return doc
+	return doc

frappe/core/doctype/file/file.py

@@ -498,7 +498,7 @@ class File(Document):
 		self.file_size = self.check_max_file_size()
 
 		if (
-			self.content_type and "image" in self.content_type
+			self.content_type and self.content_type == "image/jpeg"
 			and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
 		):
 			self.content = strip_exif_data(self.content, self.content_type)
@@ -912,7 +912,7 @@ def extract_images_from_html(doc, content):
 		return '<img src="{file_url}"'.format(file_url=file_url)
 
 	if content and isinstance(content, string_types):
-		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+		content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
 
 	return content
 

frappe/core/doctype/file/test_file.py

@@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase):
 
 class TestFile(unittest.TestCase):
 	def setUp(self):
+		frappe.set_user('Administrator')
 		self.delete_test_data()
 		self.upload_file()
 

frappe/core/doctype/report/test_report.py

@@ -106,7 +106,7 @@ class TestReport(unittest.TestCase):
 		else:
 			report = frappe.get_doc('Report', 'Test Report')
 
-		self.assertNotEquals(report.is_permitted(), True)
+		self.assertNotEqual(report.is_permitted(), True)
 		frappe.set_user('Administrator')
 
 	# test for the `_format` method if report data doesn't have sort_by parameter

frappe/core/doctype/role_profile/test_role_profile.py

@@ -5,6 +5,8 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
+test_dependencies = ['Role']
+
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
@@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase):
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
-		self.assertEqual(new_role_profile.roles, [])
+		self.assertEqual(new_role_profile.roles, [])

frappe/core/doctype/system_settings/system_settings.py

@@ -42,7 +42,7 @@ class SystemSettings(Document):
 
 	def on_update(self):
 		for df in self.meta.get("fields"):
-			if df.fieldtype not in no_value_fields:
+			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
 				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
 
 		if self.language:

frappe/core/doctype/user_permission/test_user_permission.py

@@ -46,7 +46,7 @@ class TestUserPermission(unittest.TestCase):
 		frappe.set_user('test_user_perm1@example.com')
 		doc = frappe.new_doc("Blog Post")
 
-		self.assertEquals(doc.blog_category, 'general')
+		self.assertEqual(doc.blog_category, 'general')
 		frappe.set_user('Administrator')
 
 	def test_apply_to_all(self):
@@ -54,7 +54,7 @@ class TestUserPermission(unittest.TestCase):
 		user = create_user('test_bulk_creation_update@example.com')
 		param = get_params(user, 'User', user.name)
 		is_created = add_user_permissions(param)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_for_apply_to_all_on_update_from_apply_all(self):
 		user = create_user('test_bulk_creation_update@example.com')
@@ -63,11 +63,11 @@ class TestUserPermission(unittest.TestCase):
 		# Initially create User Permission document with apply_to_all checked
 		is_created = add_user_permissions(param)
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 		is_created = add_user_permissions(param)
 
 		# User Permission should not be changed
-		self.assertEquals(is_created, 0)
+		self.assertEqual(is_created, 0)
 
 	def test_for_applicable_on_update_from_apply_to_all(self):
 		''' Update User Permission from all to some applicable Doctypes'''
@@ -77,7 +77,7 @@ class TestUserPermission(unittest.TestCase):
 		# Initially create User Permission document with apply_to_all checked
 		is_created = add_user_permissions(get_params(user, 'User', user.name))
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 		is_created = add_user_permissions(param)
 		frappe.db.commit()
@@ -92,7 +92,7 @@ class TestUserPermission(unittest.TestCase):
 		# Check that User Permissions for applicable is created
 		self.assertIsNotNone(is_created_applicable_first)
 		self.assertIsNotNone(is_created_applicable_second)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_for_apply_to_all_on_update_from_applicable(self):
 		''' Update User Permission from some to all applicable Doctypes'''
@@ -102,7 +102,7 @@ class TestUserPermission(unittest.TestCase):
 		# create User permissions that with applicable
 		is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"]))
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 		is_created = add_user_permissions(param)
 		is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
@@ -115,7 +115,7 @@ class TestUserPermission(unittest.TestCase):
 		# Check that all User Permission with applicable is removed
 		self.assertIsNone(removed_applicable_first)
 		self.assertIsNone(removed_applicable_second)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_user_perm_for_nested_doctype(self):
 		"""Test if descendants' visibility is controlled for a nested DocType."""
@@ -183,7 +183,7 @@ class TestUserPermission(unittest.TestCase):
 
 		# User perm is created on ToDo but for doctype Assignment Rule only
 		# it should not have impact on Doc A
-		self.assertEquals(new_doc.doc, "ToDo")
+		self.assertEqual(new_doc.doc, "ToDo")
 
 		frappe.set_user('Administrator')
 		remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo")
@@ -228,7 +228,7 @@ class TestUserPermission(unittest.TestCase):
 
 		# User perm is created on ToDo but for doctype Assignment Rule only
 		# it should not have impact on Doc A
-		self.assertEquals(new_doc.doc, "ToDo")
+		self.assertEqual(new_doc.doc, "ToDo")
 
 		frappe.set_user('Administrator')
 		clear_session_defaults()

frappe/core/doctype/user_permission/user_permission.py

@@ -191,7 +191,7 @@ def clear_user_permissions(user, for_doctype):
 def add_user_permissions(data):
 	''' Add and update the user permissions '''
 	frappe.only_for('System Manager')
-	if isinstance(data, frappe.string_types):
+	if isinstance(data, str):
 		data = json.loads(data)
 	data = frappe._dict(data)
 

frappe/custom/doctype/custom_field/custom_field.py

@@ -64,18 +64,19 @@ class CustomField(Document):
 			self.translatable = 0
 
 		if not self.flags.ignore_validate:
-			from frappe.core.doctype.doctype.doctype import check_if_fieldname_conflicts_with_methods
-			check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname)
+			from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts
+			check_fieldname_conflicts(self.dt, self.fieldname)
 
 	def on_update(self):
-		frappe.clear_cache(doctype=self.dt)
+		if not frappe.flags.in_setup_wizard:
+			frappe.clear_cache(doctype=self.dt)
 		if not self.flags.ignore_validate:
 			# validate field
 			from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype
 			validate_fields_for_doctype(self.dt)
 
 		# update the schema
-		if not frappe.db.get_value('DocType', self.dt, 'issingle'):
+		if not frappe.db.get_value('DocType', self.dt, 'issingle') and not frappe.flags.in_setup_wizard:
 			frappe.db.updatedb(self.dt)
 
 	def on_trash(self):
@@ -144,6 +145,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
 	'''Add / update multiple custom fields
 
 	:param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`'''
+
+	if not ignore_validate and frappe.flags.in_setup_wizard:
+		ignore_validate = True
+
 	for doctype, fields in custom_fields.items():
 		if isinstance(fields, dict):
 			# only one field
@@ -163,6 +168,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
 				custom_field.update(df)
 				custom_field.save()
 
+		frappe.clear_cache(doctype=doctype)
+		frappe.db.updatedb(doctype)
+
+
 
 @frappe.whitelist()
 def add_custom_field(doctype, df):

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -47,64 +47,64 @@ class TestCustomizeForm(unittest.TestCase):
 		self.assertEqual(len(d.get("fields")), 0)
 
 		d = self.get_customize_form("Event")
-		self.assertEquals(d.doc_type, "Event")
-		self.assertEquals(len(d.get("fields")), 36)
+		self.assertEqual(d.doc_type, "Event")
+		self.assertEqual(len(d.get("fields")), 36)
 
 		d = self.get_customize_form("Event")
-		self.assertEquals(d.doc_type, "Event")
+		self.assertEqual(d.doc_type, "Event")
 
 		self.assertEqual(len(d.get("fields")),
 			len(frappe.get_doc("DocType", d.doc_type).fields) + 1)
-		self.assertEquals(d.get("fields")[-1].fieldname, "test_custom_field")
-		self.assertEquals(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
+		self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field")
+		self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
 
 		return d
 
 	def test_save_customization_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
 
 		d.allow_copy = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), '1')
 
 		d.allow_copy = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
 
 	def test_save_customization_field_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
 
 		repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
 		repeat_this_event_field.reqd = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1')
 
 		repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
 		repeat_this_event_field.reqd = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
 
 	def test_save_customization_custom_field_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
 
 		custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0]
 		custom_field.reqd = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
 
 		custom_field = d.get("fields", {"is_custom_field": True})[0]
 		custom_field.reqd = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
 
 	def test_save_customization_new_field(self):
 		d = self.get_customize_form("Event")
@@ -115,14 +115,14 @@ class TestCustomizeForm(unittest.TestCase):
 			"is_custom_field": 1
 		})
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data")
 
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname)
 
 		frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form")
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form"), None)
 
 
@@ -142,7 +142,7 @@ class TestCustomizeForm(unittest.TestCase):
 		d.doc_type = "Event"
 		d.run_method('reset_to_defaults')
 
-		self.assertEquals(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
+		self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
 
 		frappe.local.test_objects["Property Setter"] = []
 		make_test_records_for_doctype("Property Setter")
@@ -156,7 +156,7 @@ class TestCustomizeForm(unittest.TestCase):
 		d = self.get_customize_form("Event")
 
 		# don't allow for standard fields
-		self.assertEquals(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
+		self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
 
 		# allow for custom field
 		self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1)

frappe/database/database.py

@@ -858,7 +858,7 @@ class Database(object):
 		if not datetime:
 			return '0001-01-01 00:00:00.000000'
 
-		if isinstance(datetime, frappe.string_types):
+		if isinstance(datetime, str):
 			if ':' not in datetime:
 				datetime = datetime + ' 00:00:00.000000'
 		else:

frappe/database/mariadb/database.py

@@ -1,5 +1,3 @@
-import warnings
-
 import pymysql
 from pymysql.constants import ER, FIELD_TYPE
 from pymysql.converters import conversions, escape_string
@@ -55,7 +53,6 @@ class MariaDBDatabase(Database):
 		}
 
 	def get_connection(self):
-		warnings.filterwarnings('ignore', category=pymysql.Warning)
 		usessl = 0
 		if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key:
 			usessl = 1

frappe/database/postgres/database.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import re
 import frappe
 import psycopg2
@@ -13,9 +11,9 @@ from frappe.database.postgres.schema import PostgresTable
 
 # cast decimals as floats
 DEC2FLOAT = psycopg2.extensions.new_type(
-    psycopg2.extensions.DECIMAL.values,
-    'DEC2FLOAT',
-    lambda value, curs: float(value) if value is not None else None)
+	psycopg2.extensions.DECIMAL.values,
+	'DEC2FLOAT',
+	lambda value, curs: float(value) if value is not None else None)
 
 psycopg2.extensions.register_type(DEC2FLOAT)
 
@@ -65,7 +63,6 @@ class PostgresDatabase(Database):
 		}
 
 	def get_connection(self):
-		# warnings.filterwarnings('ignore', category=psycopg2.Warning)
 		conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format(
 			self.host, self.user, self.user, self.password, self.port
 		))
@@ -114,7 +111,7 @@ class PostgresDatabase(Database):
 		if not date:
 			return '0001-01-01'
 
-		if not isinstance(date, frappe.string_types):
+		if not isinstance(date, str):
 			date = date.strftime('%Y-%m-%d')
 
 		return date

frappe/desk/doctype/notification_log/notification_log.py

@@ -46,7 +46,7 @@ def enqueue_create_notification(users, doc):
 
 	doc = frappe._dict(doc)
 
-	if isinstance(users, frappe.string_types):
+	if isinstance(users, str):
 		users = [user.strip() for user in users.split(',') if user.strip()]
 	users = list(set(users))
 

frappe/desk/doctype/todo/test_todo.py

@@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery
 from frappe.permissions import add_permission, reset_perms
 from frappe.core.doctype.doctype.doctype import clear_permissions_cache
 
-# test_records = frappe.get_test_records('ToDo')
-test_user_records = frappe.get_test_records('User')
+test_dependencies = ['User']
 
 class TestToDo(unittest.TestCase):
 	def test_delete(self):
@@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase):
 		frappe.set_user('test4@example.com')
 		#owner and assigned_by is test4
 		todo3 = create_new_todo('Test3', 'test4@example.com')
-		
+
 		# user without any role to read or write todo document
 		self.assertFalse(todo1.has_permission("read"))
 		self.assertFalse(todo1.has_permission("write"))

frappe/desk/doctype/workspace_link/workspace_link.json

@@ -8,13 +8,13 @@
   "type",
   "label",
   "icon",
+  "only_for",
   "hidden",
   "link_details_section",
   "link_type",
   "link_to",
   "column_break_7",
   "dependencies",
-  "only_for",
   "onboard",
   "is_query_report"
  ],
@@ -84,7 +84,7 @@
   {
    "fieldname": "only_for",
    "fieldtype": "Link",
-   "label": "Only for ",
+   "label": "Only for",
    "options": "Country"
   },
   {
@@ -104,7 +104,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-01-12 13:13:12.379443",
+ "modified": "2021-05-13 13:10:18.128512",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Link",

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -124,6 +124,7 @@ def handle_setup_exception(args):
 	frappe.db.rollback()
 	if args:
 		traceback = frappe.get_traceback()
+		print(traceback)
 		for hook in frappe.get_hooks("setup_wizard_exception"):
 			frappe.get_attr(hook)(traceback, args)
 

frappe/email/doctype/document_follow/test_document_follow.py

@@ -17,14 +17,14 @@ class TestDocumentFollow(unittest.TestCase):
 
 		document_follow.unfollow_document("Event", event_doc.name, user.name)
 		doc = document_follow.follow_document("Event", event_doc.name, user.name)
-		self.assertEquals(doc.user, user.name)
+		self.assertEqual(doc.user, user.name)
 
 		document_follow.send_hourly_updates()
 
 		email_queue_entry_name = frappe.get_all("Email Queue", limit=1)[0].name
 		email_queue_entry_doc = frappe.get_doc("Email Queue", email_queue_entry_name)
 
-		self.assertEquals((email_queue_entry_doc.recipients[0].recipient), user.name)
+		self.assertEqual((email_queue_entry_doc.recipients[0].recipient), user.name)
 
 		self.assertIn(event_doc.doctype, email_queue_entry_doc.message)
 		self.assertIn(event_doc.name, email_queue_entry_doc.message)

frappe/email/doctype/email_account/email_account.py

@@ -35,9 +35,6 @@ OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setu
 class SentEmailInInbox(Exception):
 	pass
 
-class InvalidEmailCredentials(frappe.ValidationError):
-	pass
-
 def cache_email_account(cache_name):
 	def decorator_cache_email_account(func):
 		@functools.wraps(func)
@@ -100,9 +97,8 @@ class EmailAccount(Document):
 					self.get_incoming_server()
 					self.no_failed = 0
 
-
 				if self.enable_outgoing:
-					self.check_smtp()
+					self.validate_smtp_conn()
 			else:
 				if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication):
 					frappe.throw(_("Password is required or select Awaiting Password"))
@@ -118,6 +114,13 @@ class EmailAccount(Document):
 			if self.append_to not in valid_doctypes:
 				frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
 
+	def validate_smtp_conn(self):
+		if not self.smtp_server:
+			frappe.throw(_("SMTP Server is required"))
+
+		server = self.get_smtp_server()
+		return server.session
+
 	def before_save(self):
 		messages = []
 		as_list = 1
@@ -179,24 +182,6 @@ class EmailAccount(Document):
 		except Exception:
 			pass
 
-	def check_smtp(self):
-		"""Checks SMTP settings."""
-		if self.enable_outgoing:
-			if not self.smtp_server:
-				frappe.throw(_("{0} is required").format("SMTP Server"))
-
-			server = SMTPServer(
-				login = getattr(self, "login_id", None) or self.email_id,
-				server=self.smtp_server,
-				port=cint(self.smtp_port),
-				use_tls=cint(self.use_tls),
-				use_ssl=cint(self.use_ssl_for_outgoing)
-			)
-			if self.password and not self.no_smtp_authentication:
-				server.password = self.get_password()
-
-			server.sess
-
 	def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
 		"""Returns logged in POP3/IMAP connection object."""
 		if frappe.cache().get_value("workers:no-internet") == True:
@@ -259,7 +244,7 @@ class EmailAccount(Document):
 				return None
 
 			elif not in_receive and any(map(lambda t: t in message, auth_error_codes)):
-				self.throw_invalid_credentials_exception()
+				SMTPServer.throw_invalid_credentials_exception()
 			else:
 				frappe.throw(cstr(e))
 
@@ -279,20 +264,18 @@ class EmailAccount(Document):
 
 	@property
 	def _password(self):
-		raise_exception = not self.no_smtp_authentication
+		raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test)
 		return self.get_password(raise_exception=raise_exception)
 
 	@property
 	def default_sender(self):
 		return email.utils.formataddr((self.name, self.get("email_id")))
 
-	@classmethod
-	def throw_invalid_credentials_exception(cls):
-		frappe.throw(
-			_("Incorrect email or password. Please check your login credentials."),
-			exc=InvalidEmailCredentials,
-			title=_("Invalid Credentials")
-		)
+	def is_exists_in_db(self):
+		"""Some of the Email Accounts we create from configs and those doesn't exists in DB.
+		This is is to check the specific email account exists in DB or not.
+		"""
+		return self.find_one_by_filters(name=self.name)
 
 	@classmethod
 	def from_record(cls, record):
@@ -402,6 +385,20 @@ class EmailAccount(Document):
 			account_details[doc_field_name] = (value and value[0]) or default
 		return account_details
 
+	def sendmail_config(self):
+		return {
+			'server': self.smtp_server,
+			'port': cint(self.smtp_port),
+			'login': getattr(self, "login_id", None) or self.email_id,
+			'password': self._password,
+			'use_ssl': cint(self.use_ssl_for_outgoing),
+			'use_tls': cint(self.use_tls)
+		}
+
+	def get_smtp_server(self):
+		config = self.sendmail_config()
+		return SMTPServer(**config)
+
 	def handle_incoming_connect_error(self, description):
 		if test_internet():
 			if self.get_failed_attempts_count() > 2:

frappe/email/doctype/email_queue/email_queue.json

@@ -24,7 +24,8 @@
   "unsubscribe_method",
   "expose_recipients",
   "attachments",
-  "retry"
+  "retry",
+  "email_account"
  ],
  "fields": [
   {
@@ -139,13 +140,19 @@
    "fieldtype": "Int",
    "label": "Retry",
    "read_only": 1
+  },
+  {
+   "fieldname": "email_account",
+   "fieldtype": "Link",
+   "label": "Email Account",
+   "options": "Email Account"
   }
  ],
  "icon": "fa fa-envelope",
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2020-07-17 15:58:15.369419",
+ "modified": "2021-04-29 06:33:25.191729",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Queue",

frappe/email/doctype/email_queue/email_queue.py

@@ -2,15 +2,26 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
+import traceback
+import json
+
+from rq.timeouts import JobTimeoutException
+import smtplib
+import quopri
+from email.parser import Parser
+
 import frappe
-from frappe import _
+from frappe import _, safe_encode, task
 from frappe.model.document import Document
-from frappe.email.queue import send_one
-from frappe.utils import now_datetime
-
+from frappe.email.queue import get_unsubcribed_url
+from frappe.email.email_body import add_attachment
+from frappe.utils import cint
+from email.policy import SMTPUTF8
 
+MAX_RETRY_COUNT = 3
 class EmailQueue(Document):
+	DOCTYPE = 'Email Queue'
+
 	def set_recipients(self, recipients):
 		self.set("recipients", [])
 		for r in recipients:
@@ -30,6 +41,241 @@ class EmailQueue(Document):
 		duplicate.set_recipients(recipients)
 		return duplicate
 
+	@classmethod
+	def find(cls, name):
+		return frappe.get_doc(cls.DOCTYPE, name)
+
+	def update_db(self, commit=False, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+		if commit:
+			frappe.db.commit()
+
+	def update_status(self, status, commit=False, **kwargs):
+		self.update_db(status = status, commit = commit, **kwargs)
+		if self.communication:
+			communication_doc = frappe.get_doc('Communication', self.communication)
+			communication_doc.set_delivery_status(commit=commit)
+
+	@property
+	def cc(self):
+		return (self.show_as_cc and self.show_as_cc.split(",")) or []
+
+	@property
+	def to(self):
+		return [r.recipient for r in self.recipients if r.recipient not in self.cc]
+
+	@property
+	def attachments_list(self):
+		return json.loads(self.attachments) if self.attachments else []
+
+	def get_email_account(self):
+		from frappe.email.doctype.email_account.email_account import EmailAccount
+
+		if self.email_account:
+			return frappe.get_doc('Email Account', self.email_account)
+
+		return EmailAccount.find_outgoing(
+			match_by_email = self.sender, match_by_doctype = self.reference_doctype)
+
+	def is_to_be_sent(self):
+		return self.status in ['Not Sent','Partially Sent']
+
+	def can_send_now(self):
+		hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1)
+		if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue:
+			return False
+
+		return True
+
+	def send(self, is_background_task=False):
+		""" Send emails to recipients.
+		"""
+		if not self.can_send_now():
+			frappe.db.rollback()
+			return
+
+		with SendMailContext(self, is_background_task) as ctx:
+			message = None
+			for recipient in self.recipients:
+				if not recipient.is_mail_to_be_sent():
+					continue
+
+				message = ctx.build_message(recipient.recipient)
+				if not frappe.flags.in_test:
+					ctx.smtp_session.sendmail(recipient.recipient, self.sender, message)
+				ctx.add_to_sent_list(recipient)
+
+			if frappe.flags.in_test:
+				frappe.flags.sent_mail = message
+				return
+
+			if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to:
+				ctx.email_account_doc.append_email_to_sent_folder(message)
+
+
+@task(queue = 'short')
+def send_mail(email_queue_name, is_background_task=False):
+	"""This is equalent to EmqilQueue.send.
+
+	This provides a way to make sending mail as a background job.
+	"""
+	record = EmailQueue.find(email_queue_name)
+	record.send(is_background_task=is_background_task)
+
+class SendMailContext:
+	def __init__(self, queue_doc: Document, is_background_task: bool = False):
+		self.queue_doc = queue_doc
+		self.is_background_task = is_background_task
+		self.email_account_doc = queue_doc.get_email_account()
+		self.smtp_server = self.email_account_doc.get_smtp_server()
+		self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()]
+
+	def __enter__(self):
+		self.queue_doc.update_status(status='Sending', commit=True)
+		return self
+
+	def __exit__(self, exc_type, exc_val, exc_tb):
+		exceptions = [
+			smtplib.SMTPServerDisconnected,
+			smtplib.SMTPAuthenticationError,
+			smtplib.SMTPRecipientsRefused,
+			smtplib.SMTPConnectError,
+			smtplib.SMTPHeloError,
+			JobTimeoutException
+		]
+
+		self.smtp_server.quit()
+		self.log_exception(exc_type, exc_val, exc_tb)
+
+		if exc_type in exceptions:
+			email_status = (self.sent_to and 'Partially Sent') or 'Not Sent'
+			self.queue_doc.update_status(status = email_status, commit = True)
+		elif exc_type:
+			if self.queue_doc.retry < MAX_RETRY_COUNT:
+				update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1}
+			else:
+				update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'}
+			self.queue_doc.update_status(**update_fields, commit = True)
+		else:
+			email_status = self.is_mail_sent_to_all() and 'Sent'
+			email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent'
+			self.queue_doc.update_status(status = email_status, commit = True)
+
+	def log_exception(self, exc_type, exc_val, exc_tb):
+		if exc_type:
+			traceback_string = "".join(traceback.format_tb(exc_tb))
+			traceback_string += f"\n Queue Name: {self.queue_doc.name}"
+
+			if self.is_background_task:
+				frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string)
+			else:
+				frappe.log_error(message = traceback_string)
+
+	@property
+	def smtp_session(self):
+		if frappe.flags.in_test:
+			return
+		return self.smtp_server.session
+
+	def add_to_sent_list(self, recipient):
+		# Update recipient status
+		recipient.update_db(status='Sent', commit=True)
+		self.sent_to.append(recipient.recipient)
+
+	def is_mail_sent_to_all(self):
+		return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients])
+
+	def get_message_object(self, message):
+		return Parser(policy=SMTPUTF8).parsestr(message)
+
+	def message_placeholder(self, placeholder_key):
+		map = {
+			'tracker': '<!--email open check-->',
+			'unsubscribe_url': '<!--unsubscribe url-->',
+			'cc': '<!--cc message-->',
+			'recipient': '<!--recipient-->',
+		}
+		return map.get(placeholder_key)
+
+	def build_message(self, recipient_email):
+		"""Build message specific to the recipient.
+		"""
+		message = self.queue_doc.message
+		if not message:
+			return ""
+
+		message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str())
+		message = message.replace(self.message_placeholder('unsubscribe_url'),
+			self.get_unsubscribe_str(recipient_email))
+		message = message.replace(self.message_placeholder('cc'), self.get_receivers_str())
+		message = message.replace(self.message_placeholder('recipient'),
+			self.get_receipient_str(recipient_email))
+		message = self.include_attachments(message)
+		return message
+
+	def get_tracker_str(self):
+		tracker_url_html = \
+			'<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'
+
+		message = ''
+		if frappe.conf.use_ssl and self.queue_doc.track_email_status:
+			message = quopri.encodestring(
+				tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
+			).decode()
+		return message
+
+	def get_unsubscribe_str(self, recipient_email):
+		unsubscribe_url = ''
+		if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype:
+			doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name
+			unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email,
+				self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param)
+
+		return quopri.encodestring(unsubscribe_url.encode()).decode()
+
+	def get_receivers_str(self):
+		message = ''
+		if self.queue_doc.expose_recipients == "footer":
+			to_str = ', '.join(self.queue_doc.to)
+			cc_str = ', '.join(self.queue_doc.cc)
+			message = f"This email was sent to {to_str}"
+			message = message + f" and copied to {cc_str}" if cc_str else message
+		return message
+
+	def get_receipient_str(self, recipient_email):
+		message = ''
+		if self.queue_doc.expose_recipients != "header":
+			message = recipient_email
+		return message
+
+	def include_attachments(self, message):
+		message_obj = self.get_message_object(message)
+		attachments = self.queue_doc.attachments_list
+
+		for attachment in attachments:
+			if attachment.get('fcontent'):
+				continue
+
+			fid = attachment.get("fid")
+			if fid:
+				_file = frappe.get_doc("File", fid)
+				fcontent = _file.get_content()
+				attachment.update({
+					'fname': _file.file_name,
+					'fcontent': fcontent,
+					'parent': message_obj
+				})
+				attachment.pop("fid", None)
+				add_attachment(**attachment)
+
+			elif attachment.get("print_format_attachment") == 1:
+				attachment.pop("print_format_attachment", None)
+				print_format_file = frappe.attach_print(**attachment)
+				print_format_file.update({"parent": message_obj})
+				add_attachment(**print_format_file)
+
+		return safe_encode(message_obj.as_string())
+
 @frappe.whitelist()
 def retry_sending(name):
 	doc = frappe.get_doc("Email Queue", name)
@@ -42,7 +288,9 @@ def retry_sending(name):
 
 @frappe.whitelist()
 def send_now(name):
-	send_one(name, now=True)
+	record = EmailQueue.find(name)
+	if record:
+		record.send()
 
 def on_doctype_update():
 	"""Add index in `tabCommunication` for `(reference_doctype, reference_name)`"""

frappe/email/doctype/email_queue_recipient/email_queue_recipient.py

@@ -7,4 +7,16 @@ import frappe
 from frappe.model.document import Document
 
 class EmailQueueRecipient(Document):
-	pass
+	DOCTYPE = 'Email Queue Recipient'
+
+	def is_mail_to_be_sent(self):
+		return self.status == 'Not Sent'
+
+	def is_main_sent(self):
+		return self.status == 'Sent'
+
+	def update_db(self, commit=False, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+		if commit:
+			frappe.db.commit()
+

frappe/email/doctype/notification/notification.json

@@ -102,7 +102,8 @@
    "default": "0",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "no_copy": 1
   },
   {
    "depends_on": "is_standard",
@@ -281,7 +282,7 @@
  "icon": "fa fa-envelope",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-11-24 14:25:43.245677",
+ "modified": "2021-05-04 11:17:11.882314",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification",

frappe/email/doctype/notification/test_notification.py

@@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler
 from frappe.desk.form import assign_to
 import unittest
 
-test_records = frappe.get_test_records('Notification')
-
-test_dependencies = ["User"]
+test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):

frappe/email/queue.py

@@ -173,19 +173,19 @@ def add(recipients, sender, subject, **kwargs):
 			if not email_queue:
 				email_queue = get_email_queue([r], sender, subject, **kwargs)
 				if kwargs.get('now'):
-					send_one(email_queue.name, now=True)
+					email_queue.send()
 			else:
 				duplicate = email_queue.get_duplicate([r])
 				duplicate.insert(ignore_permissions=True)
 
 				if kwargs.get('now'):
-					send_one(duplicate.name, now=True)
+					duplicate.send()
 
 			frappe.db.commit()
 	else:
 		email_queue = get_email_queue(recipients, sender, subject, **kwargs)
 		if kwargs.get('now'):
-			send_one(email_queue.name, now=True)
+			email_queue.send()
 
 def get_email_queue(recipients, sender, subject, **kwargs):
 	'''Make Email Queue object'''
@@ -237,6 +237,9 @@ def get_email_queue(recipients, sender, subject, **kwargs):
 			', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent')
 
 	recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', [])))
+	email_account = kwargs.get('email_account')
+	email_account_name = email_account and email_account.is_exists_in_db() and email_account.name
+
 	e.set_recipients(recipients)
 	e.reference_doctype = kwargs.get('reference_doctype')
 	e.reference_name = kwargs.get('reference_name')
@@ -248,8 +251,8 @@ def get_email_queue(recipients, sender, subject, **kwargs):
 	e.send_after = kwargs.get('send_after')
 	e.show_as_cc = ",".join(kwargs.get('cc', []))
 	e.show_as_bcc = ",".join(kwargs.get('bcc', []))
+	e.email_account = email_account_name or None
 	e.insert(ignore_permissions=True)
-
 	return e
 
 def get_emails_sent_this_month():
@@ -331,44 +334,25 @@ def return_unsubscribed_page(email, doctype, name):
 		indicator_color='green')
 
 def flush(from_test=False):
-	"""flush email queue, every time: called from scheduler"""
-	# additional check
-
-	auto_commit = not from_test
+	"""flush email queue, every time: called from scheduler
+	"""
+	from frappe.email.doctype.email_queue.email_queue import send_mail
+	# To avoid running jobs inside unit tests
 	if frappe.are_emails_muted():
 		msgprint(_("Emails are muted"))
 		from_test = True
 
-	smtpserver_dict = frappe._dict()
-
-	for email in get_queue():
-
-		if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
-			break
-
-		if email.name:
-			smtpserver = smtpserver_dict.get(email.sender)
-			if not smtpserver:
-				smtpserver = SMTPServer()
-				smtpserver_dict[email.sender] = smtpserver
+	if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
+		return
 
-			if from_test:
-				send_one(email.name, smtpserver, auto_commit)
-			else:
-				send_one_args = {
-					'email': email.name,
-					'smtpserver': smtpserver,
-					'auto_commit': auto_commit,
-				}
-				enqueue(
-					method = 'frappe.email.queue.send_one',
-					queue = 'short',
-					**send_one_args
-				)
+	for row in get_queue():
+		try:
+			func = send_mail if from_test else send_mail.enqueue
+			is_background_task = not from_test
+			func(email_queue_name = row.name, is_background_task = is_background_task)
+		except Exception:
+			frappe.log_error()
 
-		# NOTE: removing commit here because we pass auto_commit
-		# finally:
-		# 	frappe.db.commit()
 def get_queue():
 	return frappe.db.sql('''select
 			name, sender
@@ -381,213 +365,6 @@ def get_queue():
 			by priority desc, creation asc
 		limit 500''', { 'now': now_datetime() }, as_dict=True)
 
-
-def send_one(email, smtpserver=None, auto_commit=True, now=False):
-	'''Send Email Queue with given smtpserver'''
-
-	email = frappe.db.sql('''select
-			name, status, communication, message, sender, reference_doctype,
-			reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
-			show_as_cc, add_unsubscribe_link, attachments, retry
-		from
-			`tabEmail Queue`
-		where
-			name=%s
-		for update''', email, as_dict=True)
-
-	if len(email):
-		email = email[0]
-	else:
-		return
-
-	recipients_list = frappe.db.sql('''select name, recipient, status from
-		`tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1)
-
-	if frappe.are_emails_muted():
-		frappe.msgprint(_("Emails are muted"))
-		return
-
-	if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 :
-		return
-
-	if email.status not in ('Not Sent','Partially Sent') :
-		# rollback to release lock and return
-		frappe.db.rollback()
-		return
-
-	frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
-		(now_datetime(), email.name), auto_commit=auto_commit)
-
-	if email.communication:
-		frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-	email_sent_to_any_recipient = None
-
-	try:
-		message = None
-
-		if not frappe.flags.in_test:
-			if not smtpserver:
-				smtpserver = SMTPServer()
-
-			# to avoid always using default email account for outgoing
-			if getattr(frappe.local, "outgoing_email_account", None):
-				frappe.local.outgoing_email_account = {}
-
-			smtpserver.setup_email_account(email.reference_doctype, sender=email.sender)
-
-		for recipient in recipients_list:
-			if recipient.status != "Not Sent":
-				continue
-
-			message = prepare_message(email, recipient.recipient, recipients_list)
-			if not frappe.flags.in_test:
-				smtpserver.sess.sendmail(email.sender, recipient.recipient, message)
-
-			recipient.status = "Sent"
-			frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
-				(now_datetime(), recipient.name), auto_commit=auto_commit)
-
-		email_sent_to_any_recipient = any("Sent" == s.status for s in recipients_list)
-
-		#if all are sent set status
-		if email_sent_to_any_recipient:
-			frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-				where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
-		if frappe.flags.in_test:
-			frappe.flags.sent_mail = message
-			return
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient:
-			smtpserver.email_account.append_email_to_sent_folder(message)
-
-	except (smtplib.SMTPServerDisconnected,
-			smtplib.SMTPConnectError,
-			smtplib.SMTPHeloError,
-			smtplib.SMTPAuthenticationError,
-			smtplib.SMTPRecipientsRefused,
-			JobTimeoutException):
-
-		# bad connection/timeout, retry later
-
-		if email_sent_to_any_recipient:
-			frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		# no need to attempt further
-		return
-
-	except Exception as e:
-		frappe.db.rollback()
-
-		if email.retry < 3:
-			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			if email_sent_to_any_recipient:
-				frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
-					(text_type(e), email.name), auto_commit=auto_commit)
-			else:
-				frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-					where name=%s""", (text_type(e), email.name), auto_commit=auto_commit)
-
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		if now:
-			print(frappe.get_traceback())
-			raise e
-
-		else:
-			# log to Error Log
-			frappe.log_error('frappe.email.queue.flush')
-
-def prepare_message(email, recipient, recipients_list):
-	message = email.message
-	if not message:
-		return ""
-
-	# Parse "Email Account" from "Email Sender"
-	email_account = EmailAccount.find_outgoing(match_by_email=email.sender)
-	if frappe.conf.use_ssl and email_account.track_email_status:
-		# Using SSL => Publically available domain => Email Read Reciept Possible
-		message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode())
-	else:
-		# No SSL => No Email Read Reciept
-		message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode())
-
-	if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
-		unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
-		email.unsubscribe_method, email.unsubscribe_params)
-		message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
-
-	if email.expose_recipients == "header":
-		pass
-	else:
-		if email.expose_recipients == "footer":
-			if isinstance(email.show_as_cc, string_types):
-				email.show_as_cc = email.show_as_cc.split(",")
-			email_sent_to = [r.recipient for r in recipients_list]
-			email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
-			email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
-
-			if email_sent_cc:
-				email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
-			else:
-				email_sent_message = _("This email was sent to {0}").format(email_sent_to)
-			message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
-
-		message = message.replace("<!--recipient-->", recipient)
-
-	message = (message and message.encode('utf8')) or ''
-	message = safe_decode(message)
-
-	if PY3:
-		from email.policy import SMTPUTF8
-		message = Parser(policy=SMTPUTF8).parsestr(message)
-	else:
-		message = Parser().parsestr(message)
-
-	if email.attachments:
-		# On-demand attachments
-
-		attachments = json.loads(email.attachments)
-
-		for attachment in attachments:
-			if attachment.get('fcontent'):
-				continue
-
-			fid = attachment.get("fid")
-			if fid:
-				_file = frappe.get_doc("File", fid)
-				fcontent = _file.get_content()
-				attachment.update({
-					'fname': _file.file_name,
-					'fcontent': fcontent,
-					'parent': message
-				})
-				attachment.pop("fid", None)
-				add_attachment(**attachment)
-
-			elif attachment.get("print_format_attachment") == 1:
-				attachment.pop("print_format_attachment", None)
-				print_format_file = frappe.attach_print(**attachment)
-				print_format_file.update({"parent": message})
-				add_attachment(**print_format_file)
-
-	return safe_encode(message.as_string())
-
 def clear_outbox(days=None):
 	"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
 	Note: Used separate query to avoid deadlock

frappe/email/receive.py

@@ -284,7 +284,7 @@ class EmailServer:
 
 		flags = []
 		for flag in imaplib.ParseFlags(flag_string) or []:
-			pattern = re.compile("\w+")
+			pattern = re.compile(r"\w+")
 			match = re.search(pattern, frappe.as_unicode(flag))
 			flags.append(match.group(0))
 
@@ -555,7 +555,7 @@ class Email:
 
 	def get_thread_id(self):
 		"""Extract thread ID from `[]`"""
-		l = re.findall('(?<=\[)[\w/-]+', self.subject)
+		l = re.findall(r'(?<=\[)[\w/-]+', self.subject)
 		return l and l[0] or None
 
 

frappe/email/smtp.py

@@ -9,11 +9,24 @@ import _socket, sys
 from frappe import _
 from frappe.utils import cint, cstr, parse_addr
 
+CONNECTION_FAILED = _('Could not connect to outgoing email server')
+AUTH_ERROR_TITLE = _("Invalid Credentials")
+AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.")
+SOCKET_ERROR_TITLE = _("Incorrect Configuration")
+SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port")
+SEND_MAIL_FAILED = _("Unable to send emails at this time")
+EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
+
+class InvalidEmailCredentials(frappe.ValidationError):
+	pass
+
 def send(email, append_to=None, retry=1):
 	"""Deprecated: Send the message or add it to Outbox Email"""
 	def _send(retry):
+		from frappe.email.doctype.email_account.email_account import EmailAccount
 		try:
-			smtpserver = SMTPServer(append_to=append_to)
+			email_account = EmailAccount.find_outgoing(match_by_doctype=append_to)
+			smtpserver = email_account.get_smtp_server()
 
 			# validate is called in as_string
 			email_body = email.as_string()
@@ -34,102 +47,80 @@ def send(email, append_to=None, retry=1):
 
 	_send(retry)
 
-
 class SMTPServer:
-	def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None):
-		# get defaults from mail settings
-
-		self._sess = None
-		self.email_account = None
-		self.server = None
-		self.append_emails_to_sent_folder = None
-
-		if server:
-			self.server = server
-			self.port = port
-			self.use_tls = cint(use_tls)
-			self.use_ssl = cint(use_ssl)
-			self.login = login
-			self.password = password
-
-		else:
-			self.setup_email_account(append_to)
-
-	def setup_email_account(self, append_to=None, sender=None):
-		from frappe.email.doctype.email_account.email_account import EmailAccount
-		self.email_account = EmailAccount.find_outgoing(match_by_doctype=append_to, match_by_email=sender)
-		if self.email_account:
-			self.server = self.email_account.smtp_server
-			self.login = (getattr(self.email_account, "login_id", None) or self.email_account.email_id)
-			if self.email_account.no_smtp_authentication or frappe.local.flags.in_test:
-				self.password = None
-			else:
-				self.password = self.email_account._password
-			self.port = self.email_account.smtp_port
-			self.use_tls = self.email_account.use_tls
-			self.sender = self.email_account.email_id
-			self.use_ssl = self.email_account.use_ssl_for_outgoing
-			self.append_emails_to_sent_folder = self.email_account.append_emails_to_sent_folder
-			self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender"))
-			self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name"))
+	def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None):
+		self.login = login
+		self.password = password
+		self._server = server
+		self._port = port
+		self.use_tls = use_tls
+		self.use_ssl = use_ssl
+		self._session = None
+
+		if not self.server:
+			frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError)
 
 	@property
-	def sess(self):
-		"""get session"""
-		if self._sess:
-			return self._sess
+	def port(self):
+		port = self._port or (self.use_ssl and 465) or (self.use_tls and 587)
+		return cint(port)
 
-		# check if email server specified
-		if not getattr(self, 'server'):
-			err_msg = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
-			frappe.msgprint(err_msg)
-			raise frappe.OutgoingEmailError(err_msg)
-
-		try:
-			if self.use_ssl:
-				if not self.port:
-					self.port = 465
+	@property
+	def server(self):
+		return cstr(self._server or "")
 
-				self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port))
-			else:
-				if self.use_tls and not self.port:
-					self.port = 587
+	def secure_session(self, conn):
+		"""Secure the connection incase of TLS.
+		"""
+		if self.use_tls:
+			conn.ehlo()
+			conn.starttls()
+			conn.ehlo()
 
-				self._sess = smtplib.SMTP(cstr(self.server or ""),
-						cint(self.port) or None)
+	@property
+	def session(self):
+		if self.is_session_active():
+			return self._session
 
-			if not self._sess:
-				err_msg = _('Could not connect to outgoing email server')
-				frappe.msgprint(err_msg)
-				raise frappe.OutgoingEmailError(err_msg)
+		SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP
 
-			if self.use_tls:
-				self._sess.ehlo()
-				self._sess.starttls()
-				self._sess.ehlo()
+		try:
+			self._session = SMTP(self.server, self.port)
+			if not self._session:
+				frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError)
 
+			self.secure_session(self._session)
 			if self.login and self.password:
-				ret = self._sess.login(str(self.login or ""), str(self.password or ""))
+				res = self._session.login(str(self.login or ""), str(self.password or ""))
 
 				# check if logged correctly
-				if ret[0]!=235:
-					frappe.msgprint(ret[1])
-					raise frappe.OutgoingEmailError(ret[1])
+				if res[0]!=235:
+					frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError)
 
-			return self._sess
+			return self._session
 
 		except smtplib.SMTPAuthenticationError as e:
-			from frappe.email.doctype.email_account.email_account import EmailAccount
-			EmailAccount.throw_invalid_credentials_exception()
+			self.throw_invalid_credentials_exception()
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.throw(
-				_("Invalid Outgoing Mail Server or Port"),
-				exc=frappe.ValidationError,
-				title=_("Incorrect Configuration")
-			)
+			frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE)
 
 		except smtplib.SMTPException:
-			frappe.msgprint(_('Unable to send emails at this time'))
+			frappe.msgprint(SEND_MAIL_FAILED)
 			raise
+
+	def is_session_active(self):
+		if self._session:
+			try:
+				return self._session.noop()[0] == 250
+			except Exception:
+				return False
+
+	def quit(self):
+		if self.is_session_active():
+			self._session.quit()
+
+	@classmethod
+	def throw_invalid_credentials_exception(cls):
+		frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials)

frappe/email/test_email_body.py

@@ -7,10 +7,10 @@ from frappe import safe_decode
 from frappe.email.receive import Email
 from frappe.email.email_body import (replace_filename_with_cid,
 					get_email, inline_style_in_html, get_header)
-from frappe.email.queue import prepare_message, get_email_queue
+from frappe.email.queue import get_email_queue
+from frappe.email.doctype.email_queue.email_queue import SendMailContext
 from six import PY3
 
-
 class TestEmailBody(unittest.TestCase):
 	def setUp(self):
 		email_html = '''
@@ -57,7 +57,8 @@ This is the text version of this email
 			content='<h1>' + uni_chr1 + 'abcd' + uni_chr2 + '</h1>',
 			formatted='<h1>' + uni_chr1 + 'abcd' + uni_chr2 + '</h1>',
 			text_content='whatever')
-		result = prepare_message(email=email, recipient='test@test.com', recipients_list=[])
+		mail_ctx = SendMailContext(queue_doc = email)
+		result = mail_ctx.build_message(recipient_email = 'test@test.com')
 		self.assertTrue(b"<h1>=EA=80=80abcd=DE=B4</h1>" in result)
 
 	def test_prepare_message_returns_cr_lf(self):
@@ -68,8 +69,10 @@ This is the text version of this email
 			content='<h1>\n this is a test of newlines\n' + '</h1>',
 			formatted='<h1>\n this is a test of newlines\n' + '</h1>',
 			text_content='whatever')
-		result = safe_decode(prepare_message(email=email,
-						recipient='test@test.com', recipients_list=[]))
+
+		mail_ctx = SendMailContext(queue_doc = email)
+		result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com'))
+
 		if PY3:
 			self.assertTrue(result.count('\n') == result.count("\r"))
 		else:

frappe/email/test_smtp.py

@@ -75,4 +75,4 @@ def make_server(port, ssl, tls):
 		use_tls = tls
 	)
 
-	server.sess
+	server.session

frappe/handler.py

@@ -228,10 +228,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 	is_whitelisted(fn)
 	is_valid_http_method(fn)
 
-	try:
-		fnargs = inspect.getargspec(method_obj)[0]
-	except ValueError:
-		fnargs = inspect.getfullargspec(method_obj).args
+	fnargs = inspect.getfullargspec(method_obj).args
 
 	if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"):
 		response = doc.run_method(method)

frappe/hooks.py

@@ -226,7 +226,6 @@ scheduler_events = {
 		"frappe.desk.doctype.event.event.send_event_digest",
 		"frappe.sessions.clear_expired_sessions",
 		"frappe.email.doctype.notification.notification.trigger_daily_alerts",
-		"frappe.realtime.remove_old_task_logs",
 		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",

frappe/installer.py

@@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None):
 
 
 def make_site_dirs():
-	site_public_path = os.path.join(frappe.local.site_path, 'public')
-	site_private_path = os.path.join(frappe.local.site_path, 'private')
-	for dir_path in (
-			os.path.join(site_private_path, 'backups'),
-			os.path.join(site_public_path, 'files'),
-			os.path.join(site_private_path, 'files'),
-			os.path.join(frappe.local.site_path, 'logs'),
-			os.path.join(frappe.local.site_path, 'task-logs')):
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
-	locks_dir = frappe.get_site_path('locks')
-	if not os.path.exists(locks_dir):
-			os.makedirs(locks_dir)
+	for dir_path in [
+		os.path.join("public", "files"),
+		os.path.join("private", "backups"),
+		os.path.join("private", "files"),
+		"error-snapshots",
+		"locks",
+		"logs",
+	]:
+		path = frappe.get_site_path(dir_path)
+		os.makedirs(path, exist_ok=True)
 
 
 def add_module_defs(app):

frappe/integrations/doctype/connected_app/connected_app.json

@@ -54,7 +54,8 @@
    "fieldname": "client_id",
    "fieldtype": "Data",
    "in_list_view": 1,
-   "label": "Client Id"
+   "label": "Client Id",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "redirect_uri",
@@ -96,12 +97,14 @@
   {
    "fieldname": "authorization_uri",
    "fieldtype": "Data",
-   "label": "Authorization URI"
+   "label": "Authorization URI",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "token_uri",
    "fieldtype": "Data",
-   "label": "Token URI"
+   "label": "Token URI",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "revocation_uri",
@@ -136,7 +139,7 @@
    "link_fieldname": "connected_app"
   }
  ],
- "modified": "2020-11-16 16:29:50.277405",
+ "modified": "2021-05-10 05:03:06.296863",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Connected App",

frappe/integrations/doctype/connected_app/connected_app.py

@@ -26,20 +26,27 @@ class ConnectedApp(Document):
 		self.redirect_uri = urljoin(base_url, callback_path)
 
 	def get_oauth2_session(self, user=None, init=False):
+		"""Return an auto-refreshing OAuth2 session which is an extension of a requests.Session()"""
 		token = None
 		token_updater = None
+		auto_refresh_kwargs = None
 
 		if not init:
 			user = user or frappe.session.user
 			token_cache = self.get_user_token(user)
 			token = token_cache.get_json()
 			token_updater = token_cache.update_data
+			auto_refresh_kwargs = {'client_id': self.client_id}
+			client_secret = self.get_password('client_secret')
+			if client_secret:
+				auto_refresh_kwargs['client_secret'] = client_secret
 
 		return OAuth2Session(
 			client_id=self.client_id,
 			token=token,
 			token_updater=token_updater,
 			auto_refresh_url=self.token_uri,
+			auto_refresh_kwargs=auto_refresh_kwargs,
 			redirect_uri=self.redirect_uri,
 			scope=self.get_scopes()
 		)

frappe/integrations/oauth2.py

@@ -1,6 +1,5 @@
 import json
 from urllib.parse import quote, urlencode
-
 from oauthlib.oauth2 import FatalClientError, OAuth2Error
 from oauthlib.openid.connect.core.endpoints.pre_configured import (
 	Server as WebApplicationServer,

frappe/model/base_document.py

@@ -34,8 +34,9 @@ def get_controller(doctype):
 		from frappe.model.document import Document
 		from frappe.utils.nestedset import NestedSet
 
-		module_name, custom = frappe.db.get_value("DocType", doctype, ("module", "custom"), cache=True) \
-			or ["Core", False]
+		module_name, custom = frappe.db.get_value(
+			"DocType", doctype, ("module", "custom"), cache=True
+		) or ["Core", False]
 
 		if custom:
 			if frappe.db.field_exists("DocType", "is_tree"):
@@ -869,7 +870,7 @@ class BaseDocument(object):
 			from frappe.model.meta import get_default_df
 			df = get_default_df(fieldname)
 
-		if not currency:
+		if not currency and df:
 			currency = self.get(df.get("options"))
 			if not frappe.db.exists('Currency', currency, cache=True):
 				currency = None

frappe/model/db_query.py

@@ -465,7 +465,7 @@ class DatabaseQuery(object):
 
 		elif f.operator.lower() in ('in', 'not in'):
 			values = f.value or ''
-			if isinstance(values, frappe.string_types):
+			if isinstance(values, str):
 				values = values.split(",")
 
 			fallback = "''"

frappe/model/document.py

@@ -1347,6 +1347,22 @@ class Document(BaseDocument):
 		from frappe.desk.doctype.tag.tag import DocTags
 		return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
 
+	def __repr__(self):
+		name = self.name or "unsaved"
+		doctype = self.__class__.__name__
+
+		docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
+		parent = f" parent={self.parent}" if self.parent else ""
+
+		return f"<{doctype}: {name}{docstatus}{parent}>"
+
+	def __str__(self):
+		name = self.name or "unsaved"
+		doctype = self.__class__.__name__
+
+		return f"{doctype}({name})"
+
+
 def execute_action(doctype, name, action, **kwargs):
 	"""Execute an action on a document (called by background worker)"""
 	doc = frappe.get_doc(doctype, name)

frappe/model/meta.py

@@ -118,7 +118,7 @@ class Meta(Document):
 						# non standard list object, skip
 						continue
 
-				if (isinstance(value, (frappe.text_type, int, float, datetime, list, tuple))
+				if (isinstance(value, (str, int, float, datetime, list, tuple))
 					or (not no_nulls and value is None)):
 					out[key] = value
 

frappe/model/naming.py

@@ -199,10 +199,39 @@ def getseries(key, digits):
 
 
 def revert_series_if_last(key, name, doc=None):
-	if ".#" in key:
+	"""
+	Reverts the series for particular naming series:
+	* key is naming series		- SINV-.YYYY-.####
+	* name is actual name		- SINV-2021-0001
+		
+	1. This function split the key into two parts prefix (SINV-YYYY) & hashes (####).
+	2. Use prefix to get the current index of that naming series from Series table
+	3. Then revert the current index.
+
+	*For custom naming series:*
+	1. hash can exist anywhere, if it exist in hashes then it take normal flow.
+	2. If hash doesn't exit in hashes, we get the hash from prefix, then update name and prefix accordingly.
+
+	*Example:*
+		1. key = SINV-.YYYY.- 
+			* If key doesn't have hash it will add hash at the end
+			* prefix will be SINV-YYYY based on this will get current index from Series table.
+		2. key = SINV-.####.-2021
+			* now prefix = SINV-#### and hashes = 2021 (hash doesn't exist)
+			* will search hash in key then accordingly get prefix = SINV-
+		3. key = ####.-2021
+			* prefix = #### and hashes = 2021 (hash doesn't exist)
+			* will search hash in key then accordingly get prefix = "" 
+	"""
+	if ".#" in key: 
 		prefix, hashes = key.rsplit(".", 1)
 		if "#" not in hashes:
-			return
+			# get the hash part from the key
+			hash = re.search("#+", key)
+			if not hash:
+				return
+			name = name.replace(hashes, "")
+			prefix = prefix.replace(hash.group(), "")
 	else:
 		prefix = key
 
@@ -254,7 +283,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
 	filters.update({fieldname: value})
 	exists = frappe.db.exists(doctype, filters)
 
-	regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator)
+	regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator)
 
 	if exists:
 		last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}`

frappe/oauth.py

@@ -4,11 +4,9 @@ import hashlib
 import re
 from http import cookies
 from urllib.parse import unquote, urlparse
-
 import jwt
 import pytz
 from oauthlib.openid import RequestValidator
-
 import frappe
 from frappe.auth import LoginManager
 

frappe/parallel_test_runner.py

@@ -0,0 +1,282 @@
+import json
+import os
+import re
+import sys
+import time
+import unittest
+import click
+import frappe
+import requests
+
+from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config)
+
+click_ctx = click.get_current_context(True)
+if click_ctx:
+	click_ctx.color = True
+
+class ParallelTestRunner():
+	def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False):
+		self.app = app
+		self.site = site
+		self.with_coverage = with_coverage
+		self.build_number = frappe.utils.cint(build_number) or 1
+		self.total_builds = frappe.utils.cint(total_builds)
+		self.setup_test_site()
+		self.run_tests()
+
+	def setup_test_site(self):
+		frappe.init(site=self.site)
+		if not frappe.db:
+			frappe.connect()
+
+		frappe.flags.in_test = True
+		frappe.clear_cache()
+		frappe.utils.scheduler.disable_scheduler()
+		set_test_email_config()
+		self.before_test_setup()
+
+	def before_test_setup(self):
+		start_time = time.time()
+		for fn in frappe.get_hooks("before_tests", app_name=self.app):
+			frappe.get_attr(fn)()
+
+		test_module = frappe.get_module(f'{self.app}.tests')
+
+		if hasattr(test_module, "global_test_dependencies"):
+			for doctype in test_module.global_test_dependencies:
+				make_test_records(doctype)
+
+		elapsed = time.time() - start_time
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red')
+		click.echo(f'Before Test {elapsed}')
+
+	def run_tests(self):
+		self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2)
+
+		self.start_coverage()
+
+		for test_file_info in self.get_test_file_list():
+			self.run_tests_for_file(test_file_info)
+
+		self.save_coverage()
+		self.print_result()
+
+	def run_tests_for_file(self, file_info):
+		if not file_info: return
+
+		frappe.set_user('Administrator')
+		path, filename = file_info
+		module = self.get_module(path, filename)
+		self.create_test_dependency_records(module, path, filename)
+		test_suite = unittest.TestSuite()
+		module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
+		test_suite.addTest(module_test_cases)
+		test_suite(self.test_result)
+
+	def create_test_dependency_records(self, module, path, filename):
+		if hasattr(module, "test_dependencies"):
+			for doctype in module.test_dependencies:
+				make_test_records(doctype)
+
+		if os.path.basename(os.path.dirname(path)) == "doctype":
+			# test_data_migration_connector.py > data_migration_connector.json
+			test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json")
+			test_record_file_path = os.path.join(path, test_record_filename)
+			if os.path.exists(test_record_file_path):
+				with open(test_record_file_path, 'r') as f:
+					doc = json.loads(f.read())
+					doctype = doc["name"]
+					make_test_records(doctype)
+
+	def get_module(self, path, filename):
+		app_path = frappe.get_pymodule_path(self.app)
+		relative_path = os.path.relpath(path, app_path)
+		if relative_path == '.':
+			module_name = self.app
+		else:
+			relative_path = relative_path.replace('/', '.')
+			module_name = os.path.splitext(filename)[0]
+			module_name = f'{self.app}.{relative_path}.{module_name}'
+
+		return frappe.get_module(module_name)
+
+	def print_result(self):
+		self.test_result.printErrors()
+		click.echo(self.test_result)
+		if self.test_result.failures or self.test_result.errors:
+			if os.environ.get('CI'):
+				sys.exit(1)
+
+	def start_coverage(self):
+		if self.with_coverage:
+			from coverage import Coverage
+			from frappe.utils import get_bench_path
+
+			# Generate coverage report only for app that is being tested
+			source_path = os.path.join(get_bench_path(), 'apps', self.app)
+			omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss',
+				'*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*']
+
+			if self.app == 'frappe':
+				omit.append('*/commands/*')
+
+			self.coverage = Coverage(source=[source_path], omit=omit)
+			self.coverage.start()
+
+	def save_coverage(self):
+		if not self.with_coverage:
+			return
+		self.coverage.stop()
+		self.coverage.save()
+
+	def get_test_file_list(self):
+		test_list = get_all_tests(self.app)
+		split_size = frappe.utils.ceil(len(test_list) / self.total_builds)
+		# [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2
+		test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)]
+		return test_chunks[self.build_number - 1]
+
+
+class ParallelTestResult(unittest.TextTestResult):
+	def startTest(self, test):
+		self._started_at = time.time()
+		super(unittest.TextTestResult, self).startTest(test)
+		test_class = unittest.util.strclass(test.__class__)
+		if not hasattr(self, 'current_test_class') or self.current_test_class != test_class:
+			click.echo(f"\n{unittest.util.strclass(test.__class__)}")
+			self.current_test_class = test_class
+
+	def getTestMethodName(self, test):
+		return test._testMethodName if hasattr(test, '_testMethodName') else str(test)
+
+	def addSuccess(self, test):
+		super(unittest.TextTestResult, self).addSuccess(test)
+		elapsed = time.time() - self._started_at
+		threshold_passed = elapsed >= SLOW_TEST_THRESHOLD
+		elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else ''
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}")
+
+	def addError(self, test, err):
+		super(unittest.TextTestResult, self).addError(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addFailure(self, test, err):
+		super(unittest.TextTestResult, self).addFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addSkip(self, test, reason):
+		super(unittest.TextTestResult, self).addSkip(test, reason)
+		click.echo(f"  {click.style(' = ', fg='white')} {self.getTestMethodName(test)}")
+
+	def addExpectedFailure(self, test, err):
+		super(unittest.TextTestResult, self).addExpectedFailure(test, err)
+		click.echo(f"  {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
+
+	def addUnexpectedSuccess(self, test):
+		super(unittest.TextTestResult, self).addUnexpectedSuccess(test)
+		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}")
+
+	def printErrors(self):
+		click.echo('\n')
+		self.printErrorList(' ERROR ', self.errors, 'red')
+		self.printErrorList(' FAIL ', self.failures, 'red')
+
+	def printErrorList(self, flavour, errors, color):
+		for test, err in errors:
+			click.echo(self.separator1)
+			click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}")
+			click.echo(self.separator2)
+			click.echo(err)
+
+	def __str__(self):
+		return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}"
+
+def get_all_tests(app):
+	test_file_list = []
+	for path, folders, files in os.walk(frappe.get_pymodule_path(app)):
+		for dontwalk in ('locals', '.git', 'public', '__pycache__'):
+			if dontwalk in folders:
+				folders.remove(dontwalk)
+
+		# for predictability
+		folders.sort()
+		files.sort()
+
+		if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path:
+			# in /doctype/doctype/boilerplate/
+			continue
+
+		for filename in files:
+			if filename.startswith("test_") and filename.endswith(".py") \
+				and filename != 'test_runner.py':
+				test_file_list.append([path, filename])
+
+	return test_file_list
+
+
+class ParallelTestWithOrchestrator(ParallelTestRunner):
+	'''
+		This can be used to balance-out test time across multiple instances
+		This is dependent on external orchestrator which returns next test to run
+
+		orchestrator endpoints
+		- register-instance (<build_id>, <instance_id>, test_spec_list)
+		- get-next-test-spec (<build_id>, <instance_id>)
+		- test-completed (<build_id>, <instance_id>)
+	'''
+	def __init__(self, app, site, with_coverage=False):
+		self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL')
+		if not self.orchestrator_url:
+			click.echo('ORCHESTRATOR_URL environment variable not found!')
+			click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator')
+			sys.exit(1)
+
+		self.ci_build_id = os.environ.get('CI_BUILD_ID')
+		self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10)
+		if not self.ci_build_id:
+			click.echo('CI_BUILD_ID environment variable not found!')
+			sys.exit(1)
+
+		ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage)
+
+	def run_tests(self):
+		self.test_status = 'ongoing'
+		self.register_instance()
+		super().run_tests()
+
+	def get_test_file_list(self):
+		while self.test_status == 'ongoing':
+			yield self.get_next_test()
+
+	def register_instance(self):
+		test_spec_list = get_all_tests(self.app)
+		response_data = self.call_orchestrator('register-instance', data={
+			'test_spec_list': test_spec_list
+		})
+		self.is_master = response_data.get('is_master')
+
+	def get_next_test(self):
+		response_data = self.call_orchestrator('get-next-test-spec')
+		self.test_status = response_data.get('status')
+		return response_data.get('next_test')
+
+	def print_result(self):
+		self.call_orchestrator('test-completed')
+		return super().print_result()
+
+	def call_orchestrator(self, endpoint, data={}):
+		# add repo token header
+		# build id in header
+		headers = {
+			'CI-BUILD-ID': self.ci_build_id,
+			'CI-INSTANCE-ID': self.ci_instance_id,
+			'REPO-TOKEN': '2948288382838DE'
+		}
+		url = f'{self.orchestrator_url}/{endpoint}'
+		res = requests.get(url, json=data, headers=headers)
+		res.raise_for_status()
+		response_data = {}
+		if 'application/json' in res.headers.get('content-type'):
+			response_data = res.json()
+
+		return response_data

frappe/patches/v5_0/fix_text_editor_file_urls.py

@@ -33,8 +33,7 @@ def execute():
 def scrub_relative_urls(html):
 	"""prepend a slash before a relative url"""
 	try:
-		return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html)
-		# return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html)
+		return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html)
 	except:
 		print("Error", html)
 		raise

frappe/printing/doctype/print_format/test_print_format.py

@@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase):
 	def test_print_user(self, style=None):
 		print_html = frappe.get_print("User", "Administrator", style=style)
 		self.assertTrue("<label>First Name: </label>" in print_html)
-		self.assertTrue(re.findall('<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
+		self.assertTrue(re.findall(r'<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
 		return print_html
 
 	def test_print_user_standard(self):
 		print_html = self.test_print_user("Standard")
-		self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html))
-		self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html))
+		self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html))
+		self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html))
 		self.assertFalse("font-family: serif;" in print_html)
 
 	def test_print_user_modern(self):

frappe/public/js/frappe/desk.js

@@ -474,14 +474,19 @@ frappe.Application = Class.extend({
 		$('<link rel="icon" href="' + link + '" type="image/x-icon">').appendTo("head");
 	},
 	trigger_primary_action: function() {
-		if(window.cur_dialog && cur_dialog.display) {
-			// trigger primary
-			cur_dialog.get_primary_btn().trigger("click");
-		} else if(cur_frm && cur_frm.page.btn_primary.is(':visible')) {
-			cur_frm.page.btn_primary.trigger('click');
-		} else if(frappe.container.page.save_action) {
-			frappe.container.page.save_action();
-		}
+		// to trigger change event on active input before triggering primary action
+		$(document.activeElement).blur();
+		// wait for possible JS validations triggered after blur (it might change primary button)
+		setTimeout(() => {
+			if (window.cur_dialog && cur_dialog.display) {
+				// trigger primary
+				cur_dialog.get_primary_btn().trigger("click");
+			} else if (cur_frm && cur_frm.page.btn_primary.is(':visible')) {
+				cur_frm.page.btn_primary.trigger('click');
+			} else if (frappe.container.page.save_action) {
+				frappe.container.page.save_action();
+			}
+		}, 100);
 	},
 
 	set_rtl: function() {

frappe/public/js/frappe/form/controls/date.js

@@ -13,9 +13,11 @@ frappe.ui.form.ControlDate = frappe.ui.form.ControlData.extend({
 		this._super(value);
 		if (this.timepicker_only) return;
 		if (!this.datepicker) return;
-		if(!value) {
+		if (!value) {
 			this.datepicker.clear();
 			return;
+		} else if (value === "Today") {
+			value = this.get_now_date();
 		}
 
 		let should_refresh = this.last_value && this.last_value !== value;

frappe/public/js/frappe/form/grid.js

@@ -910,6 +910,10 @@ export default class Grid {
 
 	update_docfield_property(fieldname, property, value) {
 		// update the docfield of each row
+		if (!this.grid_rows) {
+			return;
+		}
+
 		for (let row of this.grid_rows) {
 			let docfield = row.docfields.find(d => d.fieldname === fieldname);
 			if (docfield) {

frappe/public/js/frappe/form/grid_row.js

@@ -7,7 +7,8 @@ export default class GridRow {
 		$.extend(this, opts);
 		if (this.doc && this.parent_df.options) {
 			frappe.meta.make_docfield_copy_for(this.parent_df.options, this.doc.name, this.docfields);
-			this.docfields = frappe.meta.get_docfields(this.parent_df.options, this.doc.name);
+			const docfields = frappe.meta.get_docfields(this.parent_df.options, this.doc.name);
+			this.docfields = docfields.length ? docfields : opts.docfields;
 		}
 		this.columns = {};
 		this.columns_list = [];

frappe/public/js/frappe/form/layout.js

@@ -510,7 +510,7 @@ frappe.ui.form.Layout = Class.extend({
 			form_obj = this;
 		}
 		if (form_obj) {
-			if (this.doc && this.doc.parent) {
+			if (this.doc && this.doc.parent && this.doc.parentfield) {
 				form_obj.setting_dependency = true;
 				form_obj.set_df_property(this.doc.parentfield, property, value, this.doc.parent, fieldname, this.doc.name);
 				form_obj.setting_dependency = false;

frappe/public/js/frappe/web_form/web_form.js

@@ -87,11 +87,13 @@ export default class WebForm extends frappe.ui.FieldGroup {
 	}
 
 	setup_delete_button() {
-		this.add_button_to_header(
-			frappe.utils.icon('delete'),
-			"danger",
-			() => this.delete()
-		);
+		frappe.has_permission(this.doc_type, "", "delete", () => {
+			this.add_button_to_header(
+				frappe.utils.icon('delete'),
+				"danger",
+				() => this.delete()
+			);
+		});
 	}
 
 	setup_print_button() {

frappe/public/js/frappe/web_form/web_form_list.js

@@ -190,9 +190,11 @@ export default class WebFormList {
 	make_actions() {
 		const actions = document.querySelector(".list-view-actions");
 
-		this.addButton(actions, "delete-rows", "danger", true, "Delete", () =>
-			this.delete_rows()
-		);
+		frappe.has_permission(this.doctype, "", "delete", () => {
+			this.addButton(actions, "delete-rows", "danger", true, "Delete", () =>
+				this.delete_rows()
+			);
+		});
 
 		this.addButton(
 			actions,

frappe/public/scss/common/quill.scss

@@ -164,7 +164,7 @@
 }
 
 .ql-editor td {
-	border: 1px solid var(--border-color);
+	border: 1px solid var(--dark-border-color);
 }
 
 .ql-editor blockquote {

frappe/public/scss/desk/report.scss

@@ -161,7 +161,8 @@
 	.summary-item {
 		// SIZE & SPACING
 		margin: 0px 30px;
-		width: 180px;
+		min-width: 180px;
+		max-width: 300px;
 		height: 62px;
 
 		// LAYOUT

frappe/realtime.py

@@ -1,56 +1,23 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-
-
 import frappe
 import os
-import time
 import redis
-from io import FileIO
-from frappe.utils import get_site_path
-from frappe import conf
 
-END_LINE = '<!-- frappe: end-file -->'
-TASK_LOG_MAX_AGE = 86400  # 1 day in seconds
 redis_server = None
 
+
 @frappe.whitelist()
 def get_pending_tasks_for_doc(doctype, docname):
 	return frappe.db.sql_list("select name from `tabAsync Task` where status in ('Queued', 'Running') and reference_doctype=%s and reference_name=%s", (doctype, docname))
 
 
-def set_task_status(task_id, status, response=None):
-	if not response:
-		response = {}
-	response.update({
-		"status": status,
-		"task_id": task_id
-	})
-	emit_via_redis("task_status_change", response, room="task:" + task_id)
-
-
-def remove_old_task_logs():
-	logs_path = get_site_path('task-logs')
-
-	def full_path(_file):
-		return os.path.join(logs_path, _file)
-
-	files_to_remove = [full_path(_file) for _file in os.listdir(logs_path)]
-	files_to_remove = [_file for _file in files_to_remove if is_file_old(_file) and os.path.isfile(_file)]
-	for _file in files_to_remove:
-		os.remove(_file)
-
-
-def is_file_old(file_path):
-	return ((time.time() - os.stat(file_path).st_mtime) > TASK_LOG_MAX_AGE)
-
 def publish_progress(percent, title=None, doctype=None, docname=None, description=None):
 	publish_realtime('progress', {'percent': percent, 'title': title, 'description': description},
 		user=frappe.session.user, doctype=doctype, docname=docname)
 
+
 def publish_realtime(event=None, message=None, room=None,
 	user=None, doctype=None, docname=None, task_id=None,
 	after_commit=False):
@@ -103,6 +70,7 @@ def publish_realtime(event=None, message=None, room=None,
 	else:
 		emit_via_redis(event, message, room)
 
+
 def emit_via_redis(event, message, room):
 	"""Publish real-time updates via redis
 
@@ -117,57 +85,17 @@ def emit_via_redis(event, message, room):
 		# print(frappe.get_traceback())
 		pass
 
-def put_log(line_no, line, task_id=None):
-	r = get_redis_server()
-	if not task_id:
-		task_id = frappe.local.task_id
-	task_progress_room = get_task_progress_room(task_id)
-	task_log_key = "task_log:" + task_id
-	publish_realtime('task_progress', {
-		"message": {
-			"lines": {line_no: line}
-		},
-		"task_id": task_id
-	}, room=task_progress_room)
-	r.hset(task_log_key, line_no, line)
-	r.expire(task_log_key, 3600)
-
 
 def get_redis_server():
 	"""returns redis_socketio connection."""
 	global redis_server
 	if not redis_server:
 		from redis import Redis
-		redis_server = Redis.from_url(conf.get("redis_socketio")
+		redis_server = Redis.from_url(frappe.conf.redis_socketio
 			or "redis://localhost:12311")
 	return redis_server
 
 
-class FileAndRedisStream(FileIO):
-	def __init__(self, *args, **kwargs):
-		ret = super(FileAndRedisStream, self).__init__(*args, **kwargs)
-		self.count = 0
-		return ret
-
-	def write(self, data):
-		ret = super(FileAndRedisStream, self).write(data)
-		if frappe.local.task_id:
-			put_log(self.count, data, task_id=frappe.local.task_id)
-			self.count += 1
-		return ret
-
-
-def get_std_streams(task_id):
-	stdout = FileAndRedisStream(get_task_log_file_path(task_id, 'stdout'), 'w')
-	# stderr = FileAndRedisStream(get_task_log_file_path(task_id, 'stderr'), 'w')
-	return stdout, stdout
-
-
-def get_task_log_file_path(task_id, stream_type):
-	logs_dir = frappe.utils.get_site_path('task-logs')
-	return os.path.join(logs_dir, task_id + '.' + stream_type)
-
-
 @frappe.whitelist(allow_guest=True)
 def can_subscribe_doc(doctype, docname):
 	if os.environ.get('CI'):
@@ -201,9 +129,7 @@ def get_site_room():
 def get_task_progress_room(task_id):
 	return "".join([frappe.local.site, ":task_progress:", task_id])
 
-# frappe.chat
 def get_chat_room(room):
 	room = ''.join([frappe.local.site, ":room:", room])
 
 	return room
-# end frappe.chat room

frappe/search/full_text_search.py

@@ -1,8 +1,8 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
 import frappe
+from frappe.utils import update_progress_bar
 
 from whoosh.index import create_in, open_dir, EmptyIndexError
 from whoosh.fields import TEXT, ID, Schema
@@ -95,9 +95,10 @@ class FullTextSearch:
 		ix = self.create_index()
 		writer = ix.writer()
 
-		for document in self.documents:
+		for i, document in enumerate(self.documents):
 			if document:
 				writer.add_document(**document)
+			update_progress_bar("Building Index", i, len(self.documents))
 
 		writer.commit(optimize=True)
 

frappe/search/website_search.py

@@ -1,14 +1,15 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
-import frappe
+import os
+
 from bs4 import BeautifulSoup
-from whoosh.fields import TEXT, ID, Schema
+from whoosh.fields import ID, TEXT, Schema
+
+import frappe
 from frappe.search.full_text_search import FullTextSearch
+from frappe.utils import set_request, update_progress_bar
 from frappe.website.render import render_page
-from frappe.utils import set_request
-import os
 
 INDEX_NAME = "web_routes"
 
@@ -30,11 +31,21 @@ class WebsiteSearch(FullTextSearch):
 		Returns:
 			self (object): FullTextSearch Instance
 		"""
-		routes = get_static_pages_from_all_apps()
-		routes += slugs_with_web_view()
 
-		documents = [self.get_document_to_index(route) for route in routes]
-		return documents
+		if getattr(self, "_items_to_index", False):
+			return self._items_to_index
+
+		routes = get_static_pages_from_all_apps() + slugs_with_web_view()
+
+		self._items_to_index = []
+
+		for i, route in enumerate(routes):
+			update_progress_bar("Retrieving Routes", i, len(routes))
+			self._items_to_index += [self.get_document_to_index(route)]
+
+		print()
+
+		return self.get_items_to_index()
 
 	def get_document_to_index(self, route):
 		"""Render a page and parse it using BeautifulSoup
@@ -114,4 +125,4 @@ def remove_document_from_index(path):
 
 def build_index_for_all_routes():
 	ws = WebsiteSearch(INDEX_NAME)
-	return ws.build()
+	return ws.build()

frappe/social/doctype/energy_point_log/test_energy_point_log.py

@@ -78,7 +78,7 @@ class TestEnergyPointLog(unittest.TestCase):
 		points_after_closing_todo = get_points('test@example.com')
 
 		# test max_points cap
-		self.assertNotEquals(points_after_closing_todo,
+		self.assertNotEqual(points_after_closing_todo,
 			energy_point_of_user + round(todo_point_rule.points * multiplier_value))
 
 		self.assertEqual(points_after_closing_todo,

frappe/test_runner.py

@@ -9,7 +9,6 @@ import time
 import xmlrunner
 import importlib
 from frappe.modules import load_doctype_module, get_module_name
-from frappe.utils import cstr
 import frappe.utils.scheduler
 import cProfile, pstats
 from six import StringIO
@@ -308,6 +307,8 @@ def get_dependencies(doctype):
 			if doctype_name in options_list:
 				options_list.remove(doctype_name)
 
+	options_list.sort()
+
 	return options_list
 
 def make_test_records_for_doctype(doctype, verbose=0, force=False):

frappe/tests/__init__.py

@@ -3,7 +3,10 @@ import frappe
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()
 
 def get_system_setting(key):
 	return frappe.db.get_single_value("System Settings", key)
+
+global_test_dependencies = ['User']

frappe/tests/test_auth.py

@@ -110,7 +110,7 @@ class TestLoginAttemptTracker(unittest.TestCase):
 	def test_account_unlock(self):
 		"""Make sure that locked account gets unlocked after lock_interval of time.
 		"""
-		lock_interval = 10 # In sec
+		lock_interval = 2 # In sec
 		tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=1, lock_interval=lock_interval)
 		# Clear the cache by setting attempt as success
 		tracker.add_success_attempt()

frappe/tests/test_boilerplate.py

@@ -0,0 +1,88 @@
+import ast
+import glob
+import os
+import shutil
+import unittest
+from unittest.mock import patch
+
+import frappe
+from frappe.utils.boilerplate import make_boilerplate
+
+
+class TestBoilerPlate(unittest.TestCase):
+	@classmethod
+	def tearDownClass(cls):
+
+		bench_path = frappe.utils.get_bench_path()
+		test_app_dir = os.path.join(bench_path, "apps", "test_app")
+		if os.path.exists(test_app_dir):
+			shutil.rmtree(test_app_dir)
+
+	def test_create_app(self):
+		title = "Test App"
+		description = "Test app for unit testing"
+		publisher = "Test Publisher"
+		email = "example@example.org"
+		icon = ""  # empty -> default
+		color = ""
+		app_license = "MIT"
+
+		user_input = [
+			title,
+			description,
+			publisher,
+			email,
+			icon,
+			color,
+			app_license,
+		]
+
+		bench_path = frappe.utils.get_bench_path()
+		apps_dir = os.path.join(bench_path, "apps")
+		app_name = "test_app"
+
+		with patch("builtins.input", side_effect=user_input):
+			make_boilerplate(apps_dir, app_name)
+
+		root_paths = [
+			app_name,
+			"requirements.txt",
+			"README.md",
+			"setup.py",
+			"license.txt",
+			".git",
+		]
+		paths_inside_app = [
+			"__init__.py",
+			"hooks.py",
+			"patches.txt",
+			"templates",
+			"www",
+			"config",
+			"modules.txt",
+			"public",
+			app_name,
+		]
+
+		new_app_dir = os.path.join(bench_path, apps_dir, app_name)
+
+		all_paths = list()
+
+		for path in root_paths:
+			all_paths.append(os.path.join(new_app_dir, path))
+
+		for path in paths_inside_app:
+			all_paths.append(os.path.join(new_app_dir, app_name, path))
+
+		for path in all_paths:
+			self.assertTrue(os.path.exists(path), msg=f"{path} should exist in new app")
+
+		# check if python files are parsable
+		python_files = glob.glob(new_app_dir + "**/*.py", recursive=True)
+
+		for python_file in python_files:
+			with open(python_file) as p:
+				try:
+					ast.parse(p.read())
+				except Exception as e:
+					self.fail(f"Can't parse python file in new app: {python_file}\n" + str(e))

frappe/tests/test_commands.py

@@ -115,12 +115,12 @@ class TestCommands(BaseTestCommands):
 	def test_execute(self):
 		# test 1: execute a command expecting a numeric output
 		self.execute("bench --site {site} execute frappe.db.get_database_size")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIsInstance(float(self.stdout), float)
 
 		# test 2: execute a command expecting an errored output as local won't exist
 		self.execute("bench --site {site} execute frappe.local.site")
-		self.assertEquals(self.returncode, 1)
+		self.assertEqual(self.returncode, 1)
 		self.assertIsNotNone(self.stderr)
 
 		# test 3: execute a command with kwargs
@@ -128,8 +128,8 @@ class TestCommands(BaseTestCommands):
 		# terminal command has been escaped to avoid .format string replacement
 		# The returned value has quotes which have been trimmed for the test
 		self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
-		self.assertEquals(self.returncode, 0)
-		self.assertEquals(self.stdout[1:-1], frappe.bold(text="DocType"))
+		self.assertEqual(self.returncode, 0)
+		self.assertEqual(self.stdout[1:-1], frappe.bold(text="DocType"))
 
 	def test_backup(self):
 		backup = {
@@ -155,7 +155,7 @@ class TestCommands(BaseTestCommands):
 		self.execute("bench --site {site} backup")
 		after_backup = fetch_latest_backups()
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("successfully completed", self.stdout)
 		self.assertNotEqual(before_backup["database"], after_backup["database"])
 
@@ -164,7 +164,7 @@ class TestCommands(BaseTestCommands):
 		self.execute("bench --site {site} backup --with-files")
 		after_backup = fetch_latest_backups()
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("successfully completed", self.stdout)
 		self.assertIn("with files", self.stdout)
 		self.assertNotEqual(before_backup, after_backup)
@@ -175,7 +175,7 @@ class TestCommands(BaseTestCommands):
 		backup_path = os.path.join(home, "backups")
 		self.execute("bench --site {site} backup --backup-path {backup_path}", {"backup_path": backup_path})
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertTrue(os.path.exists(backup_path))
 		self.assertGreaterEqual(len(os.listdir(backup_path)), 2)
 
@@ -200,19 +200,19 @@ class TestCommands(BaseTestCommands):
 			kwargs,
 		)
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		for path in kwargs.values():
 			self.assertTrue(os.path.exists(path))
 
 		# test 5: take a backup with --compress
 		self.execute("bench --site {site} backup --with-files --compress")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		compressed_files = glob.glob(site_backup_path + "/*.tgz")
 		self.assertGreater(len(compressed_files), 0)
 
 		# test 6: take a backup with --verbose
 		self.execute("bench --site {site} backup --verbose")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 
 		# test 7: take a backup with frappe.conf.backup.includes
 		self.execute(
@@ -220,7 +220,7 @@ class TestCommands(BaseTestCommands):
 			{"includes": json.dumps(backup["includes"])},
 		)
 		self.execute("bench --site {site} backup --verbose")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
 
@@ -230,7 +230,7 @@ class TestCommands(BaseTestCommands):
 			{"excludes": json.dumps(backup["excludes"])},
 		)
 		self.execute("bench --site {site} backup --verbose")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
 		self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
@@ -240,7 +240,7 @@ class TestCommands(BaseTestCommands):
 			"bench --site {site} backup --include '{include}'",
 			{"include": ",".join(backup["includes"]["includes"])},
 		)
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
 
@@ -249,13 +249,13 @@ class TestCommands(BaseTestCommands):
 			"bench --site {site} backup --exclude '{exclude}'",
 			{"exclude": ",".join(backup["excludes"]["excludes"])},
 		)
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
 
 		# test 11: take a backup with --ignore-backup-conf
 		self.execute("bench --site {site} backup --ignore-backup-conf")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups()["database"]
 		self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database))
 
@@ -296,7 +296,7 @@ class TestCommands(BaseTestCommands):
 		)
 		site_data.update({"database": json.loads(self.stdout)["database"]})
 		self.execute("bench --site {another_site} restore {database}", site_data)
-		self.assertEquals(self.returncode, 1)
+		self.assertEqual(self.returncode, 1)
 
 	def test_partial_restore(self):
 		_now = now()
@@ -319,8 +319,8 @@ class TestCommands(BaseTestCommands):
 		frappe.db.commit()
 
 		self.execute("bench --site {site} partial-restore {path}", {"path": db_path})
-		self.assertEquals(self.returncode, 0)
-		self.assertEquals(frappe.db.count("ToDo"), todo_count)
+		self.assertEqual(self.returncode, 0)
+		self.assertEqual(frappe.db.count("ToDo"), todo_count)
 
 	def test_recorder(self):
 		frappe.recorder.stop()
@@ -343,18 +343,18 @@ class TestCommands(BaseTestCommands):
 
 		# test 1: remove app from installed_apps global default
 		self.execute("bench --site {site} remove-from-installed-apps {app}", {"app": app})
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.execute("bench --site {site} list-apps")
 		self.assertNotIn(app, self.stdout)
 
 	def test_list_apps(self):
 		# test 1: sanity check for command
 		self.execute("bench --site all list-apps")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 
 		# test 2: bare functionality for single site
 		self.execute("bench --site {site} list-apps")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		list_apps = set([
 			_x.split()[0] for _x in self.stdout.split("\n")
 		])
@@ -367,7 +367,7 @@ class TestCommands(BaseTestCommands):
 
 		# test 3: parse json format
 		self.execute("bench --site all list-apps --format json")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIsInstance(json.loads(self.stdout), dict)
 
 		self.execute("bench --site {site} list-apps --format json")
@@ -379,7 +379,7 @@ class TestCommands(BaseTestCommands):
 	def test_show_config(self):
 		# test 1: sanity check for command
 		self.execute("bench --site all show-config")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 
 		# test 2: test keys in table text
 		self.execute(
@@ -387,13 +387,13 @@ class TestCommands(BaseTestCommands):
 			{"second_order": json.dumps({"test_key": "test_value"})},
 		)
 		self.execute("bench --site {site} show-config")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("test_key.test_key", self.stdout.split())
 		self.assertIn("test_value", self.stdout.split())
 
 		# test 3: parse json format
 		self.execute("bench --site all show-config --format json")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIsInstance(json.loads(self.stdout), dict)
 
 		self.execute("bench --site {site} show-config --format json")
@@ -423,6 +423,6 @@ class TestCommands(BaseTestCommands):
 	def test_frappe_site_env(self):
 		os.putenv('FRAPPE_SITE', frappe.local.site)
 		self.execute("bench execute frappe.ping")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("pong", self.stdout)
 

frappe/tests/test_db.py

@@ -18,7 +18,7 @@ class TestDB(unittest.TestCase):
 	def test_get_value(self):
 		self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator")
-		self.assertNotEquals(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest")
+		self.assertNotEqual(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["<", "Adn"]}), "Administrator")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator")
 

frappe/tests/test_document.py

@@ -6,9 +6,8 @@ import os
 import unittest
 
 import frappe
-from frappe.utils import cint, add_to_date, now
+from frappe.utils import cint
 from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series
-from frappe.exceptions import DoesNotExistError
 
 
 class TestDocument(unittest.TestCase):
@@ -87,13 +86,13 @@ class TestDocument(unittest.TestCase):
 		d.insert()
 		self.assertEqual(frappe.db.get_value("User", d.name), d.name)
 
-	def test_confict_validation(self):
+	def test_conflict_validation(self):
 		d1 = self.test_insert()
 		d2 = frappe.get_doc(d1.doctype, d1.name)
 		d1.save()
 		self.assertRaises(frappe.TimestampMismatchError, d2.save)
 
-	def test_confict_validation_single(self):
+	def test_conflict_validation_single(self):
 		d1 = frappe.get_doc("Website Settings", "Website Settings")
 		d1.home_page = "test-web-page-1"
 
@@ -110,7 +109,7 @@ class TestDocument(unittest.TestCase):
 
 	def test_permission_single(self):
 		frappe.set_user("Guest")
-		d = frappe.get_doc("Website Settings", "Website Settigns")
+		d = frappe.get_doc("Website Settings", "Website Settings")
 		self.assertRaises(frappe.PermissionError, d.save)
 		frappe.set_user("Administrator")
 
@@ -196,41 +195,6 @@ class TestDocument(unittest.TestCase):
 		self.assertTrue(xss not in d.subject)
 		self.assertTrue(escaped_xss in d.subject)
 
-	def test_link_count(self):
-		if os.environ.get('CI'):
-			# cannot run this test reliably in travis due to its handling
-			# of parallelism
-			return
-
-		from frappe.model.utils.link_count import update_link_count
-
-		update_link_count()
-
-		doctype, name = 'User', 'test@example.com'
-
-		d = self.test_insert()
-		d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name})
-
-		d.save()
-
-		link_count = frappe.cache().get_value('_link_count') or {}
-		old_count = link_count.get((doctype, name)) or 0
-
-		frappe.db.commit()
-
-		link_count = frappe.cache().get_value('_link_count') or {}
-		new_count = link_count.get((doctype, name)) or 0
-
-		self.assertEqual(old_count + 1, new_count)
-
-		before_update = frappe.db.get_value(doctype, name, 'idx')
-
-		update_link_count()
-
-		after_update = frappe.db.get_value(doctype, name, 'idx')
-
-		self.assertEqual(before_update + new_count, after_update)
-
 	def test_naming_series(self):
 		data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"]
 

frappe/tests/test_email.py

@@ -6,11 +6,7 @@ from __future__ import unicode_literals
 import unittest, frappe, re, email
 from six import PY3
 
-from frappe.test_runner import make_test_records
-
-make_test_records("User")
-make_test_records("Email Account")
-
+test_dependencies = ['Email Account']
 
 class TestEmail(unittest.TestCase):
 	def setUp(self):

frappe/tests/test_fmt_datetime.py

@@ -45,6 +45,7 @@ class TestFmtDatetime(unittest.TestCase):
 		frappe.db.set_default("time_format", self.pre_test_time_format)
 		frappe.local.user_date_format = None
 		frappe.local.user_time_format = None
+		frappe.db.rollback()
 
 	# Test utility functions
 
@@ -97,28 +98,12 @@ class TestFmtDatetime(unittest.TestCase):
 			self.assertEqual(formatdate(test_date), valid_fmt)
 
 	# Test time formatters
-
 	def test_format_time_forced(self):
 		# Test with forced time formats
 		self.assertEqual(
 			format_time(test_time, 'ss:mm:HH'),
 			test_date_obj.strftime('%S:%M:%H'))
 
-	@unittest.expectedFailure
-	def test_format_time_forced_broken_locale(self):
-		# Test with forced time formats
-		# Currently format_time defaults to HH:mm:ss if the locale is
-		# broken, so this is an expected failure.
-		lang = frappe.local.lang
-		try:
-			# Force fallback from Babel
-			frappe.local.lang = 'FAKE'
-			self.assertEqual(
-				format_time(test_time, 'ss:mm:HH'),
-				test_date_obj.strftime('%S:%M:%H'))
-		finally:
-			frappe.local.lang = lang
-
 	def test_format_time(self):
 		# Test format_time with various default time formats set
 		for fmt, valid_fmt in test_time_formats.items():
@@ -135,21 +120,6 @@ class TestFmtDatetime(unittest.TestCase):
 			format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
 			test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
 
-	@unittest.expectedFailure
-	def test_format_datetime_forced_broken_locale(self):
-		# Test with forced datetime formats
-		# Currently format_datetime defaults to yyyy-MM-dd HH:mm:ss
-		# if the locale is broken, so this is an expected failure.
-		lang = frappe.local.lang
-		# Force fallback from Babel
-		try:
-			frappe.local.lang = 'FAKE'
-			self.assertEqual(
-				format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
-				test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
-		finally:
-			frappe.local.lang = lang
-
 	def test_format_datetime(self):
 		# Test formatdate with various default date formats set
 		for date_fmt, valid_date in test_date_formats.items():

frappe/tests/test_naming.py

@@ -70,9 +70,9 @@ class TestNaming(unittest.TestCase):
 		name = 'TEST-{}-00001'.format(year)
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 1)""", (series,))
 		revert_series_if_last(key, name)
-		count = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
+		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
-		self.assertEqual(count.get('current'), 0)
+		self.assertEqual(current_index.get('current'), 0)
 		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
 
 		series = 'TEST-{}-'.format(year)
@@ -80,9 +80,9 @@ class TestNaming(unittest.TestCase):
 		name = 'TEST-{}-00002'.format(year)
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 2)""", (series,))
 		revert_series_if_last(key, name)
-		count = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
+		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
-		self.assertEqual(count.get('current'), 1)
+		self.assertEqual(current_index.get('current'), 1)
 		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
 
 		series = 'TEST-'
@@ -91,7 +91,29 @@ class TestNaming(unittest.TestCase):
 		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
 		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
 		revert_series_if_last(key, name)
-		count = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
+		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
 
-		self.assertEqual(count.get('current'), 2)
+		self.assertEqual(current_index.get('current'), 2)
+		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+
+		series = 'TEST1-'
+		key = 'TEST1-.#####.-2021-22'
+		name = 'TEST1-00003-2021-22'
+		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
+		revert_series_if_last(key, name)
+		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
+
+		self.assertEqual(current_index.get('current'), 2)
+		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)
+
+		series = ''
+		key = '.#####.-2021-22'
+		name = '00003-2021-22'
+		frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series)
+		frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,))
+		revert_series_if_last(key, name)
+		current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0]
+
+		self.assertEqual(current_index.get('current'), 2)
 		frappe.db.sql("""delete from `tabSeries` where name = %s""", series)

frappe/tests/test_seen.py

@@ -41,6 +41,7 @@ class TestSeen(unittest.TestCase):
 		self.assertTrue('test1@example.com' in json.loads(ev._seen))
 
 		ev.save()
+		ev = frappe.get_doc('Event', ev.name)
 
 		self.assertFalse('test@example.com' in json.loads(ev._seen))
 		self.assertTrue('test1@example.com' in json.loads(ev._seen))

frappe/tests/test_twofactor.py

@@ -8,7 +8,7 @@ from frappe.utils import cint
 from frappe.utils import set_request
 from frappe.auth import validate_ip_address, get_login_attempt_tracker
 from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass,
-	two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj)
+	two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, ExpiredLoginException)
 from . import update_system_settings, get_system_setting
 
 import time
@@ -111,6 +111,7 @@ class TestTwoFactor(unittest.TestCase):
 
 	def test_confirm_otp_token(self):
 		'''Ensure otp is confirmed'''
+		frappe.flags.otp_expiry = 2
 		authenticate_for_2factor(self.user)
 		tmp_id = frappe.local.response['tmp_id']
 		otp = 'wrongotp'
@@ -118,10 +119,11 @@ class TestTwoFactor(unittest.TestCase):
 			confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
 		otp = get_otp(self.user)
 		self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id))
+		frappe.flags.otp_expiry = None
 		if frappe.flags.tests_verbose:
-			print('Sleeping for 30secs to confirm token expires..')
-		time.sleep(30)
-		with self.assertRaises(frappe.AuthenticationError):
+			print('Sleeping for 2 secs to confirm token expires..')
+		time.sleep(2)
+		with self.assertRaises(ExpiredLoginException):
 			confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
 
 	def test_get_verification_obj(self):
@@ -208,12 +210,14 @@ def enable_2fa(bypass_two_factor_auth=0, bypass_restrict_ip_check=0):
 	system_settings.bypass_2fa_for_retricted_ip_users = cint(bypass_two_factor_auth)
 	system_settings.bypass_restrict_ip_check_if_2fa_enabled = cint(bypass_restrict_ip_check)
 	system_settings.two_factor_method = 'OTP App'
+	system_settings.flags.ignore_mandatory = True
 	system_settings.save(ignore_permissions=True)
 	frappe.db.commit()
 
 def disable_2fa():
 	system_settings = frappe.get_doc('System Settings')
 	system_settings.enable_two_factor_auth = 0
+	system_settings.flags.ignore_mandatory = True
 	system_settings.save(ignore_permissions=True)
 	frappe.db.commit()
 

frappe/tests/test_website.py

@@ -48,7 +48,7 @@ class TestWebsite(unittest.TestCase):
 		set_request(method='POST', path='login')
 		response = render.render()
 
-		self.assertEquals(response.status_code, 200)
+		self.assertEqual(response.status_code, 200)
 
 		html = frappe.safe_decode(response.get_data())
 
@@ -76,27 +76,27 @@ class TestWebsite(unittest.TestCase):
 
 		set_request(method='GET', path='/testfrom')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), r'://testto1')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), r'://testto1')
 
 		set_request(method='GET', path='/testfromregex/test')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), r'://testto2')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), r'://testto2')
 
 		set_request(method='GET', path='/testsub/me')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), r'://testto3/me')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), r'://testto3/me')
 
 		set_request(method='GET', path='/test404')
 		response = render.render()
-		self.assertEquals(response.status_code, 404)
+		self.assertEqual(response.status_code, 404)
 
 		set_request(method='GET', path='/testsource')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), '/testtarget')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), '/testtarget')
 
 		delattr(frappe.hooks, 'website_redirects')
 		frappe.cache().delete_key('app_hooks')

frappe/translate.py

@@ -98,6 +98,7 @@ def get_dict(fortype, name=None):
 	translation_assets = cache.hget("translation_assets", frappe.local.lang, shared=True) or {}
 
 	if not asset_key in translation_assets:
+		messages = []
 		if fortype=="doctype":
 			messages = get_messages_from_doctype(name)
 		elif fortype=="page":
@@ -109,14 +110,12 @@ def get_dict(fortype, name=None):
 		elif fortype=="jsfile":
 			messages = get_messages_from_file(name)
 		elif fortype=="boot":
-			messages = []
 			apps = frappe.get_all_apps(True)
 			for app in apps:
 				messages.extend(get_server_messages(app))
-			messages = deduplicate_messages(messages)
 
-			messages += frappe.db.sql("""select 'navbar', item_label from `tabNavbar Item` where item_label is not null""")
-			messages = get_messages_from_include_files()
+			messages += get_messages_from_navbar()
+			messages += get_messages_from_include_files()
 			messages += frappe.db.sql("select 'Print Format:', name from `tabPrint Format`")
 			messages += frappe.db.sql("select 'DocType:', name from tabDocType")
 			messages += frappe.db.sql("select 'Role:', name from tabRole")
@@ -124,6 +123,7 @@ def get_dict(fortype, name=None):
 			messages += frappe.db.sql("select '', format from `tabWorkspace Shortcut` where format is not null")
 			messages += frappe.db.sql("select '', title from `tabOnboarding Step`")
 
+		messages = deduplicate_messages(messages)
 		message_dict = make_dict_from_messages(messages, load_user_translation=False)
 		message_dict.update(get_dict_from_hooks(fortype, name))
 		# remove untranslated
@@ -320,10 +320,22 @@ def get_messages_for_app(app, deduplicate=True):
 
 	# server_messages
 	messages.extend(get_server_messages(app))
+
+	# messages from navbar settings
+	messages.extend(get_messages_from_navbar())
+
 	if deduplicate:
 		messages = deduplicate_messages(messages)
+
 	return messages
 
+
+def get_messages_from_navbar():
+	"""Return all labels from Navbar Items, as specified in Navbar Settings."""
+	labels = frappe.get_all('Navbar Item', filters={'item_label': ('is', 'set')}, pluck='item_label')
+	return [('Navbar:', label, 'Label of a Navbar Item') for label in labels]
+
+
 def get_messages_from_doctype(name):
 	"""Extract all translatable messages for a doctype. Includes labels, Python code,
 	Javascript code, html templates"""
@@ -490,8 +502,14 @@ def get_server_messages(app):
 def get_messages_from_include_files(app_name=None):
 	"""Returns messages from js files included at time of boot like desk.min.js for desk and web"""
 	messages = []
-	for file in (frappe.get_hooks("app_include_js", app_name=app_name) or []) + (frappe.get_hooks("web_include_js", app_name=app_name) or []):
-		messages.extend(get_messages_from_file(os.path.join(frappe.local.sites_path, file)))
+	app_include_js = frappe.get_hooks("app_include_js", app_name=app_name) or []
+	web_include_js = frappe.get_hooks("web_include_js", app_name=app_name) or []
+	include_js = app_include_js + web_include_js
+
+	for js_path in include_js:
+		relative_path = os.path.join(frappe.local.sites_path, js_path.lstrip('/'))
+		messages_from_file = get_messages_from_file(relative_path)
+		messages.extend(messages_from_file)
 
 	return messages
 

frappe/twofactor.py

@@ -73,11 +73,11 @@ def cache_2fa_data(user, token, otp_secret, tmp_id):
 
 	# set increased expiry time for SMS and Email
 	if verification_method in ['SMS', 'Email']:
-		expiry_time = 300
+		expiry_time = frappe.flags.token_expiry or 300
 		frappe.cache().set(tmp_id + '_token', token)
 		frappe.cache().expire(tmp_id + '_token', expiry_time)
 	else:
-		expiry_time = 180
+		expiry_time = frappe.flags.otp_expiry or 180
 	for k, v in iteritems({'_usr': user, '_pwd': pwd, '_otp_secret': otp_secret}):
 		frappe.cache().set("{0}{1}".format(tmp_id, k), v)
 		frappe.cache().expire("{0}{1}".format(tmp_id, k), expiry_time)

frappe/utils/__init__.py

@@ -66,7 +66,7 @@ def get_formatted_email(user, mail=None):
 def extract_email_id(email):
 	"""fetch only the email part of the Email Address"""
 	email_id = parse_addr(email)[1]
-	if email_id and isinstance(email_id, bytes):
+	if email_id and isinstance(email_id, str) and not isinstance(email_id, str):
 		email_id = email_id.decode("utf-8", "ignore")
 	return email_id
 
@@ -161,7 +161,7 @@ def validate_url(txt, throw=False, valid_schemes=None):
 
 		Parameters:
 			throw (`bool`): throws a validationError if URL is not valid
-			valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this 
+			valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this
 
 		Returns:
 			bool: if `txt` represents a valid URL
@@ -225,14 +225,17 @@ def get_gravatar(email):
 
 	return gravatar_url
 
-def get_traceback():
+def get_traceback() -> str:
 	"""
 		 Returns the traceback of the Exception
 	"""
 	exc_type, exc_value, exc_tb = sys.exc_info()
+
+	if not any([exc_type, exc_value, exc_tb]):
+		return ""
+
 	trace_list = traceback.format_exception(exc_type, exc_value, exc_tb)
-	body = "".join(cstr(t) for t in trace_list)
-	return body
+	return "".join(cstr(t) for t in trace_list)
 
 def log(event, details):
 	frappe.logger().info(details)
@@ -391,16 +394,15 @@ def get_site_url(site):
 
 def encode_dict(d, encoding="utf-8"):
 	for key in d:
-		if isinstance(d[key], str):
+		if isinstance(d[key], str) and isinstance(d[key], str):
 			d[key] = d[key].encode(encoding)
 
 	return d
 
 def decode_dict(d, encoding="utf-8"):
 	for key in d:
-		if isinstance(d[key], bytes):
+		if isinstance(d[key], str) and not isinstance(d[key], str):
 			d[key] = d[key].decode(encoding, "ignore")
-
 	return d
 
 @functools.lru_cache()
@@ -425,7 +427,7 @@ def get_test_client():
 	return Client(application)
 
 def get_hook_method(hook_name, fallback=None):
-	method = (frappe.get_hooks().get(hook_name))
+	method = frappe.get_hooks().get(hook_name)
 	if method:
 		method = frappe.get_attr(method[0])
 		return method
@@ -439,6 +441,16 @@ def call_hook_method(hook, *args, **kwargs):
 
 	return out
 
+def is_cli() -> bool:
+	"""Returns True if current instance is being run via a terminal
+	"""
+	invoked_from_terminal = False
+	try:
+		invoked_from_terminal = bool(os.get_terminal_size())
+	except Exception:
+		invoked_from_terminal = sys.stdin.isatty()
+	return invoked_from_terminal
+
 def update_progress_bar(txt, i, l):
 	if os.environ.get("CI"):
 		if i == 0:
@@ -448,7 +460,7 @@ def update_progress_bar(txt, i, l):
 		sys.stdout.flush()
 		return
 
-	if not getattr(frappe.local, 'request', None):
+	if not getattr(frappe.local, 'request', None) or is_cli():
 		lt = len(txt)
 		try:
 			col = 40 if os.get_terminal_size().columns > 80 else 20
@@ -834,3 +846,11 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str):
 		for item in items:
 			records.setdefault(item[key], {}).setdefault(category, []).append(item)
 	return records
+
+def validate_url(url_string):
+	try:
+		result = urlparse(url_string)
+		return result.scheme and result.scheme in ["http", "https", "ftp", "ftps"]
+	except Exception:
+		return False
+

frappe/utils/boilerplate.py

@@ -3,8 +3,6 @@
 
 from __future__ import unicode_literals, print_function
 
-from six.moves import input
-
 import frappe, os, re, git
 from frappe.utils import touch_file, cstr
 
@@ -42,7 +40,7 @@ def make_boilerplate(dest, app_name):
 			if hook_key=="app_name" and hook_val.lower().replace(" ", "_") != hook_val:
 				print("App Name must be all lowercase and without spaces")
 				hook_val = ""
-			elif hook_key=="app_title" and not re.match("^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
+			elif hook_key=="app_title" and not re.match(r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
 				print("App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores")
 				hook_val = ""
 
@@ -254,10 +252,10 @@ app_license = "{app_license}"
 # 	],
 # 	"weekly": [
 # 		"{app_name}.tasks.weekly"
-# 	]
+# 	],
 # 	"monthly": [
 # 		"{app_name}.tasks.monthly"
-# 	]
+# 	],
 # }}
 
 # Testing
@@ -287,26 +285,26 @@ app_license = "{app_license}"
 # User Data Protection
 # --------------------
 
-user_data_fields = [
-	{{
-		"doctype": "{{doctype_1}}",
-		"filter_by": "{{filter_by}}",
-		"redact_fields": ["{{field_1}}", "{{field_2}}"],
-		"partial": 1,
-	}},
-	{{
-		"doctype": "{{doctype_2}}",
-		"filter_by": "{{filter_by}}",
-		"partial": 1,
-	}},
-	{{
-		"doctype": "{{doctype_3}}",
-		"strict": False,
-	}},
-	{{
-		"doctype": "{{doctype_4}}"
-	}}
-]
+# user_data_fields = [
+# 	{{
+# 		"doctype": "{{doctype_1}}",
+# 		"filter_by": "{{filter_by}}",
+# 		"redact_fields": ["{{field_1}}", "{{field_2}}"],
+# 		"partial": 1,
+# 	}},
+# 	{{
+# 		"doctype": "{{doctype_2}}",
+# 		"filter_by": "{{filter_by}}",
+# 		"partial": 1,
+# 	}},
+# 	{{
+# 		"doctype": "{{doctype_3}}",
+# 		"strict": False,
+# 	}},
+# 	{{
+# 		"doctype": "{{doctype_4}}"
+# 	}}
+# ]
 
 # Authentication and authorization
 # --------------------------------

frappe/utils/data.py

@@ -871,7 +871,7 @@ def in_words(integer, in_million=True):
 	return ret.replace('-', ' ')
 
 def is_html(text):
-	if not isinstance(text, frappe.string_types):
+	if not isinstance(text, str):
 		return False
 	return re.search('<[^>]+>', text)
 
@@ -1278,7 +1278,9 @@ def make_filter_dict(filters):
 
 def sanitize_column(column_name):
 	from frappe import _
+	import sqlparse
 	regex = re.compile("^.*[,'();].*")
+	column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower")
 	blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or']
 
 	def _raise_exception():