Merge branch 'develop' into api-updates

.eslintrc

@@ -80,6 +80,7 @@
 		"validate_email": true,
 		"validate_name": true,
 		"validate_phone": true,
+		"validate_url": true,
 		"get_number_format": true,
 		"format_number": true,
 		"format_currency": true,

.flake8

@@ -29,4 +29,5 @@ ignore =
     B950,
     W191,
 
-max-line-length = 200
+max-line-length = 200
+exclude=.github/helper/semgrep_rules

.git-blame-ignore-revs

@@ -0,0 +1,12 @@
+# Since version 2.23 (released in August 2019), git-blame has a feature
+# to ignore or bypass certain commits.
+#
+# This file contains a list of commits that are not likely what you
+# are looking for in a blame, such as mass reformatting or renaming.
+# You can set this file as a default ignore file for blame by running
+# the following command.
+#
+# $ git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Replace use of Class.extend with native JS class
+fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85

.github/helper/semgrep_rules/frappe_correctness.py

@@ -4,25 +4,61 @@ from frappe import _, flt
 from frappe.model.document import Document
 
 
+# ruleid: frappe-modifying-but-not-comitting
 def on_submit(self):
 	if self.value_of_goods == 0:
 		frappe.throw(_('Value of goods cannot be 0'))
-	# ruleid: frappe-modifying-after-submit
 	self.status = 'Submitted'
 
-def on_submit(self):   # noqa
-	if flt(self.per_billed) < 100:
-		self.update_billing_status()
-	else:
-		# todook: frappe-modifying-after-submit
-		self.status = "Completed"
-		self.db_set("status", "Completed")
-
-class TestDoc(Document):
-	pass
-
-	def validate(self):
-		#ruleid: frappe-modifying-child-tables-while-iterating
-		for item in self.child_table:
-			if item.value < 0:
-				self.remove(item)
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	self.status = 'Submitted'
+	self.db_set('status', 'Submitted')
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	x = "y"
+	self.status = x
+	self.db_set('status', x)
+
+
+# ok: frappe-modifying-but-not-comitting
+def on_submit(self):
+	x = "y"
+	self.status = x
+	self.save()
+
+# ruleid: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "uptate"
+
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+
+	def tainted_method(self):
+		self.status = "update"
+		self.db_set("status", "update")
+
+# ok: frappe-modifying-but-not-comitting-other-method
+class DoctypeClass(Document):
+	def on_submit(self):
+		self.good_method()
+		self.tainted_method()
+		self.save()
+
+	def tainted_method(self):
+		self.status = "uptate"

.github/helper/semgrep_rules/translate.js

@@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.')
 // ruleid: frappe-translation-js-splitting
 __('You have {0} subscribers' +
     'in your mailing list', [subscribers.length])
+
+// ok: frappe-translation-js-splitting
+__("Ctrl+Enter to add comment")
+
+// ruleid: frappe-translation-js-splitting
+__('You have {0} subscribers \
+    in your mailing list', [subscribers.length])

.github/helper/semgrep_rules/translate.py

@@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool")
 _("")
 # ruleid: frappe-translation-empty-string
 _('')
+
+
+class Test:
+	# ok: frappe-translation-python-splitting
+	def __init__(
+			args
+			):
+		pass

.github/helper/semgrep_rules/translate.yml

@@ -42,10 +42,10 @@ rules:
 
 - id: frappe-translation-python-splitting
   pattern-either:
-  - pattern: _(...) + ... + _(...)
+  - pattern: _(...) + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
-  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
+  - pattern-regex: '[\s\.]_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '[\s\.]_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations
@@ -54,8 +54,8 @@ rules:
 
 - id: frappe-translation-js-splitting
   pattern-either:
-  - pattern-regex: '__\([^\)]*[\+\\]\s*'
-  - pattern: __('...' + '...')
+  - pattern-regex: '__\([^\)]*[\\]\s+'
+  - pattern: __('...' + '...', ...)
   - pattern: __('...') + __('...')
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.

.github/workflows/publish-assets-develop.yml

@@ -15,11 +15,11 @@ jobs:
           path: 'frappe'
       - uses: actions/setup-node@v1
         with:
-          python-version: '12.x'
+          node-version: 14
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Set up bench for current push
+      - name: Set up bench and build assets
         run: |
           npm install -g yarn
           pip3 install -U frappe-bench
@@ -29,7 +29,7 @@ jobs:
       - name: Package assets
         run: |
           mkdir -p $GITHUB_WORKSPACE/build
-          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/frappe/dist
 
       - name: Publish assets to S3
         uses: jakejarvis/s3-sync-action@master

.github/workflows/publish-assets-releases.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: '3.6'
-      - name: Set up bench for current push
+      - name: Set up bench and build assets
         run: |
           npm install -g yarn
           pip3 install -U frappe-bench
@@ -32,7 +32,7 @@ jobs:
       - name: Package assets
         run: |
           mkdir -p $GITHUB_WORKSPACE/build
-          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/frappe/dist
 
       - name: Get release
         id: get_release

.github/workflows/semgrep.yml

@@ -4,6 +4,8 @@ on:
   pull_request:
     branches:
       - develop
+      - version-13-hotfix
+      - version-13-pre-release
 jobs:
   semgrep:
     name: Frappe Linter

.github/workflows/ci-tests.yml → .github/workflows/server-mariadb-tests.yml

@@ -1,10 +1,8 @@
-name: CI
+name: Server
 
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
   workflow_dispatch:
-  push:
 
 jobs:
   test:
@@ -13,23 +11,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-       include:
-        - DB: "mariadb"
-          TYPE: "server"
-          JOB_NAME: "Python MariaDB"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
+       container: [1, 2]
 
-        - DB: "postgres"
-          TYPE: "server"
-          JOB_NAME: "Python PostgreSQL"
-          RUN_COMMAND: bench --site test_site run-tests --coverage
-
-        - DB: "mariadb"
-          TYPE: "ui"
-          JOB_NAME: "UI MariaDB"
-          RUN_COMMAND:  bench --site test_site run-ui-tests frappe --headless
-
-    name: ${{ matrix.JOB_NAME }}
+    name: Python Unit Tests (MariaDB)
 
     services:
       mysql:
@@ -40,18 +24,6 @@ jobs:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
 
-      postgres:
-        image: postgres:12.4
-        env:
-          POSTGRES_PASSWORD: travis
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
       - name: Clone
         uses: actions/checkout@v2
@@ -63,7 +35,7 @@ jobs:
 
       - uses: actions/setup-node@v2
         with:
-          node-version: '12'
+          node-version: 14
           check-latest: true
 
       - name: Add to Hosts
@@ -104,68 +76,54 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
 
-      - name: Cache cypress binary
-        if: matrix.TYPE == 'ui'
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: ${{ runner.os }}-cypress-
-          restore-keys: |
-            ${{ runner.os }}-cypress-
-            ${{ runner.os }}-
-
       - name: Install Dependencies
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
         env:
           BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
           AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
-          TYPE: ${{ matrix.TYPE }}
+          TYPE: server
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          DB: mariadb
+          TYPE: server
 
-      - name: Run Set-Up
-        if: matrix.TYPE == 'ui'
-        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
-        env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
-
-      - name: Setup tmate session
-        if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
-        uses: mxschmitt/action-tmate@v3
 
       - name: Run Tests
-        run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
         env:
-          DB: ${{ matrix.DB }}
-          TYPE: ${{ matrix.TYPE }}
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
-      - name: Coverage - Pull Request
-        if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
+      - name: Upload Coverage Data
         run: |
           cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github
-          
-      - name: Coverage - Push
-        if: matrix.TYPE == 'server' && github.event_name == 'push'
+          COVERALLS_FLAG_NAME: run-${{ matrix.container }}
+          COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
+          COVERALLS_PARALLEL: true
+
+  coveralls:
+    name: Coverage Wrap Up
+    needs: test
+    container: python:3-slim
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Coveralls Finished
         run: |
-          cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
           cd ${GITHUB_WORKSPACE}
-          pip install coveralls==2.2.0
-          pip install coverage==4.5.4
-          coveralls --service=github-actions
+          pip3 install coverage==5.5
+          pip3 install coveralls==3.0.1
+          coveralls --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-          COVERALLS_SERVICE_NAME: github-actions

.github/workflows/server-postgres-tests.yml

@@ -0,0 +1,100 @@
+name: Server
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       container: [1, 2]
+
+    name: Python Unit Tests (Postgres)
+
+    services:
+      postgres:
+        image: postgres:12.4
+        env:
+          POSTGRES_PASSWORD: travis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: server
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: postgres
+          TYPE: server
+
+      - name: Run Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
+        env:
+          CI_BUILD_ID: ${{ github.run_id }}
+          ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

.github/workflows/ui-tests.yml

@@ -0,0 +1,105 @@
+name: UI
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-18.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+       containers: [1, 2]
+
+    name: UI Tests (Cypress)
+
+    services:
+      mysql:
+        image: mariadb:10.3
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: YES
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
+
+    steps:
+      - name: Clone
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 14
+          check-latest: true
+
+      - name: Add to Hosts
+        run: |
+          echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+
+      - uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+
+      - name: Cache cypress binary
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache
+          key: ${{ runner.os }}-cypress-
+          restore-keys: |
+            ${{ runner.os }}-cypress-
+            ${{ runner.os }}-
+
+      - name: Install Dependencies
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
+        env:
+          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
+          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          TYPE: ui
+
+      - name: Install
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
+        env:
+          DB: mariadb
+          TYPE: ui
+
+      - name: Site Setup
+        run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
+
+      - name: UI Tests
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

.gitignore

@@ -9,6 +9,7 @@ locale
 dist/
 # build/
 frappe/docs/current
+frappe/public/dist
 .vscode
 node_modules
 .kdev4/

.mergify.yml

@@ -3,9 +3,12 @@ pull_request_rules:
     conditions:
       - status-success=Sider
       - status-success=Semantic Pull Request
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label!=squash
@@ -16,9 +19,12 @@ pull_request_rules:
   - name: Automatic squash on CI success and review
     conditions:
       - status-success=Sider
-      - status-success=Python MariaDB
-      - status-success=Python PostgreSQL
-      - status-success=UI MariaDB
+      - status-success=Python Unit Tests (MariaDB) (1)
+      - status-success=Python Unit Tests (MariaDB) (2)
+      - status-success=Python Unit Tests (Postgres) (1)
+      - status-success=Python Unit Tests (Postgres) (2)
+      - status-success=UI Tests (Cypress) (1)
+      - status-success=UI Tests (Cypress) (2)
       - status-success=security/snyk (frappe)
       - label!=dont-merge
       - label=squash

cypress/fixtures/data_field_validation_doctype.js

@@ -0,0 +1,65 @@
+export default {
+	name: 'Validation Test',
+	custom: 1,
+	actions: [],
+	creation: '2019-03-15 06:29:07.215072',
+	doctype: 'DocType',
+	editable_grid: 1,
+	engine: 'InnoDB',
+	fields: [
+		{
+			fieldname: 'email',
+			fieldtype: 'Data',
+			label: 'Email',
+			options: 'Email'
+		},
+		{
+			fieldname: 'URL',
+			fieldtype: 'Data',
+			label: 'URL',
+			options: 'URL'
+		},
+		{
+			fieldname: 'Phone',
+			fieldtype: 'Data',
+			label: 'Phone',
+			options: 'Phone'
+		},
+		{
+			fieldname: 'person_name',
+			fieldtype: 'Data',
+			label: 'Person Name',
+			options: 'Name'
+		},
+		{
+			fieldname: 'read_only_url',
+			fieldtype: 'Data',
+			label: 'Read Only URL',
+			options: 'URL',
+			read_only: '1',
+			default: 'https://frappe.io'
+		}
+	],
+	issingle: 1,
+	links: [],
+	modified: '2021-04-19 14:40:53.127615',
+	modified_by: 'Administrator',
+	module: 'Custom',
+	owner: 'Administrator',
+	permissions: [
+		{
+			create: 1,
+			delete: 1,
+			email: 1,
+			print: 1,
+			read: 1,
+			role: 'System Manager',
+			share: 1,
+			write: 1
+		}
+	],
+	quick_entry: 1,
+	sort_field: 'modified',
+	sort_order: 'ASC',
+	track_changes: 1
+};

cypress/integration/data_field_form_validation.js

@@ -0,0 +1,43 @@
+import data_field_validation_doctype from '../fixtures/data_field_validation_doctype';
+const doctype_name = data_field_validation_doctype.name;
+
+
+context('Data Field Input Validation in New Form', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+		return cy.insert_doc('DocType', data_field_validation_doctype, true);
+	});
+
+	function validateField(fieldname, invalid_value, valid_value) {
+		// Invalid, should have has-error class
+		cy.get_field(fieldname).clear().type(invalid_value).blur();
+		cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('have.class', 'has-error');
+		// Valid value, should not have has-error class
+		cy.get_field(fieldname).clear().type(valid_value);
+		cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('not.have.class', 'has-error');
+	}
+
+	describe('Data Field Options', () => {
+		it('should validate email address', () => {
+			cy.new_form(doctype_name);
+			validateField('email', 'captian', 'hello@test.com');
+		});
+
+		it('should validate URL', () => {
+			validateField('url', 'jkl', 'https://frappe.io');
+			validateField('url', 'abcd.com', 'http://google.com/home');
+			validateField('url', '&&http://google.uae', 'gopher://frappe.io');
+			validateField('url', 'ftt2:://google.in?q=news', 'ftps2://frappe.io/__/#home');
+			validateField('url', 'ftt2://', 'ntps://localhost'); // For intranet URLs
+		});
+
+		it('should validate phone number', () => {
+			validateField('phone', 'america', '89787878');
+		});
+
+		it('should validate name', () => {
+			validateField('person_name', ' 777Hello', 'James Bond');
+		});
+	});
+});

cypress/integration/recorder.js

@@ -50,7 +50,7 @@ context('Recorder', () => {
 		cy.get('.result-list').should('contain', '/api/method/frappe.desk.reportview.get');
 	});
 
-	it.only('Recorder View Request', () => {
+	it('Recorder View Request', () => {
 		cy.get('.primary-action').should('contain', 'Start').click();
 
 		cy.visit('/app/List/DocType/List');

cypress/integration/url_data_field.js

@@ -0,0 +1,43 @@
+import data_field_validation_doctype from '../fixtures/data_field_validation_doctype';
+
+const doctype_name = data_field_validation_doctype.name;
+
+context('URL Data Field Input', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+		return cy.insert_doc('DocType', data_field_validation_doctype, true);
+	});
+
+
+	describe('URL Data Field Input ', () => {
+		it('should not show URL link button without focus', () => {
+			cy.new_form(doctype_name);
+			cy.get_field('url').clear().type('https://frappe.io');
+			cy.get_field('url').blur().wait(500);
+			cy.get('.link-btn').should('not.be.visible');
+		});
+
+		it('should show URL link button on focus', () => {
+			cy.get_field('url').focus().wait(500);
+			cy.get('.link-btn').should('be.visible');
+		});
+
+		it('should not show URL link button for invalid URL', () => {
+			cy.get_field('url').clear().type('fuzzbuzz');
+			cy.get('.link-btn').should('not.be.visible');
+		});
+
+		it('should have valid URL link with target _blank', () => {
+			cy.get_field('url').clear().type('https://frappe.io');
+			cy.get('.link-btn .btn-open').should('have.attr', 'href', 'https://frappe.io');
+			cy.get('.link-btn .btn-open').should('have.attr', 'target', '_blank');
+		});
+
+		it('should inject anchor tag in read-only URL data field', () => {
+			cy.get('[data-fieldname="read_only_url"]')
+				.find('a')
+				.should('have.attr', 'target', '_blank');
+		});
+	});
+});

esbuild/esbuild.js

@@ -0,0 +1,486 @@
+/* eslint-disable no-console */
+let path = require("path");
+let fs = require("fs");
+let glob = require("fast-glob");
+let esbuild = require("esbuild");
+let vue = require("esbuild-vue");
+let yargs = require("yargs");
+let cliui = require("cliui")();
+let chalk = require("chalk");
+let html_plugin = require("./frappe-html");
+let postCssPlugin = require("esbuild-plugin-postcss2").default;
+let ignore_assets = require("./ignore-assets");
+let sass_options = require("./sass_options");
+let {
+	app_list,
+	assets_path,
+	apps_path,
+	sites_path,
+	get_app_path,
+	get_public_path,
+	log,
+	log_warn,
+	log_error,
+	bench_path,
+	get_redis_subscriber
+} = require("./utils");
+
+let argv = yargs
+	.usage("Usage: node esbuild [options]")
+	.option("apps", {
+		type: "string",
+		description: "Run build for specific apps"
+	})
+	.option("skip_frappe", {
+		type: "boolean",
+		description: "Skip building frappe assets"
+	})
+	.option("files", {
+		type: "string",
+		description: "Run build for specified bundles"
+	})
+	.option("watch", {
+		type: "boolean",
+		description: "Run in watch mode and rebuild on file changes"
+	})
+	.option("production", {
+		type: "boolean",
+		description: "Run build in production mode"
+	})
+	.option("run-build-command", {
+		type: "boolean",
+		description: "Run build command for apps"
+	})
+	.example(
+		"node esbuild --apps frappe,erpnext",
+		"Run build only for frappe and erpnext"
+	)
+	.example(
+		"node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js",
+		"Run build only for specified bundles"
+	)
+	.version(false).argv;
+
+const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter(
+	app => !(argv.skip_frappe && app == "frappe")
+);
+const FILES_TO_BUILD = argv.files ? argv.files.split(",") : [];
+const WATCH_MODE = Boolean(argv.watch);
+const PRODUCTION = Boolean(argv.production);
+const RUN_BUILD_COMMAND = !WATCH_MODE && Boolean(argv["run-build-command"]);
+
+const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`;
+const NODE_PATHS = [].concat(
+	// node_modules of apps directly importable
+	app_list
+		.map(app => path.resolve(get_app_path(app), "../node_modules"))
+		.filter(fs.existsSync),
+	// import js file of any app if you provide the full path
+	app_list
+		.map(app => path.resolve(get_app_path(app), ".."))
+		.filter(fs.existsSync)
+);
+
+execute()
+	.then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS))
+	.catch(e => console.error(e));
+
+if (WATCH_MODE) {
+	// listen for open files in editor event
+	open_in_editor();
+}
+
+async function execute() {
+	console.time(TOTAL_BUILD_TIME);
+	if (!FILES_TO_BUILD.length) {
+		await clean_dist_folders(APPS);
+	}
+
+	let result;
+	try {
+		result = await build_assets_for_apps(APPS, FILES_TO_BUILD);
+	} catch (e) {
+		log_error("There were some problems during build");
+		log();
+		log(chalk.dim(e.stack));
+		return;
+	}
+
+	if (!WATCH_MODE) {
+		log_built_assets(result.metafile);
+		console.timeEnd(TOTAL_BUILD_TIME);
+		log();
+	} else {
+		log("Watching for changes...");
+	}
+	return await write_assets_json(result.metafile);
+}
+
+function build_assets_for_apps(apps, files) {
+	let { include_patterns, ignore_patterns } = files.length
+		? get_files_to_build(files)
+		: get_all_files_to_build(apps);
+
+	return glob(include_patterns, { ignore: ignore_patterns }).then(files => {
+		let output_path = assets_path;
+
+		let file_map = {};
+		for (let file of files) {
+			let relative_app_path = path.relative(apps_path, file);
+			let app = relative_app_path.split(path.sep)[0];
+
+			let extension = path.extname(file);
+			let output_name = path.basename(file, extension);
+			if (
+				[".css", ".scss", ".less", ".sass", ".styl"].includes(extension)
+			) {
+				output_name = path.join("css", output_name);
+			} else if ([".js", ".ts"].includes(extension)) {
+				output_name = path.join("js", output_name);
+			}
+			output_name = path.join(app, "dist", output_name);
+
+			if (Object.keys(file_map).includes(output_name)) {
+				log_warn(
+					`Duplicate output file ${output_name} generated from ${file}`
+				);
+			}
+
+			file_map[output_name] = file;
+		}
+
+		return build_files({
+			files: file_map,
+			outdir: output_path
+		});
+	});
+}
+
+function get_all_files_to_build(apps) {
+	let include_patterns = [];
+	let ignore_patterns = [];
+
+	for (let app of apps) {
+		let public_path = get_public_path(app);
+		include_patterns.push(
+			path.resolve(
+				public_path,
+				"**",
+				"*.bundle.{js,ts,css,sass,scss,less,styl}"
+			)
+		);
+		ignore_patterns.push(
+			path.resolve(public_path, "node_modules"),
+			path.resolve(public_path, "dist")
+		);
+	}
+
+	return {
+		include_patterns,
+		ignore_patterns
+	};
+}
+
+function get_files_to_build(files) {
+	// files: ['frappe/website.bundle.js', 'erpnext/main.bundle.js']
+	let include_patterns = [];
+	let ignore_patterns = [];
+
+	for (let file of files) {
+		let [app, bundle] = file.split("/");
+		let public_path = get_public_path(app);
+		include_patterns.push(path.resolve(public_path, "**", bundle));
+		ignore_patterns.push(
+			path.resolve(public_path, "node_modules"),
+			path.resolve(public_path, "dist")
+		);
+	}
+
+	return {
+		include_patterns,
+		ignore_patterns
+	};
+}
+
+function build_files({ files, outdir }) {
+	return esbuild.build({
+		entryPoints: files,
+		entryNames: "[dir]/[name].[hash]",
+		outdir,
+		sourcemap: true,
+		bundle: true,
+		metafile: true,
+		minify: PRODUCTION,
+		nodePaths: NODE_PATHS,
+		define: {
+			"process.env.NODE_ENV": JSON.stringify(
+				PRODUCTION ? "production" : "development"
+			)
+		},
+		plugins: [
+			html_plugin,
+			ignore_assets,
+			vue(),
+			postCssPlugin({
+				plugins: [require("autoprefixer")],
+				sassOptions: sass_options
+			})
+		],
+		watch: get_watch_config()
+	});
+}
+
+function get_watch_config() {
+	if (WATCH_MODE) {
+		return {
+			async onRebuild(error, result) {
+				if (error) {
+					log_error("There was an error during rebuilding changes.");
+					log();
+					log(chalk.dim(error.stack));
+					notify_redis({ error });
+				} else {
+					let {
+						assets_json,
+						prev_assets_json
+					} = await write_assets_json(result.metafile);
+					if (prev_assets_json) {
+						log_rebuilt_assets(prev_assets_json, assets_json);
+					}
+					notify_redis({ success: true });
+				}
+			}
+		};
+	}
+	return null;
+}
+
+async function clean_dist_folders(apps) {
+	for (let app of apps) {
+		let public_path = get_public_path(app);
+		await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), {
+			recursive: true
+		});
+		await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), {
+			recursive: true
+		});
+	}
+}
+
+function log_built_assets(metafile) {
+	let column_widths = [60, 20];
+	cliui.div(
+		{
+			text: chalk.cyan.bold("File"),
+			width: column_widths[0]
+		},
+		{
+			text: chalk.cyan.bold("Size"),
+			width: column_widths[1]
+		}
+	);
+	cliui.div("");
+
+	let output_by_dist_path = {};
+	for (let outfile in metafile.outputs) {
+		if (outfile.endsWith(".map")) continue;
+		let data = metafile.outputs[outfile];
+		outfile = path.resolve(outfile);
+		outfile = path.relative(assets_path, outfile);
+		let filename = path.basename(outfile);
+		let dist_path = outfile.replace(filename, "");
+		output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || [];
+		output_by_dist_path[dist_path].push({
+			name: filename,
+			size: (data.bytes / 1000).toFixed(2) + " Kb"
+		});
+	}
+
+	for (let dist_path in output_by_dist_path) {
+		let files = output_by_dist_path[dist_path];
+		cliui.div({
+			text: dist_path,
+			width: column_widths[0]
+		});
+
+		for (let i in files) {
+			let file = files[i];
+			let branch = "";
+			if (i < files.length - 1) {
+				branch = "├─ ";
+			} else {
+				branch = "└─ ";
+			}
+			let color = file.name.endsWith(".js") ? "green" : "blue";
+			cliui.div(
+				{
+					text: branch + chalk[color]("" + file.name),
+					width: column_widths[0]
+				},
+				{
+					text: file.size,
+					width: column_widths[1]
+				}
+			);
+		}
+		cliui.div("");
+	}
+	log(cliui.toString());
+}
+
+// to store previous build's assets.json for comparison
+let prev_assets_json;
+let curr_assets_json;
+
+async function write_assets_json(metafile) {
+	prev_assets_json = curr_assets_json;
+	let out = {};
+	for (let output in metafile.outputs) {
+		let info = metafile.outputs[output];
+		let asset_path = "/" + path.relative(sites_path, output);
+		if (info.entryPoint) {
+			out[path.basename(info.entryPoint)] = asset_path;
+		}
+	}
+
+	let assets_json_path = path.resolve(
+		assets_path,
+		"frappe",
+		"dist",
+		"assets.json"
+	);
+	let assets_json;
+	try {
+		assets_json = await fs.promises.readFile(assets_json_path, "utf-8");
+	} catch (error) {
+		assets_json = "{}";
+	}
+	assets_json = JSON.parse(assets_json);
+	// update with new values
+	assets_json = Object.assign({}, assets_json, out);
+	curr_assets_json = assets_json;
+
+	await fs.promises.writeFile(
+		assets_json_path,
+		JSON.stringify(assets_json, null, 4)
+	);
+	await update_assets_json_in_cache(assets_json);
+	return {
+		assets_json,
+		prev_assets_json
+	};
+}
+
+function update_assets_json_in_cache(assets_json) {
+	// update assets_json cache in redis, so that it can be read directly by python
+	return new Promise(resolve => {
+		let client = get_redis_subscriber("redis_cache");
+		// handle error event to avoid printing stack traces
+		client.on("error", _ => {
+			log_warn("Cannot connect to redis_cache to update assets_json");
+		});
+		client.set("assets_json", JSON.stringify(assets_json), err => {
+			client.unref();
+			resolve();
+		});
+	});
+}
+
+function run_build_command_for_apps(apps) {
+	let cwd = process.cwd();
+	let { execSync } = require("child_process");
+
+	for (let app of apps) {
+		if (app === "frappe") continue;
+
+		let root_app_path = path.resolve(get_app_path(app), "..");
+		let package_json = path.resolve(root_app_path, "package.json");
+		if (fs.existsSync(package_json)) {
+			let { scripts } = require(package_json);
+			if (scripts && scripts.build) {
+				log("\nRunning build command for", chalk.bold(app));
+				process.chdir(root_app_path);
+				execSync("yarn build", { encoding: "utf8", stdio: "inherit" });
+			}
+		}
+	}
+
+	process.chdir(cwd);
+}
+
+async function notify_redis({ error, success }) {
+	// notify redis which in turns tells socketio to publish this to browser
+	let subscriber = get_redis_subscriber("redis_socketio");
+	subscriber.on("error", _ => {
+		log_warn("Cannot connect to redis_socketio for browser events");
+	});
+
+	let payload = null;
+	if (error) {
+		let formatted = await esbuild.formatMessages(error.errors, {
+			kind: "error",
+			terminalWidth: 100
+		});
+		let stack = error.stack.replace(new RegExp(bench_path, "g"), "");
+		payload = {
+			error,
+			formatted,
+			stack
+		};
+	}
+	if (success) {
+		payload = {
+			success: true
+		};
+	}
+
+	subscriber.publish(
+		"events",
+		JSON.stringify({
+			event: "build_event",
+			message: payload
+		})
+	);
+}
+
+function open_in_editor() {
+	let subscriber = get_redis_subscriber("redis_socketio");
+	subscriber.on("error", _ => {
+		log_warn("Cannot connect to redis_socketio for open_in_editor events");
+	});
+	subscriber.on("message", (event, file) => {
+		if (event === "open_in_editor") {
+			file = JSON.parse(file);
+			let file_path = path.resolve(file.file);
+			log("Opening file in editor:", file_path);
+			let launch = require("launch-editor");
+			launch(`${file_path}:${file.line}:${file.column}`);
+		}
+	});
+	subscriber.subscribe("open_in_editor");
+}
+
+function log_rebuilt_assets(prev_assets, new_assets) {
+	let added_files = [];
+	let old_files = Object.values(prev_assets);
+	let new_files = Object.values(new_assets);
+
+	for (let filepath of new_files) {
+		if (!old_files.includes(filepath)) {
+			added_files.push(filepath);
+		}
+	}
+
+	log(
+		chalk.yellow(
+			`${new Date().toLocaleTimeString()}: Compiled ${
+				added_files.length
+			} files...`
+		)
+	);
+	for (let filepath of added_files) {
+		let filename = path.basename(filepath);
+		log("    " + filename);
+	}
+	log();
+}

esbuild/frappe-html.js

@@ -0,0 +1,43 @@
+module.exports = {
+	name: "frappe-html",
+	setup(build) {
+		let path = require("path");
+		let fs = require("fs/promises");
+
+		build.onResolve({ filter: /\.html$/ }, args => {
+			return {
+				path: path.join(args.resolveDir, args.path),
+				namespace: "frappe-html"
+			};
+		});
+
+		build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => {
+			let filepath = args.path;
+			let filename = path.basename(filepath).split(".")[0];
+
+			return fs
+				.readFile(filepath, "utf-8")
+				.then(content => {
+					content = scrub_html_template(content);
+					return {
+						contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n`
+					};
+				})
+				.catch(() => {
+					return {
+						contents: "",
+						warnings: [
+							{
+								text: `There was an error importing ${filepath}`
+							}
+						]
+					};
+				});
+		});
+	}
+};
+
+function scrub_html_template(content) {
+	content = content.replace(/`/g, "\\`");
+	return content;
+}

esbuild/ignore-assets.js

@@ -0,0 +1,11 @@
+module.exports = {
+	name: "frappe-ignore-asset",
+	setup(build) {
+		build.onResolve({ filter: /^\/assets\// }, args => {
+			return {
+				path: args.path,
+				external: true
+			};
+		});
+	}
+};

esbuild/index.js

@@ -0,0 +1 @@
+require("./esbuild");

esbuild/sass_options.js

@@ -0,0 +1,29 @@
+let path = require("path");
+let { get_app_path, app_list } = require("./utils");
+
+let node_modules_path = path.resolve(
+	get_app_path("frappe"),
+	"..",
+	"node_modules"
+);
+let app_paths = app_list
+	.map(get_app_path)
+	.map(app_path => path.resolve(app_path, ".."));
+
+module.exports = {
+	includePaths: [node_modules_path, ...app_paths],
+	importer: function(url) {
+		if (url.startsWith("~")) {
+			// strip ~ so that it can resolve from node_modules
+			url = url.slice(1);
+		}
+		if (url.endsWith(".css")) {
+			// strip .css from end of path
+			url = url.slice(0, -4);
+		}
+		// normal file, let it go
+		return {
+			file: url
+		};
+	}
+};

esbuild/utils.js

@@ -0,0 +1,145 @@
+const path = require("path");
+const fs = require("fs");
+const chalk = require("chalk");
+
+const frappe_path = path.resolve(__dirname, "..");
+const bench_path = path.resolve(frappe_path, "..", "..");
+const sites_path = path.resolve(bench_path, "sites");
+const apps_path = path.resolve(bench_path, "apps");
+const assets_path = path.resolve(sites_path, "assets");
+const app_list = get_apps_list();
+
+const app_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(apps_path, app, app);
+	return out;
+}, {});
+const public_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(app_paths[app], "public");
+	return out;
+}, {});
+const public_js_paths = app_list.reduce((out, app) => {
+	out[app] = path.resolve(app_paths[app], "public/js");
+	return out;
+}, {});
+
+const bundle_map = app_list.reduce((out, app) => {
+	const public_js_path = public_js_paths[app];
+	if (fs.existsSync(public_js_path)) {
+		const all_files = fs.readdirSync(public_js_path);
+		const js_files = all_files.filter(file => file.endsWith(".js"));
+
+		for (let js_file of js_files) {
+			const filename = path.basename(js_file).split(".")[0];
+			out[path.join(app, "js", filename)] = path.resolve(
+				public_js_path,
+				js_file
+			);
+		}
+	}
+
+	return out;
+}, {});
+
+const get_public_path = app => public_paths[app];
+
+const get_build_json_path = app =>
+	path.resolve(get_public_path(app), "build.json");
+
+function get_build_json(app) {
+	try {
+		return require(get_build_json_path(app));
+	} catch (e) {
+		// build.json does not exist
+		return null;
+	}
+}
+
+function delete_file(path) {
+	if (fs.existsSync(path)) {
+		fs.unlinkSync(path);
+	}
+}
+
+function run_serially(tasks) {
+	let result = Promise.resolve();
+	tasks.forEach(task => {
+		if (task) {
+			result = result.then ? result.then(task) : Promise.resolve();
+		}
+	});
+	return result;
+}
+
+const get_app_path = app => app_paths[app];
+
+function get_apps_list() {
+	return fs
+		.readFileSync(path.resolve(sites_path, "apps.txt"), {
+			encoding: "utf-8"
+		})
+		.split("\n")
+		.filter(Boolean);
+}
+
+function get_cli_arg(name) {
+	let args = process.argv.slice(2);
+	let arg = `--${name}`;
+	let index = args.indexOf(arg);
+
+	let value = null;
+	if (index != -1) {
+		value = true;
+	}
+	if (value && args[index + 1]) {
+		value = args[index + 1];
+	}
+	return value;
+}
+
+function log_error(message, badge = "ERROR") {
+	badge = chalk.white.bgRed(` ${badge} `);
+	console.error(`${badge} ${message}`); // eslint-disable-line no-console
+}
+
+function log_warn(message, badge = "WARN") {
+	badge = chalk.black.bgYellowBright(` ${badge} `);
+	console.warn(`${badge} ${message}`); // eslint-disable-line no-console
+}
+
+function log(...args) {
+	console.log(...args); // eslint-disable-line no-console
+}
+
+function get_redis_subscriber(kind) {
+	// get redis subscriber that aborts after 10 connection attempts
+	let { get_redis_subscriber: get_redis } = require("../node_utils");
+	return get_redis(kind, {
+		retry_strategy: function(options) {
+			// abort after 10 connection attempts
+			if (options.attempt > 10) {
+				return undefined;
+			}
+			return Math.min(options.attempt * 100, 2000);
+		}
+	});
+}
+
+module.exports = {
+	app_list,
+	bench_path,
+	assets_path,
+	sites_path,
+	apps_path,
+	bundle_map,
+	get_public_path,
+	get_build_json_path,
+	get_build_json,
+	get_app_path,
+	delete_file,
+	run_serially,
+	get_cli_arg,
+	log,
+	log_warn,
+	log_error,
+	get_redis_subscriber
+};

frappe/__init__.py

@@ -10,11 +10,16 @@ be used to build database driven apps.
 
 Read the documentation: https://frappeframework.com/docs
 """
-from __future__ import unicode_literals, print_function
+import os, warnings
+
+_dev_server = os.environ.get('DEV_SERVER', False)
+
+if _dev_server:
+	warnings.simplefilter('always', DeprecationWarning)
+	warnings.simplefilter('always', PendingDeprecationWarning)
 
-from six import iteritems, binary_type, text_type, string_types, PY2
 from werkzeug.local import Local, release_local
-import os, sys, importlib, inspect, json
+import sys, importlib, inspect, json
 import typing
 from past.builtins import cmp
 import click
@@ -27,13 +32,6 @@ from .utils.lazy_loader import lazy_import
 # Lazy imports
 faker = lazy_import('faker')
 
-
-# Harmless for Python 3
-# For Python 2 set default encoding to utf-8
-if PY2:
-	reload(sys)
-	sys.setdefaultencoding("utf-8")
-
 __version__ = '14.0.0-dev'
 
 __title__ = "Frappe Framework"
@@ -97,14 +95,14 @@ def _(msg, lang=None, context=None):
 
 def as_unicode(text, encoding='utf-8'):
 	'''Convert to unicode if required'''
-	if isinstance(text, text_type):
+	if isinstance(text, str):
 		return text
 	elif text==None:
 		return ''
-	elif isinstance(text, binary_type):
-		return text_type(text, encoding)
+	elif isinstance(text, bytes):
+		return str(text, encoding)
 	else:
-		return text_type(text)
+		return str(text)
 
 def get_lang_dict(fortype, name=None):
 	"""Returns the translated language dict for the given type and name.
@@ -204,7 +202,7 @@ def init(site, sites_path=None, new_site=False):
 	local.meta_cache = {}
 	local.form_dict = _dict()
 	local.session = _dict()
-	local.dev_server = os.environ.get('DEV_SERVER', False)
+	local.dev_server = _dev_server
 
 	setup_module_map()
 
@@ -597,7 +595,7 @@ def is_whitelisted(method):
 		# strictly sanitize form_dict
 		# escapes html characters like <> except for predefined tags like a, b, ul etc.
 		for key, value in form_dict.items():
-			if isinstance(value, string_types):
+			if isinstance(value, str):
 				form_dict[key] = sanitize_html(value)
 
 def read_only():
@@ -721,7 +719,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc
 		user = session.user
 
 	if doc:
-		if isinstance(doc, string_types):
+		if isinstance(doc, str):
 			doc = get_doc(doctype, doc)
 
 		doctype = doc.doctype
@@ -790,7 +788,7 @@ def set_value(doctype, docname, fieldname, value=None):
 	return frappe.client.set_value(doctype, docname, fieldname, value)
 
 def get_cached_doc(*args, **kwargs):
-	if args and len(args) > 1 and isinstance(args[1], text_type):
+	if args and len(args) > 1 and isinstance(args[1], str):
 		key = get_document_cache_key(args[0], args[1])
 		# local cache
 		doc = local.document_cache.get(key)
@@ -821,7 +819,7 @@ def clear_document_cache(doctype, name):
 
 def get_cached_value(doctype, name, fieldname, as_dict=False):
 	doc = get_cached_doc(doctype, name)
-	if isinstance(fieldname, string_types):
+	if isinstance(fieldname, str):
 		if as_dict:
 			throw('Cannot make dict for single fieldname')
 		return doc.get(fieldname)
@@ -1027,7 +1025,7 @@ def get_doc_hooks():
 	if not hasattr(local, 'doc_events_hooks'):
 		hooks = get_hooks('doc_events', {})
 		out = {}
-		for key, value in iteritems(hooks):
+		for key, value in hooks.items():
 			if isinstance(key, tuple):
 				for doctype in key:
 					append_hook(out, doctype, value)
@@ -1144,7 +1142,7 @@ def get_file_json(path):
 
 def read_file(path, raise_not_found=False):
 	"""Open a file and return its content as Unicode."""
-	if isinstance(path, text_type):
+	if isinstance(path, str):
 		path = path.encode("utf-8")
 
 	if os.path.exists(path):
@@ -1167,7 +1165,7 @@ def get_attr(method_string):
 
 def call(fn, *args, **kwargs):
 	"""Call a function and match arguments."""
-	if isinstance(fn, string_types):
+	if isinstance(fn, str):
 		fn = get_attr(fn)
 
 	newargs = get_newargs(fn, kwargs)
@@ -1178,13 +1176,9 @@ def get_newargs(fn, kwargs):
 	if hasattr(fn, 'fnargs'):
 		fnargs = fn.fnargs
 	else:
-		try:
-			fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
-		except ValueError:
-			fnargs = inspect.getfullargspec(fn).args
-			varargs = inspect.getfullargspec(fn).varargs
-			varkw = inspect.getfullargspec(fn).varkw
-			defaults = inspect.getfullargspec(fn).defaults
+		fnargs = inspect.getfullargspec(fn).args
+		fnargs.extend(inspect.getfullargspec(fn).kwonlyargs)
+		varkw = inspect.getfullargspec(fn).varkw
 
 	newargs = {}
 	for a in kwargs:
@@ -1626,6 +1620,12 @@ def enqueue(*args, **kwargs):
 	import frappe.utils.background_jobs
 	return frappe.utils.background_jobs.enqueue(*args, **kwargs)
 
+def task(**task_kwargs):
+	def decorator_task(f):
+		f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs)
+		return f
+	return decorator_task
+
 def enqueue_doc(*args, **kwargs):
 	'''
 		Enqueue method to be executed using a background worker

frappe/app.py

@@ -99,17 +99,7 @@ def application(request):
 		frappe.monitor.stop(response)
 		frappe.recorder.dump()
 
-		if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
-			frappe.logger("frappe.web", allow_site=frappe.local.site).info({
-				"site": get_site_name(request.host),
-				"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
-				"base_url": getattr(request, "base_url", "NOTFOUND"),
-				"full_path": getattr(request, "full_path", "NOTFOUND"),
-				"method": getattr(request, "method", "NOTFOUND"),
-				"scheme": getattr(request, "scheme", "NOTFOUND"),
-				"http_status_code": getattr(response, "status_code", "NOTFOUND")
-			})
-
+		log_request(request, response)
 		process_response(response)
 		frappe.destroy()
 
@@ -137,6 +127,19 @@ def init_request(request):
 	if request.method != "OPTIONS":
 		frappe.local.http_request = frappe.auth.HTTPRequest()
 
+def log_request(request, response):
+	if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
+		frappe.logger("frappe.web", allow_site=frappe.local.site).info({
+			"site": get_site_name(request.host),
+			"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
+			"base_url": getattr(request, "base_url", "NOTFOUND"),
+			"full_path": getattr(request, "full_path", "NOTFOUND"),
+			"method": getattr(request, "method", "NOTFOUND"),
+			"scheme": getattr(request, "scheme", "NOTFOUND"),
+			"http_status_code": getattr(response, "status_code", "NOTFOUND")
+		})
+
+
 def process_response(response):
 	if not response:
 		return
@@ -185,7 +188,7 @@ def make_form_dict(request):
 		args = request.form or request.args
 
 	if not isinstance(args, dict):
-		frappe.throw("Invalid request arguments")
+		frappe.throw(_("Invalid request arguments"))
 
 	try:
 		frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
@@ -201,12 +204,20 @@ def handle_exception(e):
 	response = None
 	http_status_code = getattr(e, "http_status_code", 500)
 	return_as_message = False
+	accept_header = frappe.get_request_header("Accept") or ""
+	respond_as_json = (
+		frappe.get_request_header('Accept')
+		and (frappe.local.is_ajax or 'application/json' in accept_header)
+		or (
+			frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text")
+		)
+	)
 
 	if frappe.conf.get('developer_mode'):
 		# don't fail silently
 		print(frappe.get_traceback())
 
-	if frappe.get_request_header('Accept') and (frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept')):
+	if respond_as_json:
 		# handle ajax responses first
 		# if the request is ajax, send back the trace or error message
 		response = frappe.utils.response.report_error(http_status_code)
@@ -286,8 +297,9 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 	_sites_path = sites_path
 
 	from werkzeug.serving import run_simple
+	patch_werkzeug_reloader()
 
-	if profile:
+	if profile or os.environ.get('USE_PROFILER'):
 		application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))
 
 	if not os.environ.get('NO_STATICS'):
@@ -316,3 +328,23 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 		use_debugger=not in_test_env,
 		use_evalex=not in_test_env,
 		threaded=not no_threading)
+
+def patch_werkzeug_reloader():
+	"""
+	This function monkey patches Werkzeug reloader to ignore reloading files in
+	the __pycache__ directory.
+
+	To be deprecated when upgrading to Werkzeug 2.
+	"""
+
+	from werkzeug._reloader import WatchdogReloaderLoop
+
+	trigger_reload = WatchdogReloaderLoop.trigger_reload
+
+	def custom_trigger_reload(self, filename):
+		if os.path.basename(os.path.dirname(filename)) == "__pycache__":
+			return
+
+		return trigger_reload(self, filename)
+
+	WatchdogReloaderLoop.trigger_reload = custom_trigger_reload

frappe/automation/doctype/auto_repeat/auto_repeat.js

@@ -103,7 +103,7 @@ frappe.ui.form.on('Auto Repeat', {
 frappe.auto_repeat.render_schedule = function(frm) {
 	if (!frm.is_dirty() && frm.doc.status !== 'Disabled') {
 		frm.call("get_auto_repeat_schedule").then(r => {
-			frm.dashboard.wrapper.empty();
+			frm.dashboard.reset();
 			frm.dashboard.add_section(
 				frappe.render_template("auto_repeat_schedule", {
 					schedule_details: r.message || []

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -173,7 +173,7 @@ class TestAutoRepeat(unittest.TestCase):
 			fields=['docstatus'],
 			limit=1
 		)
-		self.assertEquals(docnames[0].docstatus, 1)
+		self.assertEqual(docnames[0].docstatus, 1)
 
 
 def make_auto_repeat(**args):

frappe/boot.py

@@ -42,8 +42,6 @@ def get_bootinfo():
 		bootinfo.user_info = get_user_info()
 		bootinfo.sid = frappe.session['sid']
 
-	bootinfo.user_groups = frappe.get_all('User Group', pluck="name")
-
 	bootinfo.modules = {}
 	bootinfo.module_list = []
 	load_desktop_data(bootinfo)

frappe/build.py

@@ -1,14 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import print_function, unicode_literals
-
 import os
 import re
 import json
 import shutil
-import warnings
-import tempfile
+import subprocess
+from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
 
 import frappe
@@ -16,8 +14,9 @@ from frappe.utils.minify import JavascriptMinify
 
 import click
 import psutil
-from six import iteritems, text_type
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+from simple_chalk import green
+from semantic_version import Version
 
 
 timestamps = {}
@@ -39,35 +38,36 @@ def download_file(url, prefix):
 
 
 def build_missing_files():
-	# check which files dont exist yet from the build.json and tell build.js to build only those!
+	'''Check which files dont exist yet from the assets.json and run build for those files'''
+
 	missing_assets = []
 	current_asset_files = []
-	frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json")
 
 	for type in ["css", "js"]:
-		current_asset_files.extend(
-			[
-				"{0}/{1}".format(type, name)
-				for name in os.listdir(os.path.join(sites_path, "assets", type))
-			]
-		)
+		folder = os.path.join(sites_path, "assets", "frappe", "dist", type)
+		current_asset_files.extend(os.listdir(folder))
 
-	with open(frappe_build) as f:
-		all_asset_files = json.load(f).keys()
+	development = frappe.local.conf.developer_mode or frappe.local.dev_server
+	build_mode = "development" if development else "production"
 
-	for asset in all_asset_files:
-		if asset.replace("concat:", "") not in current_asset_files:
-			missing_assets.append(asset)
+	assets_json = frappe.read_file(frappe.get_app_path('frappe', 'public', 'dist', 'assets.json'))
+	if assets_json:
+		assets_json = frappe.parse_json(assets_json)
 
-	if missing_assets:
-		from subprocess import check_call
-		from shlex import split
+		for bundle_file, output_file in assets_json.items():
+			if not output_file.startswith('/assets/frappe'):
+				continue
 
-		click.secho("\nBuilding missing assets...\n", fg="yellow")
-		command = split(
-			"node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets))
-		)
-		check_call(command, cwd=os.path.join("..", "apps", "frappe"))
+			if os.path.basename(output_file) not in current_asset_files:
+				missing_assets.append(bundle_file)
+
+		if missing_assets:
+			click.secho("\nBuilding missing assets...\n", fg="yellow")
+			files_to_build = ["frappe/" + name for name in missing_assets]
+			bundle(build_mode, files=files_to_build)
+	else:
+		# no assets.json, run full build
+		bundle(build_mode, apps="frappe")
 
 
 def get_assets_link(frappe_head):
@@ -75,8 +75,8 @@ def get_assets_link(frappe_head):
 	from requests import head
 
 	tag = getoutput(
-			"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			" refs/tags/,,' -e 's/\^{}//'"
+			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+			r" refs/tags/,,' -e 's/\^{}//'"
 			% frappe_head
 		)
 
@@ -97,9 +97,7 @@ def download_frappe_assets(verbose=True):
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from simple_chalk import green
 	from subprocess import getoutput
-	from tempfile import mkdtemp
 
 	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
@@ -166,7 +164,7 @@ def symlink(target, link_name, overwrite=False):
 
 	# Create link to target with temporary filename
 	while True:
-		temp_link_name = tempfile.mktemp(dir=link_dir)
+		temp_link_name = mktemp(dir=link_dir)
 
 		# os.* functions mimic as closely as possible system functions
 		# The POSIX symlink() returns EEXIST if link_name already exists
@@ -193,7 +191,8 @@ def symlink(target, link_name, overwrite=False):
 
 
 def setup():
-	global app_paths
+	global app_paths, assets_path
+
 	pymodules = []
 	for app in frappe.get_all_apps(True):
 		try:
@@ -201,51 +200,54 @@ def setup():
 		except ImportError:
 			pass
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
+	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
 
-def get_node_pacman():
-	exec_ = find_executable("yarn")
-	if exec_:
-		return exec_
-	raise ValueError("Yarn not found")
-
-
-def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
+def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
 	"""concat / minify js files"""
 	setup()
-	make_asset_dirs(make_copy=make_copy, restore=restore)
+	make_asset_dirs(hard_link=hard_link)
 
-	pacman = get_node_pacman()
-	mode = "build" if no_compress else "production"
-	command = "{pacman} run {mode}".format(pacman=pacman, mode=mode)
+	mode = "production" if mode == "production" else "build"
+	command = "yarn run {mode}".format(mode=mode)
 
-	if app:
-		command += " --app {app}".format(app=app)
+	if apps:
+		command += " --apps {apps}".format(apps=apps)
 
 	if skip_frappe:
 		command += " --skip_frappe"
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
-	check_yarn()
+	if files:
+		command += " --files {files}".format(files=','.join(files))
+
+	command += " --run-build-command"
+
+	check_node_executable()
+	frappe_app_path = frappe.get_app_path("frappe", "..")
 	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
-def watch(no_compress):
+def watch(apps=None):
 	"""watch and rebuild if necessary"""
 	setup()
 
-	pacman = get_node_pacman()
+	command = "yarn run watch"
+	if apps:
+		command += " --apps {apps}".format(apps=apps)
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
-	check_yarn()
+	check_node_executable()
 	frappe_app_path = frappe.get_app_path("frappe", "..")
-	frappe.commands.popen("{pacman} run watch".format(pacman=pacman),
-		cwd=frappe_app_path, env=get_node_env())
+	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
 
 
-def check_yarn():
+def check_node_executable():
+	node_version = Version(subprocess.getoutput('node -v')[1:])
+	warn = '⚠️ '
+	if node_version.major < 14:
+		click.echo(f"{warn} Please update your node version to 14")
 	if not find_executable("yarn"):
-		print("Please install yarn using below command and try again.\nnpm install -g yarn")
+		click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn")
+	click.echo()
 
 def get_node_env():
 	node_env = {
@@ -266,75 +268,109 @@ def get_safe_max_old_space_size():
 
 	return safe_max_old_space_size
 
-def make_asset_dirs(make_copy=False, restore=False):
-	# don't even think of making assets_path absolute - rm -rf ahead.
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
+def generate_assets_map():
+	symlinks = {}
 
-	for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
+	for app_name in frappe.get_all_apps():
+		app_doc_path = None
 
-	for app_name in frappe.get_all_apps(True):
 		pymodule = frappe.get_module(app_name)
 		app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
-
-		symlinks = []
 		app_public_path = os.path.join(app_base_path, "public")
-		# app/public > assets/app
-		symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
-		# app/node_modules > assets/app/node_modules
-		if os.path.exists(os.path.abspath(app_public_path)):
-			symlinks.append(
-				[
-					os.path.join(app_base_path, "..", "node_modules"),
-					os.path.join(assets_path, app_name, "node_modules"),
-				]
-			)
+		app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
+		app_docs_path = os.path.join(app_base_path, "docs")
+		app_www_docs_path = os.path.join(app_base_path, "www", "docs")
 
-		app_doc_path = None
-		if os.path.isdir(os.path.join(app_base_path, "docs")):
-			app_doc_path = os.path.join(app_base_path, "docs")
+		app_assets = os.path.abspath(app_public_path)
+		app_node_modules = os.path.abspath(app_node_modules_path)
 
-		elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
-			app_doc_path = os.path.join(app_base_path, "www", "docs")
+		# {app}/public > assets/{app}
+		if os.path.isdir(app_assets):
+			symlinks[app_assets] = os.path.join(assets_path, app_name)
 
+		# {app}/node_modules > assets/{app}/node_modules
+		if os.path.isdir(app_node_modules):
+			symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
+
+		# {app}/docs > assets/{app}_docs
+		if os.path.isdir(app_docs_path):
+			app_doc_path = os.path.join(app_base_path, "docs")
+		elif os.path.isdir(app_www_docs_path):
+			app_doc_path = os.path.join(app_base_path, "www", "docs")
 		if app_doc_path:
-			symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
-
-		for source, target in symlinks:
-			source = os.path.abspath(source)
-			if os.path.exists(source):
-				if restore:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-						shutil.copytree(source, target)
-				elif make_copy:
-					if os.path.exists(target):
-						warnings.warn("Target {target} already exists.".format(target=target))
-					else:
-						shutil.copytree(source, target)
-				else:
-					if os.path.exists(target):
-						if os.path.islink(target):
-							os.unlink(target)
-						else:
-							shutil.rmtree(target)
-					try:
-						symlink(source, target, overwrite=True)
-					except OSError:
-						print("Cannot link {} to {}".format(source, target))
-			else:
-				# warnings.warn('Source {source} does not exist.'.format(source = source))
-				pass
+			app_docs = os.path.abspath(app_doc_path)
+			symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
 
+	return symlinks
 
-def build(no_compress=False, verbose=False):
-	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
-	for target, sources in iteritems(get_build_maps()):
+def setup_assets_dirs():
+	for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
+		os.makedirs(dir_path, exist_ok=True)
+
+
+def clear_broken_symlinks():
+	for path in os.listdir(assets_path):
+		path = os.path.join(assets_path, path)
+		if os.path.islink(path) and not os.path.exists(path):
+			os.remove(path)
+
+
+
+def unstrip(message: str) -> str:
+	"""Pads input string on the right side until the last available column in the terminal
+	"""
+	_len = len(message)
+	try:
+		max_str = os.get_terminal_size().columns
+	except Exception:
+		max_str = 80
+
+	if _len < max_str:
+		_rem = max_str - _len
+	else:
+		_rem = max_str % _len
+
+	return f"{message}{' ' * _rem}"
+
+
+def make_asset_dirs(hard_link=False):
+	setup_assets_dirs()
+	clear_broken_symlinks()
+	symlinks = generate_assets_map()
+
+	for source, target in symlinks.items():
+		start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
+		fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
+
+		# Used '\r' instead of '\x1b[1K\r' to print entire lines in smaller terminal sizes
+		try:
+			print(start_message, end="\r")
+			link_assets_dir(source, target, hard_link=hard_link)
+		except Exception:
+			print(fail_message, end="\r")
+
+	print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
+
+
+def link_assets_dir(source, target, hard_link=False):
+	if not os.path.exists(source):
+		return
+
+	if os.path.exists(target):
+		if os.path.islink(target):
+			os.unlink(target)
+		else:
+			shutil.rmtree(target)
+
+	if hard_link:
+		shutil.copytree(source, target, dirs_exist_ok=True)
+	else:
+		symlink(source, target, overwrite=True)
+
+
+def build(no_compress=False, verbose=False):
+	for target, sources in get_build_maps().items():
 		pack(os.path.join(assets_path, target), sources, no_compress, verbose)
 
 
@@ -348,7 +384,7 @@ def get_build_maps():
 		if os.path.exists(path):
 			with open(path) as f:
 				try:
-					for target, sources in iteritems(json.loads(f.read())):
+					for target, sources in (json.loads(f.read() or "{}")).items():
 						# update app path
 						source_paths = []
 						for source in sources:
@@ -381,7 +417,7 @@ def pack(target, sources, no_compress, verbose):
 		timestamps[f] = os.path.getmtime(f)
 		try:
 			with open(f, "r") as sourcefile:
-				data = text_type(sourcefile.read(), "utf-8", errors="ignore")
+				data = str(sourcefile.read(), "utf-8", errors="ignore")
 
 			extn = f.rsplit(".", 1)[1]
 
@@ -396,7 +432,7 @@ def pack(target, sources, no_compress, verbose):
 				jsm.minify(tmpin, tmpout)
 				minified = tmpout.getvalue()
 				if minified:
-					outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
+					outtxt += str(minified or "", "utf-8").strip("\n") + ";"
 
 				if verbose:
 					print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@@ -426,16 +462,16 @@ def html_to_js_template(path, content):
 def scrub_html_template(content):
 	"""Returns HTML content with removed whitespace and comments"""
 	# remove whitespace to a single space
-	content = re.sub("\s+", " ", content)
+	content = re.sub(r"\s+", " ", content)
 
 	# strip comments
-	content = re.sub("(<!--.*?-->)", "", content)
+	content = re.sub(r"(<!--.*?-->)", "", content)
 
 	return content.replace("'", "\'")
 
 
 def files_dirty():
-	for target, sources in iteritems(get_build_maps()):
+	for target, sources in get_build_maps().items():
 		for f in sources:
 			if ":" in f:
 				f, suffix = f.split(":")

frappe/cache_manager.py

@@ -13,6 +13,8 @@ common_default_keys = ["__default", "__global"]
 doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map',
 	'milestone_tracker_map', 'event_consumer_document_type_map')
 
+bench_cache_keys = ('assets_json',)
+
 global_cache_keys = ("app_hooks", "installed_apps", 'all_apps',
 		"app_modules", "module_app", "system_settings",
 		'scheduler_events', 'time_zone', 'webhooks', 'active_domains',
@@ -58,6 +60,7 @@ def clear_global_cache():
 	clear_doctype_cache()
 	clear_website_cache()
 	frappe.cache().delete_value(global_cache_keys)
+	frappe.cache().delete_value(bench_cache_keys)
 	frappe.setup_module_map()
 
 def clear_defaults_cache(user=None):

frappe/change_log/v13/v13_2_0.md

@@ -0,0 +1,32 @@
+# Version 13.2.0 Release Notes
+
+### Features & Enhancements
+
+- Add option to mention a group of users ([#12844](https://github.com/frappe/frappe/pull/12844))
+- Copy DocType / documents across sites ([#12872](https://github.com/frappe/frappe/pull/12872))
+- Scheduler log in notifications ([#1135](https://github.com/frappe/frappe/pull/1135))
+- Add Enable/Disable Webhook via Check Field ([#12842](https://github.com/frappe/frappe/pull/12842))
+- Allow query/custom reports to save custom data in the json field ([#12534](https://github.com/frappe/frappe/pull/12534))
+
+### Fixes
+
+- Load server translations in boot (backport #12848) ([#12852](https://github.com/frappe/frappe/pull/12852))
+- Allow to override dashboard chart properties type/color ([#12846](https://github.com/frappe/frappe/pull/12846))
+- Multi-column paste in grid ([#12861](https://github.com/frappe/frappe/pull/12861))
+- Add log_error and FrappeClient to restricted python ([#12857](https://github.com/frappe/frappe/pull/12857))
+- Redirect Web Form user directly to success URL, if no amount is due ([#12661](https://github.com/frappe/frappe/pull/12661))
+- Attachment pill lock icon redirects to File ([#12864](https://github.com/frappe/frappe/pull/12864))
+- Redirect Web Form user directly to success URL, if no amount is due (backport #12661) ([#12856](https://github.com/frappe/frappe/pull/12856))
+- Remove events to redraw charts ([#12973](https://github.com/frappe/frappe/pull/12973))
+- Don't allow user to remove/change data source file in data import ([#12827](https://github.com/frappe/frappe/pull/12827))
+- Load server translations in boot ([#12848](https://github.com/frappe/frappe/pull/12848))
+- Newly created Workspace not being accessible unless a shortcut u… ([#12866](https://github.com/frappe/frappe/pull/12866))
+- Currency labels in grids ([#12974](https://github.com/frappe/frappe/pull/12974))
+- Handle error while session start ([#12933](https://github.com/frappe/frappe/pull/12933))
+- Add field type check in custom field validation ([#12858](https://github.com/frappe/frappe/pull/12858))
+- Make language select optional and fix breakpoint issues ([#12860](https://github.com/frappe/frappe/pull/12860))
+- Form Dashboard reference link ([#12945](https://github.com/frappe/frappe/pull/12945))
+- Invalid HTML generated by the base template ([#12953](https://github.com/frappe/frappe/pull/12953))
+- Default values were not triggering change event ([#12975](https://github.com/frappe/frappe/pull/12975))
+- Make strings translatable ([#12877](https://github.com/frappe/frappe/pull/12877))
+- Added build-message-files command ([#12950](https://github.com/frappe/frappe/pull/12950))

frappe/change_log/v13/v13_3_0.md

@@ -0,0 +1,49 @@
+# Version 13.3.0 Release Notes
+
+### Features & Enhancements
+
+- Deletion Steps in Data Deletion Tool ([#13124](https://github.com/frappe/frappe/pull/13124))
+- Format Option for list-apps in bench CLI ([#13125](https://github.com/frappe/frappe/pull/13125))
+- Add password fieldtype option for Web Form ([#13093](https://github.com/frappe/frappe/pull/13093))
+- Add simple __repr__ for DocTypes ([#13151](https://github.com/frappe/frappe/pull/13151))
+- Switch theme with left/right keys ([#13077](https://github.com/frappe/frappe/pull/13077))
+- sourceURL for injected javascript ([#13022](https://github.com/frappe/frappe/pull/13022))
+
+### Fixes
+
+- Decode uri before importing file via weblink ([#13026](https://github.com/frappe/frappe/pull/13026))
+- Respond to /api requests as JSON by default ([#13053](https://github.com/frappe/frappe/pull/13053))
+- Disabled checkbox should be disabled ([#13021](https://github.com/frappe/frappe/pull/13021))
+- Moving Site folder across different FileSystems failed ([#13038](https://github.com/frappe/frappe/pull/13038))
+- Freeze screen till the background request is complete ([#13078](https://github.com/frappe/frappe/pull/13078))
+- Added conditional rendering for content field in split section w… ([#13075](https://github.com/frappe/frappe/pull/13075))
+- Show delete button on portal if user has permission to delete document ([#13149](https://github.com/frappe/frappe/pull/13149))
+- Dont disable dialog scroll on focusing a Link/Autocomplete field ([#13119](https://github.com/frappe/frappe/pull/13119))
+- Typo in RecorderDetail.vue ([#13086](https://github.com/frappe/frappe/pull/13086))
+- Error for bench drop-site. Added missing import. ([#13064](https://github.com/frappe/frappe/pull/13064))
+- Report column context ([#13090](https://github.com/frappe/frappe/pull/13090))
+- Different service name for push and pull request events ([#13094](https://github.com/frappe/frappe/pull/13094))
+- Moving Site folder across different FileSystems failed ([#13033](https://github.com/frappe/frappe/pull/13033))
+- Consistent checkboxes on all browsers ([#13042](https://github.com/frappe/frappe/pull/13042))
+- Changed shorcut widgets color picker to dropdown ([#13073](https://github.com/frappe/frappe/pull/13073))
+- Error while exporting reports with duration field ([#13118](https://github.com/frappe/frappe/pull/13118))
+- Add margin to download backup card ([#13079](https://github.com/frappe/frappe/pull/13079))
+- Move mention list generation logic to server-side ([#13074](https://github.com/frappe/frappe/pull/13074))
+- Make strings translatable ([#13046](https://github.com/frappe/frappe/pull/13046))
+- Don't evaluate dynamic properties to check if conflicts exist ([#13186](https://github.com/frappe/frappe/pull/13186))
+- Add __ function in vue global for translation in recorder ([#13089](https://github.com/frappe/frappe/pull/13089))
+- Make strings translatable ([#13076](https://github.com/frappe/frappe/pull/13076))
+- Show config in bench CLI ([#13128](https://github.com/frappe/frappe/pull/13128))
+- Add breadcrumbs for list view ([#13091](https://github.com/frappe/frappe/pull/13091))
+- Do not skip data in save while using shortcut ([#13182](https://github.com/frappe/frappe/pull/13182))
+- Use docfields from options if no docfields are returned from meta ([#13188](https://github.com/frappe/frappe/pull/13188))
+- Disable reloading files in `__pycache__` directory ([#13109](https://github.com/frappe/frappe/pull/13109))
+- RTL stylesheet route to load RTL style on demand. ([#13007](https://github.com/frappe/frappe/pull/13007))
+- Do not show messsage when exception is handled ([#13111](https://github.com/frappe/frappe/pull/13111))
+- Replace parseFloat by Number ([#13082](https://github.com/frappe/frappe/pull/13082))
+- Add margin to download backup card ([#13050](https://github.com/frappe/frappe/pull/13050))
+- Translate report column labels ([#13083](https://github.com/frappe/frappe/pull/13083))
+- Grid row color picker field not working ([#13040](https://github.com/frappe/frappe/pull/13040))
+- Improve oauthlib implementation ([#13045](https://github.com/frappe/frappe/pull/13045))
+- Replace filter_by like with full text filter ([#13126](https://github.com/frappe/frappe/pull/13126))
+- Focus jumps to first field ([#13067](https://github.com/frappe/frappe/pull/13067))

frappe/commands/__init__.py

@@ -28,6 +28,10 @@ def pass_context(f):
 		except frappe.exceptions.SiteNotSpecifiedError as e:
 			click.secho(str(e), fg='yellow')
 			sys.exit(1)
+		except frappe.exceptions.IncorrectSitePath:
+			site = ctx.obj.get("sites", "")[0]
+			click.secho(f'Site {site} does not exist!', fg='yellow')
+			sys.exit(1)
 
 		if profile:
 			pr.disable()

frappe/commands/site.py

@@ -1,6 +1,7 @@
 # imports - standard imports
 import os
 import sys
+import shutil
 
 # imports - third party imports
 import click
@@ -202,10 +203,13 @@ def install_app(context, apps):
 
 
 @click.command("list-apps")
+@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text")
 @pass_context
-def list_apps(context):
+def list_apps(context, format):
 	"List apps in site"
 
+	summary_dict = {}
+
 	def fix_whitespaces(text):
 		if site == context.sites[-1]:
 			text = text.rstrip()
@@ -234,18 +238,23 @@ def list_apps(context):
 			]
 			applications_summary = "\n".join(installed_applications)
 			summary = f"{site_title}\n{applications_summary}\n"
+			summary_dict[site] = [app.app_name for app in apps]
 
 		else:
-			applications_summary = "\n".join(frappe.get_installed_apps())
+			installed_applications = frappe.get_installed_apps()
+			applications_summary = "\n".join(installed_applications)
 			summary = f"{site_title}\n{applications_summary}\n"
+			summary_dict[site] = installed_applications
 
 		summary = fix_whitespaces(summary)
 
-		if applications_summary and summary:
+		if format == "text" and applications_summary and summary:
 			print(summary)
 
 		frappe.destroy()
 
+	if format == "json":
+		click.echo(frappe.as_json(summary_dict))
 
 @click.command('add-system-manager')
 @click.argument('email')
@@ -547,7 +556,7 @@ def move(dest_dir, site):
 		site_dump_exists = os.path.exists(final_new_path)
 		count = int(count or 0) + 1
 
-	os.rename(old_path, final_new_path)
+	shutil.move(old_path, final_new_path)
 	frappe.destroy()
 	return final_new_path
 

frappe/commands/utils.py

@@ -16,33 +16,52 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
 
 @click.command('build')
 @click.option('--app', help='Build assets for app')
-@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
-@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
+@click.option('--apps', help='Build assets for specific apps')
+@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
+@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
+@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
+@click.option('--production', is_flag=True, default=False, help='Build assets in production mode')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
 @click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
-def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
-	"Minify + concatenate JS and CSS files, build translations"
-	import frappe.build
+def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False):
+	"Compile JS and CSS source files"
+	from frappe.build import bundle, download_frappe_assets
 	frappe.init('')
-	# don't minify in developer_mode for faster builds
-	no_compress = frappe.local.conf.developer_mode or False
+
+	if not apps and app:
+		apps = app
 
 	# dont try downloading assets if force used, app specified or running via CI
-	if not (force or app or os.environ.get('CI')):
+	if not (force or apps or os.environ.get('CI')):
 		# skip building frappe if assets exist remotely
-		skip_frappe = frappe.build.download_frappe_assets(verbose=verbose)
+		skip_frappe = download_frappe_assets(verbose=verbose)
 	else:
 		skip_frappe = False
 
-	frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
+	# don't minify in developer_mode for faster builds
+	development = frappe.local.conf.developer_mode or frappe.local.dev_server
+	mode = "development" if development else "production"
+	if production:
+		mode = "production"
+
+	if make_copy or restore:
+		hard_link = make_copy or restore
+		click.secho(
+			"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
+			fg="yellow",
+		)
+
+	bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe)
+
 
 
 @click.command('watch')
-def watch():
-	"Watch and concatenate JS and CSS files as and when they change"
-	import frappe.build
+@click.option('--apps', help='Watch assets for specific apps')
+def watch(apps=None):
+	"Watch and compile JS and CSS files as and when they change"
+	from frappe.build import watch
 	frappe.init('')
-	frappe.build.watch(True)
+	watch(apps)
 
 
 @click.command('clear-cache')
@@ -96,22 +115,54 @@ def destroy_all_sessions(context, reason=None):
 		raise SiteNotSpecifiedError
 
 @click.command('show-config')
+@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text")
 @pass_context
-def show_config(context):
-	"print configuration file"
-	print("\t\033[92m{:<50}\033[0m \033[92m{:<15}\033[0m".format('Config','Value'))
-	sites_path = os.path.join(frappe.utils.get_bench_path(), 'sites')
-	site_path = context.sites[0]
-	configuration = frappe.get_site_config(sites_path=sites_path, site_path=site_path)
-	print_config(configuration)
+def show_config(context, format):
+	"Print configuration file to STDOUT in speified format"
+
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	sites_config = {}
+	sites_path = os.getcwd()
+
+	from frappe.utils.commands import render_table
+
+	def transform_config(config, prefix=None):
+		prefix = f"{prefix}." if prefix else ""
+		site_config = []
+
+		for conf, value in config.items():
+			if isinstance(value, dict):
+				site_config += transform_config(value, prefix=f"{prefix}{conf}")
+			else:
+				log_value = json.dumps(value) if isinstance(value, list) else value
+				site_config += [[f"{prefix}{conf}", log_value]]
+
+		return site_config
+
+	for site in context.sites:
+		frappe.init(site)
+
+		if len(context.sites) != 1 and format == "text":
+			if context.sites.index(site) != 0:
+				click.echo()
+			click.secho(f"Site {site}", fg="yellow")
+
+		configuration = frappe.get_site_config(sites_path=sites_path, site_path=site)
+
+		if format == "text":
+			data = transform_config(configuration)
+			data.insert(0, ['Config','Value'])
+			render_table(data)
 
+		if format == "json":
+			sites_config[site] = configuration
 
-def print_config(config):
-	for conf, value in config.items():
-		if isinstance(value, dict):
-			print_config(value)
-		else:
-			print("\t{:<50} {:<15}".format(conf, value))
+		frappe.destroy()
+
+	if format == "json":
+		click.echo(frappe.as_json(sites_config))
 
 
 @click.command('reset-perms')
@@ -470,6 +521,7 @@ def console(context):
 			locals()[app] = __import__(app)
 		except ModuleNotFoundError:
 			failed_to_import.append(app)
+			all_apps.remove(app)
 
 	print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
 	if failed_to_import:
@@ -552,12 +604,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 	if os.environ.get('CI'):
 		sys.exit(ret)
 
+@click.command('run-parallel-tests')
+@click.option('--app', help="For App", default='frappe')
+@click.option('--build-number', help="Build number", default=1)
+@click.option('--total-builds', help="Total number of builds", default=1)
+@click.option('--with-coverage', is_flag=True, help="Build coverage file")
+@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
+@pass_context
+def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
+	site = get_site(context)
+	if use_orchestrator:
+		from frappe.parallel_test_runner import ParallelTestWithOrchestrator
+		ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
+	else:
+		from frappe.parallel_test_runner import ParallelTestRunner
+		ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
 
 @click.command('run-ui-tests')
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
+@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False):
+def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -589,6 +658,12 @@ def run_ui_tests(context, app, headless=False):
 	command = '{site_env} {password_env} {cypress} {run_or_open}'
 	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
 
+	if parallel:
+		formatted_command += ' --parallel'
+
+	if ci_build_id:
+		formatted_command += ' --ci-build-id {}'.format(ci_build_id)
+
 	click.secho("Running Cypress...", fg="yellow")
 	frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
 
@@ -657,20 +732,27 @@ def make_app(destination, app_name):
 @click.command('set-config')
 @click.argument('key')
 @click.argument('value')
-@click.option('-g', '--global', 'global_', is_flag = True, default = False, help = 'Set Global Site Config')
-@click.option('--as-dict', is_flag=True, default=False)
+@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config')
+@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object')
+@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object')
 @pass_context
-def set_config(context, key, value, global_ = False, as_dict=False):
+def set_config(context, key, value, global_=False, parse=False, as_dict=False):
 	"Insert/Update a value in site_config.json"
 	from frappe.installer import update_site_config
-	import ast
+
 	if as_dict:
+		from frappe.utils.commands import warn
+		warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning)
+		parse = as_dict
+
+	if parse:
+		import ast
 		value = ast.literal_eval(value)
 
 	if global_:
-		sites_path = os.getcwd() # big assumption.
+		sites_path = os.getcwd()
 		common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
-		update_site_config(key, value, validate = False, site_config_path = common_site_config_path)
+		update_site_config(key, value, validate=False, site_config_path=common_site_config_path)
 	else:
 		for site in context.sites:
 			frappe.init(site=site)
@@ -727,50 +809,6 @@ def rebuild_global_search(context, static_pages=False):
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
-@click.command('auto-deploy')
-@click.argument('app')
-@click.option('--migrate', is_flag=True, default=False, help='Migrate after pulling')
-@click.option('--restart', is_flag=True, default=False, help='Restart after migration')
-@click.option('--remote', default='upstream', help='Remote, default is "upstream"')
-@pass_context
-def auto_deploy(context, app, migrate=False, restart=False, remote='upstream'):
-	'''Pull and migrate sites that have new version'''
-	from frappe.utils.gitutils import get_app_branch
-	from frappe.utils import get_sites
-
-	branch = get_app_branch(app)
-	app_path = frappe.get_app_path(app)
-
-	# fetch
-	subprocess.check_output(['git', 'fetch', remote, branch], cwd = app_path)
-
-	# get diff
-	if subprocess.check_output(['git', 'diff', '{0}..{1}/{0}'.format(branch, remote)], cwd = app_path):
-		print('Updates found for {0}'.format(app))
-		if app=='frappe':
-			# run bench update
-			import shlex
-			subprocess.check_output(shlex.split('bench update --no-backup'), cwd = '..')
-		else:
-			updated = False
-			subprocess.check_output(['git', 'pull', '--rebase', remote, branch],
-				cwd = app_path)
-			# find all sites with that app
-			for site in get_sites():
-				frappe.init(site)
-				if app in frappe.get_installed_apps():
-					print('Updating {0}'.format(site))
-					updated = True
-					subprocess.check_output(['bench', '--site', site, 'clear-cache'], cwd = '..')
-					if migrate:
-						subprocess.check_output(['bench', '--site', site, 'migrate'], cwd = '..')
-				frappe.destroy()
-
-			if updated or restart:
-				subprocess.check_output(['bench', 'restart'], cwd = '..')
-	else:
-		print('No Updates')
-
 
 commands = [
 	build,
@@ -801,5 +839,6 @@ commands = [
 	watch,
 	bulk_rename,
 	add_to_email_queue,
-	rebuild_global_search
+	rebuild_global_search,
+	run_parallel_tests
 ]

frappe/contacts/doctype/contact/test_contact.py

@@ -5,7 +5,8 @@ from __future__ import unicode_literals
 
 import frappe
 import unittest
-from frappe.exceptions import ValidationError
+
+test_dependencies = ['Contact', 'Salutation']
 
 class TestContact(unittest.TestCase):
 
@@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
 	if save:
 		doc.insert()
 
-	return doc
+	return doc

frappe/core/doctype/activity_log/test_activity_log.py

@@ -65,12 +65,12 @@ class TestActivityLog(unittest.TestCase):
 		frappe.local.login_manager = LoginManager()
 
 		auth_log = self.get_auth_log()
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test user logout log
 		frappe.local.login_manager.logout()
 		auth_log = self.get_auth_log(operation='Logout')
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test invalid login
 		frappe.form_dict.update({ 'pwd': 'password' })
@@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase):
 def update_system_settings(args):
 	doc = frappe.get_doc('System Settings')
 	doc.update(args)
+	doc.flags.ignore_mandatory = 1
 	doc.save()

frappe/core/doctype/communication/email.py

@@ -272,22 +272,13 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None)
 				doc.attachments.append(a)
 
 def set_incoming_outgoing_accounts(doc):
-	doc.incoming_email_account = doc.outgoing_email_account = None
+	from frappe.email.doctype.email_account.email_account import EmailAccount
+	incoming_email_account = EmailAccount.find_incoming(
+		match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
+	doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None
 
-	if not doc.incoming_email_account and doc.sender:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"email_id": doc.sender, "enable_incoming": 1}, "email_id")
-
-	if not doc.incoming_email_account and doc.reference_doctype:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"append_to": doc.reference_doctype, }, "email_id")
-
-	if not doc.incoming_email_account:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"default_incoming": 1, "enable_incoming": 1},  "email_id")
-
-	doc.outgoing_email_account = frappe.email.smtp.get_outgoing_email_account(raise_exception_not_set=False,
-		append_to=doc.doctype, sender=doc.sender)
+	doc.outgoing_email_account = EmailAccount.find_outgoing(
+		match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
 
 	if doc.sent_or_received == "Sent":
 		doc.db_set("email_account", doc.outgoing_email_account.name)

frappe/core/doctype/data_export/exporter.py

@@ -282,7 +282,7 @@ class DataExporter:
 				try:
 					sflags = self.docs_to_export.get("flags", "I,U").upper()
 					flags = 0
-					for a in re.split('\W+',sflags):
+					for a in re.split(r'\W+', sflags):
 						flags = flags | reflags.get(a,0)
 
 					c = re.compile(names, flags)

frappe/core/doctype/data_import/data_import.js

@@ -203,7 +203,7 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	download_template(frm) {
-		frappe.require('/assets/js/data_import_tools.min.js', () => {
+		frappe.require('data_import_tools.bundle.js', () => {
 			frm.data_exporter = new frappe.data_import.DataExporter(
 				frm.doc.reference_doctype,
 				frm.doc.import_type
@@ -287,7 +287,7 @@ frappe.ui.form.on('Data Import', {
 			return;
 		}
 
-		frappe.require('/assets/js/data_import_tools.min.js', () => {
+		frappe.require('data_import_tools.bundle.js', () => {
 			frm.import_preview = new frappe.data_import.ImportPreview({
 				wrapper: frm.get_field('import_preview').$wrapper,
 				doctype: frm.doc.reference_doctype,

frappe/core/doctype/data_import/data_import.py

@@ -211,7 +211,12 @@ def export_json(
 	doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"
 ):
 	def post_process(out):
-		del_keys = ("modified_by", "creation", "owner", "idx")
+		# Note on Tree DocTypes:
+		# The tree structure is maintained in the database via the fields "lft"
+		# and "rgt". They are automatically set and kept up-to-date. Importing
+		# them would destroy any existing tree structure. For this reason they
+		# are not exported as well.
+		del_keys = ("modified_by", "creation", "owner", "idx", "lft", "rgt")
 		for doc in out:
 			for key in del_keys:
 				if key in doc:

frappe/core/doctype/data_import/importer.py

@@ -233,7 +233,7 @@ class Importer:
 			return updated_doc
 		else:
 			# throw if no changes
-			frappe.throw("No changes to update")
+			frappe.throw(_("No changes to update"))
 
 	def get_eta(self, current, total, processing_time):
 		self.last_eta = getattr(self, "last_eta", 0)
@@ -319,7 +319,7 @@ class ImportFile:
 		self.warnings = []
 
 		self.file_doc = self.file_path = self.google_sheets_url = None
-		if isinstance(file, frappe.string_types):
+		if isinstance(file, str):
 			if frappe.db.exists("File", {"file_url": file}):
 				self.file_doc = frappe.get_doc("File", {"file_url": file})
 			elif "docs.google.com/spreadsheets" in file:
@@ -626,7 +626,7 @@ class Row:
 				return
 		elif df.fieldtype in ["Date", "Datetime"]:
 			value = self.get_date(value, col)
-			if isinstance(value, frappe.string_types):
+			if isinstance(value, str):
 				# value was not parsed as datetime object
 				self.warnings.append(
 					{
@@ -641,7 +641,7 @@ class Row:
 				return
 		elif df.fieldtype == "Duration":
 			import re
-			is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
+			is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
 			if not is_valid_duration:
 				self.warnings.append(
 					{
@@ -929,10 +929,7 @@ class Column:
 				self.warnings.append(
 					{
 						"col": self.column_number,
-						"message": _(
-							"Date format could not be determined from the values in"
-							" this column. Defaulting to yyyy-mm-dd."
-						),
+						"message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."),
 						"type": "info",
 					}
 				)

frappe/core/doctype/docshare/test_docshare.py

@@ -7,6 +7,8 @@ import frappe.share
 import unittest
 from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype
 
+test_dependencies = ['User']
+
 class TestDocShare(unittest.TestCase):
 	def setUp(self):
 		self.user = "test@example.com"
@@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase):
 		self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
 		self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
 
-		frappe.share.remove(doctype, submittable_doc.name, self.user)
+		frappe.share.remove(doctype, submittable_doc.name, self.user)

frappe/core/doctype/doctype/boilerplate/controller._py

@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) {year}, {app_publisher} and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 {base_class_import}
 

frappe/core/doctype/doctype/boilerplate/test_controller._py

@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) {year}, {app_publisher} and Contributors
 # See license.txt
-from __future__ import unicode_literals
 
 # import frappe
 import unittest

frappe/core/doctype/doctype/doctype.json

@@ -662,4 +662,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
+}

frappe/core/doctype/doctype/doctype.py

@@ -83,12 +83,61 @@ class DocType(Document):
 		if not self.is_new():
 			self.before_update = frappe.get_doc('DocType', self.name)
 			self.setup_fields_to_fetch()
+			self.validate_field_name_conflicts()
 
 		check_email_append_to(self)
 
 		if self.default_print_format and not self.custom:
 			frappe.throw(_('Standard DocType cannot have default print format, use Customize Form'))
 
+		if frappe.conf.get('developer_mode'):
+			self.owner = 'Administrator'
+			self.modified_by = 'Administrator'
+
+	def validate_field_name_conflicts(self):
+		"""Check if field names dont conflict with controller properties and methods"""
+		core_doctypes = [
+			"Custom DocPerm",
+			"DocPerm",
+			"Custom Field",
+			"Customize Form Field",
+			"DocField",
+		]
+
+		if self.name in core_doctypes:
+			return
+
+		from frappe.model.base_document import get_controller
+
+		try:
+			controller = get_controller(self.name)
+		except ImportError:
+			controller = Document
+
+		available_objects = {x for x in dir(controller) if isinstance(x, str)}
+		property_set = {
+			x for x in available_objects if isinstance(getattr(controller, x, None), property)
+		}
+		method_set = {
+			x for x in available_objects if x not in property_set and callable(getattr(controller, x, None))
+		}
+
+		for docfield in self.get("fields") or []:
+			conflict_type = None
+			field = docfield.fieldname
+			field_label = docfield.label or docfield.fieldname
+
+			if docfield.fieldname in method_set:
+				conflict_type = "controller method"
+			if docfield.fieldname in property_set:
+				conflict_type = "class property"
+
+			if conflict_type:
+				frappe.throw(
+					_("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}")
+						.format(field_label, conflict_type, field, self.name)
+				)
+
 	def after_insert(self):
 		# clear user cache so that on the next reload this doctype is included in boot
 		clear_user_cache(frappe.session.user)
@@ -622,12 +671,12 @@ class DocType(Document):
 		flags = {"flags": re.ASCII} if six.PY3 else {}
 
 		# a DocType name should not start or end with an empty space
-		if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
+		if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
 			frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
 
 		# a DocType's name should not start with a number or underscore
 		# and should only contain letters, numbers and underscore
-		if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
+		if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags):
 			frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
 
 		validate_route_conflict(self.doctype, self.name)
@@ -915,7 +964,7 @@ def validate_fields(meta):
 		for field in depends_on_fields:
 			depends_on = docfield.get(field, None)
 			if depends_on and ("=" in depends_on) and \
-				re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on):
+				re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on):
 				frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError)
 
 	def check_table_multiselect_option(docfield):
@@ -1174,11 +1223,19 @@ def make_module_and_roles(doc, perm_fieldname="permissions"):
 		else:
 			raise
 
-def check_if_fieldname_conflicts_with_methods(doctype, fieldname):
-	doc = frappe.get_doc({"doctype": doctype})
-	method_list = [method for method in dir(doc) if isinstance(method, str) and callable(getattr(doc, method))]
+def check_fieldname_conflicts(doctype, fieldname):
+	"""Checks if fieldname conflicts with methods or properties"""
 
-	if fieldname in method_list:
+	doc = frappe.get_doc({"doctype": doctype})
+	available_objects = [x for x in dir(doc) if isinstance(x, str)]
+	property_list = [
+		x for x in available_objects if isinstance(getattr(type(doc), x, None), property)
+	]
+	method_list = [
+		x for x in available_objects if x not in property_list and callable(getattr(doc, x))
+	]
+
+	if fieldname in method_list + property_list:
 		frappe.throw(_("Fieldname {0} conflicting with meta object").format(fieldname))
 
 def clear_linked_doctype_cache():

frappe/core/doctype/doctype/test_doctype.py

@@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase):
 			fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\
 				"read_only_depends_on", "fieldname", "fieldtype"])
 
-		pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+"""
+		pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+'
 		for field in docfields:
 			for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]:
 				condition = field.get(depends_on)
@@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None):
 		for f in fields:
 			doc.append('fields', f)
 
-	return doc
+	return doc

frappe/core/doctype/file/file.py

@@ -498,7 +498,7 @@ class File(Document):
 		self.file_size = self.check_max_file_size()
 
 		if (
-			self.content_type and "image" in self.content_type
+			self.content_type and self.content_type == "image/jpeg"
 			and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
 		):
 			self.content = strip_exif_data(self.content, self.content_type)
@@ -912,7 +912,7 @@ def extract_images_from_html(doc, content):
 		return '<img src="{file_url}"'.format(file_url=file_url)
 
 	if content and isinstance(content, string_types):
-		content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
+		content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
 
 	return content
 

frappe/core/doctype/file/test_file.py

@@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase):
 
 class TestFile(unittest.TestCase):
 	def setUp(self):
+		frappe.set_user('Administrator')
 		self.delete_test_data()
 		self.upload_file()
 

frappe/core/doctype/report/boilerplate/controller.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2013, {app_publisher} and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 
 def execute(filters=None):

frappe/core/doctype/report/test_report.py

@@ -106,7 +106,7 @@ class TestReport(unittest.TestCase):
 		else:
 			report = frappe.get_doc('Report', 'Test Report')
 
-		self.assertNotEquals(report.is_permitted(), True)
+		self.assertNotEqual(report.is_permitted(), True)
 		frappe.set_user('Administrator')
 
 	# test for the `_format` method if report data doesn't have sort_by parameter

frappe/core/doctype/role_profile/test_role_profile.py

@@ -5,6 +5,8 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
+test_dependencies = ['Role']
+
 class TestRoleProfile(unittest.TestCase):
 	def test_make_new_role_profile(self):
 		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
@@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase):
 		# clear roles
 		new_role_profile.roles = []
 		new_role_profile.save()
-		self.assertEqual(new_role_profile.roles, [])
+		self.assertEqual(new_role_profile.roles, [])

frappe/core/doctype/system_settings/system_settings.py

@@ -42,7 +42,7 @@ class SystemSettings(Document):
 
 	def on_update(self):
 		for df in self.meta.get("fields"):
-			if df.fieldtype not in no_value_fields:
+			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
 				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
 
 		if self.language:

frappe/core/doctype/user/user.py

@@ -56,6 +56,7 @@ class User(Document):
 
 	def after_insert(self):
 		create_notification_settings(self.name)
+		frappe.cache().delete_key('users_for_mentions')
 
 	def validate(self):
 		self.check_demo()
@@ -129,6 +130,9 @@ class User(Document):
 		if self.time_zone:
 			frappe.defaults.set_default("time_zone", self.time_zone, self.name)
 
+		if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'):
+			frappe.cache().delete_key('users_for_mentions')
+
 	def has_website_permission(self, ptype, user, verbose=False):
 		"""Returns true if current user is the session user"""
 		return self.name == frappe.session.user
@@ -389,6 +393,9 @@ class User(Document):
 		# delete notification settings
 		frappe.delete_doc("Notification Settings", self.name, ignore_permissions=True)
 
+		if self.get('allow_in_mentions'):
+			frappe.cache().delete_key('users_for_mentions')
+
 
 	def before_rename(self, old_name, new_name, merge=False):
 		self.check_demo()

frappe/core/doctype/user_group/user_group.py

@@ -9,7 +9,7 @@ import frappe
 
 class UserGroup(Document):
 	def after_insert(self):
-		frappe.publish_realtime('user_group_added', self.name)
+		frappe.cache().delete_key('user_groups')
 
 	def on_trash(self):
-		frappe.publish_realtime('user_group_deleted', self.name)
+		frappe.cache().delete_key('user_groups')

frappe/core/doctype/user_permission/test_user_permission.py

@@ -46,7 +46,7 @@ class TestUserPermission(unittest.TestCase):
 		frappe.set_user('test_user_perm1@example.com')
 		doc = frappe.new_doc("Blog Post")
 
-		self.assertEquals(doc.blog_category, 'general')
+		self.assertEqual(doc.blog_category, 'general')
 		frappe.set_user('Administrator')
 
 	def test_apply_to_all(self):
@@ -54,7 +54,7 @@ class TestUserPermission(unittest.TestCase):
 		user = create_user('test_bulk_creation_update@example.com')
 		param = get_params(user, 'User', user.name)
 		is_created = add_user_permissions(param)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_for_apply_to_all_on_update_from_apply_all(self):
 		user = create_user('test_bulk_creation_update@example.com')
@@ -63,11 +63,11 @@ class TestUserPermission(unittest.TestCase):
 		# Initially create User Permission document with apply_to_all checked
 		is_created = add_user_permissions(param)
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 		is_created = add_user_permissions(param)
 
 		# User Permission should not be changed
-		self.assertEquals(is_created, 0)
+		self.assertEqual(is_created, 0)
 
 	def test_for_applicable_on_update_from_apply_to_all(self):
 		''' Update User Permission from all to some applicable Doctypes'''
@@ -77,7 +77,7 @@ class TestUserPermission(unittest.TestCase):
 		# Initially create User Permission document with apply_to_all checked
 		is_created = add_user_permissions(get_params(user, 'User', user.name))
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 		is_created = add_user_permissions(param)
 		frappe.db.commit()
@@ -92,7 +92,7 @@ class TestUserPermission(unittest.TestCase):
 		# Check that User Permissions for applicable is created
 		self.assertIsNotNone(is_created_applicable_first)
 		self.assertIsNotNone(is_created_applicable_second)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_for_apply_to_all_on_update_from_applicable(self):
 		''' Update User Permission from some to all applicable Doctypes'''
@@ -102,7 +102,7 @@ class TestUserPermission(unittest.TestCase):
 		# create User permissions that with applicable
 		is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"]))
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 		is_created = add_user_permissions(param)
 		is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
@@ -115,7 +115,7 @@ class TestUserPermission(unittest.TestCase):
 		# Check that all User Permission with applicable is removed
 		self.assertIsNone(removed_applicable_first)
 		self.assertIsNone(removed_applicable_second)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_user_perm_for_nested_doctype(self):
 		"""Test if descendants' visibility is controlled for a nested DocType."""
@@ -183,7 +183,7 @@ class TestUserPermission(unittest.TestCase):
 
 		# User perm is created on ToDo but for doctype Assignment Rule only
 		# it should not have impact on Doc A
-		self.assertEquals(new_doc.doc, "ToDo")
+		self.assertEqual(new_doc.doc, "ToDo")
 
 		frappe.set_user('Administrator')
 		remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo")
@@ -228,7 +228,7 @@ class TestUserPermission(unittest.TestCase):
 
 		# User perm is created on ToDo but for doctype Assignment Rule only
 		# it should not have impact on Doc A
-		self.assertEquals(new_doc.doc, "ToDo")
+		self.assertEqual(new_doc.doc, "ToDo")
 
 		frappe.set_user('Administrator')
 		clear_session_defaults()

frappe/core/doctype/user_permission/user_permission.py

@@ -191,7 +191,7 @@ def clear_user_permissions(user, for_doctype):
 def add_user_permissions(data):
 	''' Add and update the user permissions '''
 	frappe.only_for('System Manager')
-	if isinstance(data, frappe.string_types):
+	if isinstance(data, str):
 		data = json.loads(data)
 	data = frappe._dict(data)
 

frappe/core/page/recorder/recorder.js

@@ -1,7 +1,7 @@
 frappe.pages['recorder'].on_page_load = function(wrapper) {
 	frappe.ui.make_app_page({
 		parent: wrapper,
-		title: 'Recorder',
+		title: __('Recorder'),
 		single_column: true,
 		card_layout: true
 	});
@@ -11,7 +11,7 @@ frappe.pages['recorder'].on_page_load = function(wrapper) {
 		frappe.recorder.show();
 	});
 
-	frappe.require('/assets/js/frappe-recorder.min.js');
+	frappe.require('recorder.bundle.js');
 };
 
 class Recorder {

frappe/custom/doctype/custom_field/custom_field.py

@@ -64,18 +64,19 @@ class CustomField(Document):
 			self.translatable = 0
 
 		if not self.flags.ignore_validate:
-			from frappe.core.doctype.doctype.doctype import check_if_fieldname_conflicts_with_methods
-			check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname)
+			from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts
+			check_fieldname_conflicts(self.dt, self.fieldname)
 
 	def on_update(self):
-		frappe.clear_cache(doctype=self.dt)
+		if not frappe.flags.in_setup_wizard:
+			frappe.clear_cache(doctype=self.dt)
 		if not self.flags.ignore_validate:
 			# validate field
 			from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype
 			validate_fields_for_doctype(self.dt)
 
 		# update the schema
-		if not frappe.db.get_value('DocType', self.dt, 'issingle'):
+		if not frappe.db.get_value('DocType', self.dt, 'issingle') and not frappe.flags.in_setup_wizard:
 			frappe.db.updatedb(self.dt)
 
 	def on_trash(self):
@@ -144,6 +145,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
 	'''Add / update multiple custom fields
 
 	:param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`'''
+
+	if not ignore_validate and frappe.flags.in_setup_wizard:
+		ignore_validate = True
+
 	for doctype, fields in custom_fields.items():
 		if isinstance(fields, dict):
 			# only one field
@@ -163,6 +168,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
 				custom_field.update(df)
 				custom_field.save()
 
+		frappe.clear_cache(doctype=doctype)
+		frappe.db.updatedb(doctype)
+
+
 
 @frappe.whitelist()
 def add_custom_field(doctype, df):

frappe/custom/doctype/customize_form/customize_form.json

@@ -278,6 +278,7 @@
   },
   {
    "collapsible": 1,
+   "depends_on": "doc_type",
    "fieldname": "naming_section",
    "fieldtype": "Section Break",
    "label": "Naming"
@@ -287,6 +288,16 @@
    "fieldname": "autoname",
    "fieldtype": "Data",
    "label": "Auto Name"
+  },
+  {
+   "fieldname": "default_email_template",
+   "fieldtype": "Link",
+   "label": "Default Email Template",
+   "options": "Email Template"
+  },
+  {
+   "fieldname": "column_break_26",
+   "fieldtype": "Column Break"
   }
  ],
  "hide_toolbar": 1,
@@ -295,7 +306,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-03-22 12:27:15.462727",
+ "modified": "2021-04-29 21:21:06.476372",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",
@@ -316,4 +327,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
+}

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -47,64 +47,64 @@ class TestCustomizeForm(unittest.TestCase):
 		self.assertEqual(len(d.get("fields")), 0)
 
 		d = self.get_customize_form("Event")
-		self.assertEquals(d.doc_type, "Event")
-		self.assertEquals(len(d.get("fields")), 36)
+		self.assertEqual(d.doc_type, "Event")
+		self.assertEqual(len(d.get("fields")), 36)
 
 		d = self.get_customize_form("Event")
-		self.assertEquals(d.doc_type, "Event")
+		self.assertEqual(d.doc_type, "Event")
 
 		self.assertEqual(len(d.get("fields")),
 			len(frappe.get_doc("DocType", d.doc_type).fields) + 1)
-		self.assertEquals(d.get("fields")[-1].fieldname, "test_custom_field")
-		self.assertEquals(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
+		self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field")
+		self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
 
 		return d
 
 	def test_save_customization_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
 
 		d.allow_copy = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), '1')
 
 		d.allow_copy = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
 
 	def test_save_customization_field_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
 
 		repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
 		repeat_this_event_field.reqd = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1')
 
 		repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
 		repeat_this_event_field.reqd = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
 
 	def test_save_customization_custom_field_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
 
 		custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0]
 		custom_field.reqd = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
 
 		custom_field = d.get("fields", {"is_custom_field": True})[0]
 		custom_field.reqd = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
 
 	def test_save_customization_new_field(self):
 		d = self.get_customize_form("Event")
@@ -115,14 +115,14 @@ class TestCustomizeForm(unittest.TestCase):
 			"is_custom_field": 1
 		})
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data")
 
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname)
 
 		frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form")
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form"), None)
 
 
@@ -142,7 +142,7 @@ class TestCustomizeForm(unittest.TestCase):
 		d.doc_type = "Event"
 		d.run_method('reset_to_defaults')
 
-		self.assertEquals(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
+		self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
 
 		frappe.local.test_objects["Property Setter"] = []
 		make_test_records_for_doctype("Property Setter")
@@ -156,7 +156,7 @@ class TestCustomizeForm(unittest.TestCase):
 		d = self.get_customize_form("Event")
 
 		# don't allow for standard fields
-		self.assertEquals(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
+		self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
 
 		# allow for custom field
 		self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1)

frappe/database/database.py

@@ -858,7 +858,7 @@ class Database(object):
 		if not datetime:
 			return '0001-01-01 00:00:00.000000'
 
-		if isinstance(datetime, frappe.string_types):
+		if isinstance(datetime, str):
 			if ':' not in datetime:
 				datetime = datetime + ' 00:00:00.000000'
 		else:

frappe/database/mariadb/database.py

@@ -1,5 +1,3 @@
-import warnings
-
 import pymysql
 from pymysql.constants import ER, FIELD_TYPE
 from pymysql.converters import conversions, escape_string
@@ -55,7 +53,6 @@ class MariaDBDatabase(Database):
 		}
 
 	def get_connection(self):
-		warnings.filterwarnings('ignore', category=pymysql.Warning)
 		usessl = 0
 		if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key:
 			usessl = 1

frappe/database/postgres/database.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import re
 import frappe
 import psycopg2
@@ -13,9 +11,9 @@ from frappe.database.postgres.schema import PostgresTable
 
 # cast decimals as floats
 DEC2FLOAT = psycopg2.extensions.new_type(
-    psycopg2.extensions.DECIMAL.values,
-    'DEC2FLOAT',
-    lambda value, curs: float(value) if value is not None else None)
+	psycopg2.extensions.DECIMAL.values,
+	'DEC2FLOAT',
+	lambda value, curs: float(value) if value is not None else None)
 
 psycopg2.extensions.register_type(DEC2FLOAT)
 
@@ -65,7 +63,6 @@ class PostgresDatabase(Database):
 		}
 
 	def get_connection(self):
-		# warnings.filterwarnings('ignore', category=psycopg2.Warning)
 		conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format(
 			self.host, self.user, self.user, self.password, self.port
 		))
@@ -114,7 +111,7 @@ class PostgresDatabase(Database):
 		if not date:
 			return '0001-01-01'
 
-		if not isinstance(date, frappe.string_types):
+		if not isinstance(date, str):
 			date = date.strftime('%Y-%m-%d')
 
 		return date

frappe/desk/desktop.py

@@ -359,15 +359,18 @@ def get_desktop_page(page):
 	Returns:
 		dict: dictionary of cards, charts and shortcuts to be displayed on website
 	"""
-	wspace = Workspace(page)
-	wspace.build_workspace()
-	return {
-		'charts': wspace.charts,
-		'shortcuts': wspace.shortcuts,
-		'cards': wspace.cards,
-		'onboarding': wspace.onboarding,
-		'allow_customization': not wspace.doc.disable_user_customization
-	}
+	try:
+		wspace = Workspace(page)
+		wspace.build_workspace()
+		return {
+			'charts': wspace.charts,
+			'shortcuts': wspace.shortcuts,
+			'cards': wspace.cards,
+			'onboarding': wspace.onboarding,
+			'allow_customization': not wspace.doc.disable_user_customization
+		}
+	except DoesNotExistError:
+		return {}
 
 @frappe.whitelist()
 def get_desk_sidebar_items():
@@ -608,3 +611,4 @@ def merge_cards_based_on_label(cards):
 			cards_dict[label] = card
 
 	return list(cards_dict.values())
+

frappe/desk/doctype/notification_log/notification_log.py

@@ -46,7 +46,7 @@ def enqueue_create_notification(users, doc):
 
 	doc = frappe._dict(doc)
 
-	if isinstance(users, frappe.string_types):
+	if isinstance(users, str):
 		users = [user.strip() for user in users.split(',') if user.strip()]
 	users = list(set(users))
 

frappe/desk/doctype/todo/test_todo.py

@@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery
 from frappe.permissions import add_permission, reset_perms
 from frappe.core.doctype.doctype.doctype import clear_permissions_cache
 
-# test_records = frappe.get_test_records('ToDo')
-test_user_records = frappe.get_test_records('User')
+test_dependencies = ['User']
 
 class TestToDo(unittest.TestCase):
 	def test_delete(self):
@@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase):
 		frappe.set_user('test4@example.com')
 		#owner and assigned_by is test4
 		todo3 = create_new_todo('Test3', 'test4@example.com')
-		
+
 		# user without any role to read or write todo document
 		self.assertFalse(todo1.has_permission("read"))
 		self.assertFalse(todo1.has_permission("write"))

frappe/desk/doctype/workspace_link/workspace_link.json

@@ -8,13 +8,13 @@
   "type",
   "label",
   "icon",
+  "only_for",
   "hidden",
   "link_details_section",
   "link_type",
   "link_to",
   "column_break_7",
   "dependencies",
-  "only_for",
   "onboard",
   "is_query_report"
  ],
@@ -84,7 +84,7 @@
   {
    "fieldname": "only_for",
    "fieldtype": "Link",
-   "label": "Only for ",
+   "label": "Only for",
    "options": "Country"
   },
   {
@@ -104,7 +104,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-01-12 13:13:12.379443",
+ "modified": "2021-05-13 13:10:18.128512",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Link",

frappe/desk/page/activity/activity.js

@@ -67,8 +67,8 @@ frappe.pages['activity'].on_page_show = function () {
 }
 
 frappe.activity.last_feed_date = false;
-frappe.activity.Feed = Class.extend({
-	init: function (row, data) {
+frappe.activity.Feed = class Feed {
+	constructor(row, data) {
 		this.scrub_data(data);
 		this.add_date_separator(row, data);
 		if (!data.add_class)
@@ -97,8 +97,9 @@ frappe.activity.Feed = Class.extend({
 		$(row)
 			.append(frappe.render_template("activity_row", data))
 			.find("a").addClass("grey");
-	},
-	scrub_data: function (data) {
+	}
+
+	scrub_data(data) {
 		data.by = frappe.user.full_name(data.owner);
 		data.avatar = frappe.avatar(data.owner);
 
@@ -113,9 +114,9 @@ frappe.activity.Feed = Class.extend({
 
 		data.when = comment_when(data.creation);
 		data.feed_type = data.comment_type || data.communication_medium;
-	},
+	}
 
-	add_date_separator: function (row, data) {
+	add_date_separator(row, data) {
 		var date = frappe.datetime.str_to_obj(data.creation);
 		var last = frappe.activity.last_feed_date;
 
@@ -137,7 +138,7 @@ frappe.activity.Feed = Class.extend({
 		}
 		frappe.activity.last_feed_date = date;
 	}
-});
+};
 
 frappe.activity.render_heatmap = function (page) {
 	$('<div class="heatmap-container" style="text-align:center">\

frappe/desk/page/backups/backups.css

@@ -5,6 +5,7 @@
 .download-backup-card {
 	display: block;
 	text-decoration: none;
+	margin-bottom: var(--margin-lg);
 }
 
 .download-backup-card:hover {

frappe/desk/page/backups/backups.js

@@ -1,7 +1,7 @@
 frappe.pages['backups'].on_page_load = function(wrapper) {
 	var page = frappe.ui.make_app_page({
 		parent: wrapper,
-		title: 'Download Backups',
+		title: __('Download Backups'),
 		single_column: true
 	});
 

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -124,6 +124,7 @@ def handle_setup_exception(args):
 	frappe.db.rollback()
 	if args:
 		traceback = frappe.get_traceback()
+		print(traceback)
 		for hook in frappe.get_hooks("setup_wizard_exception"):
 			frappe.get_attr(hook)(traceback, args)
 

frappe/desk/page/translation_tool/translation_tool.js

@@ -1,7 +1,7 @@
 frappe.pages['translation-tool'].on_page_load = function(wrapper) {
 	var page = frappe.ui.make_app_page({
 		parent: wrapper,
-		title: 'Translation Tool',
+		title: __('Translation Tool'),
 		single_column: true,
 		card_layout: true,
 	});

frappe/desk/page/user_profile/user_profile.html

@@ -8,7 +8,7 @@
 				</div>
 				<div class="chart-wrapper performance-heatmap">
 					<div class="null-state">
-						<span>No Data to Show</span>
+						<span>{%=__("No Data to Show") %}</span>
 					</div>
 				</div>
 			</div>
@@ -19,7 +19,7 @@
 				</div>
 				<div class="chart-wrapper performance-percentage-chart">
 					<div class="null-state">
-						<span>No Data to Show</span>
+						<span>{%=__("No Data to Show") %}</span>
 					</div>
 				</div>
 			</div>
@@ -30,7 +30,7 @@
 				</div>
 				<div class="chart-wrapper performance-line-chart">
 					<div class="null-state">
-						<span>No Data to Show</span>
+						<span>{%=__("No Data to Show") %}</span>
 					</div>
 				</div>
 			</div>
@@ -41,4 +41,4 @@
 			<div class="recent-activity-footer"></div>
 		</div>
 	</div>
-</div>
+</div>

frappe/desk/query_report.py

@@ -377,10 +377,17 @@ def handle_duration_fieldtype_values(result, columns):
 
 		if fieldtype == "Duration":
 			for entry in range(0, len(result)):
-				val_in_seconds = result[entry][i]
-				if val_in_seconds:
-					duration_val = format_duration(val_in_seconds)
-					result[entry][i] = duration_val
+				row = result[entry]
+				if isinstance(row, dict):
+					val_in_seconds = row[col.fieldname]
+					if val_in_seconds:
+						duration_val = format_duration(val_in_seconds)
+						row[col.fieldname] = duration_val
+				else:
+					val_in_seconds = row[i]
+					if val_in_seconds:
+						duration_val = format_duration(val_in_seconds)
+						row[i] = duration_val
 
 	return result
 

frappe/desk/search.py

@@ -221,3 +221,37 @@ def validate_and_sanitize_search_inputs(fn, instance, args, kwargs):
 		return []
 
 	return fn(**kwargs)
+
+
+@frappe.whitelist()
+def get_names_for_mentions(search_term):
+	users_for_mentions = frappe.cache().get_value('users_for_mentions', get_users_for_mentions)
+	user_groups = frappe.cache().get_value('user_groups', get_user_groups)
+
+	filtered_mentions = []
+	for mention_data in users_for_mentions + user_groups:
+		if search_term.lower() not in mention_data.value.lower():
+			continue
+
+		mention_data['link'] = frappe.utils.get_url_to_form(
+			'User Group' if mention_data.get('is_group') else 'User Profile',
+			mention_data['id']
+		)
+
+		filtered_mentions.append(mention_data)
+
+	return sorted(filtered_mentions, key=lambda d: d['value'])
+
+def get_users_for_mentions():
+	return frappe.get_all('User',
+		fields=['name as id', 'full_name as value'],
+		filters={
+			'name': ['not in', ('Administrator', 'Guest')],
+			'allowed_in_mentions': True,
+			'user_type': 'System User',
+		})
+
+def get_user_groups():
+	return frappe.get_all('User Group', fields=['name as id', 'name as value'], update={
+		'is_group': True
+	})

frappe/email/doctype/document_follow/test_document_follow.py

@@ -17,14 +17,14 @@ class TestDocumentFollow(unittest.TestCase):
 
 		document_follow.unfollow_document("Event", event_doc.name, user.name)
 		doc = document_follow.follow_document("Event", event_doc.name, user.name)
-		self.assertEquals(doc.user, user.name)
+		self.assertEqual(doc.user, user.name)
 
 		document_follow.send_hourly_updates()
 
 		email_queue_entry_name = frappe.get_all("Email Queue", limit=1)[0].name
 		email_queue_entry_doc = frappe.get_doc("Email Queue", email_queue_entry_name)
 
-		self.assertEquals((email_queue_entry_doc.recipients[0].recipient), user.name)
+		self.assertEqual((email_queue_entry_doc.recipients[0].recipient), user.name)
 
 		self.assertIn(event_doc.doctype, email_queue_entry_doc.message)
 		self.assertIn(event_doc.name, email_queue_entry_doc.message)

frappe/email/doctype/email_account/email_account.py

@@ -8,9 +8,14 @@ import re
 import json
 import socket
 import time
-from frappe import _
+import functools
+
+import email.utils
+
+from frappe import _, are_emails_muted
 from frappe.model.document import Document
-from frappe.utils import validate_email_address, cint, cstr, get_datetime, DATE_FORMAT, strip, comma_or, sanitize_html, add_days
+from frappe.utils import (validate_email_address, cint, cstr, get_datetime,
+	DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr)
 from frappe.utils.user import is_system_user
 from frappe.utils.jinja import render_template
 from frappe.email.smtp import SMTPServer
@@ -21,17 +26,37 @@ from datetime import datetime, timedelta
 from frappe.desk.form import assign_to
 from frappe.utils.user import get_system_managers
 from frappe.utils.background_jobs import enqueue, get_jobs
-from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
 from frappe.utils.html_utils import clean_email_html
+from frappe.utils.error import raise_error_on_no_output
 from frappe.email.utils import get_port
 
+OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account")
+
 class SentEmailInInbox(Exception):
 	pass
 
-class InvalidEmailCredentials(frappe.ValidationError):
-	pass
+def cache_email_account(cache_name):
+	def decorator_cache_email_account(func):
+		@functools.wraps(func)
+		def wrapper_cache_email_account(*args, **kwargs):
+			if not hasattr(frappe.local, cache_name):
+				setattr(frappe.local, cache_name, {})
+
+			cached_accounts = getattr(frappe.local, cache_name)
+			match_by = list(kwargs.values()) + ['default']
+			matched_accounts = list(filter(None, [cached_accounts.get(key) for key in match_by]))
+			if matched_accounts:
+				return matched_accounts[0]
+
+			matched_accounts = func(*args, **kwargs)
+			cached_accounts.update(matched_accounts or {})
+			return matched_accounts and list(matched_accounts.values())[0]
+		return wrapper_cache_email_account
+	return decorator_cache_email_account
 
 class EmailAccount(Document):
+	DOCTYPE = 'Email Account'
+
 	def autoname(self):
 		"""Set name as `email_account_name` or make title from Email Address."""
 		if not self.email_account_name:
@@ -72,9 +97,8 @@ class EmailAccount(Document):
 					self.get_incoming_server()
 					self.no_failed = 0
 
-
 				if self.enable_outgoing:
-					self.check_smtp()
+					self.validate_smtp_conn()
 			else:
 				if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication):
 					frappe.throw(_("Password is required or select Awaiting Password"))
@@ -90,6 +114,13 @@ class EmailAccount(Document):
 			if self.append_to not in valid_doctypes:
 				frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
 
+	def validate_smtp_conn(self):
+		if not self.smtp_server:
+			frappe.throw(_("SMTP Server is required"))
+
+		server = self.get_smtp_server()
+		return server.session
+
 	def before_save(self):
 		messages = []
 		as_list = 1
@@ -151,24 +182,6 @@ class EmailAccount(Document):
 		except Exception:
 			pass
 
-	def check_smtp(self):
-		"""Checks SMTP settings."""
-		if self.enable_outgoing:
-			if not self.smtp_server:
-				frappe.throw(_("{0} is required").format("SMTP Server"))
-
-			server = SMTPServer(
-				login = getattr(self, "login_id", None) or self.email_id,
-				server=self.smtp_server,
-				port=cint(self.smtp_port),
-				use_tls=cint(self.use_tls),
-				use_ssl=cint(self.use_ssl_for_outgoing)
-			)
-			if self.password and not self.no_smtp_authentication:
-				server.password = self.get_password()
-
-			server.sess
-
 	def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
 		"""Returns logged in POP3/IMAP connection object."""
 		if frappe.cache().get_value("workers:no-internet") == True:
@@ -231,7 +244,7 @@ class EmailAccount(Document):
 				return None
 
 			elif not in_receive and any(map(lambda t: t in message, auth_error_codes)):
-				self.throw_invalid_credentials_exception()
+				SMTPServer.throw_invalid_credentials_exception()
 			else:
 				frappe.throw(cstr(e))
 
@@ -249,13 +262,142 @@ class EmailAccount(Document):
 			else:
 				raise
 
+	@property
+	def _password(self):
+		raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test)
+		return self.get_password(raise_exception=raise_exception)
+
+	@property
+	def default_sender(self):
+		return email.utils.formataddr((self.name, self.get("email_id")))
+
+	def is_exists_in_db(self):
+		"""Some of the Email Accounts we create from configs and those doesn't exists in DB.
+		This is is to check the specific email account exists in DB or not.
+		"""
+		return self.find_one_by_filters(name=self.name)
+
+	@classmethod
+	def from_record(cls, record):
+		email_account = frappe.new_doc(cls.DOCTYPE)
+		email_account.update(record)
+		return email_account
+
 	@classmethod
-	def throw_invalid_credentials_exception(cls):
-		frappe.throw(
-			_("Incorrect email or password. Please check your login credentials."),
-			exc=InvalidEmailCredentials,
-			title=_("Invalid Credentials")
-		)
+	def find(cls, name):
+		return frappe.get_doc(cls.DOCTYPE, name)
+
+	@classmethod
+	def find_one_by_filters(cls, **kwargs):
+		name = frappe.db.get_value(cls.DOCTYPE, kwargs)
+		return cls.find(name) if name else None
+
+	@classmethod
+	def find_from_config(cls):
+		config = cls.get_account_details_from_site_config()
+		return cls.from_record(config) if config else None
+
+	@classmethod
+	def create_dummy(cls):
+		return cls.from_record({"sender": "notifications@example.com"})
+
+	@classmethod
+	@raise_error_on_no_output(
+		keep_quiet = lambda: not cint(frappe.get_system_settings('setup_complete')),
+		error_message = OUTGOING_EMAIL_ACCOUNT_MISSING, error_type = frappe.OutgoingEmailError) # noqa
+	@cache_email_account('outgoing_email_account')
+	def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False):
+		"""Find the outgoing Email account to use.
+
+		:param match_by_email: Find account using emailID
+		:param match_by_doctype: Find account by matching `Append To` doctype
+		:param _raise_error: This is used by raise_error_on_no_output decorator to raise error.
+		"""
+		if match_by_email:
+			match_by_email = parse_addr(match_by_email)[1]
+			doc = cls.find_one_by_filters(enable_outgoing=1, email_id=match_by_email)
+			if doc:
+				return {match_by_email: doc}
+
+		if match_by_doctype:
+			doc = cls.find_one_by_filters(enable_outgoing=1, enable_incoming=1, append_to=match_by_doctype)
+			if doc:
+				return {match_by_doctype: doc}
+
+		doc = cls.find_default_outgoing()
+		if doc:
+			return {'default': doc}
+
+	@classmethod
+	def find_default_outgoing(cls):
+		""" Find default outgoing account.
+		"""
+		doc = cls.find_one_by_filters(enable_outgoing=1, default_outgoing=1)
+		doc = doc or cls.find_from_config()
+		return doc or (are_emails_muted() and cls.create_dummy())
+
+	@classmethod
+	def find_incoming(cls, match_by_email=None, match_by_doctype=None):
+		"""Find the incoming Email account to use.
+		:param match_by_email: Find account using emailID
+		:param match_by_doctype: Find account by matching `Append To` doctype
+		"""
+		doc = cls.find_one_by_filters(enable_incoming=1, email_id=match_by_email)
+		if doc:
+			return doc
+
+		doc = cls.find_one_by_filters(enable_incoming=1, append_to=match_by_doctype)
+		if doc:
+			return doc
+
+		doc = cls.find_default_incoming()
+		return doc
+
+	@classmethod
+	def find_default_incoming(cls):
+		doc = cls.find_one_by_filters(enable_incoming=1, default_incoming=1)
+		return doc
+
+	@classmethod
+	def get_account_details_from_site_config(cls):
+		if not frappe.conf.get("mail_server"):
+			return {}
+
+		field_to_conf_name_map = {
+			'smtp_server': {'conf_names': ('mail_server',)},
+			'smtp_port': {'conf_names': ('mail_port',)},
+			'use_tls': {'conf_names': ('use_tls', 'mail_login')},
+			'login_id': {'conf_names': ('mail_login',)},
+			'email_id': {'conf_names': ('auto_email_id', 'mail_login'), 'default': 'notifications@example.com'},
+			'password': {'conf_names': ('mail_password',)},
+			'always_use_account_email_id_as_sender':
+				{'conf_names': ('always_use_account_email_id_as_sender',), 'default': 0},
+			'always_use_account_name_as_sender_name':
+				{'conf_names': ('always_use_account_name_as_sender_name',), 'default': 0},
+			'name': {'conf_names': ('email_sender_name',), 'default': 'Frappe'},
+			'from_site_config': {'default': True}
+		}
+
+		account_details = {}
+		for doc_field_name, d in field_to_conf_name_map.items():
+			conf_names, default = d.get('conf_names') or [], d.get('default')
+			value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)]
+			account_details[doc_field_name] = (value and value[0]) or default
+		return account_details
+
+	def sendmail_config(self):
+		return {
+			'server': self.smtp_server,
+			'port': cint(self.smtp_port),
+			'login': getattr(self, "login_id", None) or self.email_id,
+			'password': self._password,
+			'use_ssl': cint(self.use_ssl_for_outgoing),
+			'use_tls': cint(self.use_tls)
+		}
+
+	def get_smtp_server(self):
+		config = self.sendmail_config()
+		return SMTPServer(**config)
 
 	def handle_incoming_connect_error(self, description):
 		if test_internet():
@@ -642,6 +784,8 @@ class EmailAccount(Document):
 
 	def send_auto_reply(self, communication, email):
 		"""Send auto reply if set."""
+		from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
+
 		if self.enable_auto_reply:
 			set_incoming_outgoing_accounts(communication)
 
@@ -653,7 +797,7 @@ class EmailAccount(Document):
 			frappe.sendmail(recipients = [email.from_email],
 				sender = self.email_id,
 				reply_to = communication.incoming_email_account,
-				subject = _("Re: ") + communication.subject,
+				subject = " ".join([_("Re:"), communication.subject]),
 				content = render_template(self.auto_reply_message or "", communication.as_dict()) or \
 					 frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()),
 				reference_doctype = communication.reference_doctype,

frappe/email/doctype/email_domain/test_records.json

@@ -10,7 +10,8 @@
   "incoming_port": "993",
   "attachment_limit": "1",
   "smtp_server": "smtp.test.com",
-  "smtp_port": "587"
+  "smtp_port": "587",
+  "password": "password"
  },
  {
   "doctype": "Email Account",
@@ -25,6 +26,7 @@
   "incoming_port": "143",
   "attachment_limit": "1",
   "smtp_server": "smtp.test.com",
-  "smtp_port": "587"
+  "smtp_port": "587",
+  "password": "password"
  }
 ]

frappe/email/doctype/email_queue/email_queue.json

@@ -24,7 +24,8 @@
   "unsubscribe_method",
   "expose_recipients",
   "attachments",
-  "retry"
+  "retry",
+  "email_account"
  ],
  "fields": [
   {
@@ -139,13 +140,19 @@
    "fieldtype": "Int",
    "label": "Retry",
    "read_only": 1
+  },
+  {
+   "fieldname": "email_account",
+   "fieldtype": "Link",
+   "label": "Email Account",
+   "options": "Email Account"
   }
  ],
  "icon": "fa fa-envelope",
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2020-07-17 15:58:15.369419",
+ "modified": "2021-04-29 06:33:25.191729",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Queue",

frappe/email/doctype/email_queue/email_queue.py

@@ -2,15 +2,26 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
+import traceback
+import json
+
+from rq.timeouts import JobTimeoutException
+import smtplib
+import quopri
+from email.parser import Parser
+
 import frappe
-from frappe import _
+from frappe import _, safe_encode, task
 from frappe.model.document import Document
-from frappe.email.queue import send_one
-from frappe.utils import now_datetime
-
+from frappe.email.queue import get_unsubcribed_url
+from frappe.email.email_body import add_attachment
+from frappe.utils import cint
+from email.policy import SMTPUTF8
 
+MAX_RETRY_COUNT = 3
 class EmailQueue(Document):
+	DOCTYPE = 'Email Queue'
+
 	def set_recipients(self, recipients):
 		self.set("recipients", [])
 		for r in recipients:
@@ -30,6 +41,241 @@ class EmailQueue(Document):
 		duplicate.set_recipients(recipients)
 		return duplicate
 
+	@classmethod
+	def find(cls, name):
+		return frappe.get_doc(cls.DOCTYPE, name)
+
+	def update_db(self, commit=False, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+		if commit:
+			frappe.db.commit()
+
+	def update_status(self, status, commit=False, **kwargs):
+		self.update_db(status = status, commit = commit, **kwargs)
+		if self.communication:
+			communication_doc = frappe.get_doc('Communication', self.communication)
+			communication_doc.set_delivery_status(commit=commit)
+
+	@property
+	def cc(self):
+		return (self.show_as_cc and self.show_as_cc.split(",")) or []
+
+	@property
+	def to(self):
+		return [r.recipient for r in self.recipients if r.recipient not in self.cc]
+
+	@property
+	def attachments_list(self):
+		return json.loads(self.attachments) if self.attachments else []
+
+	def get_email_account(self):
+		from frappe.email.doctype.email_account.email_account import EmailAccount
+
+		if self.email_account:
+			return frappe.get_doc('Email Account', self.email_account)
+
+		return EmailAccount.find_outgoing(
+			match_by_email = self.sender, match_by_doctype = self.reference_doctype)
+
+	def is_to_be_sent(self):
+		return self.status in ['Not Sent','Partially Sent']
+
+	def can_send_now(self):
+		hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1)
+		if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue:
+			return False
+
+		return True
+
+	def send(self, is_background_task=False):
+		""" Send emails to recipients.
+		"""
+		if not self.can_send_now():
+			frappe.db.rollback()
+			return
+
+		with SendMailContext(self, is_background_task) as ctx:
+			message = None
+			for recipient in self.recipients:
+				if not recipient.is_mail_to_be_sent():
+					continue
+
+				message = ctx.build_message(recipient.recipient)
+				if not frappe.flags.in_test:
+					ctx.smtp_session.sendmail(recipient.recipient, self.sender, message)
+				ctx.add_to_sent_list(recipient)
+
+			if frappe.flags.in_test:
+				frappe.flags.sent_mail = message
+				return
+
+			if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to:
+				ctx.email_account_doc.append_email_to_sent_folder(message)
+
+
+@task(queue = 'short')
+def send_mail(email_queue_name, is_background_task=False):
+	"""This is equalent to EmqilQueue.send.
+
+	This provides a way to make sending mail as a background job.
+	"""
+	record = EmailQueue.find(email_queue_name)
+	record.send(is_background_task=is_background_task)
+
+class SendMailContext:
+	def __init__(self, queue_doc: Document, is_background_task: bool = False):
+		self.queue_doc = queue_doc
+		self.is_background_task = is_background_task
+		self.email_account_doc = queue_doc.get_email_account()
+		self.smtp_server = self.email_account_doc.get_smtp_server()
+		self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()]
+
+	def __enter__(self):
+		self.queue_doc.update_status(status='Sending', commit=True)
+		return self
+
+	def __exit__(self, exc_type, exc_val, exc_tb):
+		exceptions = [
+			smtplib.SMTPServerDisconnected,
+			smtplib.SMTPAuthenticationError,
+			smtplib.SMTPRecipientsRefused,
+			smtplib.SMTPConnectError,
+			smtplib.SMTPHeloError,
+			JobTimeoutException
+		]
+
+		self.smtp_server.quit()
+		self.log_exception(exc_type, exc_val, exc_tb)
+
+		if exc_type in exceptions:
+			email_status = (self.sent_to and 'Partially Sent') or 'Not Sent'
+			self.queue_doc.update_status(status = email_status, commit = True)
+		elif exc_type:
+			if self.queue_doc.retry < MAX_RETRY_COUNT:
+				update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1}
+			else:
+				update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'}
+			self.queue_doc.update_status(**update_fields, commit = True)
+		else:
+			email_status = self.is_mail_sent_to_all() and 'Sent'
+			email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent'
+			self.queue_doc.update_status(status = email_status, commit = True)
+
+	def log_exception(self, exc_type, exc_val, exc_tb):
+		if exc_type:
+			traceback_string = "".join(traceback.format_tb(exc_tb))
+			traceback_string += f"\n Queue Name: {self.queue_doc.name}"
+
+			if self.is_background_task:
+				frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string)
+			else:
+				frappe.log_error(message = traceback_string)
+
+	@property
+	def smtp_session(self):
+		if frappe.flags.in_test:
+			return
+		return self.smtp_server.session
+
+	def add_to_sent_list(self, recipient):
+		# Update recipient status
+		recipient.update_db(status='Sent', commit=True)
+		self.sent_to.append(recipient.recipient)
+
+	def is_mail_sent_to_all(self):
+		return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients])
+
+	def get_message_object(self, message):
+		return Parser(policy=SMTPUTF8).parsestr(message)
+
+	def message_placeholder(self, placeholder_key):
+		map = {
+			'tracker': '<!--email open check-->',
+			'unsubscribe_url': '<!--unsubscribe url-->',
+			'cc': '<!--cc message-->',
+			'recipient': '<!--recipient-->',
+		}
+		return map.get(placeholder_key)
+
+	def build_message(self, recipient_email):
+		"""Build message specific to the recipient.
+		"""
+		message = self.queue_doc.message
+		if not message:
+			return ""
+
+		message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str())
+		message = message.replace(self.message_placeholder('unsubscribe_url'),
+			self.get_unsubscribe_str(recipient_email))
+		message = message.replace(self.message_placeholder('cc'), self.get_receivers_str())
+		message = message.replace(self.message_placeholder('recipient'),
+			self.get_receipient_str(recipient_email))
+		message = self.include_attachments(message)
+		return message
+
+	def get_tracker_str(self):
+		tracker_url_html = \
+			'<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'
+
+		message = ''
+		if frappe.conf.use_ssl and self.queue_doc.track_email_status:
+			message = quopri.encodestring(
+				tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
+			).decode()
+		return message
+
+	def get_unsubscribe_str(self, recipient_email):
+		unsubscribe_url = ''
+		if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype:
+			doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name
+			unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email,
+				self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param)
+
+		return quopri.encodestring(unsubscribe_url.encode()).decode()
+
+	def get_receivers_str(self):
+		message = ''
+		if self.queue_doc.expose_recipients == "footer":
+			to_str = ', '.join(self.queue_doc.to)
+			cc_str = ', '.join(self.queue_doc.cc)
+			message = f"This email was sent to {to_str}"
+			message = message + f" and copied to {cc_str}" if cc_str else message
+		return message
+
+	def get_receipient_str(self, recipient_email):
+		message = ''
+		if self.queue_doc.expose_recipients != "header":
+			message = recipient_email
+		return message
+
+	def include_attachments(self, message):
+		message_obj = self.get_message_object(message)
+		attachments = self.queue_doc.attachments_list
+
+		for attachment in attachments:
+			if attachment.get('fcontent'):
+				continue
+
+			fid = attachment.get("fid")
+			if fid:
+				_file = frappe.get_doc("File", fid)
+				fcontent = _file.get_content()
+				attachment.update({
+					'fname': _file.file_name,
+					'fcontent': fcontent,
+					'parent': message_obj
+				})
+				attachment.pop("fid", None)
+				add_attachment(**attachment)
+
+			elif attachment.get("print_format_attachment") == 1:
+				attachment.pop("print_format_attachment", None)
+				print_format_file = frappe.attach_print(**attachment)
+				print_format_file.update({"parent": message_obj})
+				add_attachment(**print_format_file)
+
+		return safe_encode(message_obj.as_string())
+
 @frappe.whitelist()
 def retry_sending(name):
 	doc = frappe.get_doc("Email Queue", name)
@@ -42,7 +288,9 @@ def retry_sending(name):
 
 @frappe.whitelist()
 def send_now(name):
-	send_one(name, now=True)
+	record = EmailQueue.find(name)
+	if record:
+		record.send()
 
 def on_doctype_update():
 	"""Add index in `tabCommunication` for `(reference_doctype, reference_name)`"""

frappe/email/doctype/email_queue_recipient/email_queue_recipient.py

@@ -7,4 +7,16 @@ import frappe
 from frappe.model.document import Document
 
 class EmailQueueRecipient(Document):
-	pass
+	DOCTYPE = 'Email Queue Recipient'
+
+	def is_mail_to_be_sent(self):
+		return self.status == 'Not Sent'
+
+	def is_main_sent(self):
+		return self.status == 'Sent'
+
+	def update_db(self, commit=False, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+		if commit:
+			frappe.db.commit()
+

frappe/email/doctype/notification/notification.json

@@ -102,7 +102,8 @@
    "default": "0",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "no_copy": 1
   },
   {
    "depends_on": "is_standard",
@@ -281,7 +282,7 @@
  "icon": "fa fa-envelope",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-11-24 14:25:43.245677",
+ "modified": "2021-05-04 11:17:11.882314",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification",

frappe/email/doctype/notification/test_notification.py

@@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler
 from frappe.desk.form import assign_to
 import unittest
 
-test_records = frappe.get_test_records('Notification')
-
-test_dependencies = ["User"]
+test_dependencies = ["User", "Notification"]
 
 class TestNotification(unittest.TestCase):
 	def setUp(self):

frappe/email/email_body.py

@@ -4,7 +4,7 @@
 from __future__ import unicode_literals
 import frappe, re, os
 from frappe.utils.pdf import get_pdf
-from frappe.email.smtp import get_outgoing_email_account
+from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint,
 	split_emails, to_markdown, markdown, random_string, parse_addr)
 import email.utils
@@ -75,7 +75,8 @@ class EMail:
 		self.bcc = bcc or []
 		self.html_set = False
 
-		self.email_account = email_account or get_outgoing_email_account(sender=sender)
+		self.email_account = email_account or \
+			EmailAccount.find_outgoing(match_by_email=sender, _raise_error=True)
 
 	def set_html(self, message, text_content = None, footer=None, print_html=None,
 		formatted=None, inline_images=None, header=None):
@@ -249,8 +250,8 @@ class EMail:
 
 def get_formatted_html(subject, message, footer=None, print_html=None,
 		email_account=None, header=None, unsubscribe_link=None, sender=None, with_container=False):
-	if not email_account:
-		email_account = get_outgoing_email_account(False, sender=sender)
+
+	email_account = email_account or EmailAccount.find_outgoing(match_by_email=sender)
 
 	signature = None
 	if "<!-- signature-included -->" not in message:
@@ -291,18 +292,12 @@ def inline_style_in_html(html):
 	''' Convert email.css and html to inline-styled html
 	'''
 	from premailer import Premailer
+	from frappe.utils.jinja_globals import bundled_asset
 
-	apps = frappe.get_installed_apps()
-
-	# add frappe email css file
-	css_files = ['assets/css/email.css']
-	if 'frappe' in apps:
-		apps.remove('frappe')
-
-	for app in apps:
-		path = 'assets/{0}/css/email.css'.format(app)
-		css_files.append(path)
-
+	# get email css files from hooks
+	css_files = frappe.get_hooks('email_css')
+	css_files = [bundled_asset(path) for path in css_files]
+	css_files = [path.lstrip('/') for path in css_files]
 	css_files = [css_file for css_file in css_files if os.path.exists(os.path.abspath(css_file))]
 
 	p = Premailer(html=html, external_styles=css_files, strip_important=False)
@@ -480,4 +475,4 @@ def sanitize_email_header(str):
 	return str.replace('\r', '').replace('\n', '')
 
 def get_brand_logo(email_account):
-	return email_account.get('brand_logo')
+	return email_account.get('brand_logo')

frappe/email/queue.py

@@ -7,7 +7,8 @@ import sys
 from six.moves import html_parser as HTMLParser
 import smtplib, quopri, json
 from frappe import msgprint, _, safe_decode, safe_encode, enqueue
-from frappe.email.smtp import SMTPServer, get_outgoing_email_account
+from frappe.email.smtp import SMTPServer
+from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.email.email_body import get_email, get_formatted_html, add_attachment
 from frappe.utils.verified_command import get_signed_params, verify_request
 from html2text import html2text
@@ -73,7 +74,9 @@ def send(recipients=None, sender=None, subject=None, message=None, text_content=
 	if isinstance(send_after, int):
 		send_after = add_days(nowdate(), send_after)
 
-	email_account = get_outgoing_email_account(True, append_to=reference_doctype, sender=sender)
+	email_account = EmailAccount.find_outgoing(
+		match_by_doctype=reference_doctype, match_by_email=sender, _raise_error=True)
+
 	if not sender or sender == "Administrator":
 		sender = email_account.default_sender
 
@@ -170,19 +173,19 @@ def add(recipients, sender, subject, **kwargs):
 			if not email_queue:
 				email_queue = get_email_queue([r], sender, subject, **kwargs)
 				if kwargs.get('now'):
-					send_one(email_queue.name, now=True)
+					email_queue.send()
 			else:
 				duplicate = email_queue.get_duplicate([r])
 				duplicate.insert(ignore_permissions=True)
 
 				if kwargs.get('now'):
-					send_one(duplicate.name, now=True)
+					duplicate.send()
 
 			frappe.db.commit()
 	else:
 		email_queue = get_email_queue(recipients, sender, subject, **kwargs)
 		if kwargs.get('now'):
-			send_one(email_queue.name, now=True)
+			email_queue.send()
 
 def get_email_queue(recipients, sender, subject, **kwargs):
 	'''Make Email Queue object'''
@@ -234,6 +237,9 @@ def get_email_queue(recipients, sender, subject, **kwargs):
 			', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent')
 
 	recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', [])))
+	email_account = kwargs.get('email_account')
+	email_account_name = email_account and email_account.is_exists_in_db() and email_account.name
+
 	e.set_recipients(recipients)
 	e.reference_doctype = kwargs.get('reference_doctype')
 	e.reference_name = kwargs.get('reference_name')
@@ -245,8 +251,8 @@ def get_email_queue(recipients, sender, subject, **kwargs):
 	e.send_after = kwargs.get('send_after')
 	e.show_as_cc = ",".join(kwargs.get('cc', []))
 	e.show_as_bcc = ",".join(kwargs.get('bcc', []))
+	e.email_account = email_account_name or None
 	e.insert(ignore_permissions=True)
-
 	return e
 
 def get_emails_sent_this_month():
@@ -328,44 +334,25 @@ def return_unsubscribed_page(email, doctype, name):
 		indicator_color='green')
 
 def flush(from_test=False):
-	"""flush email queue, every time: called from scheduler"""
-	# additional check
-
-	auto_commit = not from_test
+	"""flush email queue, every time: called from scheduler
+	"""
+	from frappe.email.doctype.email_queue.email_queue import send_mail
+	# To avoid running jobs inside unit tests
 	if frappe.are_emails_muted():
 		msgprint(_("Emails are muted"))
 		from_test = True
 
-	smtpserver_dict = frappe._dict()
-
-	for email in get_queue():
-
-		if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
-			break
-
-		if email.name:
-			smtpserver = smtpserver_dict.get(email.sender)
-			if not smtpserver:
-				smtpserver = SMTPServer()
-				smtpserver_dict[email.sender] = smtpserver
+	if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
+		return
 
-			if from_test:
-				send_one(email.name, smtpserver, auto_commit)
-			else:
-				send_one_args = {
-					'email': email.name,
-					'smtpserver': smtpserver,
-					'auto_commit': auto_commit,
-				}
-				enqueue(
-					method = 'frappe.email.queue.send_one',
-					queue = 'short',
-					**send_one_args
-				)
+	for row in get_queue():
+		try:
+			func = send_mail if from_test else send_mail.enqueue
+			is_background_task = not from_test
+			func(email_queue_name = row.name, is_background_task = is_background_task)
+		except Exception:
+			frappe.log_error()
 
-		# NOTE: removing commit here because we pass auto_commit
-		# finally:
-		# 	frappe.db.commit()
 def get_queue():
 	return frappe.db.sql('''select
 			name, sender
@@ -378,213 +365,6 @@ def get_queue():
 			by priority desc, creation asc
 		limit 500''', { 'now': now_datetime() }, as_dict=True)
 
-
-def send_one(email, smtpserver=None, auto_commit=True, now=False):
-	'''Send Email Queue with given smtpserver'''
-
-	email = frappe.db.sql('''select
-			name, status, communication, message, sender, reference_doctype,
-			reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
-			show_as_cc, add_unsubscribe_link, attachments, retry
-		from
-			`tabEmail Queue`
-		where
-			name=%s
-		for update''', email, as_dict=True)
-
-	if len(email):
-		email = email[0]
-	else:
-		return
-
-	recipients_list = frappe.db.sql('''select name, recipient, status from
-		`tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1)
-
-	if frappe.are_emails_muted():
-		frappe.msgprint(_("Emails are muted"))
-		return
-
-	if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 :
-		return
-
-	if email.status not in ('Not Sent','Partially Sent') :
-		# rollback to release lock and return
-		frappe.db.rollback()
-		return
-
-	frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
-		(now_datetime(), email.name), auto_commit=auto_commit)
-
-	if email.communication:
-		frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-	email_sent_to_any_recipient = None
-
-	try:
-		message = None
-
-		if not frappe.flags.in_test:
-			if not smtpserver:
-				smtpserver = SMTPServer()
-
-			# to avoid always using default email account for outgoing
-			if getattr(frappe.local, "outgoing_email_account", None):
-				frappe.local.outgoing_email_account = {}
-
-			smtpserver.setup_email_account(email.reference_doctype, sender=email.sender)
-
-		for recipient in recipients_list:
-			if recipient.status != "Not Sent":
-				continue
-
-			message = prepare_message(email, recipient.recipient, recipients_list)
-			if not frappe.flags.in_test:
-				smtpserver.sess.sendmail(email.sender, recipient.recipient, message)
-
-			recipient.status = "Sent"
-			frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
-				(now_datetime(), recipient.name), auto_commit=auto_commit)
-
-		email_sent_to_any_recipient = any("Sent" == s.status for s in recipients_list)
-
-		#if all are sent set status
-		if email_sent_to_any_recipient:
-			frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-				where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
-		if frappe.flags.in_test:
-			frappe.flags.sent_mail = message
-			return
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient:
-			smtpserver.email_account.append_email_to_sent_folder(message)
-
-	except (smtplib.SMTPServerDisconnected,
-			smtplib.SMTPConnectError,
-			smtplib.SMTPHeloError,
-			smtplib.SMTPAuthenticationError,
-			smtplib.SMTPRecipientsRefused,
-			JobTimeoutException):
-
-		# bad connection/timeout, retry later
-
-		if email_sent_to_any_recipient:
-			frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		# no need to attempt further
-		return
-
-	except Exception as e:
-		frappe.db.rollback()
-
-		if email.retry < 3:
-			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			if email_sent_to_any_recipient:
-				frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
-					(text_type(e), email.name), auto_commit=auto_commit)
-			else:
-				frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-					where name=%s""", (text_type(e), email.name), auto_commit=auto_commit)
-
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		if now:
-			print(frappe.get_traceback())
-			raise e
-
-		else:
-			# log to Error Log
-			frappe.log_error('frappe.email.queue.flush')
-
-def prepare_message(email, recipient, recipients_list):
-	message = email.message
-	if not message:
-		return ""
-
-	# Parse "Email Account" from "Email Sender"
-	email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender)
-	if frappe.conf.use_ssl and email_account.track_email_status:
-		# Using SSL => Publically available domain => Email Read Reciept Possible
-		message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode())
-	else:
-		# No SSL => No Email Read Reciept
-		message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode())
-
-	if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
-		unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
-		email.unsubscribe_method, email.unsubscribe_params)
-		message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
-
-	if email.expose_recipients == "header":
-		pass
-	else:
-		if email.expose_recipients == "footer":
-			if isinstance(email.show_as_cc, string_types):
-				email.show_as_cc = email.show_as_cc.split(",")
-			email_sent_to = [r.recipient for r in recipients_list]
-			email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
-			email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
-
-			if email_sent_cc:
-				email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
-			else:
-				email_sent_message = _("This email was sent to {0}").format(email_sent_to)
-			message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
-
-		message = message.replace("<!--recipient-->", recipient)
-
-	message = (message and message.encode('utf8')) or ''
-	message = safe_decode(message)
-
-	if PY3:
-		from email.policy import SMTPUTF8
-		message = Parser(policy=SMTPUTF8).parsestr(message)
-	else:
-		message = Parser().parsestr(message)
-
-	if email.attachments:
-		# On-demand attachments
-
-		attachments = json.loads(email.attachments)
-
-		for attachment in attachments:
-			if attachment.get('fcontent'):
-				continue
-
-			fid = attachment.get("fid")
-			if fid:
-				_file = frappe.get_doc("File", fid)
-				fcontent = _file.get_content()
-				attachment.update({
-					'fname': _file.file_name,
-					'fcontent': fcontent,
-					'parent': message
-				})
-				attachment.pop("fid", None)
-				add_attachment(**attachment)
-
-			elif attachment.get("print_format_attachment") == 1:
-				attachment.pop("print_format_attachment", None)
-				print_format_file = frappe.attach_print(**attachment)
-				print_format_file.update({"parent": message})
-				add_attachment(**print_format_file)
-
-	return safe_encode(message.as_string())
-
 def clear_outbox(days=None):
 	"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
 	Note: Used separate query to avoid deadlock

frappe/email/receive.py

@@ -284,7 +284,7 @@ class EmailServer:
 
 		flags = []
 		for flag in imaplib.ParseFlags(flag_string) or []:
-			pattern = re.compile("\w+")
+			pattern = re.compile(r"\w+")
 			match = re.search(pattern, frappe.as_unicode(flag))
 			flags.append(match.group(0))
 
@@ -555,7 +555,7 @@ class Email:
 
 	def get_thread_id(self):
 		"""Extract thread ID from `[]`"""
-		l = re.findall('(?<=\[)[\w/-]+', self.subject)
+		l = re.findall(r'(?<=\[)[\w/-]+', self.subject)
 		return l and l[0] or None
 
 

frappe/email/smtp.py

@@ -9,11 +9,24 @@ import _socket, sys
 from frappe import _
 from frappe.utils import cint, cstr, parse_addr
 
+CONNECTION_FAILED = _('Could not connect to outgoing email server')
+AUTH_ERROR_TITLE = _("Invalid Credentials")
+AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.")
+SOCKET_ERROR_TITLE = _("Incorrect Configuration")
+SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port")
+SEND_MAIL_FAILED = _("Unable to send emails at this time")
+EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
+
+class InvalidEmailCredentials(frappe.ValidationError):
+	pass
+
 def send(email, append_to=None, retry=1):
 	"""Deprecated: Send the message or add it to Outbox Email"""
 	def _send(retry):
+		from frappe.email.doctype.email_account.email_account import EmailAccount
 		try:
-			smtpserver = SMTPServer(append_to=append_to)
+			email_account = EmailAccount.find_outgoing(match_by_doctype=append_to)
+			smtpserver = email_account.get_smtp_server()
 
 			# validate is called in as_string
 			email_body = email.as_string()
@@ -34,224 +47,80 @@ def send(email, append_to=None, retry=1):
 
 	_send(retry)
 
-def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None):
-	"""Returns outgoing email account based on `append_to` or the default
-		outgoing account. If default outgoing account is not found, it will
-		try getting settings from `site_config.json`."""
-
-	sender_email_id = None
-	_email_account = None
-
-	if sender:
-		sender_email_id = parse_addr(sender)[1]
-
-	if not getattr(frappe.local, "outgoing_email_account", None):
-		frappe.local.outgoing_email_account = {}
-
-	if not (frappe.local.outgoing_email_account.get(append_to)
-		or frappe.local.outgoing_email_account.get(sender_email_id)
-		or frappe.local.outgoing_email_account.get("default")):
-		email_account = None
-
-		if sender_email_id:
-			# check if the sender has an email account with enable_outgoing
-			email_account = _get_email_account({"enable_outgoing": 1,
-					"email_id": sender_email_id})
-
-		if not email_account and append_to:
-			# append_to is only valid when enable_incoming is checked
-			email_accounts = frappe.db.get_values("Email Account", {
-				"enable_outgoing": 1,
-				"enable_incoming": 1,
-				"append_to": append_to,
-			}, cache=True)
-
-			if email_accounts:
-				_email_account = email_accounts[0]
-
-			else:
-				email_account = _get_email_account({
-					"enable_outgoing": 1,
-					"enable_incoming": 1,
-					"append_to": append_to
-				})
-
-		if not email_account:
-			# sender don't have the outging email account
-			sender_email_id = None
-			email_account = get_default_outgoing_email_account(raise_exception_not_set=raise_exception_not_set)
-
-		if not email_account and _email_account:
-			# if default email account is not configured then setup first email account based on append to
-			email_account = _email_account
-
-		if not email_account and raise_exception_not_set and cint(frappe.db.get_single_value('System Settings', 'setup_complete')):
-			frappe.throw(_("Please setup default Email Account from Setup > Email > Email Account"),
-				frappe.OutgoingEmailError)
-
-		if email_account:
-			if email_account.enable_outgoing and not getattr(email_account, 'from_site_config', False):
-				raise_exception = True
-				if email_account.smtp_server in ['localhost','127.0.0.1'] or email_account.no_smtp_authentication:
-					raise_exception = False
-				email_account.password = email_account.get_password(raise_exception=raise_exception)
-			email_account.default_sender = email.utils.formataddr((email_account.name, email_account.get("email_id")))
-
-		frappe.local.outgoing_email_account[append_to or sender_email_id or "default"] = email_account
-
-	return frappe.local.outgoing_email_account.get(append_to) \
-		or frappe.local.outgoing_email_account.get(sender_email_id) \
-		or frappe.local.outgoing_email_account.get("default")
-
-def get_default_outgoing_email_account(raise_exception_not_set=True):
-	'''conf should be like:
-		{
-		 "mail_server": "smtp.example.com",
-		 "mail_port": 587,
-		 "use_tls": 1,
-		 "mail_login": "emails@example.com",
-		 "mail_password": "Super.Secret.Password",
-		 "auto_email_id": "emails@example.com",
-		 "email_sender_name": "Example Notifications",
-		 "always_use_account_email_id_as_sender": 0,
-		 "always_use_account_name_as_sender_name": 0
-		}
-	'''
-	email_account = _get_email_account({"enable_outgoing": 1, "default_outgoing": 1})
-	if email_account:
-		email_account.password = email_account.get_password(raise_exception=False)
-
-	if not email_account and frappe.conf.get("mail_server"):
-		# from site_config.json
-		email_account = frappe.new_doc("Email Account")
-		email_account.update({
-			"smtp_server": frappe.conf.get("mail_server"),
-			"smtp_port": frappe.conf.get("mail_port"),
-
-			# legacy: use_ssl was used in site_config instead of use_tls, but meant the same thing
-			"use_tls": cint(frappe.conf.get("use_tls") or 0) or cint(frappe.conf.get("use_ssl") or 0),
-			"login_id": frappe.conf.get("mail_login"),
-			"email_id": frappe.conf.get("auto_email_id") or frappe.conf.get("mail_login") or 'notifications@example.com',
-			"password": frappe.conf.get("mail_password"),
-			"always_use_account_email_id_as_sender": frappe.conf.get("always_use_account_email_id_as_sender", 0),
-			"always_use_account_name_as_sender_name": frappe.conf.get("always_use_account_name_as_sender_name", 0)
-		})
-		email_account.from_site_config = True
-		email_account.name = frappe.conf.get("email_sender_name") or "Frappe"
-
-	if not email_account and not raise_exception_not_set:
-		return None
-
-	if frappe.are_emails_muted():
-		# create a stub
-		email_account = frappe.new_doc("Email Account")
-		email_account.update({
-			"email_id": "notifications@example.com"
-		})
-
-	return email_account
-
-def _get_email_account(filters):
-	name = frappe.db.get_value("Email Account", filters)
-	return frappe.get_doc("Email Account", name) if name else None
-
 class SMTPServer:
-	def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None):
-		# get defaults from mail settings
-
-		self._sess = None
-		self.email_account = None
-		self.server = None
-		self.append_emails_to_sent_folder = None
+	def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None):
+		self.login = login
+		self.password = password
+		self._server = server
+		self._port = port
+		self.use_tls = use_tls
+		self.use_ssl = use_ssl
+		self._session = None
+
+		if not self.server:
+			frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError)
 
-		if server:
-			self.server = server
-			self.port = port
-			self.use_tls = cint(use_tls)
-			self.use_ssl = cint(use_ssl)
-			self.login = login
-			self.password = password
+	@property
+	def port(self):
+		port = self._port or (self.use_ssl and 465) or (self.use_tls and 587)
+		return cint(port)
 
-		else:
-			self.setup_email_account(append_to)
+	@property
+	def server(self):
+		return cstr(self._server or "")
 
-	def setup_email_account(self, append_to=None, sender=None):
-		self.email_account = get_outgoing_email_account(raise_exception_not_set=False, append_to=append_to, sender=sender)
-		if self.email_account:
-			self.server = self.email_account.smtp_server
-			self.login = (getattr(self.email_account, "login_id", None) or self.email_account.email_id)
-			if not self.email_account.no_smtp_authentication:
-				if self.email_account.ascii_encode_password:
-					self.password = frappe.safe_encode(self.email_account.password, 'ascii')
-				else:
-					self.password = self.email_account.password
-			else:
-				self.password = None
-			self.port = self.email_account.smtp_port
-			self.use_tls = self.email_account.use_tls
-			self.sender = self.email_account.email_id
-			self.use_ssl = self.email_account.use_ssl_for_outgoing
-			self.append_emails_to_sent_folder = self.email_account.append_emails_to_sent_folder
-			self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender"))
-			self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name"))
+	def secure_session(self, conn):
+		"""Secure the connection incase of TLS.
+		"""
+		if self.use_tls:
+			conn.ehlo()
+			conn.starttls()
+			conn.ehlo()
 
 	@property
-	def sess(self):
-		"""get session"""
-		if self._sess:
-			return self._sess
+	def session(self):
+		if self.is_session_active():
+			return self._session
 
-		# check if email server specified
-		if not getattr(self, 'server'):
-			err_msg = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
-			frappe.msgprint(err_msg)
-			raise frappe.OutgoingEmailError(err_msg)
+		SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP
 
 		try:
-			if self.use_ssl:
-				if not self.port:
-					self.port = 465
-
-				self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port))
-			else:
-				if self.use_tls and not self.port:
-					self.port = 587
-
-				self._sess = smtplib.SMTP(cstr(self.server or ""),
-						cint(self.port) or None)
-
-			if not self._sess:
-				err_msg = _('Could not connect to outgoing email server')
-				frappe.msgprint(err_msg)
-				raise frappe.OutgoingEmailError(err_msg)
-
-			if self.use_tls:
-				self._sess.ehlo()
-				self._sess.starttls()
-				self._sess.ehlo()
+			self._session = SMTP(self.server, self.port)
+			if not self._session:
+				frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError)
 
+			self.secure_session(self._session)
 			if self.login and self.password:
-				ret = self._sess.login(str(self.login or ""), str(self.password or ""))
+				res = self._session.login(str(self.login or ""), str(self.password or ""))
 
 				# check if logged correctly
-				if ret[0]!=235:
-					frappe.msgprint(ret[1])
-					raise frappe.OutgoingEmailError(ret[1])
+				if res[0]!=235:
+					frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError)
 
-			return self._sess
+			return self._session
 
 		except smtplib.SMTPAuthenticationError as e:
-			from frappe.email.doctype.email_account.email_account import EmailAccount
-			EmailAccount.throw_invalid_credentials_exception()
+			self.throw_invalid_credentials_exception()
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.throw(
-				_("Invalid Outgoing Mail Server or Port"),
-				exc=frappe.ValidationError,
-				title=_("Incorrect Configuration")
-			)
+			frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE)
 
 		except smtplib.SMTPException:
-			frappe.msgprint(_('Unable to send emails at this time'))
+			frappe.msgprint(SEND_MAIL_FAILED)
 			raise
+
+	def is_session_active(self):
+		if self._session:
+			try:
+				return self._session.noop()[0] == 250
+			except Exception:
+				return False
+
+	def quit(self):
+		if self.is_session_active():
+			self._session.quit()
+
+	@classmethod
+	def throw_invalid_credentials_exception(cls):
+		frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials)

frappe/email/test_email_body.py

@@ -7,10 +7,10 @@ from frappe import safe_decode
 from frappe.email.receive import Email
 from frappe.email.email_body import (replace_filename_with_cid,
 					get_email, inline_style_in_html, get_header)
-from frappe.email.queue import prepare_message, get_email_queue
+from frappe.email.queue import get_email_queue
+from frappe.email.doctype.email_queue.email_queue import SendMailContext
 from six import PY3
 
-
 class TestEmailBody(unittest.TestCase):
 	def setUp(self):
 		email_html = '''
@@ -57,7 +57,8 @@ This is the text version of this email
 			content='<h1>' + uni_chr1 + 'abcd' + uni_chr2 + '</h1>',
 			formatted='<h1>' + uni_chr1 + 'abcd' + uni_chr2 + '</h1>',
 			text_content='whatever')
-		result = prepare_message(email=email, recipient='test@test.com', recipients_list=[])
+		mail_ctx = SendMailContext(queue_doc = email)
+		result = mail_ctx.build_message(recipient_email = 'test@test.com')
 		self.assertTrue(b"<h1>=EA=80=80abcd=DE=B4</h1>" in result)
 
 	def test_prepare_message_returns_cr_lf(self):
@@ -68,8 +69,10 @@ This is the text version of this email
 			content='<h1>\n this is a test of newlines\n' + '</h1>',
 			formatted='<h1>\n this is a test of newlines\n' + '</h1>',
 			text_content='whatever')
-		result = safe_decode(prepare_message(email=email,
-						recipient='test@test.com', recipients_list=[]))
+
+		mail_ctx = SendMailContext(queue_doc = email)
+		result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com'))
+
 		if PY3:
 			self.assertTrue(result.count('\n') == result.count("\r"))
 		else:

frappe/email/test_smtp.py

@@ -4,7 +4,7 @@
 import unittest
 import frappe
 from frappe.email.smtp import SMTPServer
-from frappe.email.smtp import get_outgoing_email_account
+from frappe.email.doctype.email_account.email_account import EmailAccount
 
 class TestSMTP(unittest.TestCase):
 	def test_smtp_ssl_session(self):
@@ -33,13 +33,13 @@ class TestSMTP(unittest.TestCase):
 
 		frappe.local.outgoing_email_account = {}
 		# lowest preference given to email account with default incoming enabled
-		create_email_account(email_id="default_outgoing_enabled@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1)
-		self.assertEqual(get_outgoing_email_account().email_id, "default_outgoing_enabled@gmail.com")
+		create_email_account(email_id="default_outgoing_enabled@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1)
+		self.assertEqual(EmailAccount.find_outgoing().email_id, "default_outgoing_enabled@gmail.com")
 
 		frappe.local.outgoing_email_account = {}
 		# highest preference given to email account with append_to matching
-		create_email_account(email_id="append_to@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
-		self.assertEqual(get_outgoing_email_account(append_to="Blog Post").email_id, "append_to@gmail.com")
+		create_email_account(email_id="append_to@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
+		self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com")
 
 		# add back the mail_server
 		frappe.conf['mail_server'] = mail_server
@@ -75,4 +75,4 @@ def make_server(port, ssl, tls):
 		use_tls = tls
 	)
 
-	server.sess
+	server.session

frappe/event_streaming/doctype/event_producer/event_producer.py

@@ -55,8 +55,8 @@ class EventProducer(Document):
 			self.reload()
 
 	def check_url(self):
-		if not frappe.utils.validate_url(self.producer_url):
-			frappe.throw(_('Invalid URL'))
+		valid_url_schemes = ("http", "https")
+		frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes)
 
 		# remove '/' from the end of the url like http://test_site.com/
 		# to prevent mismatch in get_url() results

frappe/handler.py

@@ -228,10 +228,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 	is_whitelisted(fn)
 	is_valid_http_method(fn)
 
-	try:
-		fnargs = inspect.getargspec(method_obj)[0]
-	except ValueError:
-		fnargs = inspect.getfullargspec(method_obj).args
+	fnargs = inspect.getfullargspec(method_obj).args
 
 	if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"):
 		response = doc.run_method(method)

frappe/hooks.py

@@ -29,16 +29,16 @@ page_js = {
 
 # website
 app_include_js = [
-	"/assets/js/libs.min.js",
-	"/assets/js/desk.min.js",
-	"/assets/js/list.min.js",
-	"/assets/js/form.min.js",
-	"/assets/js/control.min.js",
-	"/assets/js/report.min.js",
+	"libs.bundle.js",
+	"desk.bundle.js",
+	"list.bundle.js",
+	"form.bundle.js",
+	"controls.bundle.js",
+	"report.bundle.js",
 ]
 app_include_css = [
-	"/assets/css/desk.min.css",
-	"/assets/css/report.min.css",
+	"desk.bundle.css",
+	"report.bundle.css",
 ]
 
 doctype_js = {
@@ -52,6 +52,8 @@ web_include_js = [
 
 web_include_css = []
 
+email_css = ['email.bundle.css']
+
 website_route_rules = [
 	{"from_route": "/blog/<category>", "to_route": "Blog Post"},
 	{"from_route": "/kb/<category>", "to_route": "Help Article"},
@@ -226,7 +228,6 @@ scheduler_events = {
 		"frappe.desk.doctype.event.event.send_event_digest",
 		"frappe.sessions.clear_expired_sessions",
 		"frappe.email.doctype.notification.notification.trigger_daily_alerts",
-		"frappe.realtime.remove_old_task_logs",
 		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",

frappe/installer.py

@@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None):
 
 
 def make_site_dirs():
-	site_public_path = os.path.join(frappe.local.site_path, 'public')
-	site_private_path = os.path.join(frappe.local.site_path, 'private')
-	for dir_path in (
-			os.path.join(site_private_path, 'backups'),
-			os.path.join(site_public_path, 'files'),
-			os.path.join(site_private_path, 'files'),
-			os.path.join(frappe.local.site_path, 'logs'),
-			os.path.join(frappe.local.site_path, 'task-logs')):
-		if not os.path.exists(dir_path):
-			os.makedirs(dir_path)
-	locks_dir = frappe.get_site_path('locks')
-	if not os.path.exists(locks_dir):
-			os.makedirs(locks_dir)
+	for dir_path in [
+		os.path.join("public", "files"),
+		os.path.join("private", "backups"),
+		os.path.join("private", "files"),
+		"error-snapshots",
+		"locks",
+		"logs",
+	]:
+		path = frappe.get_site_path(dir_path)
+		os.makedirs(path, exist_ok=True)
 
 
 def add_module_defs(app):

frappe/integrations/doctype/connected_app/connected_app.json

@@ -54,7 +54,8 @@
    "fieldname": "client_id",
    "fieldtype": "Data",
    "in_list_view": 1,
-   "label": "Client Id"
+   "label": "Client Id",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "redirect_uri",
@@ -96,12 +97,14 @@
   {
    "fieldname": "authorization_uri",
    "fieldtype": "Data",
-   "label": "Authorization URI"
+   "label": "Authorization URI",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "token_uri",
    "fieldtype": "Data",
-   "label": "Token URI"
+   "label": "Token URI",
+   "mandatory_depends_on": "eval:doc.redirect_uri"
   },
   {
    "fieldname": "revocation_uri",
@@ -136,7 +139,7 @@
    "link_fieldname": "connected_app"
   }
  ],
- "modified": "2020-11-16 16:29:50.277405",
+ "modified": "2021-05-10 05:03:06.296863",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Connected App",

frappe/integrations/doctype/connected_app/connected_app.py

@@ -26,20 +26,27 @@ class ConnectedApp(Document):
 		self.redirect_uri = urljoin(base_url, callback_path)
 
 	def get_oauth2_session(self, user=None, init=False):
+		"""Return an auto-refreshing OAuth2 session which is an extension of a requests.Session()"""
 		token = None
 		token_updater = None
+		auto_refresh_kwargs = None
 
 		if not init:
 			user = user or frappe.session.user
 			token_cache = self.get_user_token(user)
 			token = token_cache.get_json()
 			token_updater = token_cache.update_data
+			auto_refresh_kwargs = {'client_id': self.client_id}
+			client_secret = self.get_password('client_secret')
+			if client_secret:
+				auto_refresh_kwargs['client_secret'] = client_secret
 
 		return OAuth2Session(
 			client_id=self.client_id,
 			token=token,
 			token_updater=token_updater,
 			auto_refresh_url=self.token_uri,
+			auto_refresh_kwargs=auto_refresh_kwargs,
 			redirect_uri=self.redirect_uri,
 			scope=self.get_scopes()
 		)

frappe/integrations/oauth2.py

@@ -1,6 +1,5 @@
 import json
 from urllib.parse import quote, urlencode
-
 from oauthlib.oauth2 import FatalClientError, OAuth2Error
 from oauthlib.openid.connect.core.endpoints.pre_configured import (
 	Server as WebApplicationServer,