Merge pull request #13002 from frappe/version-13-hotfix

.github/helper/semgrep_rules/frappe_correctness.py

@@ -0,0 +1,28 @@
+import frappe
+from frappe import _, flt
+
+from frappe.model.document import Document
+
+
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	# ruleid: frappe-modifying-after-submit
+	self.status = 'Submitted'
+
+def on_submit(self):   # noqa
+	if flt(self.per_billed) < 100:
+		self.update_billing_status()
+	else:
+		# todook: frappe-modifying-after-submit
+		self.status = "Completed"
+		self.db_set("status", "Completed")
+
+class TestDoc(Document):
+	pass
+
+	def validate(self):
+		#ruleid: frappe-modifying-child-tables-while-iterating
+		for item in self.child_table:
+			if item.value < 0:
+				self.remove(item)

.github/helper/semgrep_rules/frappe_correctness.yml

@@ -0,0 +1,135 @@
+# This file specifies rules for correctness according to how frappe doctype data model works.
+
+rules:
+- id: frappe-modifying-but-not-comitting
+  patterns:
+    - pattern: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.save()
+    - metavariable-regex:
+        metavariable: '$ATTR'
+        # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me)
+        regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$'
+    - metavariable-regex:
+        metavariable: "$METHOD"
+        regex: "(on_submit|on_cancel)"
+  message: |
+    DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-modifying-but-not-comitting-other-method
+  patterns:
+  - pattern: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+          self.save()
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - metavariable-regex:
+      metavariable: "$METHOD"
+      regex: "(on_submit|on_cancel)"
+  message: |
+    self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-print-function-in-doctypes
+  pattern: print(...)
+  message: |
+      Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement.
+  languages: [python]
+  severity: WARNING
+  paths:
+      exclude:
+        - test_*.py
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-modifying-child-tables-while-iterating
+  pattern-either:
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.remove(...)
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.append(...)
+  message: |
+      Child table being modified while iterating on it.
+  languages: [python]
+  severity: ERROR
+  paths:
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-same-key-assigned-twice
+  pattern-either:
+    - pattern: |
+        {..., $X: $A, ..., $X: $B, ...}
+    - pattern: |
+        dict(..., ($X, $A), ..., ($X, $B), ...)
+    - pattern: |
+        _dict(..., ($X, $A), ..., ($X, $B), ...)
+  message: |
+      key `$X` is uselessly assigned twice. This could be a potential bug.
+  languages: [python]
+  severity: ERROR

.github/helper/semgrep_rules/security.yml

@@ -12,3 +12,18 @@ rules:
     exclude:
       - frappe/__init__.py
       - frappe/commands/utils.py
+
+- id: frappe-sqli-format-strings
+  patterns:
+    - pattern-inside: |
+        @frappe.whitelist()
+        def $FUNC(...):
+            ...
+    - pattern-either:
+        - pattern: frappe.db.sql("..." % ...)
+        - pattern: frappe.db.sql(f"...", ...)
+        - pattern: frappe.db.sql("...".format(...), ...)
+  message: |
+      Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines
+  languages: [python]
+  severity: WARNING

.github/helper/semgrep_rules/translate.yml

@@ -44,7 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + ... + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'
+  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations

.github/helper/semgrep_rules/ux.py

@@ -0,0 +1,31 @@
+import frappe
+from frappe import msgprint, throw, _
+
+
+# ruleid: frappe-missing-translate-function
+throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.msgprint("Useful message")
+
+# ruleid: frappe-missing-translate-function
+msgprint("Useful message")
+
+
+# ok: frappe-missing-translate-function
+translatedmessage = _("Hello")
+
+# ok: frappe-missing-translate-function
+throw(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(_("Helpful message"))
+
+# ok: frappe-missing-translate-function
+frappe.throw(_("Error occured"))

.github/helper/semgrep_rules/ux.yml

@@ -0,0 +1,15 @@
+rules:
+- id: frappe-missing-translate-function
+  pattern-either:
+  - patterns:
+      - pattern: frappe.msgprint("...", ...)
+      - pattern-not: frappe.msgprint(_("..."), ...)
+      - pattern-not: frappe.msgprint(__("..."), ...)
+  - patterns:
+      - pattern: frappe.throw("...", ...)
+      - pattern-not: frappe.throw(_("..."), ...)
+      - pattern-not: frappe.throw(__("..."), ...)
+  message: |
+      All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
+  languages: [python, javascript, json]
+  severity: ERROR

.github/workflows/ci-tests.yml

@@ -151,7 +151,8 @@ jobs:
           cd ${GITHUB_WORKSPACE}
           pip install coveralls==2.2.0
           pip install coverage==4.5.4
-          coveralls
+          coveralls --service=github
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
+          COVERALLS_SERVICE_NAME: github

.github/workflows/semgrep.yml

@@ -14,9 +14,19 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Run semgrep
+
+    - name: Setup semgrep
       run: |
         python -m pip install -q semgrep
         git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
+
+    - name: Semgrep errors
+      run: |
+        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files
+        semgrep --config="r/python.lang.correctness" --quiet --error $files
+
+    - name: Semgrep warnings
+      run: |
         files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files

README.md

@@ -39,7 +39,8 @@ Full-stack web application framework that uses Python and MariaDB on the server
 
 ### Installation
 
-[Install via Frappe Bench](https://github.com/frappe/bench)
+* [Install via Docker](https://github.com/frappe/frappe_docker)
+* [Install via Frappe Bench](https://github.com/frappe/bench)
 
 ## Contributing
 

cypress/integration/form.js

@@ -8,7 +8,7 @@ context('Form', () => {
 	});
 	it('create a new form', () => {
 		cy.visit('/app/todo/new');
-		cy.fill_field('description', 'this is a test todo', 'Text Editor').blur();
+		cy.fill_field('description', 'this is a test todo', 'Text Editor');
 		cy.wait(300);
 		cy.get('.page-title').should('contain', 'Not Saved');
 		cy.intercept({

cypress/integration/relative_time_filters.js

@@ -1,7 +1,4 @@
 context('Relative Timeframe', () => {
-	beforeEach(() => {
-		cy.login();
-	});
 	before(() => {
 		cy.login();
 		cy.visit('/app/website');

cypress/integration/table_multiselect.js

@@ -1,5 +1,5 @@
 context('Table MultiSelect', () => {
-	beforeEach(() => {
+	before(() => {
 		cy.login();
 	});
 

frappe/__init__.py

@@ -15,6 +15,7 @@ from __future__ import unicode_literals, print_function
 from six import iteritems, binary_type, text_type, string_types, PY2
 from werkzeug.local import Local, release_local
 import os, sys, importlib, inspect, json
+import typing
 from past.builtins import cmp
 import click
 
@@ -134,6 +135,14 @@ message_log = local("message_log")
 
 lang = local("lang")
 
+# This if block is never executed when running the code. It is only used for
+# telling static code analyzer where to find dynamically defined attributes.
+if typing.TYPE_CHECKING:
+	from frappe.database.mariadb.database import MariaDBDatabase
+	from frappe.database.postgres.database import PostgresDatabase
+	db: typing.Union[MariaDBDatabase, PostgresDatabase]
+# end: static analysis hack
+
 def init(site, sites_path=None, new_site=False):
 	"""Initialize frappe for the current site. Reset thread locals `frappe.local`"""
 	if getattr(local, "initialised", None):

frappe/api.py

@@ -1,12 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-from __future__ import unicode_literals
 
 import base64
 import binascii
 import json
-
-from six.moves.urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode, urlparse
 
 import frappe
 import frappe.client
@@ -14,6 +12,7 @@ import frappe.handler
 from frappe import _
 from frappe.utils.response import build_response
 
+
 def handle():
 	"""
 	Handler for `/api` methods
@@ -38,9 +37,6 @@ def handle():
 	`/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method
 	"""
 
-
-	validate_auth()
-
 	parts = frappe.request.path[1:].split("/",3)
 	call = doctype = name = None
 
@@ -116,7 +112,7 @@ def handle():
 						frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields'])
 					frappe.local.form_dict.setdefault('limit_page_length', 20)
 					frappe.local.response.update({
-						"data":  frappe.call(
+						"data": frappe.call(
 							frappe.client.get_list,
 							doctype,
 							**frappe.local.form_dict
@@ -140,6 +136,7 @@ def handle():
 
 	return build_response("json")
 
+
 def get_request_form_data():
 	if frappe.local.form_dict.data is None:
 		data = frappe.safe_decode(frappe.local.request.get_data())
@@ -148,25 +145,18 @@ def get_request_form_data():
 
 	return frappe.parse_json(data)
 
-def validate_auth():
-	if frappe.get_request_header("Authorization") is None:
-		return
-
-	VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token']
-	VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title()
 
+def validate_auth():
+	"""
+	Authenticate and sets user for the request.
+	"""
 	authorization_header = frappe.get_request_header("Authorization", str()).split(" ")
-	authorization_type = authorization_header[0].lower()
-
-	if len(authorization_header) == 1:
-		frappe.throw(_('Invalid Authorization headers, add a token with a prefix from one of the following: {0}.').format(VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationHeader)
 
-	if authorization_type == "bearer":
+	if len(authorization_header) == 2:
 		validate_oauth(authorization_header)
-	elif authorization_type in VALID_AUTH_PREFIX_TYPES:
 		validate_auth_via_api_keys(authorization_header)
-	else:
-		frappe.throw(_('Invalid Authorization Type {0}, must be one of {1}.').format(authorization_type, VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationPrefix)
+
+	validate_auth_via_hooks()
 
 
 def validate_oauth(authorization_header):
@@ -177,8 +167,8 @@ def validate_oauth(authorization_header):
 		authorization_header (list of str): The 'Authorization' header containing the prefix and token
 	"""
 
-	from frappe.oauth import get_url_delimiter
 	from frappe.integrations.oauth2 import get_oauth_server
+	from frappe.oauth import get_url_delimiter
 
 	form_dict = frappe.local.form_dict
 	token = authorization_header[1]
@@ -187,19 +177,20 @@ def validate_oauth(authorization_header):
 	access_token = {"access_token": token}
 	uri = parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token)
 	http_method = req.method
-	body = req.get_data()
 	headers = req.headers
+	body = req.get_data()
+	if req.content_type and "multipart/form-data" in req.content_type:
+		body = None
 
 	try:
 		required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter())
+		valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes)
+		if valid:
+			frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
+			frappe.local.form_dict = form_dict
 	except AttributeError:
-		frappe.throw(_("Invalid Bearer token, please provide a valid access token with prefix 'Bearer'."), frappe.InvalidAuthorizationToken)
+		pass
 
-	valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes)
-
-	if valid:
-		frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
-		frappe.local.form_dict = form_dict
 
 
 def validate_auth_via_api_keys(authorization_header):
@@ -222,8 +213,7 @@ def validate_auth_via_api_keys(authorization_header):
 	except binascii.Error:
 		frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken)
 	except (AttributeError, TypeError, ValueError):
-		frappe.throw(_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."), frappe.InvalidAuthorizationToken)
-
+		pass
 
 
 def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None):
@@ -248,3 +238,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non
 		if frappe.local.login_manager.user in ('', 'Guest'):
 			frappe.set_user(user)
 		frappe.local.form_dict = form_dict
+
+
+def validate_auth_via_hooks():
+	for auth_hook in frappe.get_hooks('auth_hooks', []):
+		frappe.get_attr(auth_hook)()

frappe/app.py

@@ -56,6 +56,7 @@ def application(request):
 		frappe.recorder.record()
 		frappe.monitor.start()
 		frappe.rate_limiter.apply()
+		frappe.api.validate_auth()
 
 		if request.method == "OPTIONS":
 			response = Response()

frappe/boot.py

@@ -42,6 +42,8 @@ def get_bootinfo():
 		bootinfo.user_info = get_user_info()
 		bootinfo.sid = frappe.session['sid']
 
+	bootinfo.user_groups = frappe.get_all('User Group', pluck="name")
+
 	bootinfo.modules = {}
 	bootinfo.module_list = []
 	load_desktop_data(bootinfo)

frappe/cache_manager.py

@@ -18,7 +18,7 @@ global_cache_keys = ("app_hooks", "installed_apps", 'all_apps',
 		'scheduler_events', 'time_zone', 'webhooks', 'active_domains',
 		'active_modules', 'assignment_rule', 'server_script_map', 'wkhtmltopdf_version',
 		'domain_restricted_doctypes', 'domain_restricted_pages', 'information_schema:counts',
-		'sitemap_routes', 'db_tables') + doctype_map_keys
+		'sitemap_routes', 'db_tables', 'server_script_autocompletion_items') + doctype_map_keys
 
 user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang",
 		"defaults", "user_permissions", "home_page", "linked_with",

frappe/commands/__init__.py

@@ -62,11 +62,24 @@ def popen(command, *args, **kwargs):
 	if env:
 		env = dict(environ, **env)
 
+	def set_low_prio():
+		import psutil
+		if psutil.LINUX:
+			psutil.Process().nice(19)
+			psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE)
+		elif psutil.WINDOWS:
+			psutil.Process().nice(psutil.IDLE_PRIORITY_CLASS)
+			psutil.Process().ionice(psutil.IOPRIO_VERYLOW)
+		else:
+			psutil.Process().nice(19)
+			# ionice not supported
+
 	proc = subprocess.Popen(command,
 		stdout=None if output else subprocess.PIPE,
 		stderr=None if output else subprocess.PIPE,
 		shell=shell,
 		cwd=cwd,
+		preexec_fn=set_low_prio,
 		env=env
 	)
 

frappe/commands/scheduler.py

@@ -18,22 +18,33 @@ def _is_scheduler_enabled():
 
 	return enable_scheduler
 
-@click.command('trigger-scheduler-event')
-@click.argument('event')
+
+@click.command("trigger-scheduler-event", help="Trigger a scheduler event")
+@click.argument("event")
 @pass_context
 def trigger_scheduler_event(context, event):
-	"Trigger a scheduler event"
 	import frappe.utils.scheduler
+
+	exit_code = 0
+
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			frappe.utils.scheduler.trigger(site, event, now=True)
+			try:
+				frappe.get_doc("Scheduled Job Type", {"method": event}).execute()
+			except frappe.DoesNotExistError:
+				click.secho(f"Event {event} does not exist!", fg="red")
+				exit_code = 1
 		finally:
 			frappe.destroy()
+
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
+	sys.exit(exit_code)
+
+
 @click.command('enable-scheduler')
 @pass_context
 def enable_scheduler(context):

frappe/commands/site.py

@@ -676,10 +676,8 @@ def start_ngrok(context):
 	frappe.init(site=site)
 
 	port = frappe.conf.http_port or frappe.conf.webserver_port
-	public_url = ngrok.connect(port=port, options={
-		'host_header': site
-	})
-	print(f'Public URL: {public_url}')
+	tunnel = ngrok.connect(addr=str(port), host_header=site)
+	print(f'Public URL: {tunnel.public_url}')
 	print('Inspect logs at http://localhost:4040')
 
 	ngrok_process = ngrok.get_ngrok_process()

frappe/commands/utils.py

@@ -11,7 +11,7 @@ import click
 import frappe
 from frappe.commands import get_site, pass_context
 from frappe.exceptions import SiteNotSpecifiedError
-from frappe.utils import get_bench_path, update_progress_bar
+from frappe.utils import get_bench_path, update_progress_bar, cint
 
 
 @click.command('build')
@@ -567,11 +567,14 @@ def run_ui_tests(context, app, headless=False):
 
 	node_bin = subprocess.getoutput("npm bin")
 	cypress_path = "{0}/cypress".format(node_bin)
-	plugin_path = "{0}/cypress-file-upload".format(node_bin)
+	plugin_path = "{0}/../cypress-file-upload".format(node_bin)
 
 	# check if cypress in path...if not, install it.
-	if not (os.path.exists(cypress_path) or os.path.exists(plugin_path)) \
-		or not subprocess.getoutput("npm view cypress version").startswith("6."):
+	if not (
+		os.path.exists(cypress_path)
+		and os.path.exists(plugin_path)
+		and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6
+	):
 		# install cypress
 		click.secho("Installing Cypress...", fg="yellow")
 		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 --no-lockfile")

frappe/core/doctype/communication/communication_list.js

@@ -20,6 +20,6 @@ frappe.listview_settings['Communication'] = {
 	},
 
 	primary_action: function() {
-		new frappe.views.CommunicationComposer({ doc: {} });
+		new frappe.views.CommunicationComposer();
 	}
 };

frappe/core/doctype/data_import/data_import.json

@@ -53,7 +53,8 @@
    "fieldname": "import_file",
    "fieldtype": "Attach",
    "in_list_view": 1,
-   "label": "Import File"
+   "label": "Import File",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
   },
   {
    "fieldname": "import_preview",
@@ -156,10 +157,11 @@
    "description": "Must be a publicly accessible Google Sheets URL",
    "fieldname": "google_sheets_url",
    "fieldtype": "Data",
-   "label": "Import from Google Sheets"
+   "label": "Import from Google Sheets",
+   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
   },
   {
-   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved",
+   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
    "fieldname": "refresh_google_sheet",
    "fieldtype": "Button",
    "label": "Refresh Google Sheet"
@@ -167,7 +169,7 @@
  ],
  "hide_toolbar": 1,
  "links": [],
- "modified": "2020-06-24 14:33:03.173876",
+ "modified": "2021-04-11 01:50:42.074623",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Data Import",

frappe/core/doctype/doctype/doctype.json

@@ -56,6 +56,8 @@
   "show_preview_popup",
   "show_name_in_global_search",
   "email_settings_sb",
+  "default_email_template",
+  "column_break_51",
   "email_append_to",
   "sender_field",
   "subject_field",
@@ -535,6 +537,16 @@
    "fieldname": "is_virtual",
    "fieldtype": "Check",
    "label": "Is Virtual"
+  },
+  {
+   "fieldname": "default_email_template",
+   "fieldtype": "Link",
+   "label": "Default Email Template",
+   "options": "Email Template"
+  },
+  {
+   "fieldname": "column_break_51",
+   "fieldtype": "Column Break"
   }
  ],
  "icon": "fa fa-bolt",
@@ -616,7 +628,7 @@
    "link_fieldname": "reference_doctype"
   }
  ],
- "modified": "2021-02-17 20:18:06.212232",
+ "modified": "2021-03-22 12:26:41.031135",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",
@@ -650,4 +662,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
+}

frappe/core/doctype/prepared_report/prepared_report.py

@@ -37,7 +37,10 @@ def run_background(prepared_report):
 			custom_report_doc = report
 			reference_report = custom_report_doc.reference_report
 			report = frappe.get_doc("Report", reference_report)
-			report.custom_columns = custom_report_doc.json
+			if custom_report_doc.json:
+				data = json.loads(custom_report_doc.json)
+				if data:
+					report.custom_columns = data["columns"]
 
 		result = generate_report_result(
 			report=report,

frappe/core/doctype/report/report.py

@@ -325,9 +325,8 @@ def get_group_by_field(args, doctype):
 	if args['aggregate_function'] == 'count':
 		group_by_field = 'count(*) as _aggregate_column'
 	else:
-		group_by_field = '{0}(`tab{1}`.{2}) as _aggregate_column'.format(
+		group_by_field = '{0}({1}) as _aggregate_column'.format(
 			args.aggregate_function,
-			doctype,
 			args.aggregate_on
 		)
 

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -2,14 +2,15 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
+import json
+from datetime import datetime
 from typing import Dict, List
 
-import frappe, json
-from frappe.model.document import Document
-from frappe.utils import now_datetime, get_datetime
-from datetime import datetime
 from croniter import croniter
+
+import frappe
+from frappe.model.document import Document
+from frappe.utils import get_datetime, now_datetime
 from frappe.utils.background_jobs import enqueue, get_jobs
 
 

frappe/core/doctype/server_script/server_script.js

@@ -9,6 +9,12 @@ frappe.ui.form.on('Server Script', {
 		if (frm.doc.script_type != 'Scheduler Event') {
 			frm.dashboard.hide();
 		}
+
+		frm.call('get_autocompletion_items')
+			.then(r => r.message)
+			.then(items => {
+				frm.set_df_property('script', 'autocompletions', items);
+			});
 	},
 
 	setup_help(frm) {

frappe/core/doctype/server_script/server_script.py

@@ -5,11 +5,12 @@
 from __future__ import unicode_literals
 
 import ast
+from types import FunctionType, MethodType, ModuleType
 from typing import Dict, List
 
 import frappe
 from frappe.model.document import Document
-from frappe.utils.safe_exec import safe_exec
+from frappe.utils.safe_exec import get_safe_globals, safe_exec, NamespaceDict
 from frappe import _
 
 
@@ -122,6 +123,51 @@ class ServerScript(Document):
 		if locals["conditions"]:
 			return locals["conditions"]
 
+	@frappe.whitelist()
+	def get_autocompletion_items(self):
+		"""Generates a list of a autocompletion strings from the context dict
+		that is used while executing a Server Script.
+
+		Returns:
+			list: Returns list of autocompletion items.
+			For e.g., ["frappe.utils.cint", "frappe.db.get_all", ...]
+		"""
+		def get_keys(obj):
+			out = []
+			for key in obj:
+				if key.startswith('_'):
+					continue
+				value = obj[key]
+				if isinstance(value, (NamespaceDict, dict)) and value:
+					if key == 'form_dict':
+						out.append(['form_dict', 7])
+						continue
+					for subkey, score in get_keys(value):
+						fullkey = f'{key}.{subkey}'
+						out.append([fullkey, score])
+				else:
+					if isinstance(value, type) and issubclass(value, Exception):
+						score = 0
+					elif isinstance(value, ModuleType):
+						score = 10
+					elif isinstance(value, (FunctionType, MethodType)):
+						score = 9
+					elif isinstance(value, type):
+						score = 8
+					elif isinstance(value, dict):
+						score = 7
+					else:
+						score = 6
+					out.append([key, score])
+			return out
+
+		items = frappe.cache().get_value('server_script_autocompletion_items')
+		if not items:
+			items = get_keys(get_safe_globals())
+			items = [{'value': d[0], 'score': d[1]} for d in items]
+			frappe.cache().set_value('server_script_autocompletion_items', items)
+		return items
+
 
 @frappe.whitelist()
 def setup_scheduler_events(script_name, frequency):

frappe/core/doctype/user/test_user.py

@@ -229,6 +229,28 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(extract_mentions(comment)[0], "test_user@example.com")
 		self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com")
 
+		doc = frappe.get_doc({
+			'doctype': 'User Group',
+			'name': 'Team',
+			'user_group_members': [{
+				'user': 'test@example.com'
+			}, {
+				'user': 'test1@example.com'
+			}]
+		})
+		doc.insert(ignore_if_duplicate=True)
+
+		comment = '''
+			<div>
+				Testing comment for
+				<span class="mention" data-id="Team" data-value="Team" data-is-group="true" data-denotation-char="@">
+					<span><span class="ql-mention-denotation-char">@</span>Team</span>
+				</span>
+				please check
+			</div>
+		'''
+		self.assertListEqual(extract_mentions(comment), ['test@example.com', 'test1@example.com'])
+
 	def test_rate_limiting_for_reset_password(self):
 		# Allow only one reset request for a day
 		frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 1)

frappe/core/doctype/user/user.py

@@ -1018,8 +1018,16 @@ def extract_mentions(txt):
 	soup = BeautifulSoup(txt, 'html.parser')
 	emails = []
 	for mention in soup.find_all(class_='mention'):
+		if mention.get('data-is-group') == 'true':
+			try:
+				user_group = frappe.get_cached_doc('User Group', mention['data-id'])
+				emails += [d.user for d in user_group.user_group_members]
+			except frappe.DoesNotExistError:
+				pass
+			continue
 		email = mention['data-id']
 		emails.append(email)
+
 	return emails
 
 def handle_password_test_fail(result):

frappe/core/doctype/user_group/test_user_group.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestUserGroup(unittest.TestCase):
+	pass

frappe/core/doctype/user_group/user_group.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('User Group', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/user_group/user_group.json

@@ -0,0 +1,48 @@
+{
+ "actions": [],
+ "autoname": "Prompt",
+ "creation": "2021-04-12 15:17:24.751710",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "user_group_members"
+ ],
+ "fields": [
+  {
+   "fieldname": "user_group_members",
+   "fieldtype": "Table MultiSelect",
+   "label": "User Group Members",
+   "options": "User Group Member",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-04-15 16:12:31.455401",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Group",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "read": 1,
+   "role": "All"
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/user_group/user_group.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+import frappe
+
+class UserGroup(Document):
+	def after_insert(self):
+		frappe.publish_realtime('user_group_added', self.name)
+
+	def on_trash(self):
+		frappe.publish_realtime('user_group_deleted', self.name)

frappe/core/doctype/user_group_member/test_user_group_member.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestUserGroupMember(unittest.TestCase):
+	pass

frappe/core/doctype/user_group_member/user_group_member.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('User Group Member', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/user_group_member/user_group_member.json

@@ -0,0 +1,32 @@
+{
+ "actions": [],
+ "creation": "2021-04-12 15:16:29.279107",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "user"
+ ],
+ "fields": [
+  {
+   "fieldname": "user",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "User",
+   "options": "User",
+   "reqd": 1
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2021-04-12 15:17:18.773046",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "User Group Member",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/user_group_member/user_group_member.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class UserGroupMember(Document):
+	pass

frappe/custom/doctype/custom_field/custom_field.py

@@ -40,6 +40,8 @@ class CustomField(Document):
 			frappe.throw(_("A field with the name '{}' already exists in doctype {}.").format(self.fieldname, self.dt))
 
 	def validate(self):
+		from frappe.custom.doctype.customize_form.customize_form import CustomizeForm
+
 		meta = frappe.get_meta(self.dt, cached=False)
 		fieldnames = [df.fieldname for df in meta.get("fields")]
 
@@ -49,7 +51,11 @@ class CustomField(Document):
 		if self.insert_after and self.insert_after in fieldnames:
 			self.idx = fieldnames.index(self.insert_after) + 1
 
-		self._old_fieldtype = self.db_get('fieldtype')
+		old_fieldtype = self.db_get('fieldtype')
+		is_fieldtype_changed = (not self.is_new()) and (old_fieldtype != self.fieldtype)
+
+		if is_fieldtype_changed and not CustomizeForm.allow_fieldtype_change(old_fieldtype, self.fieldtype):
+			frappe.throw(_("Fieldtype cannot be changed from {0} to {1}").format(old_fieldtype, self.fieldtype))
 
 		if not self.fieldname:
 			frappe.throw(_("Fieldname not set for Custom Field"))

frappe/custom/doctype/customize_form/customize_form.json

@@ -33,6 +33,8 @@
   "show_preview_popup",
   "image_view",
   "email_settings_section",
+  "default_email_template",
+  "column_break_26",
   "email_append_to",
   "sender_field",
   "subject_field",
@@ -264,6 +266,16 @@
    "label": "Actions",
    "options": "DocType Action"
   },
+  {
+   "fieldname": "default_email_template",
+   "fieldtype": "Link",
+   "label": "Default Email Template",
+   "options": "Email Template"
+  },
+  {
+   "fieldname": "column_break_26",
+   "fieldtype": "Column Break"
+  },
   {
    "collapsible": 1,
    "fieldname": "naming_section",
@@ -275,6 +287,16 @@
    "fieldname": "autoname",
    "fieldtype": "Data",
    "label": "Auto Name"
+  },
+  {
+   "fieldname": "default_email_template",
+   "fieldtype": "Link",
+   "label": "Default Email Template",
+   "options": "Email Template"
+  },
+  {
+   "fieldname": "column_break_26",
+   "fieldtype": "Column Break"
   }
  ],
  "hide_toolbar": 1,
@@ -283,7 +305,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-02-16 15:22:11.108256",
+ "modified": "2021-03-22 12:27:15.462727",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",
@@ -304,4 +326,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
+}

frappe/custom/doctype/customize_form/customize_form.py

@@ -401,22 +401,18 @@ class CustomizeForm(Document):
 		return property_value
 
 	def validate_fieldtype_change(self, df, old_value, new_value):
-		allowed = False
-		self.check_length_for_fieldtypes = []
-		for allowed_changes in ALLOWED_FIELDTYPE_CHANGE:
-			if (old_value in allowed_changes and new_value in allowed_changes):
-				allowed = True
-				old_value_length = cint(frappe.db.type_map.get(old_value)[1])
-				new_value_length = cint(frappe.db.type_map.get(new_value)[1])
-
-				# Ignore fieldtype check validation if new field type has unspecified maxlength
-				# Changes like DATA to TEXT, where new_value_lenth equals 0 will not be validated
-				if new_value_length and (old_value_length > new_value_length):
-					self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value})
-					self.validate_fieldtype_length()
-				else:
-					self.flags.update_db = True
-				break
+		allowed = self.allow_fieldtype_change(old_value, new_value)
+		if allowed:
+			old_value_length = cint(frappe.db.type_map.get(old_value)[1])
+			new_value_length = cint(frappe.db.type_map.get(new_value)[1])
+
+			# Ignore fieldtype check validation if new field type has unspecified maxlength
+			# Changes like DATA to TEXT, where new_value_lenth equals 0 will not be validated
+			if new_value_length and (old_value_length > new_value_length):
+				self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value})
+				self.validate_fieldtype_length()
+			else:
+				self.flags.update_db = True
 		if not allowed:
 			frappe.throw(_("Fieldtype cannot be changed from {0} to {1} in row {2}").format(old_value, new_value, df.idx))
 
@@ -458,6 +454,14 @@ class CustomizeForm(Document):
 		reset_customization(self.doc_type)
 		self.fetch_to_customize()
 
+	@classmethod
+	def allow_fieldtype_change(self, old_type: str, new_type: str) -> bool:
+		""" allow type change, if both old_type and new_type are in same field group.
+		field groups are defined in ALLOWED_FIELDTYPE_CHANGE variables.
+		"""
+		in_field_group = lambda group: (old_type in group) and (new_type in group)
+		return any(map(in_field_group, ALLOWED_FIELDTYPE_CHANGE))
+
 def reset_customization(doctype):
 	setters = frappe.get_all("Property Setter", filters={
 		'doc_type': doctype,
@@ -487,6 +491,7 @@ doctype_properties = {
 	'allow_auto_repeat': 'Check',
 	'allow_import': 'Check',
 	'show_preview_popup': 'Check',
+	'default_email_template': 'Data',
 	'email_append_to': 'Check',
 	'subject_field': 'Data',
 	'sender_field': 'Data',

frappe/database/mariadb/database.py

@@ -1,17 +1,13 @@
-from __future__ import unicode_literals
-
-import frappe
 import warnings
 
 import pymysql
-from pymysql.times import TimeDelta
-from pymysql.constants 	import ER, FIELD_TYPE
-from pymysql.converters import conversions
+from pymysql.constants import ER, FIELD_TYPE
+from pymysql.converters import conversions, escape_string
 
-from frappe.utils import get_datetime, cstr, UnicodeWithAttrs
+import frappe
 from frappe.database.database import Database
-from six import PY2, binary_type, text_type, string_types
 from frappe.database.mariadb.schema import MariaDBTable
+from frappe.utils import UnicodeWithAttrs, cstr, get_datetime
 
 
 class MariaDBDatabase(Database):
@@ -72,22 +68,20 @@ class MariaDBDatabase(Database):
 		conversions.update({
 			FIELD_TYPE.NEWDECIMAL: float,
 			FIELD_TYPE.DATETIME: get_datetime,
-			UnicodeWithAttrs: conversions[text_type]
+			UnicodeWithAttrs: conversions[str]
 		})
 
-		if PY2:
-			conversions.update({
-				TimeDelta: conversions[binary_type]
-			})
-
-		if usessl:
-			conn = pymysql.connect(self.host, self.user or '', self.password or '',
-				port=self.port, charset='utf8mb4', use_unicode = True, ssl=ssl_params,
-				conv = conversions, local_infile = frappe.conf.local_infile)
-		else:
-			conn = pymysql.connect(self.host, self.user or '', self.password or '',
-				port=self.port, charset='utf8mb4', use_unicode = True, conv = conversions,
-				local_infile = frappe.conf.local_infile)
+		conn = pymysql.connect(
+			user=self.user or '',
+			password=self.password or '',
+			host=self.host,
+			port=self.port,
+			charset='utf8mb4',
+			use_unicode=True,
+			ssl=ssl_params if usessl else None,
+			conv=conversions,
+			local_infile=frappe.conf.local_infile
+		)
 
 		# MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1
 		# # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF)
@@ -111,7 +105,7 @@ class MariaDBDatabase(Database):
 	def escape(s, percent=True):
 		"""Excape quotes and percent in given string."""
 		# pymysql expects unicode argument to escape_string with Python 3
-		s = frappe.as_unicode(pymysql.escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`")
+		s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`")
 
 		# NOTE separating % escape, because % escape should only be done when using LIKE operator
 		# or when you use python format string to generate query that already has a %s
@@ -260,7 +254,7 @@ class MariaDBDatabase(Database):
 				ADD INDEX `%s`(%s)""" % (table_name, index_name, ", ".join(fields)))
 
 	def add_unique(self, doctype, fields, constraint_name=None):
-		if isinstance(fields, string_types):
+		if isinstance(fields, str):
 			fields = [fields]
 		if not constraint_name:
 			constraint_name = "unique_" + "_".join(fields)

frappe/desk/doctype/notification_settings/notification_settings.js

@@ -19,7 +19,7 @@ frappe.ui.form.on('Notification Settings', {
 
 	refresh: (frm) => {
 		if (frappe.user.has_role('System Manager')) {
-			frm.add_custom_button('Go to Notification Settings List', () => {
+			frm.add_custom_button(__('Go to Notification Settings List'), () => {
 				frappe.set_route('List', 'Notification Settings');
 			});
 		}

frappe/desk/query_report.py

@@ -36,7 +36,10 @@ def get_report_doc(report_name):
 		reference_report = custom_report_doc.reference_report
 		doc = frappe.get_doc("Report", reference_report)
 		doc.custom_report = report_name
-		doc.custom_columns = custom_report_doc.json
+		if custom_report_doc.json:
+			data = json.loads(custom_report_doc.json)
+			if data:
+				doc.custom_columns = data["columns"]
 		doc.is_custom_report = True
 
 	if not doc.is_permitted():
@@ -83,7 +86,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None)
 
 	if report.custom_columns:
 		# saved columns (with custom columns / with different column order)
-		columns = json.loads(report.custom_columns)
+		columns = report.custom_columns
 
 	# unsaved custom_columns
 	if custom_columns:
@@ -524,9 +527,12 @@ def save_report(reference_report, report_name, columns):
 			"report_type": "Custom Report",
 		},
 	)
+
 	if docname:
 		report = frappe.get_doc("Report", docname)
-		report.update({"json": columns})
+		existing_jd = json.loads(report.json)
+		existing_jd["columns"] = json.loads(columns)
+		report.update({"json": json.dumps(existing_jd, separators=(',', ':'))})
 		report.save()
 		frappe.msgprint(_("Report updated successfully"))
 
@@ -536,7 +542,7 @@ def save_report(reference_report, report_name, columns):
 			{
 				"doctype": "Report",
 				"report_name": report_name,
-				"json": columns,
+				"json": f'{{"columns":{columns}}}',
 				"ref_doctype": report_doc.ref_doctype,
 				"is_standard": "No",
 				"report_type": "Custom Report",

frappe/desk/treeview.py

@@ -66,7 +66,7 @@ def add_node():
 	doc.save()
 
 def make_tree_args(**kwarg):
-	del kwarg['cmd']
+	kwarg.pop('cmd', None)
 
 	doctype = kwarg['doctype']
 	parent_field = 'parent_' + doctype.lower().replace(' ', '_')

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -252,7 +252,7 @@ def make_links(columns, data):
 			elif col.fieldtype == "Dynamic Link":
 				if col.options and row.get(col.fieldname) and row.get(col.options):
 					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
-			elif col.fieldtype == "Currency":
+			elif col.fieldtype == "Currency" and row.get(col.fieldname):
 				row[col.fieldname] = frappe.format_value(row[col.fieldname], col)
 
 	return columns, data

frappe/email/doctype/newsletter/newsletter.py

@@ -24,6 +24,7 @@ class Newsletter(WebsiteGenerator):
 		if self.send_from:
 			validate_email_address(self.send_from, True)
 
+	@frappe.whitelist()
 	def test_send(self, doctype="Lead"):
 		self.recipients = frappe.utils.split_emails(self.test_email_id)
 		self.queue_all(test_email=True)

frappe/email/receive.py

@@ -1,18 +1,27 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
+import datetime
+import email
+import email.utils
+import imaplib
+import poplib
+import re
+import time
+from email.header import decode_header
+
+import _socket
+import chardet
 import six
-from six import iteritems, text_type
-from six.moves import range
-import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re
 from email_reply_parser import EmailReplyParser
-from email.header import decode_header
+
 import frappe
 from frappe import _, safe_decode, safe_encode
-from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now,
-	cint, cstr, strip, markdown, parse_addr)
-from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError
+from frappe.core.doctype.file.file import (MaxFileSizeReachedError,
+	get_random_filename)
+from frappe.utils import (cint, convert_utc_to_user_timezone, cstr,
+	extract_email_id, markdown, now, parse_addr, strip)
+
 
 class EmailSizeExceededError(frappe.ValidationError): pass
 class EmailTimeoutError(frappe.ValidationError): pass
@@ -337,7 +346,7 @@ class EmailServer:
 			return
 
 		self.imap.select("Inbox")
-		for uid, operation in iteritems(uid_list):
+		for uid, operation in uid_list.items():
 			if not uid: continue
 
 			op = "+FLAGS" if operation == "Read" else "-FLAGS"
@@ -473,7 +482,7 @@ class Email:
 			self.html_content += markdown(text_content)
 
 	def get_charset(self, part):
-		"""Detect chartset."""
+		"""Detect charset."""
 		charset = part.get_content_charset()
 		if not charset:
 			charset = chardet.detect(safe_encode(cstr(part)))['encoding']
@@ -484,7 +493,7 @@ class Email:
 		charset = self.get_charset(part)
 
 		try:
-			return text_type(part.get_payload(decode=True), str(charset), "ignore")
+			return str(part.get_payload(decode=True), str(charset), "ignore")
 		except LookupError:
 			return part.get_payload()
 

frappe/email/test_smtp.py

@@ -2,7 +2,9 @@
 # License: The MIT License
 
 import unittest
+import frappe
 from frappe.email.smtp import SMTPServer
+from frappe.email.smtp import get_outgoing_email_account
 
 class TestSMTP(unittest.TestCase):
 	def test_smtp_ssl_session(self):
@@ -13,6 +15,57 @@ class TestSMTP(unittest.TestCase):
 		for port in [None, 0, 587, "587"]:
 			make_server(port, 0, 1)
 
+	def test_get_email_account(self):
+		existing_email_accounts = frappe.get_all("Email Account", fields = ["name", "enable_outgoing", "default_outgoing", "append_to"])
+		unset_details = {
+			"enable_outgoing": 0,
+			"default_outgoing": 0,
+			"append_to": None
+		}
+		for email_account in existing_email_accounts:
+			frappe.db.set_value('Email Account', email_account['name'], unset_details)
+
+		# remove mail_server config so that test@example.com is not created
+		mail_server = frappe.conf.get('mail_server')
+		del frappe.conf['mail_server']
+
+		frappe.local.outgoing_email_account = {}
+
+		frappe.local.outgoing_email_account = {}
+		# lowest preference given to email account with default incoming enabled
+		create_email_account(email_id="default_outgoing_enabled@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1)
+		self.assertEqual(get_outgoing_email_account().email_id, "default_outgoing_enabled@gmail.com")
+
+		frappe.local.outgoing_email_account = {}
+		# highest preference given to email account with append_to matching
+		create_email_account(email_id="append_to@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
+		self.assertEqual(get_outgoing_email_account(append_to="Blog Post").email_id, "append_to@gmail.com")
+
+		# add back the mail_server
+		frappe.conf['mail_server'] = mail_server
+		for email_account in existing_email_accounts:
+			set_details = {
+				"enable_outgoing": email_account['enable_outgoing'],
+				"default_outgoing": email_account['default_outgoing'],
+				"append_to": email_account['append_to']
+			}
+			frappe.db.set_value('Email Account', email_account['name'], set_details)
+
+def create_email_account(email_id, password, enable_outgoing, default_outgoing=0, append_to=None):
+	email_dict = {
+		"email_id": email_id,
+		"passsword": password,
+		"enable_outgoing":enable_outgoing ,
+		"default_outgoing":default_outgoing ,
+		"enable_incoming": 1,
+		"append_to":append_to,
+		"is_dummy_password": 1,
+		"smtp_server": "localhost"
+	}
+
+	email_account = frappe.new_doc('Email Account')
+	email_account.update(email_dict)
+	email_account.save()
 
 def make_server(port, ssl, tls):
 	server = SMTPServer(
@@ -22,4 +75,4 @@ def make_server(port, ssl, tls):
 		use_tls = tls
 	)
 
-	server.sess
+	server.sess

frappe/handler.py

@@ -9,7 +9,6 @@ import frappe
 import frappe.utils
 import frappe.sessions
 from frappe.utils import cint
-from frappe.api import validate_auth
 from frappe import _, is_whitelisted
 from frappe.utils.response import build_response
 from frappe.utils.csvutils import build_csv_response
@@ -24,7 +23,7 @@ ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/
 
 def handle():
 	"""handle request"""
-	validate_auth()
+
 	cmd = frappe.local.form_dict.cmd
 	data = None
 

frappe/integrations/doctype/dropbox_settings/dropbox_settings.py

@@ -2,22 +2,23 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import dropbox
 import json
-import frappe
 import os
+from urllib.parse import parse_qs, urlparse
+
+import dropbox
+from rq.timeouts import JobTimeoutException
+
+import frappe
 from frappe import _
-from frappe.model.document import Document
-from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
+from frappe.integrations.offsite_backup_utils import (get_chunk_site,
+	get_latest_backup_file, send_email, validate_file_size)
 from frappe.integrations.utils import make_post_request
-from frappe.utils import (cint, get_request_site_address,
-	get_files_path, get_backups_path, get_url, encode)
-from frappe.utils.backups import new_backup
+from frappe.model.document import Document
+from frappe.utils import (cint, encode, get_backups_path, get_files_path,
+	get_request_site_address, get_url)
 from frappe.utils.background_jobs import enqueue
-from six.moves.urllib.parse import urlparse, parse_qs
-from rq.timeouts import JobTimeoutException
-from six import text_type
+from frappe.utils.backups import new_backup
 
 ignore_list = [".DS_Store"]
 
@@ -91,7 +92,10 @@ def backup_to_dropbox(upload_db_backup=True):
 		dropbox_settings['access_token'] = access_token['oauth2_token']
 		set_dropbox_access_token(access_token['oauth2_token'])
 
-	dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'], timeout=None)
+	dropbox_client = dropbox.Dropbox(
+		oauth2_access_token=dropbox_settings['access_token'],
+		timeout=None
+	)
 
 	if upload_db_backup:
 		if frappe.flags.create_new_backup:
@@ -127,7 +131,7 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not
 	else:
 		response = frappe._dict({"entries": []})
 
-	path = text_type(path)
+	path = str(path)
 
 	for f in frappe.get_all("File", filters={"is_folder": 0, "is_private": is_private,
 		"uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']):
@@ -286,11 +290,11 @@ def get_redirect_url():
 def get_dropbox_authorize_url():
 	app_details = get_dropbox_settings(redirect_uri=True)
 	dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
-		app_details["app_key"],
-		app_details["app_secret"],
-		app_details["redirect_uri"],
-		{},
-		"dropbox-auth-csrf-token"
+		consumer_key=app_details["app_key"],
+		redirect_uri=app_details["redirect_uri"],
+		session={},
+		csrf_token_session_key="dropbox-auth-csrf-token",
+		consumer_secret=app_details["app_secret"]
 	)
 
 	auth_url = dropbox_oauth_flow.start()
@@ -307,13 +311,13 @@ def dropbox_auth_finish(return_access_token=False):
 	close = '<p class="text-muted">' + _('Please close this window') + '</p>'
 
 	dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
-		app_details["app_key"],
-		app_details["app_secret"],
-		app_details["redirect_uri"],
-		{
+		consumer_key=app_details["app_key"],
+		redirect_uri=app_details["redirect_uri"],
+		session={
 			'dropbox-auth-csrf-token': callback.state
 		},
-		"dropbox-auth-csrf-token"
+		csrf_token_session_key="dropbox-auth-csrf-token",
+		consumer_secret=app_details["app_secret"]
 	)
 
 	if callback.state or callback.code:

frappe/integrations/doctype/google_calendar/google_calendar.py

@@ -2,22 +2,23 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
+
+from datetime import datetime, timedelta
+from urllib.parse import quote
+
 import google.oauth2.credentials
+import requests
+from dateutil import parser
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 
+import frappe
 from frappe import _
+from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
 from frappe.model.document import Document
-from frappe.utils import get_request_site_address
-from googleapiclient.errors import HttpError
+from frappe.utils import (add_days, add_to_date, get_datetime,
+	get_request_site_address, get_time_zone, get_weekdays, now_datetime)
 from frappe.utils.password import set_encrypted_password
-from frappe.utils import add_days, get_datetime, get_weekdays, now_datetime, add_to_date, get_time_zone
-from dateutil import parser
-from datetime import datetime, timedelta
-from six.moves.urllib.parse import quote
-from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
 
 SCOPES = "https://www.googleapis.com/auth/calendar"
 
@@ -171,7 +172,12 @@ def get_google_calendar_object(g_calendar):
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_calendar = googleapiclient.discovery.build("calendar", "v3", credentials=credentials)
+	google_calendar = build(
+		serviceName="calendar",
+		version="v3",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	check_google_calendar(account, google_calendar)
 

frappe/integrations/doctype/google_contacts/google_contacts.py

@@ -2,17 +2,17 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
+
 import google.oauth2.credentials
+import requests
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 
-from frappe.model.document import Document
+import frappe
 from frappe import _
-from googleapiclient.errors import HttpError
-from frappe.utils import get_request_site_address
 from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.model.document import Document
+from frappe.utils import get_request_site_address
 
 SCOPES = "https://www.googleapis.com/auth/contacts"
 
@@ -118,7 +118,12 @@ def get_google_contacts_object(g_contact):
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_contacts = googleapiclient.discovery.build("people", "v1", credentials=credentials)
+	google_contacts = build(
+		serviceName="people",
+		version="v1",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	return google_contacts, account
 

frappe/integrations/doctype/google_drive/google_drive.py

@@ -2,27 +2,29 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
-import google.oauth2.credentials
 import os
+from urllib.parse import quote
 
-from frappe import _
+import google.oauth2.credentials
+import requests
+from apiclient.http import MediaFileUpload
+from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
+
+import frappe
+from frappe import _
+from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.integrations.offsite_backup_utils import (get_latest_backup_file,
+	send_email, validate_file_size)
 from frappe.model.document import Document
-from frappe.utils import get_request_site_address
+from frappe.utils import (get_backups_path, get_bench_path,
+	get_request_site_address)
 from frappe.utils.background_jobs import enqueue
-from six.moves.urllib.parse import quote
-from apiclient.http import MediaFileUpload
-from frappe.utils import get_backups_path, get_bench_path
 from frappe.utils.backups import new_backup
-from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
-from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size
 
 SCOPES = "https://www.googleapis.com/auth/drive"
 
+
 class GoogleDrive(Document):
 
 	def validate(self):
@@ -126,7 +128,12 @@ def get_google_drive_object():
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_drive = googleapiclient.discovery.build("drive", "v3", credentials=credentials)
+	google_drive = build(
+		serviceName="drive",
+		version="v3",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	return google_drive, account
 

frappe/integrations/doctype/token_cache/test_token_cache.py

@@ -13,7 +13,7 @@ class TestTokenCache(unittest.TestCase):
 	def setUp(self):
 		self.token_cache = frappe.get_last_doc('Token Cache')
 		self.token_cache.update({'connected_app': frappe.get_last_doc('Connected App').name})
-		self.token_cache.save()
+		self.token_cache.save(ignore_permissions=True)
 
 	def test_get_auth_header(self):
 		self.token_cache.get_auth_header()

frappe/integrations/doctype/webhook/__init__.py

@@ -21,7 +21,9 @@ def run_webhooks(doc, method):
 		if webhooks is None:
 			# query webhooks
 			webhooks_list = frappe.get_all('Webhook',
-				fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"])
+						fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"], 
+						filters={"enabled": True}
+					)
 
 			# make webhooks map for cache
 			webhooks = {}

frappe/integrations/doctype/webhook/test_webhook.py

@@ -10,6 +10,44 @@ from frappe.integrations.doctype.webhook.webhook import get_webhook_headers, get
 
 
 class TestWebhook(unittest.TestCase):
+	@classmethod
+	def setUpClass(cls):
+		# delete any existing webhooks
+		frappe.db.sql("DELETE FROM tabWebhook")
+		# create test webhooks
+		cls.create_sample_webhooks()
+
+	@classmethod
+	def create_sample_webhooks(cls):
+		samples_webhooks_data = [
+			{
+				"webhook_doctype": "User",
+				"webhook_docevent": "after_insert",
+				"request_url": "https://httpbin.org/post",
+				"condition": "doc.email",
+				"enabled": True
+			},
+			{
+				"webhook_doctype": "User",
+				"webhook_docevent": "after_insert",
+				"request_url": "https://httpbin.org/post",
+				"condition": "doc.first_name",
+				"enabled": False
+			}
+		]
+
+		cls.sample_webhooks = []
+		for wh_fields in samples_webhooks_data:
+			wh = frappe.new_doc("Webhook")
+			wh.update(wh_fields)
+			wh.insert()
+			cls.sample_webhooks.append(wh)
+
+	@classmethod
+	def tearDownClass(cls):
+		# delete any existing webhooks
+		frappe.db.sql("DELETE FROM tabWebhook")
+
 	def setUp(self):
 		# retrieve or create a User webhook for `after_insert`
 		webhook_fields = {
@@ -30,10 +68,37 @@ class TestWebhook(unittest.TestCase):
 		self.user.email = frappe.mock("email")
 		self.user.save()
 
+		# Create another test user specific to this test
+		self.test_user = frappe.new_doc("User")
+		self.test_user.email = "user1@integration.webhooks.test.com"
+		self.test_user.first_name = "user1"
+
 	def tearDown(self) -> None:
 		self.user.delete()
+		self.test_user.delete()
 		super().tearDown()
 
+	def test_webhook_trigger_with_enabled_webhooks(self):
+		"""Test webhook trigger for enabled webhooks"""
+
+		frappe.cache().delete_value('webhooks')
+		frappe.flags.webhooks = None
+
+		# Insert the user to db
+		self.test_user.insert()
+		
+		self.assertTrue("User" in frappe.flags.webhooks)
+		# only 1 hook (enabled) must be queued
+		self.assertEqual(
+			len(frappe.flags.webhooks.get("User")),
+			1
+		)
+		self.assertTrue(self.test_user.email in frappe.flags.webhooks_executed)
+		self.assertEqual(
+			frappe.flags.webhooks_executed.get(self.test_user.email)[0], 
+			self.sample_webhooks[0].name
+		)
+
 	def test_validate_doc_events(self):
 		"Test creating a submit-related webhook for a non-submittable DocType"
 

frappe/integrations/doctype/webhook/webhook.json

@@ -11,6 +11,7 @@
   "webhook_doctype",
   "cb_doc_events",
   "webhook_docevent",
+  "enabled",
   "sb_condition",
   "condition",
   "cb_condition",
@@ -147,10 +148,16 @@
    "fieldname": "webhook_secret",
    "fieldtype": "Password",
    "label": "Webhook Secret"
+  },
+  {
+   "default": "1",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "label": "Enabled"
   }
  ],
  "links": [],
- "modified": "2020-01-13 01:53:04.459968",
+ "modified": "2021-04-14 05:35:28.532049",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Webhook",

frappe/integrations/oauth2.py

@@ -133,7 +133,7 @@ def get_token(*args, **kwargs):
 			}
 
 			id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header)
-			out.update({"id_token": str(id_token_encoded)})
+			out.update({"id_token": frappe.safe_decode(id_token_encoded)})
 
 		frappe.local.response = out
 

frappe/patches.txt

@@ -334,3 +334,4 @@ frappe.patches.v13_0.delete_package_publish_tool
 frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings
 frappe.patches.v13_0.remove_twilio_settings
 frappe.patches.v12_0.rename_uploaded_files_with_proper_name
+frappe.patches.v13_0.queryreport_columns

frappe/patches/v13_0/queryreport_columns.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+import json
+
+def execute():
+	"""Convert Query Report json to support other content"""
+	records = frappe.get_all('Report',
+		filters={
+			"json": ["!=", ""]
+		},
+		fields=["name", "json"]
+	)
+	for record in records:
+		jstr = record["json"]
+		data = json.loads(jstr)
+		if isinstance(data, list):
+			# double escape braces
+			jstr = f'{{"columns":{jstr}}}'
+			frappe.db.update('Report', record["name"], "json", jstr)

frappe/public/js/frappe/desk.js

@@ -51,6 +51,7 @@ frappe.Application = Class.extend({
 		this.set_fullwidth_if_enabled();
 		this.add_browser_class();
 		this.setup_energy_point_listeners();
+		this.setup_copy_doc_listener();
 
 		frappe.ui.keys.setup();
 
@@ -113,7 +114,7 @@ frappe.Application = Class.extend({
 			dialog.get_close_btn().toggle(false);
 		});
 
-		this.setup_social_listeners();
+		this.setup_user_group_listeners();
 
 		// listen to build errors
 		this.setup_build_error_listener();
@@ -592,11 +593,12 @@ frappe.Application = Class.extend({
 		}
 	},
 
-	setup_social_listeners() {
-		frappe.realtime.on('mention', (message) => {
-			if (frappe.get_route()[0] !== 'social') {
-				frappe.show_alert(message);
-			}
+	setup_user_group_listeners() {
+		frappe.realtime.on('user_group_added', (user_group) => {
+			frappe.boot.user_groups && frappe.boot.user_groups.push(user_group);
+		});
+		frappe.realtime.on('user_group_deleted', (user_group) => {
+			frappe.boot.user_groups = (frappe.boot.user_groups || []).filter(el => el !== user_group);
 		});
 	},
 
@@ -605,6 +607,39 @@ frappe.Application = Class.extend({
 			frappe.show_alert(message);
 		});
 	},
+
+	setup_copy_doc_listener() {
+		$('body').on('paste', (e) => {
+			try {
+				let clipboard_data = e.clipboardData || window.clipboardData || e.originalEvent.clipboardData;
+				let pasted_data = clipboard_data.getData('Text');
+				let doc = JSON.parse(pasted_data);
+				if (doc.doctype) {
+					e.preventDefault();
+					let sleep = (time) => {
+						return new Promise((resolve) => setTimeout(resolve, time));
+					};
+
+					frappe.dom.freeze(__('Creating {0}', [doc.doctype]) + '...');
+					// to avoid abrupt UX
+					// wait for activity feedback
+					sleep(500).then(() => {
+						let res = frappe.model.with_doctype(doc.doctype, () => {
+							let newdoc = frappe.model.copy_doc(doc);
+							newdoc.__newname = doc.name;
+							newdoc.idx = null;
+							newdoc.__run_link_triggers = false;
+							frappe.set_route('Form', newdoc.doctype, newdoc.name);
+							frappe.dom.unfreeze();
+						});
+						res && res.fail(frappe.dom.unfreeze);
+					});
+				}
+			} catch (e) {
+				//
+			}
+		});
+	}
 });
 
 frappe.get_module = function(m, default_module) {

frappe/public/js/frappe/form/controls/button.js

@@ -6,7 +6,10 @@ frappe.ui.form.ControlButton = frappe.ui.form.ControlData.extend({
 	make_input: function() {
 		var me = this;
 		const btn_type = this.df.primary ? 'btn-primary': 'btn-default';
-		this.$input = $(`<button class="btn btn-xs ${btn_type}">`)
+		const btn_size = this.df.btn_size
+			? `btn-${this.df.btn_size}`
+			: "btn-xs";
+		this.$input = $(`<button class="btn ${btn_size} ${btn_type}">`)
 			.prependTo(me.input_area)
 			.on("click", function() {
 				me.onclick();

frappe/public/js/frappe/form/controls/code.js

@@ -31,6 +31,57 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({
 			const input_value = this.get_input_value();
 			this.parse_validate_and_set_in_model(input_value);
 		}, 300));
+
+		// setup autocompletion when it is set the first time
+		Object.defineProperty(this.df, 'autocompletions', {
+			get() {
+				return this._autocompletions || [];
+			},
+			set: (value) => {
+				this.setup_autocompletion();
+				this.df._autocompletions = value;
+			}
+		});
+	},
+
+	setup_autocompletion() {
+		if (this._autocompletion_setup) return;
+
+		const ace = window.ace;
+		const get_autocompletions = () => this.df.autocompletions;
+
+		ace.config.loadModule("ace/ext/language_tools", langTools => {
+			this.editor.setOptions({
+				enableBasicAutocompletion: true,
+				enableLiveAutocompletion: true
+			});
+
+			langTools.addCompleter({
+				getCompletions: function(editor, session, pos, prefix, callback) {
+					if (prefix.length === 0) {
+						callback(null, []);
+						return;
+					}
+					let autocompletions = get_autocompletions();
+					if (autocompletions.length) {
+						callback(
+							null,
+							autocompletions.map(a => {
+								if (typeof a === 'string') {
+									a = { value: a };
+								}
+								return {
+									name: 'frappe',
+									value: a.value,
+									score: a.score
+								};
+							})
+						);
+					}
+				}
+			});
+		});
+		this._autocompletion_setup = true;
 	},
 
 	refresh_height() {

frappe/public/js/frappe/form/controls/multiselect.js

@@ -68,7 +68,7 @@ frappe.ui.form.ControlMultiSelect = frappe.ui.form.ControlAutocomplete.extend({
 		let data;
 		if(this.df.get_data) {
 			data = this.df.get_data();
-			this.set_data(data);
+			if (data) this.set_data(data);
 		} else {
 			data = this._super();
 		}

frappe/public/js/frappe/form/controls/quill-mention/blots/mention.js

@@ -15,6 +15,7 @@ class MentionBlot extends Embed {
     node.dataset.id = data.id;
     node.dataset.value = data.value;
     node.dataset.denotationChar = data.denotationChar;
+	node.dataset.isGroup = data.isGroup;
     if (data.link) {
       node.dataset.link = data.link;
     }
@@ -27,6 +28,7 @@ class MentionBlot extends Embed {
       value: domNode.dataset.value,
       link: domNode.dataset.link || null,
       denotationChar: domNode.dataset.denotationChar,
+      isGroup: domNode.dataset.isGroup,
     };
   }
 }

frappe/public/js/frappe/form/controls/quill-mention/quill.mention.js

@@ -149,6 +149,7 @@ class Mention {
         this.mentionList.childNodes[this.itemIndex].dataset.value,
       link: itemLink || null,
       denotationChar: this.mentionList.childNodes[this.itemIndex].dataset.denotationChar,
+      isGroup: this.mentionList.childNodes[this.itemIndex].dataset.isGroup,
     };
   }
 
@@ -197,6 +198,7 @@ class Mention {
         li.dataset.index = i;
         li.dataset.id = data[i].id;
         li.dataset.value = data[i].value;
+        li.dataset.isGroup = Boolean(data[i].is_group);
         li.dataset.denotationChar = mentionChar;
         if (data[i].link) {
           li.dataset.link = data[i].link;

frappe/public/js/frappe/form/dashboard.js

@@ -535,14 +535,14 @@ frappe.ui.form.Dashboard = class FormDashboard {
 	render_graph(args) {
 		this.chart_area.show();
 		this.chart_area.body.empty();
-		$.extend(args, {
+		$.extend({
 			type: 'line',
 			colors: ['green'],
 			truncateLegends: 1,
 			axisOptions: {
 				shortenYAxisNumbers: 1
 			}
-		});
+		}, args);
 		this.show();
 
 		this.chart = new frappe.Chart('.form-graph', args);

frappe/public/js/frappe/form/grid.js

@@ -387,6 +387,8 @@ export default class Grid {
 			this.wrapper.find('.grid-footer').toggle(false);
 		}
 
+		this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable());
+
 	}
 
 	truncate_rows() {

frappe/public/js/frappe/form/grid_row.js

@@ -557,13 +557,10 @@ export default class GridRow {
 		this.row.toggle(false);
 		// this.form_panel.toggle(true);
 
-		if (this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows)) {
-			this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row')
-				.addClass('hidden');
-		} else {
-			this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row')
-				.removeClass('hidden');
-		}
+		let cannot_add_rows = this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows);
+		this.wrapper
+			.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row')
+			.toggle(!cannot_add_rows);
 
 		frappe.dom.freeze("", "dark");
 		if (cur_frm) cur_frm.cur_grid = this;

frappe/public/js/frappe/form/grid_row_form.js

@@ -119,7 +119,7 @@ export default class GridRowForm {
 		});
 	}
 	toggle_add_delete_button_display($parent) {
-		$parent.find(".row-actions")
+		$parent.find(".row-actions, .grid-append-row")
 			.toggle(this.row.grid.is_editable());
 	}
 	refresh_field(fieldname) {

frappe/public/js/frappe/form/toolbar.js

@@ -277,13 +277,18 @@ frappe.ui.form.Toolbar = class Toolbar {
 			}, true)
 		}
 
-		// copy
+		// duplicate
 		if(in_list(frappe.boot.user.can_create, me.frm.doctype) && !me.frm.meta.allow_copy) {
 			this.page.add_menu_item(__("Duplicate"), function() {
 				me.frm.copy_doc();
 			}, true);
 		}
 
+		// copy doc to clipboard
+		this.page.add_menu_item(__("Copy to Clipboard"), function() {
+			frappe.utils.copy_to_clipboard(JSON.stringify(me.frm.doc));
+		}, true);
+
 		// rename
 		if(this.can_rename()) {
 			this.page.add_menu_item(__("Rename"), function() {

frappe/public/js/frappe/utils/utils.js

@@ -1285,6 +1285,16 @@ Object.assign(frappe.utils, {
 					value: frappe.boot.user_info[user].fullname,
 				};
 			});
+
+		frappe.boot.user_groups && frappe.boot.user_groups.map(group => {
+			names_for_mentions.push({
+				id: group,
+				value: group,
+				is_group: true,
+				link: frappe.utils.get_form_link('User Group', group)
+			});
+		});
+
 		return names_for_mentions;
 	},
 	print(doctype, docname, print_format, letterhead, lang_code) {

frappe/public/js/frappe/views/communication.js

@@ -2,73 +2,55 @@
 // MIT License. See license.txt
 
 frappe.last_edited_communication = {};
-frappe.standard_replies = {};
-frappe.separator_element = '<div>---</div>';
+const separator_element = '<div>---</div>';
 
-frappe.views.CommunicationComposer = Class.extend({
-	init: function(opts) {
+frappe.views.CommunicationComposer = class {
+	constructor(opts) {
 		$.extend(this, opts);
+		if (!this.doc) {
+			this.doc = this.frm && this.frm.doc || {};
+		}
+
 		this.make();
-	},
-	make: function() {
-		var me = this;
+	}
+
+	make() {
+		const me = this;
 
 		this.dialog = new frappe.ui.Dialog({
 			title: (this.title || this.subject || __("New Email")),
 			no_submit_on_enter: true,
 			fields: this.get_fields(),
 			primary_action_label: __("Send"),
-			size: 'large',
-			primary_action: function() {
-				me.delete_saved_draft();
+			primary_action() {
 				me.send_action();
 			},
+			secondary_action_label: __("Discard"),
+			secondary_action() {
+				me.dialog.hide();
+				me.clear_cache();
+			},
+			size: 'large',
 			minimizable: true
 		});
 
 		this.dialog.sections[0].wrapper.addClass('to_section');
 
-		['recipients', 'cc', 'bcc'].forEach(field => {
-			this.dialog.fields_dict[field].get_data = function() {
-				const data = me.dialog.fields_dict[field].get_value();
-				const txt = data.match(/[^,\s*]*$/)[0] || '';
-				let options = [];
-
-				frappe.call({
-					method: "frappe.email.get_contact_list",
-					args: {
-						txt: txt,
-					},
-					callback: (r) => {
-						options = r.message;
-						me.dialog.fields_dict[field].set_data(options);
-					}
-				});
-				return options;
-			}
-		});
-
 		this.prepare();
 		this.dialog.show();
 
 		if (this.frm) {
 			$(document).trigger('form-typing', [this.frm]);
 		}
+	}
 
-		if (this.cc || this.bcc) {
-			this.toggle_more_options(true);
-		}
-	},
-
-	get_fields: function() {
-		let contactList = [];
-		let fields = [
+	get_fields() {
+		const fields = [
 			{
 				label: __("To"),
 				fieldtype: "MultiSelect",
 				reqd: 0,
 				fieldname: "recipients",
-				options: contactList
 			},
 			{
 				fieldtype: "Button",
@@ -87,13 +69,11 @@ frappe.views.CommunicationComposer = Class.extend({
 				label: __("CC"),
 				fieldtype: "MultiSelect",
 				fieldname: "cc",
-				options: contactList
 			},
 			{
 				label: __("BCC"),
 				fieldtype: "MultiSelect",
 				fieldname: "bcc",
-				options: contactList
 			},
 			{
 				label: __("Email Template"),
@@ -163,78 +143,83 @@ frappe.views.CommunicationComposer = Class.extend({
 			);
 		});
 
-		if (frappe.boot.email_accounts && email_accounts.length > 1) {
-			fields = [
-				{
-					label: __("From"),
-					fieldtype: "Select",
-					reqd: 1,
-					fieldname: "sender",
-					options: email_accounts.map(function(e) {
-						return e.email_id;
-					})
-				}
-			].concat(fields);
+		if (email_accounts.length > 1) {
+			fields.unshift({
+				label: __("From"),
+				fieldtype: "Select",
+				reqd: 1,
+				fieldname: "sender",
+				options: email_accounts.map(function(e) {
+					return e.email_id;
+				})
+			});
 		}
 
 		return fields;
-	},
+	}
 
 	toggle_more_options(show_options) {
 		show_options = show_options || this.dialog.fields_dict.more_options.df.hidden;
 		this.dialog.set_df_property('more_options', 'hidden', !show_options);
-		let label = frappe.utils.icon(show_options ? 'up-line': 'down');
+
+		const label = frappe.utils.icon(show_options ? 'up-line': 'down');
 		this.dialog.get_field('option_toggle_button').set_label(label);
-	},
+	}
 
-	prepare: function() {
+	prepare() {
+		this.setup_multiselect_queries();
 		this.setup_subject_and_recipients();
 		this.setup_print_language();
 		this.setup_print();
 		this.setup_attach();
 		this.setup_email();
-		this.setup_last_edited_communication();
 		this.setup_email_template();
+		this.setup_last_edited_communication();
+		this.set_values();
+	}
 
-		this.dialog.set_value("recipients", this.recipients || '');
-		this.dialog.set_value("cc", this.cc || '');
-		this.dialog.set_value("bcc", this.bcc || '');
-
-		if(this.dialog.fields_dict.sender) {
-			this.dialog.fields_dict.sender.set_value(this.sender || '');
-		}
-		this.dialog.fields_dict.subject.set_value(
-			frappe.utils.html2text(this.subject) || ''
-		);
+	setup_multiselect_queries() {
+		['recipients', 'cc', 'bcc'].forEach(field => {
+			this.dialog.fields_dict[field].get_data = () => {
+				const data = this.dialog.fields_dict[field].get_value();
+				const txt = data.match(/[^,\s*]*$/)[0] || '';
 
-		this.setup_earlier_reply();
-	},
+				frappe.call({
+					method: "frappe.email.get_contact_list",
+					args: {txt},
+					callback: (r) => {
+						this.dialog.fields_dict[field].set_data(r.message);
+					}
+				});
+			};
+		});
+	}
 
-	setup_subject_and_recipients: function() {
+	setup_subject_and_recipients() {
 		this.subject = this.subject || "";
 
-		if(!this.forward && !this.recipients && this.last_email) {
+		if (!this.forward && !this.recipients && this.last_email) {
 			this.recipients = this.last_email.sender;
 			this.cc = this.last_email.cc;
 			this.bcc = this.last_email.bcc;
 		}
 
-		if(!this.forward && !this.recipients) {
+		if (!this.forward && !this.recipients) {
 			this.recipients = this.frm && this.frm.timeline.get_recipient();
 		}
 
-		if(!this.subject && this.frm) {
+		if (!this.subject && this.frm) {
 			// get subject from last communication
-			var last = this.frm.timeline.get_last_email();
+			const last = this.frm.timeline.get_last_email();
 
-			if(last) {
+			if (last) {
 				this.subject = last.subject;
-				if(!this.recipients) {
+				if (!this.recipients) {
 					this.recipients = last.sender;
 				}
 
 				// prepend "Re:"
-				if(strip(this.subject.toLowerCase().split(":")[0])!="re") {
+				if (strip(this.subject.toLowerCase().split(":")[0])!="re") {
 					this.subject = __("Re: {0}", [this.subject]);
 				}
 			}
@@ -251,7 +236,7 @@ frappe.views.CommunicationComposer = Class.extend({
 			// always add an identifier to catch a reply
 			// some email clients (outlook) may not send the message id to identify
 			// the thread. So as a backup we use the name of the document as identifier
-			let identifier = `#${this.frm.doc.name}`;
+			const identifier = `#${this.frm.doc.name}`;
 			if (!this.subject.includes(identifier)) {
 				this.subject = `${this.subject} (${identifier})`;
 			}
@@ -260,33 +245,25 @@ frappe.views.CommunicationComposer = Class.extend({
 		if (this.frm && !this.recipients) {
 			this.recipients = this.frm.doc[this.frm.email_field];
 		}
-	},
+	}
 
-	setup_email_template: function() {
-		var me = this;
+	setup_email_template() {
+		const me = this;
 
 		this.dialog.fields_dict["email_template"].df.onchange = () => {
-			var email_template = me.dialog.fields_dict.email_template.get_value();
+			const email_template = me.dialog.fields_dict.email_template.get_value();
+			if (!email_template) return;
 
-			var prepend_reply = function(reply) {
-				if(me.reply_added===email_template) {
-					return;
-				}
-				var content_field = me.dialog.fields_dict.content;
-				var subject_field = me.dialog.fields_dict.subject;
-				var content = content_field.get_value() || "";
-				var subject = subject_field.get_value() || "";
+			function prepend_reply(reply) {
+				if (me.reply_added === email_template) return;
 
-				var parts = content.split('<!-- salutation-ends -->');
+				const content_field = me.dialog.fields_dict.content;
+				const subject_field = me.dialog.fields_dict.subject;
 
-				if(parts.length===2) {
-					content = [reply.message, "<br>", parts[1]];
-				} else {
-					content = [reply.message, "<br>", content];
-				}
-
-				content_field.set_value(content.join(''));
+				let content = content_field.get_value() || "";
+				content = content.split('<!-- salutation-ends -->')[1] || content;
 
+				content_field.set_value(`${reply.message}<br>${content}`);
 				subject_field.set_value(reply.subject);
 
 				me.reply_added = email_template;
@@ -296,86 +273,107 @@ frappe.views.CommunicationComposer = Class.extend({
 				method: 'frappe.email.doctype.email_template.email_template.get_email_template',
 				args: {
 					template_name: email_template,
-					doc: me.frm.doc,
+					doc: me.doc,
 					_lang: me.dialog.get_value("language_sel")
 				},
-				callback: function(r) {
+				callback(r) {
 					prepend_reply(r.message);
 				},
 			});
-		}
-	},
+		};
+	}
 
-	setup_last_edited_communication: function() {
-		var me = this;
-		if (!this.doc){
-			if (cur_frm){
-				this.doc = cur_frm.doctype;
-			}else{
-				this.doc = "Inbox";
-			}
-		}
-		if (cur_frm && cur_frm.docname) {
-			this.key = cur_frm.docname;
+	setup_last_edited_communication() {
+		if (this.frm) {
+			this.doctype = this.frm.doctype;
+			this.key = this.frm.docname;
 		} else {
-			this.key = "Inbox";
+			this.doctype = this.key = "Inbox";
 		}
-		if(this.last_email) {
+
+		if (this.last_email) {
 			this.key = this.key + ":" + this.last_email.name;
 		}
-		if(this.subject){
+
+		if (this.subject) {
 			this.key = this.key + ":" + this.subject;
 		}
-		this.dialog.onhide = function() {
-			var last_edited_communication = me.get_last_edited_communication();
-			$.extend(last_edited_communication, {
-				sender: me.dialog.get_value("sender"),
-				recipients: me.dialog.get_value("recipients"),
-				cc: me.dialog.get_value("cc"),
-				bcc: me.dialog.get_value("bcc"),
-				subject: me.dialog.get_value("subject"),
-				content: me.dialog.get_value("content"),
-			});
 
-			if (me.frm) {
-				$(document).trigger("form-stopped-typing", [me.frm]);
+		this.dialog.on_hide = () => {
+			$.extend(
+				this.get_last_edited_communication(true),
+				this.dialog.get_values(true)
+			);
+
+			if (this.frm) {
+				$(document).trigger("form-stopped-typing", [this.frm]);
 			}
+		};
+	}
+
+	get_last_edited_communication(clear) {
+		if (!frappe.last_edited_communication[this.doctype]) {
+			frappe.last_edited_communication[this.doctype] = {};
 		}
 
-		this.dialog.on_page_show = function() {
-			if (!me.txt) {
-				var last_edited_communication = me.get_last_edited_communication();
-				if(last_edited_communication.content) {
-					me.dialog.set_value("sender", last_edited_communication.sender || "");
-					me.dialog.set_value("subject", last_edited_communication.subject || "");
-					me.dialog.set_value("recipients", last_edited_communication.recipients || "");
-					me.dialog.set_value("cc", last_edited_communication.cc || "");
-					me.dialog.set_value("bcc", last_edited_communication.bcc || "");
-					me.dialog.set_value("content", last_edited_communication.content || "");
-				}
-			}
+		if (clear || !frappe.last_edited_communication[this.doctype][this.key]) {
+			frappe.last_edited_communication[this.doctype][this.key] = {};
+		}
+
+		return frappe.last_edited_communication[this.doctype][this.key];
+	}
 
+	async set_values() {
+		for (const fieldname of ["recipients", "cc", "bcc", "sender"]) {
+			await this.dialog.set_value(fieldname, this[fieldname] || "");
 		}
 
-	},
+		const subject = frappe.utils.html2text(this.subject) || '';
+		await this.dialog.set_value("subject", subject);
+
+		await this.set_values_from_last_edited_communication();
+		await this.set_content();
 
-	get_last_edited_communication: function() {
-		if (!frappe.last_edited_communication[this.doc]) {
-			frappe.last_edited_communication[this.doc] = {};
+		// set default email template for the first email in a document
+		if (this.frm && !this.is_a_reply && !this.content_set) {
+			const email_template = this.frm.meta.default_email_template || '';
+			await this.dialog.set_value("email_template", email_template);
 		}
 
-		if(!frappe.last_edited_communication[this.doc][this.key]) {
-			frappe.last_edited_communication[this.doc][this.key] = {};
+		for (const fieldname of ['email_template', 'cc', 'bcc']) {
+			if (this.dialog.get_value(fieldname)) {
+				this.toggle_more_options(true);
+				break;
+			}
 		}
+	}
+
+	async set_values_from_last_edited_communication() {
+		if (this.txt) return;
+
+		const last_edited = this.get_last_edited_communication();
+		if (!last_edited.content) return;
 
-		return frappe.last_edited_communication[this.doc][this.key];
-	},
+		// prevent re-triggering of email template
+		if (last_edited.email_template) {
+			const template_field = this.dialog.fields_dict.email_template;
+			await template_field.set_model_value(last_edited.email_template);
+			delete last_edited.email_template;
+		}
 
-	selected_format: function() {
-		return this.dialog.fields_dict.select_print_format.input.value || (this.frm && this.frm.meta.default_print_format) || "Standard";
-	},
+		await this.dialog.set_values(last_edited);
+		this.content_set = true;
+	}
 
-	get_print_format: function(format) {
+	selected_format() {
+		return (
+			this.dialog.fields_dict.select_print_format.input.value
+			|| this.frm && this.frm.meta.default_print_format
+			|| "Standard"
+		);
+	}
+
+	get_print_format(format) {
 		if (!format) {
 			format = this.selected_format();
 		}
@@ -385,21 +383,18 @@ frappe.views.CommunicationComposer = Class.extend({
 		} else {
 			return {};
 		}
-	},
+	}
 
-	setup_print_language: function() {
-		var doc = this.doc || cur_frm.doc;
-		var fields = this.dialog.fields_dict;
+	setup_print_language() {
+		const fields = this.dialog.fields_dict;
 
 		//Load default print language from doctype
-		this.lang_code = doc.language
-
-		if (!this.lang_code && this.get_print_format().default_print_language) {
-			this.lang_code = this.get_print_format().default_print_language;
-		}
+		this.lang_code = this.doc.language
+			|| this.get_print_format().default_print_language
+			|| frappe.boot.lang;
 
 		//On selection of language retrieve language code
-		var me = this;
+		const me = this;
 		$(fields.language_sel.input).change(function(){
 			me.lang_code = this.value
 		})
@@ -412,11 +407,11 @@ frappe.views.CommunicationComposer = Class.extend({
 		if (this.lang_code) {
 			$(fields.language_sel.input).val(this.lang_code);
 		}
-	},
+	}
 
-	setup_print: function() {
+	setup_print() {
 		// print formats
-		var fields = this.dialog.fields_dict;
+		const fields = this.dialog.fields_dict;
 
 		// toggle print format
 		$(fields.attach_document_print.input).click(function() {
@@ -426,8 +421,8 @@ frappe.views.CommunicationComposer = Class.extend({
 		// select print format
 		$(fields.select_print_format.wrapper).toggle(false);
 
-		if (cur_frm) {
-			const print_formats = frappe.meta.get_print_formats(cur_frm.meta.name);
+		if (this.frm) {
+			const print_formats = frappe.meta.get_print_formats(this.frm.meta.name);
 			$(fields.select_print_format.input)
 				.empty()
 				.add_options(print_formats)
@@ -436,11 +431,11 @@ frappe.views.CommunicationComposer = Class.extend({
 			$(fields.attach_document_print.wrapper).toggle(false);
 		}
 
-	},
+	}
 
-	setup_attach: function() {
-		var fields = this.dialog.fields_dict;
-		var attach = $(fields.select_attachments.wrapper);
+	setup_attach() {
+		const fields = this.dialog.fields_dict;
+		const attach = $(fields.select_attachments.wrapper);
 
 		if (!this.attachments) {
 			this.attachments = [];
@@ -483,9 +478,9 @@ frappe.views.CommunicationComposer = Class.extend({
 			.find(".add-more-attachments button")
 			.on('click', () => new frappe.ui.FileUploader(args));
 		this.render_attachment_rows();
-	},
+	}
 
-	render_attachment_rows: function(attachment) {
+	render_attachment_rows(attachment) {
 		const select_attachments = this.dialog.fields_dict.select_attachments;
 		const attachment_rows = $(select_attachments.wrapper).find(".attach-list");
 		if (attachment) {
@@ -509,7 +504,7 @@ frappe.views.CommunicationComposer = Class.extend({
 				});
 			}
 		}
-	},
+	}
 
 	get_attachment_row(attachment, checked) {
 		return $(`<p class="checkbox flex">
@@ -526,56 +521,55 @@ frappe.views.CommunicationComposer = Class.extend({
 				${frappe.utils.icon('link-url')}
 			</a>
 		</p>`);
-	},
+	}
 
-	setup_email: function() {
+	setup_email() {
 		// email
-		var fields = this.dialog.fields_dict;
+		const fields = this.dialog.fields_dict;
 
-		if(this.attach_document_print) {
+		if (this.attach_document_print) {
 			$(fields.attach_document_print.input).click();
 			$(fields.select_print_format.wrapper).toggle(true);
 		}
 
 		$(fields.send_me_a_copy.input).on('click', () => {
 			// update send me a copy (make it sticky)
-			let val = fields.send_me_a_copy.get_value();
+			const val = fields.send_me_a_copy.get_value();
 			frappe.db.set_value('User', frappe.session.user, 'send_me_a_copy', val);
 			frappe.boot.user.send_me_a_copy = val;
 		});
 
-	},
-
-	send_action: function() {
-		var me = this;
-		var btn = me.dialog.get_primary_btn();
+	}
 
-		var form_values = this.get_values();
-		if(!form_values) return;
+	send_action() {
+		const me = this;
+		const btn = me.dialog.get_primary_btn();
+		const form_values = this.get_values();
+		if (!form_values) return;
 
-		var selected_attachments =
+		const selected_attachments =
 			$.map($(me.dialog.wrapper).find("[data-file-name]:checked"), function (element) {
 				return $(element).attr("data-file-name");
 			});
 
 
-		if(form_values.attach_document_print) {
+		if (form_values.attach_document_print) {
 			me.send_email(btn, form_values, selected_attachments, null, form_values.select_print_format || "");
 		} else {
 			me.send_email(btn, form_values, selected_attachments);
 		}
-	},
+	}
 
-	get_values: function() {
-		var form_values = this.dialog.get_values();
+	get_values() {
+		const form_values = this.dialog.get_values();
 
 		// cc
-		for ( var i=0, l=this.dialog.fields.length; i < l; i++ ) {
-			var df = this.dialog.fields[i];
+		for (let i = 0, l = this.dialog.fields.length; i < l; i++) {
+			const df = this.dialog.fields[i];
 
-			if ( df.is_cc_checkbox ) {
+			if (df.is_cc_checkbox) {
 				// concat in cc
-				if ( form_values[df.fieldname] ) {
+				if (form_values[df.fieldname]) {
 					form_values.cc = ( form_values.cc ? (form_values.cc + ", ") : "" ) + df.fieldname;
 					form_values.bcc = ( form_values.bcc ? (form_values.bcc + ", ") : "" ) + df.fieldname;
 				}
@@ -585,22 +579,27 @@ frappe.views.CommunicationComposer = Class.extend({
 		}
 
 		return form_values;
-	},
+	}
 
-	save_as_draft: function() {
+	save_as_draft() {
 		if (this.dialog && this.frm) {
 			let message = this.dialog.get_value('content');
-			message = message.split(frappe.separator_element)[0];
+			message = message.split(separator_element)[0];
 			localforage.setItem(this.frm.doctype + this.frm.docname, message).catch(e => {
 				if (e) {
 					// silently fail
 					console.log(e); // eslint-disable-line
-					console.warn('[Communication] localStorage is full. Cannot save message as draft'); // eslint-disable-line
+					console.warn('[Communication] IndexedDB is full. Cannot save message as draft'); // eslint-disable-line
 				}
 			});
 
 		}
-	},
+	}
+
+	clear_cache() {
+		this.delete_saved_draft();
+		this.get_last_edited_communication(true);
+	}
 
 	delete_saved_draft() {
 		if (this.dialog && this.frm) {
@@ -608,28 +607,28 @@ frappe.views.CommunicationComposer = Class.extend({
 				if (e) {
 					// silently fail
 					console.log(e); // eslint-disable-line
-					console.warn('[Communication] localStorage is full. Cannot save message as draft'); // eslint-disable-line
+					console.warn('[Communication] IndexedDB is full. Cannot save message as draft'); // eslint-disable-line
 				}
 			});
 		}
-	},
+	}
 
-	send_email: function(btn, form_values, selected_attachments, print_html, print_format) {
-		var me = this;
-		me.dialog.hide();
+	send_email(btn, form_values, selected_attachments, print_html, print_format) {
+		const me = this;
+		this.dialog.hide();
 
-		if(!form_values.recipients) {
+		if (!form_values.recipients) {
 			frappe.msgprint(__("Enter Email Recipient(s)"));
 			return;
 		}
 
-		if(!form_values.attach_document_print) {
+		if (!form_values.attach_document_print) {
 			print_html = null;
 			print_format = null;
 		}
 
 
-		if(cur_frm && !frappe.model.can_email(me.doc.doctype, cur_frm)) {
+		if (this.frm && !frappe.model.can_email(this.doc.doctype, this.frm)) {
 			frappe.msgprint(__("You are not allowed to send emails related to this document"));
 			return;
 		}
@@ -650,28 +649,29 @@ frappe.views.CommunicationComposer = Class.extend({
 				send_me_a_copy: form_values.send_me_a_copy,
 				print_format: print_format,
 				sender: form_values.sender,
-				sender_full_name: form_values.sender?frappe.user.full_name():undefined,
+				sender_full_name: form_values.sender
+					? frappe.user.full_name()
+					: undefined,
 				email_template: form_values.email_template,
 				attachments: selected_attachments,
 				_lang : me.lang_code,
 				read_receipt:form_values.send_read_receipt,
 				print_letterhead: me.is_print_letterhead_checked(),
 			},
-			btn: btn,
-			callback: function(r) {
-				if(!r.exc) {
+			btn,
+			callback(r) {
+				if (!r.exc) {
 					frappe.utils.play_sound("email");
 
-					if(r.message["emails_not_sent_to"]) {
+					if (r.message["emails_not_sent_to"]) {
 						frappe.msgprint(__("Email not sent to {0} (unsubscribed / disabled)",
 							[ frappe.utils.escape_html(r.message["emails_not_sent_to"]) ]) );
 					}
 
-					if ((frappe.last_edited_communication[me.doc] || {})[me.key]) {
-						delete frappe.last_edited_communication[me.doc][me.key];
-					}
-					if (cur_frm) {
-						cur_frm.reload_doc();
+					me.clear_cache();
+
+					if (me.frm) {
+						me.frm.reload_doc();
 					}
 
 					// try the success callback if it exists
@@ -679,7 +679,7 @@ frappe.views.CommunicationComposer = Class.extend({
 						try {
 							me.success(r);
 						} catch (e) {
-							console.log(e);
+							console.log(e); // eslint-disable-line
 						}
 					}
 
@@ -691,113 +691,115 @@ frappe.views.CommunicationComposer = Class.extend({
 						try {
 							me.error(r);
 						} catch (e) {
-							console.log(e);
+							console.log(e); // eslint-disable-line
 						}
 					}
 				}
 			}
 		});
-	},
+	}
 
-	is_print_letterhead_checked: function() {
+	is_print_letterhead_checked() {
 		if (this.frm && $(this.frm.wrapper).find('.form-print-wrapper').is(':visible')){
 			return $(this.frm.wrapper).find('.print-letterhead').prop('checked') ? 1 : 0;
 		} else {
 			return (frappe.model.get_doc(":Print Settings", "Print Settings") ||
 				{ with_letterhead: 1 }).with_letterhead ? 1 : 0;
 		}
-	},
-
-	get_default_outgoing_email_account_signature: function() {
-		return frappe.db.get_value('Email Account', { 'default_outgoing': 1, 'add_signature': 1 }, 'signature');
-	},
+	}
 
-	setup_earlier_reply: async function() {
-		let fields = this.dialog.fields_dict;
-		let signature = frappe.boot.user.email_signature || "";
+	async set_content() {
+		if (this.content_set) return;
 
-		if (!signature) {
-			const res = await this.get_default_outgoing_email_account_signature();
-			signature = "<!-- signature-included -->" + res.message.signature;
+		let message = this.txt || "";
+		if (!message && this.frm) {
+			const { doctype, docname } = this.frm;
+			message = await localforage.getItem(doctype + docname) || "";
 		}
 
-		if (signature && !frappe.utils.is_html(signature)) {
-			signature = signature.replace(/\n/g, "<br>");
+		if (message) {
+			this.content_set = true;
 		}
 
-		if(this.txt) {
-			this.message = this.txt + (this.message ? ("<br><br>" + this.message) : "");
-		} else {
-			// saved draft in localStorage
-			const { doctype, docname } = this.frm || {};
-			if (doctype && docname) {
-				this.message = await localforage.getItem(doctype + docname) || '';
-			}
+		message += await this.get_signature();
+		if (this.real_name && !message.includes("<!-- salutation-ends -->")) {
+			message = `<p>${__('Dear')} ${this.real_name},</p>
+				<!-- salutation-ends --><br>${message}`;
 		}
 
-		if(this.real_name) {
-			this.message = '<p>'+__('Dear') +' '
-				+ this.real_name + ",</p><!-- salutation-ends --><br>" + (this.message || "");
+		if (this.is_a_reply) {
+			message += this.get_earlier_reply();
 		}
 
-		if(this.message && signature && this.message.includes(signature)) {
-			signature = "";
-		}
+		await this.dialog.set_value("content", message);
+	}
 
-		let reply = (this.message || "") + (signature ? ("<br>" + signature) : "");
-		let content = '';
+	async get_signature() {
+		let signature = frappe.boot.user.email_signature;
 
-		if (this.is_a_reply === 'undefined') {
-			this.is_a_reply = true;
+		if (!signature) {
+			const response = await frappe.db.get_value(
+				'Email Account',
+				{'default_outgoing': 1, 'add_signature': 1},
+				'signature'
+			);
+
+			signature = response.message.signature;
 		}
 
-		if (this.is_a_reply) {
-			let last_email = this.last_email;
+		if (!signature) return "";
 
-			if (!last_email) {
-				last_email = this.frm && this.frm.timeline.get_last_email(true);
-			}
+		if (!frappe.utils.is_html(signature)) {
+			signature = signature.replace(/\n/g, "<br>");
+		}
 
-			if (!last_email) return;
+		return "<br><!-- signature-included -->" + signature;
+	}
 
-			let last_email_content = last_email.original_comment || last_email.content;
+	get_earlier_reply() {
+		const last_email = (
+			this.last_email
+			|| this.frm && this.frm.timeline.get_last_email(true)
+		);
 
-			// convert the email context to text as we are enclosing
-			// this inside <blockquote>
-			last_email_content = this.html2text(last_email_content).replace(/\n/g, '<br>');
+		if (!last_email) return "";
+		let last_email_content = last_email.original_comment || last_email.content;
 
-			// clip last email for a maximum of 20k characters
-			// to prevent the email content from getting too large
-			if (last_email_content.length > 20 * 1024) {
-				last_email_content += '<div>' + __('Message clipped') + '</div>' + last_email_content;
-				last_email_content = last_email_content.slice(0, 20 * 1024);
-			}
+		// convert the email context to text as we are enclosing
+		// this inside <blockquote>
+		last_email_content = this.html2text(last_email_content).replace(/\n/g, '<br>');
 
-			let communication_date = last_email.communication_date || last_email.creation;
-			content = `
-				${reply}
-				<div><br></div>
-				${frappe.separator_element || ''}
-				<p>${__("On {0}, {1} wrote:", [frappe.datetime.global_date_format(communication_date) , last_email.sender])}</p>
-				<blockquote>
-				${last_email_content}
-				</blockquote>
-			`;
-		} else {
-			content = reply;
+		// clip last email for a maximum of 20k characters
+		// to prevent the email content from getting too large
+		if (last_email_content.length > 20 * 1024) {
+			last_email_content += '<div>' + __('Message clipped') + '</div>' + last_email_content;
+			last_email_content = last_email_content.slice(0, 20 * 1024);
 		}
-		fields.content.set_value(content);
-	},
 
-	html2text: function(html) {
+		const communication_date = frappe.datetime.global_date_format(
+			last_email.communication_date || last_email.creation
+		);
+
+		return `
+			<div><br></div>
+			${separator_element || ''}
+			<p>
+			${__("On {0}, {1} wrote:", [communication_date, last_email.sender])}
+			</p>
+			<blockquote>
+			${last_email_content}
+			</blockquote>
+		`;
+	}
+
+	html2text(html) {
 		// convert HTML to text and try and preserve whitespace
-		var d = document.createElement( 'div' );
+		const d = document.createElement( 'div' );
 		d.innerHTML = html.replace(/<\/div>/g, '<br></div>')  // replace end of blocks
 			.replace(/<\/p>/g, '<br></p>') // replace end of paragraphs
 			.replace(/<br>/g, '\n');
-		let text = d.textContent;
 
 		// replace multiple empty lines with just one
-		return text.replace(/\n{3,}/g, '\n\n');
+		return d.textContent.replace(/\n{3,}/g, '\n\n');
 	}
-});
+};

frappe/public/js/frappe/views/inbox/inbox_view.js

@@ -204,9 +204,7 @@ frappe.views.InboxView = class InboxView extends frappe.views.ListView {
 			};
 			frappe.new_doc('Email Account');
 		} else {
-			new frappe.views.CommunicationComposer({
-				doc: {}
-			});
+			new frappe.views.CommunicationComposer();
 		}
 	}
 };

frappe/public/js/frappe/widgets/chart_widget.js

@@ -25,7 +25,6 @@ export default class ChartWidget extends Widget {
 		delete this.dashboard_chart;
 		this.set_body();
 		this.make_chart();
-		this.setup_events();
 	}
 
 	set_chart_title() {
@@ -747,18 +746,4 @@ export default class ChartWidget extends Widget {
 				}
 			});
 	}
-
-	setup_events() {
-		$(document.body).on('toggleSidebar', () => {
-			this.dashboard_chart && this.dashboard_chart.draw(true);
-		});
-
-		$(document.body).on('toggleListSidebar', () => {
-			this.dashboard_chart && this.dashboard_chart.draw(true);
-		});
-
-		$(document.body).on('toggleFullWidth', () => {
-			this.dashboard_chart && this.dashboard_chart.draw(true);
-		});
-	}
 }

frappe/public/scss/common/css_variables.scss

@@ -169,6 +169,9 @@
 	// Other Colors
 	--sidebar-select-color: var(--gray-200);
 
+	--scrollbar-thumb-color: var(--gray-400);
+	--scrollbar-track-color: var(--gray-200);
+
 	--shadow-inset: inset 0px -1px 0px var(--gray-300);
 	--border-color: var(--gray-100);
 	--dark-border-color: var(--gray-300);

frappe/public/scss/common/quill.scss

@@ -119,7 +119,10 @@
 	border: 1px solid var(--border-color);
 	padding: 2px 5px;
 	font-size: var(--text-sm);
-	background-color: var(--fg-color);
+	background-color: var(--user-mention-bg-color);
+	a[href] {
+		text-decoration: none;
+	}
 }
 
 // table
@@ -174,7 +177,7 @@
 .ql-editor.read-mode {
 	padding: 0;
 	.mention {
-		background-color: var(--control-bg);
+		--user-mention-bg-color: var(--control-bg);
 	}
 }
 
@@ -190,4 +193,8 @@
 
 .mention>span {
 	margin: 0 3px;
-}
+}
+
+.mention[data-is-group="true"] {
+	background-color: var(--group-mention-bg-color);
+}

frappe/public/scss/desk/css_variables.scss

@@ -59,6 +59,10 @@ $input-height: 28px !default;
 	--timeline-content-max-width: 700px;
 	--timeline-left-padding: calc(var(--padding-xl) + var(--timeline-item-icon-size) / 2);
 
+	// mentions
+	--user-mention-bg-color: var(--fg-color);
+	--group-mention-bg-color: var(--bg-purple);
+
 	// skeleton
 	--skeleton-bg: var(--gray-100);
 

frappe/public/scss/desk/dark.scss

@@ -65,6 +65,9 @@
 
 	--sidebar-select-color: var(--gray-800);
 
+	--scrollbar-thumb-color: var(--gray-600);
+	--scrollbar-track-color: var(--gray-700);
+
 	--shadow-inset: var(--fg-color);
 	--border-color: var(--gray-700);
 	--dark-border-color: var(--gray-600);
@@ -75,6 +78,8 @@
 	// input
 	--input-disabled-bg: none;
 
+	color-scheme: dark;
+
 	.frappe-card {
 		.btn-default {
 			background-color: var(--bg-color);
@@ -99,7 +104,7 @@
 	.ql-editor {
 		color: var(--text-on-gray);
 		&.read-mode {
-			span,
+			span:not(.mention),
 			p,
 			u,
 			strong {

frappe/public/scss/desk/desktop.scss

@@ -754,7 +754,28 @@ body {
 			.layout-side-section, .layout-main-section-wrapper {
 				height: 100%;
 				overflow-y: auto;
+				padding-right: 25px;
+				scrollbar-color: var(--gray-200) transparent;
+				[data-theme="dark"] & {
+					scrollbar-color: var(--gray-800) transparent;
+				}
+
+				&::-webkit-scrollbar-track {
+					background: transparent;
+				}
+
+				&::-webkit-scrollbar-thumb {
+					background: var(--gray-200);
+					[data-theme="dark"] & {
+						background: var(--gray-800);
+					}
+				}
 			}
+
+			.layout-side-section {
+				margin-right: 20px;
+			}
+
 			.desk-sidebar {
 				margin-bottom: var(--margin-2xl);
 			}

frappe/public/scss/desk/index.scss

@@ -10,6 +10,7 @@
 @import "mobile";
 @import "form";
 @import "print_preview";
+@import "scrollbar";
 @import "navbar";
 @import "../common/modal";
 @import "slides";

frappe/public/scss/desk/scrollbar.scss

@@ -0,0 +1,29 @@
+/* Works on Firefox */
+* {
+	scrollbar-width: thin;
+	scrollbar-color: var(--scrollbar-thumb-color) var(--scrollbar-track-color);
+}
+
+html {
+	scrollbar-width: auto;
+}
+
+/* Works on Chrome, Edge, and Safari */
+*::-webkit-scrollbar {
+	width: 6px;
+	height: 6px;
+}
+
+*::-webkit-scrollbar-thumb {
+	background: var(--scrollbar-thumb-color);
+}
+
+*::-webkit-scrollbar-track,
+*::-webkit-scrollbar-corner {
+	background: var(--scrollbar-track-color);
+}
+
+body::-webkit-scrollbar {
+	width: unset;
+	height: unset;
+}

frappe/public/scss/desk/timeline.scss

@@ -77,6 +77,7 @@ $threshold: 34;
 		}
 	}
 	.document-email-link-container {
+		@extend .ellipsis;
 		position: relative;
 		padding: var(--padding-sm);
 		font-size: var(--text-sm);
@@ -141,4 +142,4 @@ $threshold: 34;
 			--icon-stroke: var(--text-color);
 		}
 	}
-}
+}

frappe/translate.py

@@ -606,11 +606,23 @@ def write_csv_file(path, app_messages, lang_dict):
 	from csv import writer
 	with open(path, 'w', newline='') as msgfile:
 		w = writer(msgfile, lineterminator='\n')
-		for p, m in app_messages:
-			t = lang_dict.get(m, '')
+
+		for app_message in app_messages:
+			context = None
+			if len(app_message) == 2:
+				path, message = app_message
+			elif len(app_message) == 3:
+				path, message, lineno = app_message
+			elif len(app_message) == 4:
+				path, message, context, lineno = app_message
+			else:
+				continue
+
+			t = lang_dict.get(message, '')
 			# strip whitespaces
-			t = re.sub('{\s?([0-9]+)\s?}', "{\g<1>}", t)
-			w.writerow([p if p else '', m, t])
+			translated_string = re.sub('{\s?([0-9]+)\s?}', "{\g<1>}", t)
+			if translated_string:
+				w.writerow([message, translated_string, context])
 
 def get_untranslated(lang, untranslated_file, get_all=False):
 	"""Returns all untranslated strings for a language and writes in a file

frappe/utils/__init__.py

@@ -307,14 +307,23 @@ def unesc(s, esc_chars):
 		s = s.replace(esc_str, c)
 	return s
 
-def execute_in_shell(cmd, verbose=0):
+def execute_in_shell(cmd, verbose=0, low_priority=False):
 	# using Popen instead of os.system - as recommended by python docs
 	import tempfile
 	from subprocess import Popen
 
 	with tempfile.TemporaryFile() as stdout:
 		with tempfile.TemporaryFile() as stderr:
-			p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr)
+			kwargs = {
+				"shell": True,
+				"stdout": stdout,
+				"stderr": stderr
+			}
+
+			if low_priority:
+				kwargs["preexec_fn"] = lambda: os.nice(10)
+
+			p = Popen(cmd, **kwargs)
 			p.wait()
 
 			stdout.seek(0)

frappe/utils/backups.py

@@ -15,7 +15,7 @@ import click
 # imports - module imports
 import frappe
 from frappe import _, conf
-from frappe.utils import get_file_size, get_url, now, now_datetime
+from frappe.utils import get_file_size, get_url, now, now_datetime, cint
 
 # backup variable for backwards compatibility
 verbose = False
@@ -315,8 +315,6 @@ class BackupGenerator:
 			print(template.format(_type.title(), info["path"], info["size"]))
 
 	def backup_files(self):
-		import subprocess
-
 		for folder in ("public", "private"):
 			files_path = frappe.get_site_path(folder, "files")
 			backup_path = (
@@ -327,12 +325,12 @@ class BackupGenerator:
 				cmd_string = "tar cf - {1} | gzip > {0}"
 			else:
 				cmd_string = "tar -cf {0} {1}"
-			output = subprocess.check_output(
-				cmd_string.format(backup_path, files_path), shell=True
-			)
 
-			if self.verbose and output:
-				print(output.decode("utf8"))
+			frappe.utils.execute_in_shell(
+				cmd_string.format(backup_path, files_path),
+				verbose=self.verbose,
+				low_priority=True
+			)
 
 	def copy_site_config(self):
 		site_config_backup_path = self.backup_path_conf
@@ -436,7 +434,7 @@ class BackupGenerator:
 		if self.verbose:
 			print(command + "\n")
 
-		err, out = frappe.utils.execute_in_shell(command)
+		frappe.utils.execute_in_shell(command, low_priority=True)
 
 	def send_email(self):
 		"""
@@ -474,29 +472,6 @@ download only after 24 hours.""" % {
 		return recipient_list
 
 
-@frappe.whitelist()
-def get_backup():
-	"""
-	This function is executed when the user clicks on
-	Toos > Download Backup
-	"""
-	delete_temp_backups()
-	odb = BackupGenerator(
-		frappe.conf.db_name,
-		frappe.conf.db_name,
-		frappe.conf.db_password,
-		db_host=frappe.db.host,
-		db_type=frappe.conf.db_type,
-		db_port=frappe.conf.db_port,
-	)
-	odb.get_backup()
-	recipient_list = odb.send_email()
-	frappe.msgprint(
-		_(
-			"Download link for your backup will be emailed on the following email address: {0}"
-		).format(", ".join(recipient_list))
-	)
-
 @frappe.whitelist()
 def fetch_latest_backups(partial=False):
 	"""Fetches paths of the latest backup taken in the last 30 days
@@ -570,7 +545,7 @@ def new_backup(
 	force=False,
 	verbose=False,
 ):
-	delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
+	delete_temp_backups()
 	odb = BackupGenerator(
 		frappe.conf.db_name,
 		frappe.conf.db_name,
@@ -595,8 +570,9 @@ def new_backup(
 
 def delete_temp_backups(older_than=24):
 	"""
-	Cleans up the backup_link_path directory by deleting files older than 24 hours
+	Cleans up the backup_link_path directory by deleting older files
 	"""
+	older_than = cint(frappe.conf.keep_backups_for_hours) or older_than
 	backup_path = get_backup_path()
 	if os.path.exists(backup_path):
 		file_list = os.listdir(get_backup_path())

frappe/utils/boilerplate.py

@@ -303,6 +303,13 @@ user_data_fields = [
 	}}
 ]
 
+# Authentication and authorization
+# --------------------------------
+
+# auth_hooks = [
+# 	"{app_name}.auth.validate"
+# ]
+
 """
 
 desktop_template = """# -*- coding: utf-8 -*-

frappe/utils/html_utils.py

@@ -177,7 +177,7 @@ acceptable_attributes = [
 	'data-value', 'role', 'frameborder', 'allowfullscreen', 'spellcheck',
 	'data-mode', 'data-gramm', 'data-placeholder', 'data-comment',
 	'data-id', 'data-denotation-char', 'itemprop', 'itemscope',
-	'itemtype', 'itemid', 'itemref', 'datetime'
+	'itemtype', 'itemid', 'itemref', 'datetime', 'data-is-group'
 ]
 
 mathml_attributes = [

frappe/utils/oauth.py

@@ -64,8 +64,6 @@ def get_oauth2_authorize_url(provider, redirect_to):
 
 	state = { "site": frappe.utils.get_url(), "token": frappe.generate_hash(), "redirect_to": redirect_to 	}
 
-	frappe.cache().set_value("{0}:{1}".format(provider, state["token"]), True, expires_in_sec=120)
-
 	# relative to absolute url
 	data = {
 		"redirect_uri": get_redirect_uri(provider),
@@ -176,11 +174,6 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No
 		frappe.respond_as_web_page(_("Invalid Request"), _("Token is missing"), http_status_code=417)
 		return
 
-	token = frappe.cache().get_value("{0}:{1}".format(provider, state["token"]), expires=True)
-	if not token:
-		frappe.respond_as_web_page(_("Invalid Request"), _("Invalid Token"), http_status_code=417)
-		return
-
 	user = get_email(data)
 
 	if not user:

frappe/utils/safe_exec.py

@@ -12,6 +12,7 @@ from frappe.modules import scrub
 from frappe.www.printview import get_visible_columns
 import frappe.exceptions
 import frappe.integrations.utils
+from frappe.frappeclient import FrappeClient
 
 class ServerScriptNotEnabled(frappe.PermissionError):
 	pass
@@ -104,8 +105,10 @@ def get_safe_globals():
 			make_post_request = frappe.integrations.utils.make_post_request,
 			socketio_port=frappe.conf.socketio_port,
 			get_hooks=frappe.get_hooks,
-			sanitize_html=frappe.utils.sanitize_html
+			sanitize_html=frappe.utils.sanitize_html,
+			log_error=frappe.log_error
 		),
+		FrappeClient=FrappeClient,
 		style=frappe._dict(
 			border_color='#d1d8dd'
 		),
@@ -297,4 +300,4 @@ VALID_UTILS = (
 "formatdate",
 "get_user_info_for_avatar",
 "get_abbr"
-)
+)

frappe/utils/xlsxutils.py

@@ -1,18 +1,19 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
-from __future__ import unicode_literals
-
-import frappe
+import re
+from io import BytesIO
 
 import openpyxl
 import xlrd
-import re
-from openpyxl.styles import Font
 from openpyxl import load_workbook
+from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
-from six import BytesIO, string_types
+
+import frappe
 
 ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')
+
+
 # return xlsx file object
 def make_xlsx(data, sheet_name, wb=None, column_widths=None):
 	column_widths = column_widths or []
@@ -31,12 +32,12 @@ def make_xlsx(data, sheet_name, wb=None, column_widths=None):
 	for row in data:
 		clean_row = []
 		for item in row:
-			if isinstance(item, string_types) and (sheet_name not in ['Data Import Template', 'Data Export']):
+			if isinstance(item, str) and (sheet_name not in ['Data Import Template', 'Data Export']):
 				value = handle_html(item)
 			else:
 				value = item
 
-			if isinstance(item, string_types) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
+			if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
 				# Remove illegal characters from the string
 				value = re.sub(ILLEGAL_CHARACTERS_RE, '', value)
 
@@ -80,12 +81,12 @@ def handle_html(data):
 
 	return value
 
+
 def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=None):
 	if file_url:
 		_file = frappe.get_doc("File", {"file_url": file_url})
 		filename = _file.get_full_path()
 	elif fcontent:
-		from io import BytesIO
 		filename = BytesIO(fcontent)
 	elif filepath:
 		filename = filepath
@@ -102,6 +103,7 @@ def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=Non
 		rows.append(tmp_list)
 	return rows
 
+
 def read_xls_file_from_attached_file(content):
 	book = xlrd.open_workbook(file_contents=content)
 	sheets = book.sheets()
@@ -111,6 +113,7 @@ def read_xls_file_from_attached_file(content):
 		rows.append(sheet.row_values(i))
 	return rows
 
+
 def build_xlsx_response(data, filename):
 	xlsx_file = make_xlsx(data, filename)
 	# write out response as a xlsx type

frappe/website/doctype/web_form/web_form.py

@@ -215,6 +215,11 @@ def get_context(context):
 			amount = self.amount
 			if self.amount_based_on_field:
 				amount = doc.get(self.amount_field)
+
+			from decimal import Decimal
+			if amount is None or Decimal(amount) <= 0:
+				return frappe.utils.get_url(self.success_url or self.route)
+				
 			payment_details = {
 				"amount": amount,
 				"title": title,

frappe/website/doctype/web_page/web_page.py

@@ -242,11 +242,11 @@ def extract_script_and_style_tags(html):
 	styles = []
 
 	for script in soup.find_all('script'):
-		scripts.append(script.text)
+		scripts.append(script.string)
 		script.extract()
 
 	for style in soup.find_all('style'):
-		styles.append(style.text)
+		styles.append(style.string)
 		style.extract()
 
 	return str(soup), scripts, styles

frappe/website/doctype/website_settings/google_indexing.py

@@ -2,17 +2,18 @@
 # Copyright (c) 2020, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
-import frappe
-import requests
-import googleapiclient.discovery
+
+from urllib.parse import quote
+
 import google.oauth2.credentials
+import requests
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 
+import frappe
 from frappe import _
-from googleapiclient.errors import HttpError
-from frappe.utils import get_request_site_address
-from six.moves.urllib.parse import quote
 from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.utils import get_request_site_address
 
 SCOPES = "https://www.googleapis.com/auth/indexing"
 
@@ -82,7 +83,12 @@ def get_google_indexing_object():
 	}
 
 	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_indexing = googleapiclient.discovery.build("indexing", "v3", credentials=credentials)
+	google_indexing = build(
+		serviceName="indexing",
+		version="v3",
+		credentials=credentials,
+		static_discovery=False
+	)
 
 	return google_indexing
 

frappe/www/login.py

@@ -95,14 +95,6 @@ def login_via_frappe(code, state):
 def login_via_office365(code, state):
 	login_via_oauth2_id_token("office_365", code, state, decoder=decoder_compat)
 
-@frappe.whitelist(allow_guest=True)
-def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=None, generate_login_token=False):
-	if not ((data and provider and state) or (email_id and key)):
-		frappe.respond_as_web_page(_("Invalid Request"), _("Missing parameters for login"), http_status_code=417)
-		return
-
-	_login_oauth_user(data, provider, state, email_id, key, generate_login_token)
-
 @frappe.whitelist(allow_guest=True)
 def login_via_token(login_token):
 	sid = frappe.cache().get_value("login_token:{0}".format(login_token), expires=True)

requirements.txt

@@ -1,79 +1,79 @@
-Babel==2.6.0
-beautifulsoup4==4.8.2
-bleach-whitelist==0.0.10
-bleach==3.3.0
-boto3==1.10.18
-braintree==3.57.1
-chardet==3.0.4
-Click==7.0
-coverage==4.5.4
-croniter==0.3.31
-cryptography==3.3.2
-dropbox==9.1.0
-email-reply-parser==0.5.9
-Faker==2.0.4
+Babel~=2.9.0
+beautifulsoup4~=4.9.3
+bleach-whitelist~=0.0.11
+bleach~=3.3.0
+boto3~=1.17.53
+braintree~=4.8.0
+chardet~=4.0.0
+Click~=7.1.2
+coverage~=4.5.4
+croniter~=1.0.11
+cryptography~=3.4.7
+dropbox~=11.7.0
+email-reply-parser~=0.5.12
+Faker~=8.1.0
 future==0.18.2
-gitdb2==2.0.6;python_version<'3.4'
-GitPython==2.1.15
-git-url-parse==1.2.2
-google-api-python-client==1.9.3
-google-auth-httplib2==0.0.3
-google-auth-oauthlib==0.4.1
-google-auth==1.18.0
-googlemaps==3.1.1
-gunicorn==19.10.0
-html2text==2016.9.19
-html5lib==1.0.1
-ipython==7.14.0
-jedi==0.17.2  # not directly required. Pinned to fix upstream issue with ipython.
-Jinja2==2.11.3
-ldap3==2.7
-markdown2==2.4.0
+git-url-parse~=1.2.2
+gitdb~=4.0.7
+GitPython~=3.1.14
+google-api-python-client~=2.2.0
+google-auth-httplib2~=0.1.0
+google-auth-oauthlib~=0.4.4
+google-auth~=1.29.0
+googlemaps~=4.4.5
+gunicorn~=20.1.0
+html2text==2020.1.16
+html5lib~=1.1
+ipython~=7.16.1
+jedi==0.17.2  # not directly required. Pinned to fix upstream IPython issue (https://github.com/ipython/ipython/issues/12740)
+Jinja2~=2.11.3
+ldap3~=2.9
+markdown2~=2.4.0
 maxminddb-geolite2==2018.703
-ndg-httpsclient==0.5.1
-num2words==0.5.10
-oauthlib==3.1.0
-openpyxl==2.6.4
-passlib==1.7.3
-pdfkit==0.6.1
-Pillow>=8.0.0
-premailer==3.6.1
-psutil==5.7.2
-psycopg2-binary==2.8.4
-pyasn1==0.4.8
-PyJWT==1.7.1
-PyMySQL==0.9.3
-pyngrok==4.1.6
-pyOpenSSL==19.1.0
-pyotp==2.3.0
-PyPDF2==1.26.0
-pypng==0.0.20
-PyQRCode==1.2.1
-python-dateutil==2.8.1
-pytz==2019.3
-PyYAML==5.4
-rauth==0.7.3
-redis==3.5.3
-requests-oauthlib==1.3.0
-requests==2.23.0
-RestrictedPython==5.0
-rq>=1.1.0
-schedule==0.6.0
-semantic-version==2.8.4
-simple-chalk==0.1.0
-six==1.14.0
-sqlparse==0.2.4
-stripe==2.40.0
-terminaltables==3.1.0
-unittest-xml-reporting==2.5.2
-urllib3==1.25.9
-watchdog==0.8.0
-Werkzeug==0.16.1
-Whoosh==2.7.4
-xlrd==1.2.0
-zxcvbn-python==4.4.24
-pycryptodome==3.9.8
-paytmchecksum==1.7.0
-wrapt==1.10.11
-razorpay==1.2.0
+ndg-httpsclient~=0.5.1
+num2words~=0.5.10
+oauthlib~=3.1.0
+openpyxl~=3.0.7
+passlib~=1.7.4
+paytmchecksum~=1.7.0
+pdfkit~=0.6.1
+Pillow~=8.2.0
+premailer~=3.8.0
+psutil~=5.8.0
+psycopg2-binary~=2.8.6
+pyasn1~=0.4.8
+pycryptodome~=3.10.1
+PyJWT~=1.7.1
+PyMySQL~=1.0.2
+pyngrok~=5.0.5
+pyOpenSSL~=20.0.1
+pyotp~=2.6.0
+PyPDF2~=1.26.0
+pypng~=0.0.20
+PyQRCode~=1.2.1
+python-dateutil~=2.8.1
+pytz==2021.1
+PyYAML~=5.4.1
+rauth~=0.7.3
+razorpay~=1.2.0
+redis~=3.5.3
+requests-oauthlib~=1.3.0
+requests~=2.25.1
+RestrictedPython~=5.1
+rq~=1.8.0
 rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability
+schedule~=1.1.0
+semantic-version~=2.8.5
+simple-chalk~=0.1.0
+six~=1.15.0
+sqlparse~=0.4.1
+stripe~=2.56.0
+terminaltables~=3.1.0
+unittest-xml-reporting~=3.0.4
+urllib3~=1.26.4
+watchdog~=2.0.2
+Werkzeug~=0.16.1
+Whoosh~=2.7.4
+wrapt~=1.12.1
+xlrd~=2.0.1
+zxcvbn-python~=4.4.24