Merge remote-tracking branch 'upstream/develop' into esbuild

.github/helper/semgrep_rules/translate.yml

@@ -42,7 +42,7 @@ rules:
 
 - id: frappe-translation-python-splitting
   pattern-either:
-  - pattern: _(...) + ... + _(...)
+  - pattern: _(...) + _(...)
   - pattern: _("..." + "...")
   - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
   - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )

frappe/__init__.py

@@ -10,11 +10,9 @@ be used to build database driven apps.
 
 Read the documentation: https://frappeframework.com/docs
 """
-from __future__ import unicode_literals, print_function
 
-from six import iteritems, binary_type, text_type, string_types, PY2
 from werkzeug.local import Local, release_local
-import os, sys, importlib, inspect, json
+import os, sys, importlib, inspect, json, warnings
 import typing
 from past.builtins import cmp
 import click
@@ -27,19 +25,14 @@ from .utils.lazy_loader import lazy_import
 # Lazy imports
 faker = lazy_import('faker')
 
-
-# Harmless for Python 3
-# For Python 2 set default encoding to utf-8
-if PY2:
-	reload(sys)
-	sys.setdefaultencoding("utf-8")
-
 __version__ = '14.0.0-dev'
 
 __title__ = "Frappe Framework"
 
 local = Local()
 controllers = {}
+warnings.simplefilter('always', DeprecationWarning)
+warnings.simplefilter('always', PendingDeprecationWarning)
 
 class _dict(dict):
 	"""dict like object that exposes keys as attributes"""
@@ -97,14 +90,14 @@ def _(msg, lang=None, context=None):
 
 def as_unicode(text, encoding='utf-8'):
 	'''Convert to unicode if required'''
-	if isinstance(text, text_type):
+	if isinstance(text, str):
 		return text
 	elif text==None:
 		return ''
-	elif isinstance(text, binary_type):
-		return text_type(text, encoding)
+	elif isinstance(text, bytes):
+		return str(text, encoding)
 	else:
-		return text_type(text)
+		return str(text)
 
 def get_lang_dict(fortype, name=None):
 	"""Returns the translated language dict for the given type and name.
@@ -597,7 +590,7 @@ def is_whitelisted(method):
 		# strictly sanitize form_dict
 		# escapes html characters like <> except for predefined tags like a, b, ul etc.
 		for key, value in form_dict.items():
-			if isinstance(value, string_types):
+			if isinstance(value, str):
 				form_dict[key] = sanitize_html(value)
 
 def read_only():
@@ -721,7 +714,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc
 		user = session.user
 
 	if doc:
-		if isinstance(doc, string_types):
+		if isinstance(doc, str):
 			doc = get_doc(doctype, doc)
 
 		doctype = doc.doctype
@@ -790,7 +783,7 @@ def set_value(doctype, docname, fieldname, value=None):
 	return frappe.client.set_value(doctype, docname, fieldname, value)
 
 def get_cached_doc(*args, **kwargs):
-	if args and len(args) > 1 and isinstance(args[1], text_type):
+	if args and len(args) > 1 and isinstance(args[1], str):
 		key = get_document_cache_key(args[0], args[1])
 		# local cache
 		doc = local.document_cache.get(key)
@@ -821,7 +814,7 @@ def clear_document_cache(doctype, name):
 
 def get_cached_value(doctype, name, fieldname, as_dict=False):
 	doc = get_cached_doc(doctype, name)
-	if isinstance(fieldname, string_types):
+	if isinstance(fieldname, str):
 		if as_dict:
 			throw('Cannot make dict for single fieldname')
 		return doc.get(fieldname)
@@ -1027,7 +1020,7 @@ def get_doc_hooks():
 	if not hasattr(local, 'doc_events_hooks'):
 		hooks = get_hooks('doc_events', {})
 		out = {}
-		for key, value in iteritems(hooks):
+		for key, value in hooks.items():
 			if isinstance(key, tuple):
 				for doctype in key:
 					append_hook(out, doctype, value)
@@ -1144,7 +1137,7 @@ def get_file_json(path):
 
 def read_file(path, raise_not_found=False):
 	"""Open a file and return its content as Unicode."""
-	if isinstance(path, text_type):
+	if isinstance(path, str):
 		path = path.encode("utf-8")
 
 	if os.path.exists(path):
@@ -1167,7 +1160,7 @@ def get_attr(method_string):
 
 def call(fn, *args, **kwargs):
 	"""Call a function and match arguments."""
-	if isinstance(fn, string_types):
+	if isinstance(fn, str):
 		fn = get_attr(fn)
 
 	newargs = get_newargs(fn, kwargs)
@@ -1178,13 +1171,9 @@ def get_newargs(fn, kwargs):
 	if hasattr(fn, 'fnargs'):
 		fnargs = fn.fnargs
 	else:
-		try:
-			fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
-		except ValueError:
-			fnargs = inspect.getfullargspec(fn).args
-			varargs = inspect.getfullargspec(fn).varargs
-			varkw = inspect.getfullargspec(fn).varkw
-			defaults = inspect.getfullargspec(fn).defaults
+		fnargs = inspect.getfullargspec(fn).args
+		fnargs.extend(inspect.getfullargspec(fn).kwonlyargs)
+		varkw = inspect.getfullargspec(fn).varkw
 
 	newargs = {}
 	for a in kwargs:
@@ -1626,6 +1615,12 @@ def enqueue(*args, **kwargs):
 	import frappe.utils.background_jobs
 	return frappe.utils.background_jobs.enqueue(*args, **kwargs)
 
+def task(**task_kwargs):
+	def decorator_task(f):
+		f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs)
+		return f
+	return decorator_task
+
 def enqueue_doc(*args, **kwargs):
 	'''
 		Enqueue method to be executed using a background worker

frappe/app.py

@@ -185,7 +185,7 @@ def make_form_dict(request):
 		args = request.form or request.args
 
 	if not isinstance(args, dict):
-		frappe.throw("Invalid request arguments")
+		frappe.throw(_("Invalid request arguments"))
 
 	try:
 		frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
@@ -294,8 +294,9 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 	_sites_path = sites_path
 
 	from werkzeug.serving import run_simple
+	patch_werkzeug_reloader()
 
-	if profile:
+	if profile or os.environ.get('USE_PROFILER'):
 		application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))
 
 	if not os.environ.get('NO_STATICS'):
@@ -324,3 +325,23 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
 		use_debugger=not in_test_env,
 		use_evalex=not in_test_env,
 		threaded=not no_threading)
+
+def patch_werkzeug_reloader():
+	"""
+	This function monkey patches Werkzeug reloader to ignore reloading files in
+	the __pycache__ directory.
+
+	To be deprecated when upgrading to Werkzeug 2.
+	"""
+
+	from werkzeug._reloader import WatchdogReloaderLoop
+
+	trigger_reload = WatchdogReloaderLoop.trigger_reload
+
+	def custom_trigger_reload(self, filename):
+		if os.path.basename(os.path.dirname(filename)) == "__pycache__":
+			return
+
+		return trigger_reload(self, filename)
+
+	WatchdogReloaderLoop.trigger_reload = custom_trigger_reload

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -173,7 +173,7 @@ class TestAutoRepeat(unittest.TestCase):
 			fields=['docstatus'],
 			limit=1
 		)
-		self.assertEquals(docnames[0].docstatus, 1)
+		self.assertEqual(docnames[0].docstatus, 1)
 
 
 def make_auto_repeat(**args):

frappe/build.py

@@ -323,7 +323,7 @@ def make_asset_dirs(make_copy=False, restore=False):
 					except OSError:
 						print("Cannot link {} to {}".format(source, target))
 			else:
-				# warnings.warn('Source {source} does not exist.'.format(source = source))
+				warnings.warn('Source {source} does not exist.'.format(source = source))
 				pass
 
 

frappe/commands/site.py

@@ -254,9 +254,7 @@ def list_apps(context, format):
 		frappe.destroy()
 
 	if format == "json":
-		import json
-
-		click.echo(json.dumps(summary_dict))
+		click.echo(frappe.as_json(summary_dict))
 
 @click.command('add-system-manager')
 @click.argument('email')

frappe/commands/utils.py

@@ -107,22 +107,54 @@ def destroy_all_sessions(context, reason=None):
 		raise SiteNotSpecifiedError
 
 @click.command('show-config')
+@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text")
 @pass_context
-def show_config(context):
-	"print configuration file"
-	print("\t\033[92m{:<50}\033[0m \033[92m{:<15}\033[0m".format('Config','Value'))
-	sites_path = os.path.join(frappe.utils.get_bench_path(), 'sites')
-	site_path = context.sites[0]
-	configuration = frappe.get_site_config(sites_path=sites_path, site_path=site_path)
-	print_config(configuration)
+def show_config(context, format):
+	"Print configuration file to STDOUT in speified format"
 
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	sites_config = {}
+	sites_path = os.getcwd()
+
+	from frappe.utils.commands import render_table
+
+	def transform_config(config, prefix=None):
+		prefix = f"{prefix}." if prefix else ""
+		site_config = []
+
+		for conf, value in config.items():
+			if isinstance(value, dict):
+				site_config += transform_config(value, prefix=f"{prefix}{conf}")
+			else:
+				log_value = json.dumps(value) if isinstance(value, list) else value
+				site_config += [[f"{prefix}{conf}", log_value]]
+
+		return site_config
+
+	for site in context.sites:
+		frappe.init(site)
+
+		if len(context.sites) != 1 and format == "text":
+			if context.sites.index(site) != 0:
+				click.echo()
+			click.secho(f"Site {site}", fg="yellow")
 
-def print_config(config):
-	for conf, value in config.items():
-		if isinstance(value, dict):
-			print_config(value)
-		else:
-			print("\t{:<50} {:<15}".format(conf, value))
+		configuration = frappe.get_site_config(sites_path=sites_path, site_path=site)
+
+		if format == "text":
+			data = transform_config(configuration)
+			data.insert(0, ['Config','Value'])
+			render_table(data)
+
+		if format == "json":
+			sites_config[site] = configuration
+
+		frappe.destroy()
+
+	if format == "json":
+		click.echo(frappe.as_json(sites_config))
 
 
 @click.command('reset-perms')
@@ -481,6 +513,7 @@ def console(context):
 			locals()[app] = __import__(app)
 		except ModuleNotFoundError:
 			failed_to_import.append(app)
+			all_apps.remove(app)
 
 	print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
 	if failed_to_import:
@@ -668,20 +701,27 @@ def make_app(destination, app_name):
 @click.command('set-config')
 @click.argument('key')
 @click.argument('value')
-@click.option('-g', '--global', 'global_', is_flag = True, default = False, help = 'Set Global Site Config')
-@click.option('--as-dict', is_flag=True, default=False)
+@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config')
+@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object')
+@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object')
 @pass_context
-def set_config(context, key, value, global_ = False, as_dict=False):
+def set_config(context, key, value, global_=False, parse=False, as_dict=False):
 	"Insert/Update a value in site_config.json"
 	from frappe.installer import update_site_config
-	import ast
+
 	if as_dict:
+		from frappe.utils.commands import warn
+		warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning)
+		parse = as_dict
+
+	if parse:
+		import ast
 		value = ast.literal_eval(value)
 
 	if global_:
-		sites_path = os.getcwd() # big assumption.
+		sites_path = os.getcwd()
 		common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
-		update_site_config(key, value, validate = False, site_config_path = common_site_config_path)
+		update_site_config(key, value, validate=False, site_config_path=common_site_config_path)
 	else:
 		for site in context.sites:
 			frappe.init(site=site)
@@ -738,50 +778,6 @@ def rebuild_global_search(context, static_pages=False):
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
-@click.command('auto-deploy')
-@click.argument('app')
-@click.option('--migrate', is_flag=True, default=False, help='Migrate after pulling')
-@click.option('--restart', is_flag=True, default=False, help='Restart after migration')
-@click.option('--remote', default='upstream', help='Remote, default is "upstream"')
-@pass_context
-def auto_deploy(context, app, migrate=False, restart=False, remote='upstream'):
-	'''Pull and migrate sites that have new version'''
-	from frappe.utils.gitutils import get_app_branch
-	from frappe.utils import get_sites
-
-	branch = get_app_branch(app)
-	app_path = frappe.get_app_path(app)
-
-	# fetch
-	subprocess.check_output(['git', 'fetch', remote, branch], cwd = app_path)
-
-	# get diff
-	if subprocess.check_output(['git', 'diff', '{0}..{1}/{0}'.format(branch, remote)], cwd = app_path):
-		print('Updates found for {0}'.format(app))
-		if app=='frappe':
-			# run bench update
-			import shlex
-			subprocess.check_output(shlex.split('bench update --no-backup'), cwd = '..')
-		else:
-			updated = False
-			subprocess.check_output(['git', 'pull', '--rebase', remote, branch],
-				cwd = app_path)
-			# find all sites with that app
-			for site in get_sites():
-				frappe.init(site)
-				if app in frappe.get_installed_apps():
-					print('Updating {0}'.format(site))
-					updated = True
-					subprocess.check_output(['bench', '--site', site, 'clear-cache'], cwd = '..')
-					if migrate:
-						subprocess.check_output(['bench', '--site', site, 'migrate'], cwd = '..')
-				frappe.destroy()
-
-			if updated or restart:
-				subprocess.check_output(['bench', 'restart'], cwd = '..')
-	else:
-		print('No Updates')
-
 
 commands = [
 	build,

frappe/core/doctype/activity_log/test_activity_log.py

@@ -65,12 +65,12 @@ class TestActivityLog(unittest.TestCase):
 		frappe.local.login_manager = LoginManager()
 
 		auth_log = self.get_auth_log()
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test user logout log
 		frappe.local.login_manager.logout()
 		auth_log = self.get_auth_log(operation='Logout')
-		self.assertEquals(auth_log.status, 'Success')
+		self.assertEqual(auth_log.status, 'Success')
 
 		# test invalid login
 		frappe.form_dict.update({ 'pwd': 'password' })

frappe/core/doctype/communication/email.py

@@ -272,22 +272,13 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None)
 				doc.attachments.append(a)
 
 def set_incoming_outgoing_accounts(doc):
-	doc.incoming_email_account = doc.outgoing_email_account = None
+	from frappe.email.doctype.email_account.email_account import EmailAccount
+	incoming_email_account = EmailAccount.find_incoming(
+		match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
+	doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None
 
-	if not doc.incoming_email_account and doc.sender:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"email_id": doc.sender, "enable_incoming": 1}, "email_id")
-
-	if not doc.incoming_email_account and doc.reference_doctype:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"append_to": doc.reference_doctype, }, "email_id")
-
-	if not doc.incoming_email_account:
-		doc.incoming_email_account = frappe.db.get_value("Email Account",
-			{"default_incoming": 1, "enable_incoming": 1},  "email_id")
-
-	doc.outgoing_email_account = frappe.email.smtp.get_outgoing_email_account(raise_exception_not_set=False,
-		append_to=doc.doctype, sender=doc.sender)
+	doc.outgoing_email_account = EmailAccount.find_outgoing(
+		match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
 
 	if doc.sent_or_received == "Sent":
 		doc.db_set("email_account", doc.outgoing_email_account.name)

frappe/core/doctype/data_import/importer.py

@@ -233,7 +233,7 @@ class Importer:
 			return updated_doc
 		else:
 			# throw if no changes
-			frappe.throw("No changes to update")
+			frappe.throw(_("No changes to update"))
 
 	def get_eta(self, current, total, processing_time):
 		self.last_eta = getattr(self, "last_eta", 0)
@@ -319,7 +319,7 @@ class ImportFile:
 		self.warnings = []
 
 		self.file_doc = self.file_path = self.google_sheets_url = None
-		if isinstance(file, frappe.string_types):
+		if isinstance(file, str):
 			if frappe.db.exists("File", {"file_url": file}):
 				self.file_doc = frappe.get_doc("File", {"file_url": file})
 			elif "docs.google.com/spreadsheets" in file:
@@ -626,7 +626,7 @@ class Row:
 				return
 		elif df.fieldtype in ["Date", "Datetime"]:
 			value = self.get_date(value, col)
-			if isinstance(value, frappe.string_types):
+			if isinstance(value, str):
 				# value was not parsed as datetime object
 				self.warnings.append(
 					{

frappe/core/doctype/doctype/boilerplate/controller._py

@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) {year}, {app_publisher} and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 {base_class_import}
 

frappe/core/doctype/doctype/boilerplate/test_controller._py

@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) {year}, {app_publisher} and Contributors
 # See license.txt
-from __future__ import unicode_literals
 
 # import frappe
 import unittest

frappe/core/doctype/report/boilerplate/controller.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2013, {app_publisher} and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
 # import frappe
 
 def execute(filters=None):

frappe/core/doctype/report/test_report.py

@@ -106,7 +106,7 @@ class TestReport(unittest.TestCase):
 		else:
 			report = frappe.get_doc('Report', 'Test Report')
 
-		self.assertNotEquals(report.is_permitted(), True)
+		self.assertNotEqual(report.is_permitted(), True)
 		frappe.set_user('Administrator')
 
 	# test for the `_format` method if report data doesn't have sort_by parameter

frappe/core/doctype/user_permission/test_user_permission.py

@@ -46,7 +46,7 @@ class TestUserPermission(unittest.TestCase):
 		frappe.set_user('test_user_perm1@example.com')
 		doc = frappe.new_doc("Blog Post")
 
-		self.assertEquals(doc.blog_category, 'general')
+		self.assertEqual(doc.blog_category, 'general')
 		frappe.set_user('Administrator')
 
 	def test_apply_to_all(self):
@@ -54,7 +54,7 @@ class TestUserPermission(unittest.TestCase):
 		user = create_user('test_bulk_creation_update@example.com')
 		param = get_params(user, 'User', user.name)
 		is_created = add_user_permissions(param)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_for_apply_to_all_on_update_from_apply_all(self):
 		user = create_user('test_bulk_creation_update@example.com')
@@ -63,11 +63,11 @@ class TestUserPermission(unittest.TestCase):
 		# Initially create User Permission document with apply_to_all checked
 		is_created = add_user_permissions(param)
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 		is_created = add_user_permissions(param)
 
 		# User Permission should not be changed
-		self.assertEquals(is_created, 0)
+		self.assertEqual(is_created, 0)
 
 	def test_for_applicable_on_update_from_apply_to_all(self):
 		''' Update User Permission from all to some applicable Doctypes'''
@@ -77,7 +77,7 @@ class TestUserPermission(unittest.TestCase):
 		# Initially create User Permission document with apply_to_all checked
 		is_created = add_user_permissions(get_params(user, 'User', user.name))
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 		is_created = add_user_permissions(param)
 		frappe.db.commit()
@@ -92,7 +92,7 @@ class TestUserPermission(unittest.TestCase):
 		# Check that User Permissions for applicable is created
 		self.assertIsNotNone(is_created_applicable_first)
 		self.assertIsNotNone(is_created_applicable_second)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_for_apply_to_all_on_update_from_applicable(self):
 		''' Update User Permission from some to all applicable Doctypes'''
@@ -102,7 +102,7 @@ class TestUserPermission(unittest.TestCase):
 		# create User permissions that with applicable
 		is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"]))
 
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 		is_created = add_user_permissions(param)
 		is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
@@ -115,7 +115,7 @@ class TestUserPermission(unittest.TestCase):
 		# Check that all User Permission with applicable is removed
 		self.assertIsNone(removed_applicable_first)
 		self.assertIsNone(removed_applicable_second)
-		self.assertEquals(is_created, 1)
+		self.assertEqual(is_created, 1)
 
 	def test_user_perm_for_nested_doctype(self):
 		"""Test if descendants' visibility is controlled for a nested DocType."""
@@ -183,7 +183,7 @@ class TestUserPermission(unittest.TestCase):
 
 		# User perm is created on ToDo but for doctype Assignment Rule only
 		# it should not have impact on Doc A
-		self.assertEquals(new_doc.doc, "ToDo")
+		self.assertEqual(new_doc.doc, "ToDo")
 
 		frappe.set_user('Administrator')
 		remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo")
@@ -228,7 +228,7 @@ class TestUserPermission(unittest.TestCase):
 
 		# User perm is created on ToDo but for doctype Assignment Rule only
 		# it should not have impact on Doc A
-		self.assertEquals(new_doc.doc, "ToDo")
+		self.assertEqual(new_doc.doc, "ToDo")
 
 		frappe.set_user('Administrator')
 		clear_session_defaults()

frappe/core/doctype/user_permission/user_permission.py

@@ -191,7 +191,7 @@ def clear_user_permissions(user, for_doctype):
 def add_user_permissions(data):
 	''' Add and update the user permissions '''
 	frappe.only_for('System Manager')
-	if isinstance(data, frappe.string_types):
+	if isinstance(data, str):
 		data = json.loads(data)
 	data = frappe._dict(data)
 

frappe/custom/doctype/custom_field/custom_field.py

@@ -68,14 +68,15 @@ class CustomField(Document):
 			check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname)
 
 	def on_update(self):
-		frappe.clear_cache(doctype=self.dt)
+		if not frappe.flags.in_setup_wizard:
+			frappe.clear_cache(doctype=self.dt)
 		if not self.flags.ignore_validate:
 			# validate field
 			from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype
 			validate_fields_for_doctype(self.dt)
 
 		# update the schema
-		if not frappe.db.get_value('DocType', self.dt, 'issingle'):
+		if not frappe.db.get_value('DocType', self.dt, 'issingle') and not frappe.flags.in_setup_wizard:
 			frappe.db.updatedb(self.dt)
 
 	def on_trash(self):
@@ -144,6 +145,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
 	'''Add / update multiple custom fields
 
 	:param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`'''
+
+	if not ignore_validate and frappe.flags.in_setup_wizard:
+		ignore_validate = True
+
 	for doctype, fields in custom_fields.items():
 		if isinstance(fields, dict):
 			# only one field
@@ -163,6 +168,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
 				custom_field.update(df)
 				custom_field.save()
 
+		frappe.clear_cache(doctype=doctype)
+		frappe.db.updatedb(doctype)
+
+
 
 @frappe.whitelist()
 def add_custom_field(doctype, df):

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -47,64 +47,64 @@ class TestCustomizeForm(unittest.TestCase):
 		self.assertEqual(len(d.get("fields")), 0)
 
 		d = self.get_customize_form("Event")
-		self.assertEquals(d.doc_type, "Event")
-		self.assertEquals(len(d.get("fields")), 36)
+		self.assertEqual(d.doc_type, "Event")
+		self.assertEqual(len(d.get("fields")), 36)
 
 		d = self.get_customize_form("Event")
-		self.assertEquals(d.doc_type, "Event")
+		self.assertEqual(d.doc_type, "Event")
 
 		self.assertEqual(len(d.get("fields")),
 			len(frappe.get_doc("DocType", d.doc_type).fields) + 1)
-		self.assertEquals(d.get("fields")[-1].fieldname, "test_custom_field")
-		self.assertEquals(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
+		self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field")
+		self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
 
 		return d
 
 	def test_save_customization_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
 
 		d.allow_copy = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), '1')
 
 		d.allow_copy = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
 
 	def test_save_customization_field_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
 
 		repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
 		repeat_this_event_field.reqd = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1')
 
 		repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
 		repeat_this_event_field.reqd = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Property Setter",
+		self.assertEqual(frappe.db.get_value("Property Setter",
 			{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
 
 	def test_save_customization_custom_field_property(self):
 		d = self.get_customize_form("Event")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
 
 		custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0]
 		custom_field.reqd = 1
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
 
 		custom_field = d.get("fields", {"is_custom_field": True})[0]
 		custom_field.reqd = 0
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+		self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
 
 	def test_save_customization_new_field(self):
 		d = self.get_customize_form("Event")
@@ -115,14 +115,14 @@ class TestCustomizeForm(unittest.TestCase):
 			"is_custom_field": 1
 		})
 		d.run_method("save_customization")
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data")
 
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname)
 
 		frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form")
-		self.assertEquals(frappe.db.get_value("Custom Field",
+		self.assertEqual(frappe.db.get_value("Custom Field",
 			"Event-test_add_custom_field_via_customize_form"), None)
 
 
@@ -142,7 +142,7 @@ class TestCustomizeForm(unittest.TestCase):
 		d.doc_type = "Event"
 		d.run_method('reset_to_defaults')
 
-		self.assertEquals(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
+		self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
 
 		frappe.local.test_objects["Property Setter"] = []
 		make_test_records_for_doctype("Property Setter")
@@ -156,7 +156,7 @@ class TestCustomizeForm(unittest.TestCase):
 		d = self.get_customize_form("Event")
 
 		# don't allow for standard fields
-		self.assertEquals(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
+		self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
 
 		# allow for custom field
 		self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1)

frappe/database/database.py

@@ -858,7 +858,7 @@ class Database(object):
 		if not datetime:
 			return '0001-01-01 00:00:00.000000'
 
-		if isinstance(datetime, frappe.string_types):
+		if isinstance(datetime, str):
 			if ':' not in datetime:
 				datetime = datetime + ' 00:00:00.000000'
 		else:

frappe/database/mariadb/database.py

@@ -1,5 +1,3 @@
-import warnings
-
 import pymysql
 from pymysql.constants import ER, FIELD_TYPE
 from pymysql.converters import conversions, escape_string
@@ -55,7 +53,6 @@ class MariaDBDatabase(Database):
 		}
 
 	def get_connection(self):
-		warnings.filterwarnings('ignore', category=pymysql.Warning)
 		usessl = 0
 		if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key:
 			usessl = 1

frappe/database/postgres/database.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import re
 import frappe
 import psycopg2
@@ -13,9 +11,9 @@ from frappe.database.postgres.schema import PostgresTable
 
 # cast decimals as floats
 DEC2FLOAT = psycopg2.extensions.new_type(
-    psycopg2.extensions.DECIMAL.values,
-    'DEC2FLOAT',
-    lambda value, curs: float(value) if value is not None else None)
+	psycopg2.extensions.DECIMAL.values,
+	'DEC2FLOAT',
+	lambda value, curs: float(value) if value is not None else None)
 
 psycopg2.extensions.register_type(DEC2FLOAT)
 
@@ -65,7 +63,6 @@ class PostgresDatabase(Database):
 		}
 
 	def get_connection(self):
-		# warnings.filterwarnings('ignore', category=psycopg2.Warning)
 		conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format(
 			self.host, self.user, self.user, self.password, self.port
 		))
@@ -114,7 +111,7 @@ class PostgresDatabase(Database):
 		if not date:
 			return '0001-01-01'
 
-		if not isinstance(date, frappe.string_types):
+		if not isinstance(date, str):
 			date = date.strftime('%Y-%m-%d')
 
 		return date

frappe/desk/doctype/notification_log/notification_log.py

@@ -46,7 +46,7 @@ def enqueue_create_notification(users, doc):
 
 	doc = frappe._dict(doc)
 
-	if isinstance(users, frappe.string_types):
+	if isinstance(users, str):
 		users = [user.strip() for user in users.split(',') if user.strip()]
 	users = list(set(users))
 

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -124,6 +124,7 @@ def handle_setup_exception(args):
 	frappe.db.rollback()
 	if args:
 		traceback = frappe.get_traceback()
+		print(traceback)
 		for hook in frappe.get_hooks("setup_wizard_exception"):
 			frappe.get_attr(hook)(traceback, args)
 

frappe/desk/query_report.py

@@ -377,10 +377,17 @@ def handle_duration_fieldtype_values(result, columns):
 
 		if fieldtype == "Duration":
 			for entry in range(0, len(result)):
-				val_in_seconds = result[entry][i]
-				if val_in_seconds:
-					duration_val = format_duration(val_in_seconds)
-					result[entry][i] = duration_val
+				row = result[entry]
+				if isinstance(row, dict):
+					val_in_seconds = row[col.fieldname]
+					if val_in_seconds:
+						duration_val = format_duration(val_in_seconds)
+						row[col.fieldname] = duration_val
+				else:
+					val_in_seconds = row[i]
+					if val_in_seconds:
+						duration_val = format_duration(val_in_seconds)
+						row[i] = duration_val
 
 	return result
 

frappe/email/doctype/document_follow/test_document_follow.py

@@ -17,14 +17,14 @@ class TestDocumentFollow(unittest.TestCase):
 
 		document_follow.unfollow_document("Event", event_doc.name, user.name)
 		doc = document_follow.follow_document("Event", event_doc.name, user.name)
-		self.assertEquals(doc.user, user.name)
+		self.assertEqual(doc.user, user.name)
 
 		document_follow.send_hourly_updates()
 
 		email_queue_entry_name = frappe.get_all("Email Queue", limit=1)[0].name
 		email_queue_entry_doc = frappe.get_doc("Email Queue", email_queue_entry_name)
 
-		self.assertEquals((email_queue_entry_doc.recipients[0].recipient), user.name)
+		self.assertEqual((email_queue_entry_doc.recipients[0].recipient), user.name)
 
 		self.assertIn(event_doc.doctype, email_queue_entry_doc.message)
 		self.assertIn(event_doc.name, email_queue_entry_doc.message)

frappe/email/doctype/email_account/email_account.py

@@ -8,9 +8,14 @@ import re
 import json
 import socket
 import time
-from frappe import _
+import functools
+
+import email.utils
+
+from frappe import _, are_emails_muted
 from frappe.model.document import Document
-from frappe.utils import validate_email_address, cint, cstr, get_datetime, DATE_FORMAT, strip, comma_or, sanitize_html, add_days
+from frappe.utils import (validate_email_address, cint, cstr, get_datetime,
+	DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr)
 from frappe.utils.user import is_system_user
 from frappe.utils.jinja import render_template
 from frappe.email.smtp import SMTPServer
@@ -21,17 +26,37 @@ from datetime import datetime, timedelta
 from frappe.desk.form import assign_to
 from frappe.utils.user import get_system_managers
 from frappe.utils.background_jobs import enqueue, get_jobs
-from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
 from frappe.utils.html_utils import clean_email_html
+from frappe.utils.error import raise_error_on_no_output
 from frappe.email.utils import get_port
 
+OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account")
+
 class SentEmailInInbox(Exception):
 	pass
 
-class InvalidEmailCredentials(frappe.ValidationError):
-	pass
+def cache_email_account(cache_name):
+	def decorator_cache_email_account(func):
+		@functools.wraps(func)
+		def wrapper_cache_email_account(*args, **kwargs):
+			if not hasattr(frappe.local, cache_name):
+				setattr(frappe.local, cache_name, {})
+
+			cached_accounts = getattr(frappe.local, cache_name)
+			match_by = list(kwargs.values()) + ['default']
+			matched_accounts = list(filter(None, [cached_accounts.get(key) for key in match_by]))
+			if matched_accounts:
+				return matched_accounts[0]
+
+			matched_accounts = func(*args, **kwargs)
+			cached_accounts.update(matched_accounts or {})
+			return matched_accounts and list(matched_accounts.values())[0]
+		return wrapper_cache_email_account
+	return decorator_cache_email_account
 
 class EmailAccount(Document):
+	DOCTYPE = 'Email Account'
+
 	def autoname(self):
 		"""Set name as `email_account_name` or make title from Email Address."""
 		if not self.email_account_name:
@@ -72,9 +97,8 @@ class EmailAccount(Document):
 					self.get_incoming_server()
 					self.no_failed = 0
 
-
 				if self.enable_outgoing:
-					self.check_smtp()
+					self.validate_smtp_conn()
 			else:
 				if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication):
 					frappe.throw(_("Password is required or select Awaiting Password"))
@@ -90,6 +114,13 @@ class EmailAccount(Document):
 			if self.append_to not in valid_doctypes:
 				frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
 
+	def validate_smtp_conn(self):
+		if not self.smtp_server:
+			frappe.throw(_("SMTP Server is required"))
+
+		server = self.get_smtp_server()
+		return server.session
+
 	def before_save(self):
 		messages = []
 		as_list = 1
@@ -151,24 +182,6 @@ class EmailAccount(Document):
 		except Exception:
 			pass
 
-	def check_smtp(self):
-		"""Checks SMTP settings."""
-		if self.enable_outgoing:
-			if not self.smtp_server:
-				frappe.throw(_("{0} is required").format("SMTP Server"))
-
-			server = SMTPServer(
-				login = getattr(self, "login_id", None) or self.email_id,
-				server=self.smtp_server,
-				port=cint(self.smtp_port),
-				use_tls=cint(self.use_tls),
-				use_ssl=cint(self.use_ssl_for_outgoing)
-			)
-			if self.password and not self.no_smtp_authentication:
-				server.password = self.get_password()
-
-			server.sess
-
 	def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
 		"""Returns logged in POP3/IMAP connection object."""
 		if frappe.cache().get_value("workers:no-internet") == True:
@@ -231,7 +244,7 @@ class EmailAccount(Document):
 				return None
 
 			elif not in_receive and any(map(lambda t: t in message, auth_error_codes)):
-				self.throw_invalid_credentials_exception()
+				SMTPServer.throw_invalid_credentials_exception()
 			else:
 				frappe.throw(cstr(e))
 
@@ -249,13 +262,142 @@ class EmailAccount(Document):
 			else:
 				raise
 
+	@property
+	def _password(self):
+		raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test)
+		return self.get_password(raise_exception=raise_exception)
+
+	@property
+	def default_sender(self):
+		return email.utils.formataddr((self.name, self.get("email_id")))
+
+	def is_exists_in_db(self):
+		"""Some of the Email Accounts we create from configs and those doesn't exists in DB.
+		This is is to check the specific email account exists in DB or not.
+		"""
+		return self.find_one_by_filters(name=self.name)
+
+	@classmethod
+	def from_record(cls, record):
+		email_account = frappe.new_doc(cls.DOCTYPE)
+		email_account.update(record)
+		return email_account
+
 	@classmethod
-	def throw_invalid_credentials_exception(cls):
-		frappe.throw(
-			_("Incorrect email or password. Please check your login credentials."),
-			exc=InvalidEmailCredentials,
-			title=_("Invalid Credentials")
-		)
+	def find(cls, name):
+		return frappe.get_doc(cls.DOCTYPE, name)
+
+	@classmethod
+	def find_one_by_filters(cls, **kwargs):
+		name = frappe.db.get_value(cls.DOCTYPE, kwargs)
+		return cls.find(name) if name else None
+
+	@classmethod
+	def find_from_config(cls):
+		config = cls.get_account_details_from_site_config()
+		return cls.from_record(config) if config else None
+
+	@classmethod
+	def create_dummy(cls):
+		return cls.from_record({"sender": "notifications@example.com"})
+
+	@classmethod
+	@raise_error_on_no_output(
+		keep_quiet = lambda: not cint(frappe.get_system_settings('setup_complete')),
+		error_message = OUTGOING_EMAIL_ACCOUNT_MISSING, error_type = frappe.OutgoingEmailError) # noqa
+	@cache_email_account('outgoing_email_account')
+	def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False):
+		"""Find the outgoing Email account to use.
+
+		:param match_by_email: Find account using emailID
+		:param match_by_doctype: Find account by matching `Append To` doctype
+		:param _raise_error: This is used by raise_error_on_no_output decorator to raise error.
+		"""
+		if match_by_email:
+			match_by_email = parse_addr(match_by_email)[1]
+			doc = cls.find_one_by_filters(enable_outgoing=1, email_id=match_by_email)
+			if doc:
+				return {match_by_email: doc}
+
+		if match_by_doctype:
+			doc = cls.find_one_by_filters(enable_outgoing=1, enable_incoming=1, append_to=match_by_doctype)
+			if doc:
+				return {match_by_doctype: doc}
+
+		doc = cls.find_default_outgoing()
+		if doc:
+			return {'default': doc}
+
+	@classmethod
+	def find_default_outgoing(cls):
+		""" Find default outgoing account.
+		"""
+		doc = cls.find_one_by_filters(enable_outgoing=1, default_outgoing=1)
+		doc = doc or cls.find_from_config()
+		return doc or (are_emails_muted() and cls.create_dummy())
+
+	@classmethod
+	def find_incoming(cls, match_by_email=None, match_by_doctype=None):
+		"""Find the incoming Email account to use.
+		:param match_by_email: Find account using emailID
+		:param match_by_doctype: Find account by matching `Append To` doctype
+		"""
+		doc = cls.find_one_by_filters(enable_incoming=1, email_id=match_by_email)
+		if doc:
+			return doc
+
+		doc = cls.find_one_by_filters(enable_incoming=1, append_to=match_by_doctype)
+		if doc:
+			return doc
+
+		doc = cls.find_default_incoming()
+		return doc
+
+	@classmethod
+	def find_default_incoming(cls):
+		doc = cls.find_one_by_filters(enable_incoming=1, default_incoming=1)
+		return doc
+
+	@classmethod
+	def get_account_details_from_site_config(cls):
+		if not frappe.conf.get("mail_server"):
+			return {}
+
+		field_to_conf_name_map = {
+			'smtp_server': {'conf_names': ('mail_server',)},
+			'smtp_port': {'conf_names': ('mail_port',)},
+			'use_tls': {'conf_names': ('use_tls', 'mail_login')},
+			'login_id': {'conf_names': ('mail_login',)},
+			'email_id': {'conf_names': ('auto_email_id', 'mail_login'), 'default': 'notifications@example.com'},
+			'password': {'conf_names': ('mail_password',)},
+			'always_use_account_email_id_as_sender':
+				{'conf_names': ('always_use_account_email_id_as_sender',), 'default': 0},
+			'always_use_account_name_as_sender_name':
+				{'conf_names': ('always_use_account_name_as_sender_name',), 'default': 0},
+			'name': {'conf_names': ('email_sender_name',), 'default': 'Frappe'},
+			'from_site_config': {'default': True}
+		}
+
+		account_details = {}
+		for doc_field_name, d in field_to_conf_name_map.items():
+			conf_names, default = d.get('conf_names') or [], d.get('default')
+			value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)]
+			account_details[doc_field_name] = (value and value[0]) or default
+		return account_details
+
+	def sendmail_config(self):
+		return {
+			'server': self.smtp_server,
+			'port': cint(self.smtp_port),
+			'login': getattr(self, "login_id", None) or self.email_id,
+			'password': self._password,
+			'use_ssl': cint(self.use_ssl_for_outgoing),
+			'use_tls': cint(self.use_tls)
+		}
+
+	def get_smtp_server(self):
+		config = self.sendmail_config()
+		return SMTPServer(**config)
 
 	def handle_incoming_connect_error(self, description):
 		if test_internet():
@@ -642,6 +784,8 @@ class EmailAccount(Document):
 
 	def send_auto_reply(self, communication, email):
 		"""Send auto reply if set."""
+		from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
+
 		if self.enable_auto_reply:
 			set_incoming_outgoing_accounts(communication)
 
@@ -653,7 +797,7 @@ class EmailAccount(Document):
 			frappe.sendmail(recipients = [email.from_email],
 				sender = self.email_id,
 				reply_to = communication.incoming_email_account,
-				subject = _("Re: ") + communication.subject,
+				subject = " ".join([_("Re:"), communication.subject]),
 				content = render_template(self.auto_reply_message or "", communication.as_dict()) or \
 					 frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()),
 				reference_doctype = communication.reference_doctype,

frappe/email/doctype/email_domain/test_records.json

@@ -10,7 +10,8 @@
   "incoming_port": "993",
   "attachment_limit": "1",
   "smtp_server": "smtp.test.com",
-  "smtp_port": "587"
+  "smtp_port": "587",
+  "password": "password"
  },
  {
   "doctype": "Email Account",
@@ -25,6 +26,7 @@
   "incoming_port": "143",
   "attachment_limit": "1",
   "smtp_server": "smtp.test.com",
-  "smtp_port": "587"
+  "smtp_port": "587",
+  "password": "password"
  }
 ]

frappe/email/doctype/email_queue/email_queue.json

@@ -24,7 +24,8 @@
   "unsubscribe_method",
   "expose_recipients",
   "attachments",
-  "retry"
+  "retry",
+  "email_account"
  ],
  "fields": [
   {
@@ -139,13 +140,19 @@
    "fieldtype": "Int",
    "label": "Retry",
    "read_only": 1
+  },
+  {
+   "fieldname": "email_account",
+   "fieldtype": "Link",
+   "label": "Email Account",
+   "options": "Email Account"
   }
  ],
  "icon": "fa fa-envelope",
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2020-07-17 15:58:15.369419",
+ "modified": "2021-04-29 06:33:25.191729",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Queue",

frappe/email/doctype/email_queue/email_queue.py

@@ -2,15 +2,26 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # For license information, please see license.txt
 
-from __future__ import unicode_literals
+import traceback
+import json
+
+from rq.timeouts import JobTimeoutException
+import smtplib
+import quopri
+from email.parser import Parser
+
 import frappe
-from frappe import _
+from frappe import _, safe_encode, task
 from frappe.model.document import Document
-from frappe.email.queue import send_one
-from frappe.utils import now_datetime
-
+from frappe.email.queue import get_unsubcribed_url
+from frappe.email.email_body import add_attachment
+from frappe.utils import cint
+from email.policy import SMTPUTF8
 
+MAX_RETRY_COUNT = 3
 class EmailQueue(Document):
+	DOCTYPE = 'Email Queue'
+
 	def set_recipients(self, recipients):
 		self.set("recipients", [])
 		for r in recipients:
@@ -30,6 +41,241 @@ class EmailQueue(Document):
 		duplicate.set_recipients(recipients)
 		return duplicate
 
+	@classmethod
+	def find(cls, name):
+		return frappe.get_doc(cls.DOCTYPE, name)
+
+	def update_db(self, commit=False, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+		if commit:
+			frappe.db.commit()
+
+	def update_status(self, status, commit=False, **kwargs):
+		self.update_db(status = status, commit = commit, **kwargs)
+		if self.communication:
+			communication_doc = frappe.get_doc('Communication', self.communication)
+			communication_doc.set_delivery_status(commit=commit)
+
+	@property
+	def cc(self):
+		return (self.show_as_cc and self.show_as_cc.split(",")) or []
+
+	@property
+	def to(self):
+		return [r.recipient for r in self.recipients if r.recipient not in self.cc]
+
+	@property
+	def attachments_list(self):
+		return json.loads(self.attachments) if self.attachments else []
+
+	def get_email_account(self):
+		from frappe.email.doctype.email_account.email_account import EmailAccount
+
+		if self.email_account:
+			return frappe.get_doc('Email Account', self.email_account)
+
+		return EmailAccount.find_outgoing(
+			match_by_email = self.sender, match_by_doctype = self.reference_doctype)
+
+	def is_to_be_sent(self):
+		return self.status in ['Not Sent','Partially Sent']
+
+	def can_send_now(self):
+		hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1)
+		if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue:
+			return False
+
+		return True
+
+	def send(self, is_background_task=False):
+		""" Send emails to recipients.
+		"""
+		if not self.can_send_now():
+			frappe.db.rollback()
+			return
+
+		with SendMailContext(self, is_background_task) as ctx:
+			message = None
+			for recipient in self.recipients:
+				if not recipient.is_mail_to_be_sent():
+					continue
+
+				message = ctx.build_message(recipient.recipient)
+				if not frappe.flags.in_test:
+					ctx.smtp_session.sendmail(recipient.recipient, self.sender, message)
+				ctx.add_to_sent_list(recipient)
+
+			if frappe.flags.in_test:
+				frappe.flags.sent_mail = message
+				return
+
+			if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to:
+				ctx.email_account_doc.append_email_to_sent_folder(message)
+
+
+@task(queue = 'short')
+def send_mail(email_queue_name, is_background_task=False):
+	"""This is equalent to EmqilQueue.send.
+
+	This provides a way to make sending mail as a background job.
+	"""
+	record = EmailQueue.find(email_queue_name)
+	record.send(is_background_task=is_background_task)
+
+class SendMailContext:
+	def __init__(self, queue_doc: Document, is_background_task: bool = False):
+		self.queue_doc = queue_doc
+		self.is_background_task = is_background_task
+		self.email_account_doc = queue_doc.get_email_account()
+		self.smtp_server = self.email_account_doc.get_smtp_server()
+		self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()]
+
+	def __enter__(self):
+		self.queue_doc.update_status(status='Sending', commit=True)
+		return self
+
+	def __exit__(self, exc_type, exc_val, exc_tb):
+		exceptions = [
+			smtplib.SMTPServerDisconnected,
+			smtplib.SMTPAuthenticationError,
+			smtplib.SMTPRecipientsRefused,
+			smtplib.SMTPConnectError,
+			smtplib.SMTPHeloError,
+			JobTimeoutException
+		]
+
+		self.smtp_server.quit()
+		self.log_exception(exc_type, exc_val, exc_tb)
+
+		if exc_type in exceptions:
+			email_status = (self.sent_to and 'Partially Sent') or 'Not Sent'
+			self.queue_doc.update_status(status = email_status, commit = True)
+		elif exc_type:
+			if self.queue_doc.retry < MAX_RETRY_COUNT:
+				update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1}
+			else:
+				update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'}
+			self.queue_doc.update_status(**update_fields, commit = True)
+		else:
+			email_status = self.is_mail_sent_to_all() and 'Sent'
+			email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent'
+			self.queue_doc.update_status(status = email_status, commit = True)
+
+	def log_exception(self, exc_type, exc_val, exc_tb):
+		if exc_type:
+			traceback_string = "".join(traceback.format_tb(exc_tb))
+			traceback_string += f"\n Queue Name: {self.queue_doc.name}"
+
+			if self.is_background_task:
+				frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string)
+			else:
+				frappe.log_error(message = traceback_string)
+
+	@property
+	def smtp_session(self):
+		if frappe.flags.in_test:
+			return
+		return self.smtp_server.session
+
+	def add_to_sent_list(self, recipient):
+		# Update recipient status
+		recipient.update_db(status='Sent', commit=True)
+		self.sent_to.append(recipient.recipient)
+
+	def is_mail_sent_to_all(self):
+		return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients])
+
+	def get_message_object(self, message):
+		return Parser(policy=SMTPUTF8).parsestr(message)
+
+	def message_placeholder(self, placeholder_key):
+		map = {
+			'tracker': '<!--email open check-->',
+			'unsubscribe_url': '<!--unsubscribe url-->',
+			'cc': '<!--cc message-->',
+			'recipient': '<!--recipient-->',
+		}
+		return map.get(placeholder_key)
+
+	def build_message(self, recipient_email):
+		"""Build message specific to the recipient.
+		"""
+		message = self.queue_doc.message
+		if not message:
+			return ""
+
+		message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str())
+		message = message.replace(self.message_placeholder('unsubscribe_url'),
+			self.get_unsubscribe_str(recipient_email))
+		message = message.replace(self.message_placeholder('cc'), self.get_receivers_str())
+		message = message.replace(self.message_placeholder('recipient'),
+			self.get_receipient_str(recipient_email))
+		message = self.include_attachments(message)
+		return message
+
+	def get_tracker_str(self):
+		tracker_url_html = \
+			'<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'
+
+		message = ''
+		if frappe.conf.use_ssl and self.queue_doc.track_email_status:
+			message = quopri.encodestring(
+				tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
+			).decode()
+		return message
+
+	def get_unsubscribe_str(self, recipient_email):
+		unsubscribe_url = ''
+		if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype:
+			doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name
+			unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email,
+				self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param)
+
+		return quopri.encodestring(unsubscribe_url.encode()).decode()
+
+	def get_receivers_str(self):
+		message = ''
+		if self.queue_doc.expose_recipients == "footer":
+			to_str = ', '.join(self.queue_doc.to)
+			cc_str = ', '.join(self.queue_doc.cc)
+			message = f"This email was sent to {to_str}"
+			message = message + f" and copied to {cc_str}" if cc_str else message
+		return message
+
+	def get_receipient_str(self, recipient_email):
+		message = ''
+		if self.queue_doc.expose_recipients != "header":
+			message = recipient_email
+		return message
+
+	def include_attachments(self, message):
+		message_obj = self.get_message_object(message)
+		attachments = self.queue_doc.attachments_list
+
+		for attachment in attachments:
+			if attachment.get('fcontent'):
+				continue
+
+			fid = attachment.get("fid")
+			if fid:
+				_file = frappe.get_doc("File", fid)
+				fcontent = _file.get_content()
+				attachment.update({
+					'fname': _file.file_name,
+					'fcontent': fcontent,
+					'parent': message_obj
+				})
+				attachment.pop("fid", None)
+				add_attachment(**attachment)
+
+			elif attachment.get("print_format_attachment") == 1:
+				attachment.pop("print_format_attachment", None)
+				print_format_file = frappe.attach_print(**attachment)
+				print_format_file.update({"parent": message_obj})
+				add_attachment(**print_format_file)
+
+		return safe_encode(message_obj.as_string())
+
 @frappe.whitelist()
 def retry_sending(name):
 	doc = frappe.get_doc("Email Queue", name)
@@ -42,7 +288,9 @@ def retry_sending(name):
 
 @frappe.whitelist()
 def send_now(name):
-	send_one(name, now=True)
+	record = EmailQueue.find(name)
+	if record:
+		record.send()
 
 def on_doctype_update():
 	"""Add index in `tabCommunication` for `(reference_doctype, reference_name)`"""

frappe/email/doctype/email_queue_recipient/email_queue_recipient.py

@@ -7,4 +7,16 @@ import frappe
 from frappe.model.document import Document
 
 class EmailQueueRecipient(Document):
-	pass
+	DOCTYPE = 'Email Queue Recipient'
+
+	def is_mail_to_be_sent(self):
+		return self.status == 'Not Sent'
+
+	def is_main_sent(self):
+		return self.status == 'Sent'
+
+	def update_db(self, commit=False, **kwargs):
+		frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+		if commit:
+			frappe.db.commit()
+

frappe/email/doctype/notification/notification.json

@@ -102,7 +102,8 @@
    "default": "0",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "label": "Is Standard"
+   "label": "Is Standard",
+   "no_copy": 1
   },
   {
    "depends_on": "is_standard",
@@ -281,7 +282,7 @@
  "icon": "fa fa-envelope",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-11-24 14:25:43.245677",
+ "modified": "2021-05-04 11:17:11.882314",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification",

frappe/email/email_body.py

@@ -4,7 +4,7 @@
 from __future__ import unicode_literals
 import frappe, re, os
 from frappe.utils.pdf import get_pdf
-from frappe.email.smtp import get_outgoing_email_account
+from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint,
 	split_emails, to_markdown, markdown, random_string, parse_addr)
 import email.utils
@@ -75,7 +75,8 @@ class EMail:
 		self.bcc = bcc or []
 		self.html_set = False
 
-		self.email_account = email_account or get_outgoing_email_account(sender=sender)
+		self.email_account = email_account or \
+			EmailAccount.find_outgoing(match_by_email=sender, _raise_error=True)
 
 	def set_html(self, message, text_content = None, footer=None, print_html=None,
 		formatted=None, inline_images=None, header=None):
@@ -249,8 +250,8 @@ class EMail:
 
 def get_formatted_html(subject, message, footer=None, print_html=None,
 		email_account=None, header=None, unsubscribe_link=None, sender=None, with_container=False):
-	if not email_account:
-		email_account = get_outgoing_email_account(False, sender=sender)
+
+	email_account = email_account or EmailAccount.find_outgoing(match_by_email=sender)
 
 	signature = None
 	if "<!-- signature-included -->" not in message:

frappe/email/queue.py

@@ -7,7 +7,8 @@ import sys
 from six.moves import html_parser as HTMLParser
 import smtplib, quopri, json
 from frappe import msgprint, _, safe_decode, safe_encode, enqueue
-from frappe.email.smtp import SMTPServer, get_outgoing_email_account
+from frappe.email.smtp import SMTPServer
+from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.email.email_body import get_email, get_formatted_html, add_attachment
 from frappe.utils.verified_command import get_signed_params, verify_request
 from html2text import html2text
@@ -73,7 +74,9 @@ def send(recipients=None, sender=None, subject=None, message=None, text_content=
 	if isinstance(send_after, int):
 		send_after = add_days(nowdate(), send_after)
 
-	email_account = get_outgoing_email_account(True, append_to=reference_doctype, sender=sender)
+	email_account = EmailAccount.find_outgoing(
+		match_by_doctype=reference_doctype, match_by_email=sender, _raise_error=True)
+
 	if not sender or sender == "Administrator":
 		sender = email_account.default_sender
 
@@ -170,19 +173,19 @@ def add(recipients, sender, subject, **kwargs):
 			if not email_queue:
 				email_queue = get_email_queue([r], sender, subject, **kwargs)
 				if kwargs.get('now'):
-					send_one(email_queue.name, now=True)
+					email_queue.send()
 			else:
 				duplicate = email_queue.get_duplicate([r])
 				duplicate.insert(ignore_permissions=True)
 
 				if kwargs.get('now'):
-					send_one(duplicate.name, now=True)
+					duplicate.send()
 
 			frappe.db.commit()
 	else:
 		email_queue = get_email_queue(recipients, sender, subject, **kwargs)
 		if kwargs.get('now'):
-			send_one(email_queue.name, now=True)
+			email_queue.send()
 
 def get_email_queue(recipients, sender, subject, **kwargs):
 	'''Make Email Queue object'''
@@ -234,6 +237,9 @@ def get_email_queue(recipients, sender, subject, **kwargs):
 			', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent')
 
 	recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', [])))
+	email_account = kwargs.get('email_account')
+	email_account_name = email_account and email_account.is_exists_in_db() and email_account.name
+
 	e.set_recipients(recipients)
 	e.reference_doctype = kwargs.get('reference_doctype')
 	e.reference_name = kwargs.get('reference_name')
@@ -245,8 +251,8 @@ def get_email_queue(recipients, sender, subject, **kwargs):
 	e.send_after = kwargs.get('send_after')
 	e.show_as_cc = ",".join(kwargs.get('cc', []))
 	e.show_as_bcc = ",".join(kwargs.get('bcc', []))
+	e.email_account = email_account_name or None
 	e.insert(ignore_permissions=True)
-
 	return e
 
 def get_emails_sent_this_month():
@@ -328,44 +334,25 @@ def return_unsubscribed_page(email, doctype, name):
 		indicator_color='green')
 
 def flush(from_test=False):
-	"""flush email queue, every time: called from scheduler"""
-	# additional check
-
-	auto_commit = not from_test
+	"""flush email queue, every time: called from scheduler
+	"""
+	from frappe.email.doctype.email_queue.email_queue import send_mail
+	# To avoid running jobs inside unit tests
 	if frappe.are_emails_muted():
 		msgprint(_("Emails are muted"))
 		from_test = True
 
-	smtpserver_dict = frappe._dict()
-
-	for email in get_queue():
-
-		if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
-			break
-
-		if email.name:
-			smtpserver = smtpserver_dict.get(email.sender)
-			if not smtpserver:
-				smtpserver = SMTPServer()
-				smtpserver_dict[email.sender] = smtpserver
+	if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
+		return
 
-			if from_test:
-				send_one(email.name, smtpserver, auto_commit)
-			else:
-				send_one_args = {
-					'email': email.name,
-					'smtpserver': smtpserver,
-					'auto_commit': auto_commit,
-				}
-				enqueue(
-					method = 'frappe.email.queue.send_one',
-					queue = 'short',
-					**send_one_args
-				)
+	for row in get_queue():
+		try:
+			func = send_mail if from_test else send_mail.enqueue
+			is_background_task = not from_test
+			func(email_queue_name = row.name, is_background_task = is_background_task)
+		except Exception:
+			frappe.log_error()
 
-		# NOTE: removing commit here because we pass auto_commit
-		# finally:
-		# 	frappe.db.commit()
 def get_queue():
 	return frappe.db.sql('''select
 			name, sender
@@ -378,213 +365,6 @@ def get_queue():
 			by priority desc, creation asc
 		limit 500''', { 'now': now_datetime() }, as_dict=True)
 
-
-def send_one(email, smtpserver=None, auto_commit=True, now=False):
-	'''Send Email Queue with given smtpserver'''
-
-	email = frappe.db.sql('''select
-			name, status, communication, message, sender, reference_doctype,
-			reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
-			show_as_cc, add_unsubscribe_link, attachments, retry
-		from
-			`tabEmail Queue`
-		where
-			name=%s
-		for update''', email, as_dict=True)
-
-	if len(email):
-		email = email[0]
-	else:
-		return
-
-	recipients_list = frappe.db.sql('''select name, recipient, status from
-		`tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1)
-
-	if frappe.are_emails_muted():
-		frappe.msgprint(_("Emails are muted"))
-		return
-
-	if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 :
-		return
-
-	if email.status not in ('Not Sent','Partially Sent') :
-		# rollback to release lock and return
-		frappe.db.rollback()
-		return
-
-	frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
-		(now_datetime(), email.name), auto_commit=auto_commit)
-
-	if email.communication:
-		frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-	email_sent_to_any_recipient = None
-
-	try:
-		message = None
-
-		if not frappe.flags.in_test:
-			if not smtpserver:
-				smtpserver = SMTPServer()
-
-			# to avoid always using default email account for outgoing
-			if getattr(frappe.local, "outgoing_email_account", None):
-				frappe.local.outgoing_email_account = {}
-
-			smtpserver.setup_email_account(email.reference_doctype, sender=email.sender)
-
-		for recipient in recipients_list:
-			if recipient.status != "Not Sent":
-				continue
-
-			message = prepare_message(email, recipient.recipient, recipients_list)
-			if not frappe.flags.in_test:
-				smtpserver.sess.sendmail(email.sender, recipient.recipient, message)
-
-			recipient.status = "Sent"
-			frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
-				(now_datetime(), recipient.name), auto_commit=auto_commit)
-
-		email_sent_to_any_recipient = any("Sent" == s.status for s in recipients_list)
-
-		#if all are sent set status
-		if email_sent_to_any_recipient:
-			frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-				where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
-		if frappe.flags.in_test:
-			frappe.flags.sent_mail = message
-			return
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient:
-			smtpserver.email_account.append_email_to_sent_folder(message)
-
-	except (smtplib.SMTPServerDisconnected,
-			smtplib.SMTPConnectError,
-			smtplib.SMTPHeloError,
-			smtplib.SMTPAuthenticationError,
-			smtplib.SMTPRecipientsRefused,
-			JobTimeoutException):
-
-		# bad connection/timeout, retry later
-
-		if email_sent_to_any_recipient:
-			frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		# no need to attempt further
-		return
-
-	except Exception as e:
-		frappe.db.rollback()
-
-		if email.retry < 3:
-			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""",
-				(now_datetime(), email.name), auto_commit=auto_commit)
-		else:
-			if email_sent_to_any_recipient:
-				frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
-					(text_type(e), email.name), auto_commit=auto_commit)
-			else:
-				frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-					where name=%s""", (text_type(e), email.name), auto_commit=auto_commit)
-
-		if email.communication:
-			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
-		if now:
-			print(frappe.get_traceback())
-			raise e
-
-		else:
-			# log to Error Log
-			frappe.log_error('frappe.email.queue.flush')
-
-def prepare_message(email, recipient, recipients_list):
-	message = email.message
-	if not message:
-		return ""
-
-	# Parse "Email Account" from "Email Sender"
-	email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender)
-	if frappe.conf.use_ssl and email_account.track_email_status:
-		# Using SSL => Publically available domain => Email Read Reciept Possible
-		message = message.replace("<!--email open check-->", quopri.encodestring('<img src="https://{}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={}"/>'.format(frappe.local.site, email.communication).encode()).decode())
-	else:
-		# No SSL => No Email Read Reciept
-		message = message.replace("<!--email open check-->", quopri.encodestring("".encode()).decode())
-
-	if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
-		unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
-		email.unsubscribe_method, email.unsubscribe_params)
-		message = message.replace("<!--unsubscribe url-->", quopri.encodestring(unsubscribe_url.encode()).decode())
-
-	if email.expose_recipients == "header":
-		pass
-	else:
-		if email.expose_recipients == "footer":
-			if isinstance(email.show_as_cc, string_types):
-				email.show_as_cc = email.show_as_cc.split(",")
-			email_sent_to = [r.recipient for r in recipients_list]
-			email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
-			email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
-
-			if email_sent_cc:
-				email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
-			else:
-				email_sent_message = _("This email was sent to {0}").format(email_sent_to)
-			message = message.replace("<!--cc message-->", quopri.encodestring(email_sent_message.encode()).decode())
-
-		message = message.replace("<!--recipient-->", recipient)
-
-	message = (message and message.encode('utf8')) or ''
-	message = safe_decode(message)
-
-	if PY3:
-		from email.policy import SMTPUTF8
-		message = Parser(policy=SMTPUTF8).parsestr(message)
-	else:
-		message = Parser().parsestr(message)
-
-	if email.attachments:
-		# On-demand attachments
-
-		attachments = json.loads(email.attachments)
-
-		for attachment in attachments:
-			if attachment.get('fcontent'):
-				continue
-
-			fid = attachment.get("fid")
-			if fid:
-				_file = frappe.get_doc("File", fid)
-				fcontent = _file.get_content()
-				attachment.update({
-					'fname': _file.file_name,
-					'fcontent': fcontent,
-					'parent': message
-				})
-				attachment.pop("fid", None)
-				add_attachment(**attachment)
-
-			elif attachment.get("print_format_attachment") == 1:
-				attachment.pop("print_format_attachment", None)
-				print_format_file = frappe.attach_print(**attachment)
-				print_format_file.update({"parent": message})
-				add_attachment(**print_format_file)
-
-	return safe_encode(message.as_string())
-
 def clear_outbox(days=None):
 	"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
 	Note: Used separate query to avoid deadlock

frappe/email/smtp.py

@@ -9,11 +9,24 @@ import _socket, sys
 from frappe import _
 from frappe.utils import cint, cstr, parse_addr
 
+CONNECTION_FAILED = _('Could not connect to outgoing email server')
+AUTH_ERROR_TITLE = _("Invalid Credentials")
+AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.")
+SOCKET_ERROR_TITLE = _("Incorrect Configuration")
+SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port")
+SEND_MAIL_FAILED = _("Unable to send emails at this time")
+EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
+
+class InvalidEmailCredentials(frappe.ValidationError):
+	pass
+
 def send(email, append_to=None, retry=1):
 	"""Deprecated: Send the message or add it to Outbox Email"""
 	def _send(retry):
+		from frappe.email.doctype.email_account.email_account import EmailAccount
 		try:
-			smtpserver = SMTPServer(append_to=append_to)
+			email_account = EmailAccount.find_outgoing(match_by_doctype=append_to)
+			smtpserver = email_account.get_smtp_server()
 
 			# validate is called in as_string
 			email_body = email.as_string()
@@ -34,224 +47,80 @@ def send(email, append_to=None, retry=1):
 
 	_send(retry)
 
-def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None):
-	"""Returns outgoing email account based on `append_to` or the default
-		outgoing account. If default outgoing account is not found, it will
-		try getting settings from `site_config.json`."""
-
-	sender_email_id = None
-	_email_account = None
-
-	if sender:
-		sender_email_id = parse_addr(sender)[1]
-
-	if not getattr(frappe.local, "outgoing_email_account", None):
-		frappe.local.outgoing_email_account = {}
-
-	if not (frappe.local.outgoing_email_account.get(append_to)
-		or frappe.local.outgoing_email_account.get(sender_email_id)
-		or frappe.local.outgoing_email_account.get("default")):
-		email_account = None
-
-		if sender_email_id:
-			# check if the sender has an email account with enable_outgoing
-			email_account = _get_email_account({"enable_outgoing": 1,
-					"email_id": sender_email_id})
-
-		if not email_account and append_to:
-			# append_to is only valid when enable_incoming is checked
-			email_accounts = frappe.db.get_values("Email Account", {
-				"enable_outgoing": 1,
-				"enable_incoming": 1,
-				"append_to": append_to,
-			}, cache=True)
-
-			if email_accounts:
-				_email_account = email_accounts[0]
-
-			else:
-				email_account = _get_email_account({
-					"enable_outgoing": 1,
-					"enable_incoming": 1,
-					"append_to": append_to
-				})
-
-		if not email_account:
-			# sender don't have the outging email account
-			sender_email_id = None
-			email_account = get_default_outgoing_email_account(raise_exception_not_set=raise_exception_not_set)
-
-		if not email_account and _email_account:
-			# if default email account is not configured then setup first email account based on append to
-			email_account = _email_account
-
-		if not email_account and raise_exception_not_set and cint(frappe.db.get_single_value('System Settings', 'setup_complete')):
-			frappe.throw(_("Please setup default Email Account from Setup > Email > Email Account"),
-				frappe.OutgoingEmailError)
-
-		if email_account:
-			if email_account.enable_outgoing and not getattr(email_account, 'from_site_config', False):
-				raise_exception = True
-				if email_account.smtp_server in ['localhost','127.0.0.1'] or email_account.no_smtp_authentication:
-					raise_exception = False
-				email_account.password = email_account.get_password(raise_exception=raise_exception)
-			email_account.default_sender = email.utils.formataddr((email_account.name, email_account.get("email_id")))
-
-		frappe.local.outgoing_email_account[append_to or sender_email_id or "default"] = email_account
-
-	return frappe.local.outgoing_email_account.get(append_to) \
-		or frappe.local.outgoing_email_account.get(sender_email_id) \
-		or frappe.local.outgoing_email_account.get("default")
-
-def get_default_outgoing_email_account(raise_exception_not_set=True):
-	'''conf should be like:
-		{
-		 "mail_server": "smtp.example.com",
-		 "mail_port": 587,
-		 "use_tls": 1,
-		 "mail_login": "emails@example.com",
-		 "mail_password": "Super.Secret.Password",
-		 "auto_email_id": "emails@example.com",
-		 "email_sender_name": "Example Notifications",
-		 "always_use_account_email_id_as_sender": 0,
-		 "always_use_account_name_as_sender_name": 0
-		}
-	'''
-	email_account = _get_email_account({"enable_outgoing": 1, "default_outgoing": 1})
-	if email_account:
-		email_account.password = email_account.get_password(raise_exception=False)
-
-	if not email_account and frappe.conf.get("mail_server"):
-		# from site_config.json
-		email_account = frappe.new_doc("Email Account")
-		email_account.update({
-			"smtp_server": frappe.conf.get("mail_server"),
-			"smtp_port": frappe.conf.get("mail_port"),
-
-			# legacy: use_ssl was used in site_config instead of use_tls, but meant the same thing
-			"use_tls": cint(frappe.conf.get("use_tls") or 0) or cint(frappe.conf.get("use_ssl") or 0),
-			"login_id": frappe.conf.get("mail_login"),
-			"email_id": frappe.conf.get("auto_email_id") or frappe.conf.get("mail_login") or 'notifications@example.com',
-			"password": frappe.conf.get("mail_password"),
-			"always_use_account_email_id_as_sender": frappe.conf.get("always_use_account_email_id_as_sender", 0),
-			"always_use_account_name_as_sender_name": frappe.conf.get("always_use_account_name_as_sender_name", 0)
-		})
-		email_account.from_site_config = True
-		email_account.name = frappe.conf.get("email_sender_name") or "Frappe"
-
-	if not email_account and not raise_exception_not_set:
-		return None
-
-	if frappe.are_emails_muted():
-		# create a stub
-		email_account = frappe.new_doc("Email Account")
-		email_account.update({
-			"email_id": "notifications@example.com"
-		})
-
-	return email_account
-
-def _get_email_account(filters):
-	name = frappe.db.get_value("Email Account", filters)
-	return frappe.get_doc("Email Account", name) if name else None
-
 class SMTPServer:
-	def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None):
-		# get defaults from mail settings
-
-		self._sess = None
-		self.email_account = None
-		self.server = None
-		self.append_emails_to_sent_folder = None
+	def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None):
+		self.login = login
+		self.password = password
+		self._server = server
+		self._port = port
+		self.use_tls = use_tls
+		self.use_ssl = use_ssl
+		self._session = None
+
+		if not self.server:
+			frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError)
 
-		if server:
-			self.server = server
-			self.port = port
-			self.use_tls = cint(use_tls)
-			self.use_ssl = cint(use_ssl)
-			self.login = login
-			self.password = password
+	@property
+	def port(self):
+		port = self._port or (self.use_ssl and 465) or (self.use_tls and 587)
+		return cint(port)
 
-		else:
-			self.setup_email_account(append_to)
+	@property
+	def server(self):
+		return cstr(self._server or "")
 
-	def setup_email_account(self, append_to=None, sender=None):
-		self.email_account = get_outgoing_email_account(raise_exception_not_set=False, append_to=append_to, sender=sender)
-		if self.email_account:
-			self.server = self.email_account.smtp_server
-			self.login = (getattr(self.email_account, "login_id", None) or self.email_account.email_id)
-			if not self.email_account.no_smtp_authentication:
-				if self.email_account.ascii_encode_password:
-					self.password = frappe.safe_encode(self.email_account.password, 'ascii')
-				else:
-					self.password = self.email_account.password
-			else:
-				self.password = None
-			self.port = self.email_account.smtp_port
-			self.use_tls = self.email_account.use_tls
-			self.sender = self.email_account.email_id
-			self.use_ssl = self.email_account.use_ssl_for_outgoing
-			self.append_emails_to_sent_folder = self.email_account.append_emails_to_sent_folder
-			self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender"))
-			self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name"))
+	def secure_session(self, conn):
+		"""Secure the connection incase of TLS.
+		"""
+		if self.use_tls:
+			conn.ehlo()
+			conn.starttls()
+			conn.ehlo()
 
 	@property
-	def sess(self):
-		"""get session"""
-		if self._sess:
-			return self._sess
+	def session(self):
+		if self.is_session_active():
+			return self._session
 
-		# check if email server specified
-		if not getattr(self, 'server'):
-			err_msg = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
-			frappe.msgprint(err_msg)
-			raise frappe.OutgoingEmailError(err_msg)
+		SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP
 
 		try:
-			if self.use_ssl:
-				if not self.port:
-					self.port = 465
-
-				self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port))
-			else:
-				if self.use_tls and not self.port:
-					self.port = 587
-
-				self._sess = smtplib.SMTP(cstr(self.server or ""),
-						cint(self.port) or None)
-
-			if not self._sess:
-				err_msg = _('Could not connect to outgoing email server')
-				frappe.msgprint(err_msg)
-				raise frappe.OutgoingEmailError(err_msg)
-
-			if self.use_tls:
-				self._sess.ehlo()
-				self._sess.starttls()
-				self._sess.ehlo()
+			self._session = SMTP(self.server, self.port)
+			if not self._session:
+				frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError)
 
+			self.secure_session(self._session)
 			if self.login and self.password:
-				ret = self._sess.login(str(self.login or ""), str(self.password or ""))
+				res = self._session.login(str(self.login or ""), str(self.password or ""))
 
 				# check if logged correctly
-				if ret[0]!=235:
-					frappe.msgprint(ret[1])
-					raise frappe.OutgoingEmailError(ret[1])
+				if res[0]!=235:
+					frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError)
 
-			return self._sess
+			return self._session
 
 		except smtplib.SMTPAuthenticationError as e:
-			from frappe.email.doctype.email_account.email_account import EmailAccount
-			EmailAccount.throw_invalid_credentials_exception()
+			self.throw_invalid_credentials_exception()
 
 		except _socket.error as e:
 			# Invalid mail server -- due to refusing connection
-			frappe.throw(
-				_("Invalid Outgoing Mail Server or Port"),
-				exc=frappe.ValidationError,
-				title=_("Incorrect Configuration")
-			)
+			frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE)
 
 		except smtplib.SMTPException:
-			frappe.msgprint(_('Unable to send emails at this time'))
+			frappe.msgprint(SEND_MAIL_FAILED)
 			raise
+
+	def is_session_active(self):
+		if self._session:
+			try:
+				return self._session.noop()[0] == 250
+			except Exception:
+				return False
+
+	def quit(self):
+		if self.is_session_active():
+			self._session.quit()
+
+	@classmethod
+	def throw_invalid_credentials_exception(cls):
+		frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials)

frappe/email/test_email_body.py

@@ -7,10 +7,10 @@ from frappe import safe_decode
 from frappe.email.receive import Email
 from frappe.email.email_body import (replace_filename_with_cid,
 					get_email, inline_style_in_html, get_header)
-from frappe.email.queue import prepare_message, get_email_queue
+from frappe.email.queue import get_email_queue
+from frappe.email.doctype.email_queue.email_queue import SendMailContext
 from six import PY3
 
-
 class TestEmailBody(unittest.TestCase):
 	def setUp(self):
 		email_html = '''
@@ -57,7 +57,8 @@ This is the text version of this email
 			content='<h1>' + uni_chr1 + 'abcd' + uni_chr2 + '</h1>',
 			formatted='<h1>' + uni_chr1 + 'abcd' + uni_chr2 + '</h1>',
 			text_content='whatever')
-		result = prepare_message(email=email, recipient='test@test.com', recipients_list=[])
+		mail_ctx = SendMailContext(queue_doc = email)
+		result = mail_ctx.build_message(recipient_email = 'test@test.com')
 		self.assertTrue(b"<h1>=EA=80=80abcd=DE=B4</h1>" in result)
 
 	def test_prepare_message_returns_cr_lf(self):
@@ -68,8 +69,10 @@ This is the text version of this email
 			content='<h1>\n this is a test of newlines\n' + '</h1>',
 			formatted='<h1>\n this is a test of newlines\n' + '</h1>',
 			text_content='whatever')
-		result = safe_decode(prepare_message(email=email,
-						recipient='test@test.com', recipients_list=[]))
+
+		mail_ctx = SendMailContext(queue_doc = email)
+		result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com'))
+
 		if PY3:
 			self.assertTrue(result.count('\n') == result.count("\r"))
 		else:

frappe/email/test_smtp.py

@@ -4,7 +4,7 @@
 import unittest
 import frappe
 from frappe.email.smtp import SMTPServer
-from frappe.email.smtp import get_outgoing_email_account
+from frappe.email.doctype.email_account.email_account import EmailAccount
 
 class TestSMTP(unittest.TestCase):
 	def test_smtp_ssl_session(self):
@@ -33,13 +33,13 @@ class TestSMTP(unittest.TestCase):
 
 		frappe.local.outgoing_email_account = {}
 		# lowest preference given to email account with default incoming enabled
-		create_email_account(email_id="default_outgoing_enabled@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1)
-		self.assertEqual(get_outgoing_email_account().email_id, "default_outgoing_enabled@gmail.com")
+		create_email_account(email_id="default_outgoing_enabled@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1)
+		self.assertEqual(EmailAccount.find_outgoing().email_id, "default_outgoing_enabled@gmail.com")
 
 		frappe.local.outgoing_email_account = {}
 		# highest preference given to email account with append_to matching
-		create_email_account(email_id="append_to@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
-		self.assertEqual(get_outgoing_email_account(append_to="Blog Post").email_id, "append_to@gmail.com")
+		create_email_account(email_id="append_to@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
+		self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com")
 
 		# add back the mail_server
 		frappe.conf['mail_server'] = mail_server
@@ -75,4 +75,4 @@ def make_server(port, ssl, tls):
 		use_tls = tls
 	)
 
-	server.sess
+	server.session

frappe/handler.py

@@ -228,10 +228,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 	is_whitelisted(fn)
 	is_valid_http_method(fn)
 
-	try:
-		fnargs = inspect.getargspec(method_obj)[0]
-	except ValueError:
-		fnargs = inspect.getfullargspec(method_obj).args
+	fnargs = inspect.getfullargspec(method_obj).args
 
 	if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"):
 		response = doc.run_method(method)

frappe/model/db_query.py

@@ -465,7 +465,7 @@ class DatabaseQuery(object):
 
 		elif f.operator.lower() in ('in', 'not in'):
 			values = f.value or ''
-			if isinstance(values, frappe.string_types):
+			if isinstance(values, str):
 				values = values.split(",")
 
 			fallback = "''"

frappe/model/document.py

@@ -1347,6 +1347,22 @@ class Document(BaseDocument):
 		from frappe.desk.doctype.tag.tag import DocTags
 		return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
 
+	def __repr__(self):
+		name = self.name or "unsaved"
+		doctype = self.__class__.__name__
+
+		docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
+		parent = f" parent={self.parent}" if self.parent else ""
+
+		return f"<{doctype}: {name}{docstatus}{parent}>"
+
+	def __str__(self):
+		name = self.name or "unsaved"
+		doctype = self.__class__.__name__
+
+		return f"{doctype}({name})"
+
+
 def execute_action(doctype, name, action, **kwargs):
 	"""Execute an action on a document (called by background worker)"""
 	doc = frappe.get_doc(doctype, name)

frappe/model/meta.py

@@ -118,7 +118,7 @@ class Meta(Document):
 						# non standard list object, skip
 						continue
 
-				if (isinstance(value, (frappe.text_type, int, float, datetime, list, tuple))
+				if (isinstance(value, (str, int, float, datetime, list, tuple))
 					or (not no_nulls and value is None)):
 					out[key] = value
 

frappe/patches.txt

@@ -336,3 +336,4 @@ frappe.patches.v13_0.remove_twilio_settings
 frappe.patches.v12_0.rename_uploaded_files_with_proper_name
 frappe.patches.v13_0.queryreport_columns
 frappe.patches.v13_0.jinja_hook
+frappe.patches.v13_0.update_notification_channel_if_empty

frappe/patches/v13_0/update_notification_channel_if_empty.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+def execute():
+
+	frappe.reload_doc("Email", "doctype", "Notification")
+
+	notifications = frappe.get_all('Notification', {'is_standard': 1}, {'name', 'channel'})
+	for notification in notifications:
+		if not notification.channel:
+			frappe.db.set_value("Notification", notification.name, "channel", "Email", update_modified=False)
+			frappe.db.commit()

frappe/public/js/frappe/dom.js

@@ -319,7 +319,7 @@ frappe.get_data_pill = (label, target_id=null, remove_action=null, image=null) =
 
 frappe.get_modal = function(title, content) {
 	return $(`<div class="modal fade" style="overflow: auto;" tabindex="-1">
-		<div class="modal-dialog modal-dialog-scrollable">
+		<div class="modal-dialog">
 			<div class="modal-content">
 				<div class="modal-header">
 					<div class="fill-width flex title-section">

frappe/public/js/frappe/form/controls/autocomplete.js

@@ -90,16 +90,10 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 		});
 
 		this.$input.on("awesomplete-open", () => {
-			this.toggle_container_scroll('.modal-dialog', 'modal-dialog-scrollable');
-			this.toggle_container_scroll('.grid-form-body .form-area', 'scrollable');
-
 			this.autocomplete_open = true;
 		});
 
 		this.$input.on("awesomplete-close", () => {
-			this.toggle_container_scroll('.modal-dialog', 'modal-dialog-scrollable', true);
-			this.toggle_container_scroll('.grid-form-body .form-area', 'scrollable', true);
-
 			this.autocomplete_open = false;
 		});
 

frappe/public/js/frappe/form/controls/link.js

@@ -241,16 +241,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 		});
 
 		this.$input.on("awesomplete-open", () => {
-			this.toggle_container_scroll('.modal-dialog', 'modal-dialog-scrollable');
-			this.toggle_container_scroll('.grid-form-body .form-area', 'scrollable');
-
 			this.autocomplete_open = true;
 		});
 
 		this.$input.on("awesomplete-close", () => {
-			this.toggle_container_scroll('.modal-dialog', 'modal-dialog-scrollable', true);
-			this.toggle_container_scroll('.grid-form-body .form-area', 'scrollable', true);
-
 			this.autocomplete_open = false;
 		});
 

frappe/public/js/frappe/form/grid_row_form.js

@@ -66,7 +66,7 @@ export default class GridRowForm {
 				</div>
 			</div>
 			<div class="grid-form-body">
-				<div class="form-area scrollable"></div>
+				<div class="form-area"></div>
 				<div class="grid-footer-toolbar hidden-xs flex justify-between">
 					<div class="grid-shortcuts">
 						<span> ${frappe.utils.icon("keyboard", "md")} </span>

frappe/public/js/frappe/ui/filters/field_select.js

@@ -36,18 +36,6 @@ frappe.ui.FieldSelect = class FieldSelect {
 			var item = me.awesomplete.get_item(value);
 			me.$input.val(item.label);
 		});
-		this.$input.on("awesomplete-open", () => {
-			let modal = this.$input.parents('.modal-dialog')[0];
-			if (modal) {
-				$(modal).removeClass("modal-dialog-scrollable");
-			}
-		});
-		this.$input.on("awesomplete-close", () => {
-			let modal = this.$input.parents('.modal-dialog')[0];
-			if (modal) {
-				$(modal).addClass("modal-dialog-scrollable");
-			}
-		});
 
 		if(this.filter_fields) {
 			for(var i in this.filter_fields)

frappe/public/scss/common/modal.scss

@@ -2,25 +2,50 @@ h5.modal-title {
 	margin: 0px !important;
 }
 
-body.modal-open {
-	overflow: auto;
-	height: auto;
-	min-height: 100%;
+// Hack to fix incorrect padding applied by Bootstrap
+body.modal-open[style^="padding-right"] {
+	padding-right: 12px !important;
+
+	header.navbar {
+		padding-right: 12px !important;
+		margin-right: -12px !important;
+	}
 }
 
 .modal {
+	// Same scrollbar as body
+	scrollbar-width: auto;
+	&::-webkit-scrollbar {
+		width: 12px;
+		height: 12px;
+	}
+
+	// Hide scrollbar on touch devices
+	@media(max-width: 991px) {
+		scrollbar-width: none;
+		&::-webkit-scrollbar {
+			width: 0;
+			height: 0;
+		}
+	}
+
 	.modal-content {
 		border-color: var(--border-color);
 	}
 	.modal-header {
+		position: sticky;
+		top: 0;
+		z-index: 3;
+		background: inherit;
 		padding: var(--padding-md) var(--padding-lg);
-		padding-bottom: 0;
-		border-bottom: 0;
+		// padding-bottom: 0;
+		border-bottom: 1px solid var(--border-color);
 
 		.modal-title {
 			font-weight: 500;
 			line-height: 2em;
 			font-size: $font-size-lg;
+			max-width: calc(100% - 80px);
 		}
 
 		.modal-actions {
@@ -60,9 +85,17 @@ body.modal-open {
 		}
 	}
 
+	.awesomplete ul {
+		z-index: 2;
+	}
+
 	.modal-footer {
+		position: sticky;
+		bottom: 0;
+		z-index: 1;
+		background: inherit;
 		padding: var(--padding-md) var(--padding-lg);
-		border-top: 0;
+		border-top: 1px solid var(--border-color);
 		justify-content: space-between;
 
 		button {

frappe/public/scss/common/quill.scss

@@ -164,7 +164,7 @@
 }
 
 .ql-editor td {
-	border: 1px solid var(--border-color);
+	border: 1px solid var(--dark-border-color);
 }
 
 .ql-editor blockquote {

frappe/public/scss/desk/global.scss

@@ -442,6 +442,11 @@ kbd {
 /*rtl styles*/
 
 .frappe-rtl {
+	text-align: right;
+	.modal-actions {
+		right: auto !important;
+		left: 5px;
+	}
 	input, textarea {
 		direction: rtl !important;
 	}

frappe/public/scss/desk/report.scss

@@ -161,7 +161,8 @@
 	.summary-item {
 		// SIZE & SPACING
 		margin: 0px 30px;
-		width: 180px;
+		min-width: 180px;
+		max-width: 300px;
 		height: 62px;
 
 		// LAYOUT

frappe/public/scss/desk/scrollbar.scss

@@ -9,11 +9,6 @@ html {
 }
 
 /* Works on Chrome, Edge, and Safari */
-*::-webkit-scrollbar {
-	width: 6px;
-	height: 6px;
-}
-
 *::-webkit-scrollbar-thumb {
 	background: var(--scrollbar-thumb-color);
 }
@@ -23,7 +18,12 @@ html {
 	background: var(--scrollbar-track-color);
 }
 
+*::-webkit-scrollbar {
+	width: 6px;
+	height: 6px;
+}
+
 body::-webkit-scrollbar {
-	width: unset;
-	height: unset;
+	width: 12px;
+	height: 12px;
 }

frappe/social/doctype/energy_point_log/test_energy_point_log.py

@@ -78,7 +78,7 @@ class TestEnergyPointLog(unittest.TestCase):
 		points_after_closing_todo = get_points('test@example.com')
 
 		# test max_points cap
-		self.assertNotEquals(points_after_closing_todo,
+		self.assertNotEqual(points_after_closing_todo,
 			energy_point_of_user + round(todo_point_rule.points * multiplier_value))
 
 		self.assertEqual(points_after_closing_todo,

frappe/tests/test_commands.py

@@ -115,12 +115,12 @@ class TestCommands(BaseTestCommands):
 	def test_execute(self):
 		# test 1: execute a command expecting a numeric output
 		self.execute("bench --site {site} execute frappe.db.get_database_size")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIsInstance(float(self.stdout), float)
 
 		# test 2: execute a command expecting an errored output as local won't exist
 		self.execute("bench --site {site} execute frappe.local.site")
-		self.assertEquals(self.returncode, 1)
+		self.assertEqual(self.returncode, 1)
 		self.assertIsNotNone(self.stderr)
 
 		# test 3: execute a command with kwargs
@@ -128,8 +128,8 @@ class TestCommands(BaseTestCommands):
 		# terminal command has been escaped to avoid .format string replacement
 		# The returned value has quotes which have been trimmed for the test
 		self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
-		self.assertEquals(self.returncode, 0)
-		self.assertEquals(self.stdout[1:-1], frappe.bold(text="DocType"))
+		self.assertEqual(self.returncode, 0)
+		self.assertEqual(self.stdout[1:-1], frappe.bold(text="DocType"))
 
 	def test_backup(self):
 		backup = {
@@ -155,7 +155,7 @@ class TestCommands(BaseTestCommands):
 		self.execute("bench --site {site} backup")
 		after_backup = fetch_latest_backups()
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("successfully completed", self.stdout)
 		self.assertNotEqual(before_backup["database"], after_backup["database"])
 
@@ -164,7 +164,7 @@ class TestCommands(BaseTestCommands):
 		self.execute("bench --site {site} backup --with-files")
 		after_backup = fetch_latest_backups()
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("successfully completed", self.stdout)
 		self.assertIn("with files", self.stdout)
 		self.assertNotEqual(before_backup, after_backup)
@@ -175,7 +175,7 @@ class TestCommands(BaseTestCommands):
 		backup_path = os.path.join(home, "backups")
 		self.execute("bench --site {site} backup --backup-path {backup_path}", {"backup_path": backup_path})
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertTrue(os.path.exists(backup_path))
 		self.assertGreaterEqual(len(os.listdir(backup_path)), 2)
 
@@ -200,37 +200,37 @@ class TestCommands(BaseTestCommands):
 			kwargs,
 		)
 
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		for path in kwargs.values():
 			self.assertTrue(os.path.exists(path))
 
 		# test 5: take a backup with --compress
 		self.execute("bench --site {site} backup --with-files --compress")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		compressed_files = glob.glob(site_backup_path + "/*.tgz")
 		self.assertGreater(len(compressed_files), 0)
 
 		# test 6: take a backup with --verbose
 		self.execute("bench --site {site} backup --verbose")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 
 		# test 7: take a backup with frappe.conf.backup.includes
 		self.execute(
-			"bench --site {site} set-config backup '{includes}' --as-dict",
+			"bench --site {site} set-config backup '{includes}' --parse",
 			{"includes": json.dumps(backup["includes"])},
 		)
 		self.execute("bench --site {site} backup --verbose")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
 
 		# test 8: take a backup with frappe.conf.backup.excludes
 		self.execute(
-			"bench --site {site} set-config backup '{excludes}' --as-dict",
+			"bench --site {site} set-config backup '{excludes}' --parse",
 			{"excludes": json.dumps(backup["excludes"])},
 		)
 		self.execute("bench --site {site} backup --verbose")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
 		self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
@@ -240,7 +240,7 @@ class TestCommands(BaseTestCommands):
 			"bench --site {site} backup --include '{include}'",
 			{"include": ",".join(backup["includes"]["includes"])},
 		)
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertTrue(exists_in_backup(backup["includes"]["includes"], database))
 
@@ -249,13 +249,13 @@ class TestCommands(BaseTestCommands):
 			"bench --site {site} backup --exclude '{exclude}'",
 			{"exclude": ",".join(backup["excludes"]["excludes"])},
 		)
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups(partial=True)["database"]
 		self.assertFalse(exists_in_backup(backup["excludes"]["excludes"], database))
 
 		# test 11: take a backup with --ignore-backup-conf
 		self.execute("bench --site {site} backup --ignore-backup-conf")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		database = fetch_latest_backups()["database"]
 		self.assertTrue(exists_in_backup(backup["excludes"]["excludes"], database))
 
@@ -296,7 +296,7 @@ class TestCommands(BaseTestCommands):
 		)
 		site_data.update({"database": json.loads(self.stdout)["database"]})
 		self.execute("bench --site {another_site} restore {database}", site_data)
-		self.assertEquals(self.returncode, 1)
+		self.assertEqual(self.returncode, 1)
 
 	def test_partial_restore(self):
 		_now = now()
@@ -319,8 +319,8 @@ class TestCommands(BaseTestCommands):
 		frappe.db.commit()
 
 		self.execute("bench --site {site} partial-restore {path}", {"path": db_path})
-		self.assertEquals(self.returncode, 0)
-		self.assertEquals(frappe.db.count("ToDo"), todo_count)
+		self.assertEqual(self.returncode, 0)
+		self.assertEqual(frappe.db.count("ToDo"), todo_count)
 
 	def test_recorder(self):
 		frappe.recorder.stop()
@@ -343,18 +343,18 @@ class TestCommands(BaseTestCommands):
 
 		# test 1: remove app from installed_apps global default
 		self.execute("bench --site {site} remove-from-installed-apps {app}", {"app": app})
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.execute("bench --site {site} list-apps")
 		self.assertNotIn(app, self.stdout)
 
 	def test_list_apps(self):
 		# test 1: sanity check for command
 		self.execute("bench --site all list-apps")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 
 		# test 2: bare functionality for single site
 		self.execute("bench --site {site} list-apps")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		list_apps = set([
 			_x.split()[0] for _x in self.stdout.split("\n")
 		])
@@ -367,7 +367,7 @@ class TestCommands(BaseTestCommands):
 
 		# test 3: parse json format
 		self.execute("bench --site all list-apps --format json")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIsInstance(json.loads(self.stdout), dict)
 
 		self.execute("bench --site {site} list-apps --format json")
@@ -376,6 +376,32 @@ class TestCommands(BaseTestCommands):
 		self.execute("bench --site {site} list-apps -f json")
 		self.assertIsInstance(json.loads(self.stdout), dict)
 
+	def test_show_config(self):
+		# test 1: sanity check for command
+		self.execute("bench --site all show-config")
+		self.assertEqual(self.returncode, 0)
+
+		# test 2: test keys in table text
+		self.execute(
+			"bench --site {site} set-config test_key '{second_order}' --parse",
+			{"second_order": json.dumps({"test_key": "test_value"})},
+		)
+		self.execute("bench --site {site} show-config")
+		self.assertEqual(self.returncode, 0)
+		self.assertIn("test_key.test_key", self.stdout.split())
+		self.assertIn("test_value", self.stdout.split())
+
+		# test 3: parse json format
+		self.execute("bench --site all show-config --format json")
+		self.assertEqual(self.returncode, 0)
+		self.assertIsInstance(json.loads(self.stdout), dict)
+
+		self.execute("bench --site {site} show-config --format json")
+		self.assertIsInstance(json.loads(self.stdout), dict)
+
+		self.execute("bench --site {site} show-config -f json")
+		self.assertIsInstance(json.loads(self.stdout), dict)
+
 	def test_get_bench_relative_path(self):
 		bench_path = frappe.utils.get_bench_path()
 		test1_path = os.path.join(bench_path, "test1.txt")
@@ -397,6 +423,6 @@ class TestCommands(BaseTestCommands):
 	def test_frappe_site_env(self):
 		os.putenv('FRAPPE_SITE', frappe.local.site)
 		self.execute("bench execute frappe.ping")
-		self.assertEquals(self.returncode, 0)
+		self.assertEqual(self.returncode, 0)
 		self.assertIn("pong", self.stdout)
 

frappe/tests/test_db.py

@@ -18,7 +18,7 @@ class TestDB(unittest.TestCase):
 	def test_get_value(self):
 		self.assertEqual(frappe.db.get_value("User", {"name": ["=", "Administrator"]}), "Administrator")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["like", "Admin%"]}), "Administrator")
-		self.assertNotEquals(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest")
+		self.assertNotEqual(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["<", "Adn"]}), "Administrator")
 		self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator")
 

frappe/tests/test_website.py

@@ -48,7 +48,7 @@ class TestWebsite(unittest.TestCase):
 		set_request(method='POST', path='login')
 		response = render.render()
 
-		self.assertEquals(response.status_code, 200)
+		self.assertEqual(response.status_code, 200)
 
 		html = frappe.safe_decode(response.get_data())
 
@@ -76,27 +76,27 @@ class TestWebsite(unittest.TestCase):
 
 		set_request(method='GET', path='/testfrom')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), r'://testto1')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), r'://testto1')
 
 		set_request(method='GET', path='/testfromregex/test')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), r'://testto2')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), r'://testto2')
 
 		set_request(method='GET', path='/testsub/me')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), r'://testto3/me')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), r'://testto3/me')
 
 		set_request(method='GET', path='/test404')
 		response = render.render()
-		self.assertEquals(response.status_code, 404)
+		self.assertEqual(response.status_code, 404)
 
 		set_request(method='GET', path='/testsource')
 		response = render.render()
-		self.assertEquals(response.status_code, 301)
-		self.assertEquals(response.headers.get('Location'), '/testtarget')
+		self.assertEqual(response.status_code, 301)
+		self.assertEqual(response.headers.get('Location'), '/testtarget')
 
 		delattr(frappe.hooks, 'website_redirects')
 		frappe.cache().delete_key('app_hooks')

frappe/translate.py

@@ -548,8 +548,12 @@ def extract_messages_from_code(code):
 
 	try:
 		code = frappe.as_unicode(render_include(code))
-	except (TemplateError, ImportError, InvalidIncludePath, IOError):
-		# Exception will occur when it encounters John Resig's microtemplating code
+
+	# Exception will occur when it encounters John Resig's microtemplating code
+	except (TemplateError, ImportError, InvalidIncludePath, IOError) as e:
+		if isinstance(e, InvalidIncludePath):
+			frappe.clear_last_message()
+
 		pass
 
 	messages = []

frappe/utils/boilerplate.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
 from __future__ import unicode_literals, print_function
@@ -126,16 +126,12 @@ recursive-include {app_name} *.svg
 recursive-include {app_name} *.txt
 recursive-exclude {app_name} *.pyc"""
 
-init_template = """# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
+init_template = """
 __version__ = '0.0.1'
 
 """
 
-hooks_template = """# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-from . import __version__ as app_version
+hooks_template = """from . import __version__ as app_version
 
 app_name = "{app_name}"
 app_title = "{app_title}"
@@ -258,10 +254,10 @@ app_license = "{app_license}"
 # 	],
 # 	"weekly": [
 # 		"{app_name}.tasks.weekly"
-# 	]
+# 	],
 # 	"monthly": [
 # 		"{app_name}.tasks.monthly"
-# 	]
+# 	],
 # }}
 
 # Testing
@@ -291,26 +287,26 @@ app_license = "{app_license}"
 # User Data Protection
 # --------------------
 
-user_data_fields = [
-	{{
-		"doctype": "{{doctype_1}}",
-		"filter_by": "{{filter_by}}",
-		"redact_fields": ["{{field_1}}", "{{field_2}}"],
-		"partial": 1,
-	}},
-	{{
-		"doctype": "{{doctype_2}}",
-		"filter_by": "{{filter_by}}",
-		"partial": 1,
-	}},
-	{{
-		"doctype": "{{doctype_3}}",
-		"strict": False,
-	}},
-	{{
-		"doctype": "{{doctype_4}}"
-	}}
-]
+# user_data_fields = [
+# 	{{
+# 		"doctype": "{{doctype_1}}",
+# 		"filter_by": "{{filter_by}}",
+# 		"redact_fields": ["{{field_1}}", "{{field_2}}"],
+# 		"partial": 1,
+# 	}},
+# 	{{
+# 		"doctype": "{{doctype_2}}",
+# 		"filter_by": "{{filter_by}}",
+# 		"partial": 1,
+# 	}},
+# 	{{
+# 		"doctype": "{{doctype_3}}",
+# 		"strict": False,
+# 	}},
+# 	{{
+# 		"doctype": "{{doctype_4}}"
+# 	}}
+# ]
 
 # Authentication and authorization
 # --------------------------------
@@ -321,9 +317,7 @@ user_data_fields = [
 
 """
 
-desktop_template = """# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-from frappe import _
+desktop_template = """from frappe import _
 
 def get_data():
 	return [
@@ -337,8 +331,7 @@ def get_data():
 	]
 """
 
-setup_template = """# -*- coding: utf-8 -*-
-from setuptools import setup, find_packages
+setup_template = """from setuptools import setup, find_packages
 
 with open('requirements.txt') as f:
 	install_requires = f.read().strip().split('\\n')

frappe/utils/commands.py

@@ -1,11 +1,10 @@
 import functools
-import requests
-from terminaltables import AsciiTable
-
 
 @functools.lru_cache(maxsize=1024)
 def get_first_party_apps():
 	"""Get list of all apps under orgs: frappe. erpnext from GitHub"""
+	import requests
+
 	apps = []
 	for org in ["frappe", "erpnext"]:
 		req = requests.get(f"https://api.github.com/users/{org}/repos", {"type": "sources", "per_page": 200})
@@ -15,6 +14,8 @@ def get_first_party_apps():
 
 
 def render_table(data):
+	from terminaltables import AsciiTable
+
 	print(AsciiTable(data).table)
 
 
@@ -49,3 +50,9 @@ def log(message, colour=''):
 	colour = colours.get(colour, "")
 	end_line = '\033[0m'
 	print(colour + message + end_line)
+
+
+def warn(message, category=None):
+	from warnings import warn
+
+	warn(message=message, category=category, stacklevel=2)

frappe/utils/data.py

@@ -871,7 +871,7 @@ def in_words(integer, in_million=True):
 	return ret.replace('-', ' ')
 
 def is_html(text):
-	if not isinstance(text, frappe.string_types):
+	if not isinstance(text, str):
 		return False
 	return re.search('<[^>]+>', text)
 

frappe/utils/error.py

@@ -4,12 +4,14 @@
 
 from __future__ import unicode_literals
 
-import frappe
-from frappe.utils import cstr, encode
 import os
 import sys
-import inspect
 import traceback
+import functools
+
+import frappe
+from frappe.utils import cstr, encode
+import inspect
 import linecache
 import pydoc
 import cgitb
@@ -190,3 +192,45 @@ def clear_old_snapshots():
 
 def get_error_snapshot_path():
 	return frappe.get_site_path('error-snapshots')
+
+def get_default_args(func):
+	"""Get default arguments of a function from its signature.
+	"""
+	signature = inspect.signature(func)
+	return {k: v.default
+		for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty}
+
+def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None):
+	"""Decorate any function to throw error incase of missing output.
+
+	TODO: Remove keep_quiet flag after testing and fixing sendmail flow.
+
+	:param error_message: error message to raise
+	:param error_type: type of error to raise
+	:param keep_quiet: control error raising with external factor.
+	:type error_message: str
+	:type error_type: Exception Class
+	:type keep_quiet: function
+
+	>>> @raise_error_on_no_output("Ingradients missing")
+	... def get_indradients(_raise_error=1): return
+	...
+	>>> get_indradients()
+	`Exception Name`: Ingradients missing
+	"""
+	def decorator_raise_error_on_no_output(func):
+		@functools.wraps(func)
+		def wrapper_raise_error_on_no_output(*args, **kwargs):
+			response = func(*args, **kwargs)
+			if callable(keep_quiet) and keep_quiet():
+				return response
+
+			default_kwargs = get_default_args(func)
+			default_raise_error = default_kwargs.get('_raise_error')
+			raise_error = kwargs.get('_raise_error') if '_raise_error' in kwargs else default_raise_error
+
+			if (not response) and raise_error:
+				frappe.throw(error_message, error_type or Exception)
+			return response
+		return wrapper_raise_error_on_no_output
+	return decorator_raise_error_on_no_output

frappe/utils/jinja_globals.py

@@ -10,7 +10,7 @@ def resolve_class(classes):
 	if classes is None:
 		return ""
 
-	if isinstance(classes, frappe.string_types):
+	if isinstance(classes, str):
 		return classes
 
 	if isinstance(classes, (list, tuple)):

frappe/website/context.py

@@ -52,7 +52,7 @@ def update_controller_context(context, controller):
 		if hasattr(module, "get_context"):
 			import inspect
 			try:
-				if inspect.getargspec(module.get_context).args:
+				if inspect.getfullargspec(module.get_context).args:
 					ret = module.get_context(context)
 				else:
 					ret = module.get_context()

frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py

@@ -163,11 +163,11 @@ class PersonalDataDeletionRequest(Document):
 
 	def redact_full_match_data(self, ref, email):
 		"""Replaces the entire field value by the values set in the anonymization_value_map"""
-		filter_by = ref["filter_by"]
+		filter_by = ref.get("filter_by", "owner")
 
 		docs = frappe.get_all(
 			ref["doctype"],
-			filters={filter_by: ("like", "%" + email + "%")},
+			filters={filter_by: email},
 			fields=["name", filter_by],
 		)
 
@@ -205,7 +205,7 @@ class PersonalDataDeletionRequest(Document):
 		return anonymize_fields_dict
 
 	def redact_doc(self, doc, ref):
-		filter_by = ref["filter_by"]
+		filter_by = ref.get("filter_by", "owner")
 		meta = frappe.get_meta(ref["doctype"])
 		filter_by_meta = meta.get_field(filter_by)
 

frappe/website/doctype/web_form/test_web_form.py

@@ -42,7 +42,7 @@ class TestWebForm(unittest.TestCase):
 			'name': self.event_name
 		}
 
-		self.assertNotEquals(frappe.db.get_value("Event",
+		self.assertNotEqual(frappe.db.get_value("Event",
 			self.event_name, "description"), doc.get('description'))
 
 		accept(web_form='manage-events', docname=self.event_name, data=json.dumps(doc))

frappe/website/js/bootstrap-4.js

@@ -21,7 +21,7 @@ $('.dropdown-menu a.dropdown-toggle').on('click', function (e) {
 frappe.get_modal = function (title, content) {
 	return $(
 		`<div class="modal" tabindex="-1" role="dialog">
-			<div class="modal-dialog modal-dialog-scrollable" role="document">
+			<div class="modal-dialog" role="document">
 				<div class="modal-content">
 					<div class="modal-header">
 						<h5 class="modal-title">${title}</h5>