diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml
index df55089b9f..7754b52efc 100644
--- a/.github/helper/semgrep_rules/translate.yml
+++ b/.github/helper/semgrep_rules/translate.yml
@@ -42,7 +42,7 @@ rules:
- id: frappe-translation-python-splitting
pattern-either:
- - pattern: _(...) + ... + _(...)
+ - pattern: _(...) + _(...)
- pattern: _("..." + "...")
- pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\`
- pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( )
diff --git a/frappe/__init__.py b/frappe/__init__.py
index 5680ba86b5..02b8d71e40 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -10,11 +10,9 @@ be used to build database driven apps.
Read the documentation: https://frappeframework.com/docs
"""
-from __future__ import unicode_literals, print_function
-from six import iteritems, binary_type, text_type, string_types, PY2
from werkzeug.local import Local, release_local
-import os, sys, importlib, inspect, json
+import os, sys, importlib, inspect, json, warnings
import typing
from past.builtins import cmp
import click
@@ -27,19 +25,14 @@ from .utils.lazy_loader import lazy_import
# Lazy imports
faker = lazy_import('faker')
-
-# Harmless for Python 3
-# For Python 2 set default encoding to utf-8
-if PY2:
- reload(sys)
- sys.setdefaultencoding("utf-8")
-
__version__ = '14.0.0-dev'
__title__ = "Frappe Framework"
local = Local()
controllers = {}
+warnings.simplefilter('always', DeprecationWarning)
+warnings.simplefilter('always', PendingDeprecationWarning)
class _dict(dict):
"""dict like object that exposes keys as attributes"""
@@ -97,14 +90,14 @@ def _(msg, lang=None, context=None):
def as_unicode(text, encoding='utf-8'):
'''Convert to unicode if required'''
- if isinstance(text, text_type):
+ if isinstance(text, str):
return text
elif text==None:
return ''
- elif isinstance(text, binary_type):
- return text_type(text, encoding)
+ elif isinstance(text, bytes):
+ return str(text, encoding)
else:
- return text_type(text)
+ return str(text)
def get_lang_dict(fortype, name=None):
"""Returns the translated language dict for the given type and name.
@@ -597,7 +590,7 @@ def is_whitelisted(method):
# strictly sanitize form_dict
# escapes html characters like <> except for predefined tags like a, b, ul etc.
for key, value in form_dict.items():
- if isinstance(value, string_types):
+ if isinstance(value, str):
form_dict[key] = sanitize_html(value)
def read_only():
@@ -721,7 +714,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc
user = session.user
if doc:
- if isinstance(doc, string_types):
+ if isinstance(doc, str):
doc = get_doc(doctype, doc)
doctype = doc.doctype
@@ -790,7 +783,7 @@ def set_value(doctype, docname, fieldname, value=None):
return frappe.client.set_value(doctype, docname, fieldname, value)
def get_cached_doc(*args, **kwargs):
- if args and len(args) > 1 and isinstance(args[1], text_type):
+ if args and len(args) > 1 and isinstance(args[1], str):
key = get_document_cache_key(args[0], args[1])
# local cache
doc = local.document_cache.get(key)
@@ -821,7 +814,7 @@ def clear_document_cache(doctype, name):
def get_cached_value(doctype, name, fieldname, as_dict=False):
doc = get_cached_doc(doctype, name)
- if isinstance(fieldname, string_types):
+ if isinstance(fieldname, str):
if as_dict:
throw('Cannot make dict for single fieldname')
return doc.get(fieldname)
@@ -1027,7 +1020,7 @@ def get_doc_hooks():
if not hasattr(local, 'doc_events_hooks'):
hooks = get_hooks('doc_events', {})
out = {}
- for key, value in iteritems(hooks):
+ for key, value in hooks.items():
if isinstance(key, tuple):
for doctype in key:
append_hook(out, doctype, value)
@@ -1144,7 +1137,7 @@ def get_file_json(path):
def read_file(path, raise_not_found=False):
"""Open a file and return its content as Unicode."""
- if isinstance(path, text_type):
+ if isinstance(path, str):
path = path.encode("utf-8")
if os.path.exists(path):
@@ -1167,7 +1160,7 @@ def get_attr(method_string):
def call(fn, *args, **kwargs):
"""Call a function and match arguments."""
- if isinstance(fn, string_types):
+ if isinstance(fn, str):
fn = get_attr(fn)
newargs = get_newargs(fn, kwargs)
@@ -1178,13 +1171,9 @@ def get_newargs(fn, kwargs):
if hasattr(fn, 'fnargs'):
fnargs = fn.fnargs
else:
- try:
- fnargs, varargs, varkw, defaults = inspect.getargspec(fn)
- except ValueError:
- fnargs = inspect.getfullargspec(fn).args
- varargs = inspect.getfullargspec(fn).varargs
- varkw = inspect.getfullargspec(fn).varkw
- defaults = inspect.getfullargspec(fn).defaults
+ fnargs = inspect.getfullargspec(fn).args
+ fnargs.extend(inspect.getfullargspec(fn).kwonlyargs)
+ varkw = inspect.getfullargspec(fn).varkw
newargs = {}
for a in kwargs:
@@ -1626,6 +1615,12 @@ def enqueue(*args, **kwargs):
import frappe.utils.background_jobs
return frappe.utils.background_jobs.enqueue(*args, **kwargs)
+def task(**task_kwargs):
+ def decorator_task(f):
+ f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs)
+ return f
+ return decorator_task
+
def enqueue_doc(*args, **kwargs):
'''
Enqueue method to be executed using a background worker
diff --git a/frappe/app.py b/frappe/app.py
index 794d0f18af..a72f343532 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -185,7 +185,7 @@ def make_form_dict(request):
args = request.form or request.args
if not isinstance(args, dict):
- frappe.throw("Invalid request arguments")
+ frappe.throw(_("Invalid request arguments"))
try:
frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
@@ -294,8 +294,9 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
_sites_path = sites_path
from werkzeug.serving import run_simple
+ patch_werkzeug_reloader()
- if profile:
+ if profile or os.environ.get('USE_PROFILER'):
application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))
if not os.environ.get('NO_STATICS'):
@@ -324,3 +325,23 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No
use_debugger=not in_test_env,
use_evalex=not in_test_env,
threaded=not no_threading)
+
+def patch_werkzeug_reloader():
+ """
+ This function monkey patches Werkzeug reloader to ignore reloading files in
+ the __pycache__ directory.
+
+ To be deprecated when upgrading to Werkzeug 2.
+ """
+
+ from werkzeug._reloader import WatchdogReloaderLoop
+
+ trigger_reload = WatchdogReloaderLoop.trigger_reload
+
+ def custom_trigger_reload(self, filename):
+ if os.path.basename(os.path.dirname(filename)) == "__pycache__":
+ return
+
+ return trigger_reload(self, filename)
+
+ WatchdogReloaderLoop.trigger_reload = custom_trigger_reload
diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py
index f41f31f3bb..6ceb4dba72 100644
--- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py
+++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py
@@ -173,7 +173,7 @@ class TestAutoRepeat(unittest.TestCase):
fields=['docstatus'],
limit=1
)
- self.assertEquals(docnames[0].docstatus, 1)
+ self.assertEqual(docnames[0].docstatus, 1)
def make_auto_repeat(**args):
diff --git a/frappe/build.py b/frappe/build.py
index fc5d78e306..763199dc12 100644
--- a/frappe/build.py
+++ b/frappe/build.py
@@ -323,7 +323,7 @@ def make_asset_dirs(make_copy=False, restore=False):
except OSError:
print("Cannot link {} to {}".format(source, target))
else:
- # warnings.warn('Source {source} does not exist.'.format(source = source))
+ warnings.warn('Source {source} does not exist.'.format(source = source))
pass
diff --git a/frappe/commands/site.py b/frappe/commands/site.py
index ebd2700c9c..22a063651c 100755
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@@ -254,9 +254,7 @@ def list_apps(context, format):
frappe.destroy()
if format == "json":
- import json
-
- click.echo(json.dumps(summary_dict))
+ click.echo(frappe.as_json(summary_dict))
@click.command('add-system-manager')
@click.argument('email')
diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py
index dbca28a3a3..f82c94999b 100644
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@@ -107,22 +107,54 @@ def destroy_all_sessions(context, reason=None):
raise SiteNotSpecifiedError
@click.command('show-config')
+@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text")
@pass_context
-def show_config(context):
- "print configuration file"
- print("\t\033[92m{:<50}\033[0m \033[92m{:<15}\033[0m".format('Config','Value'))
- sites_path = os.path.join(frappe.utils.get_bench_path(), 'sites')
- site_path = context.sites[0]
- configuration = frappe.get_site_config(sites_path=sites_path, site_path=site_path)
- print_config(configuration)
+def show_config(context, format):
+ "Print configuration file to STDOUT in speified format"
+ if not context.sites:
+ raise SiteNotSpecifiedError
+
+ sites_config = {}
+ sites_path = os.getcwd()
+
+ from frappe.utils.commands import render_table
+
+ def transform_config(config, prefix=None):
+ prefix = f"{prefix}." if prefix else ""
+ site_config = []
+
+ for conf, value in config.items():
+ if isinstance(value, dict):
+ site_config += transform_config(value, prefix=f"{prefix}{conf}")
+ else:
+ log_value = json.dumps(value) if isinstance(value, list) else value
+ site_config += [[f"{prefix}{conf}", log_value]]
+
+ return site_config
+
+ for site in context.sites:
+ frappe.init(site)
+
+ if len(context.sites) != 1 and format == "text":
+ if context.sites.index(site) != 0:
+ click.echo()
+ click.secho(f"Site {site}", fg="yellow")
-def print_config(config):
- for conf, value in config.items():
- if isinstance(value, dict):
- print_config(value)
- else:
- print("\t{:<50} {:<15}".format(conf, value))
+ configuration = frappe.get_site_config(sites_path=sites_path, site_path=site)
+
+ if format == "text":
+ data = transform_config(configuration)
+ data.insert(0, ['Config','Value'])
+ render_table(data)
+
+ if format == "json":
+ sites_config[site] = configuration
+
+ frappe.destroy()
+
+ if format == "json":
+ click.echo(frappe.as_json(sites_config))
@click.command('reset-perms')
@@ -481,6 +513,7 @@ def console(context):
locals()[app] = __import__(app)
except ModuleNotFoundError:
failed_to_import.append(app)
+ all_apps.remove(app)
print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
if failed_to_import:
@@ -668,20 +701,27 @@ def make_app(destination, app_name):
@click.command('set-config')
@click.argument('key')
@click.argument('value')
-@click.option('-g', '--global', 'global_', is_flag = True, default = False, help = 'Set Global Site Config')
-@click.option('--as-dict', is_flag=True, default=False)
+@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config')
+@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object')
+@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object')
@pass_context
-def set_config(context, key, value, global_ = False, as_dict=False):
+def set_config(context, key, value, global_=False, parse=False, as_dict=False):
"Insert/Update a value in site_config.json"
from frappe.installer import update_site_config
- import ast
+
if as_dict:
+ from frappe.utils.commands import warn
+ warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning)
+ parse = as_dict
+
+ if parse:
+ import ast
value = ast.literal_eval(value)
if global_:
- sites_path = os.getcwd() # big assumption.
+ sites_path = os.getcwd()
common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
- update_site_config(key, value, validate = False, site_config_path = common_site_config_path)
+ update_site_config(key, value, validate=False, site_config_path=common_site_config_path)
else:
for site in context.sites:
frappe.init(site=site)
@@ -738,50 +778,6 @@ def rebuild_global_search(context, static_pages=False):
if not context.sites:
raise SiteNotSpecifiedError
-@click.command('auto-deploy')
-@click.argument('app')
-@click.option('--migrate', is_flag=True, default=False, help='Migrate after pulling')
-@click.option('--restart', is_flag=True, default=False, help='Restart after migration')
-@click.option('--remote', default='upstream', help='Remote, default is "upstream"')
-@pass_context
-def auto_deploy(context, app, migrate=False, restart=False, remote='upstream'):
- '''Pull and migrate sites that have new version'''
- from frappe.utils.gitutils import get_app_branch
- from frappe.utils import get_sites
-
- branch = get_app_branch(app)
- app_path = frappe.get_app_path(app)
-
- # fetch
- subprocess.check_output(['git', 'fetch', remote, branch], cwd = app_path)
-
- # get diff
- if subprocess.check_output(['git', 'diff', '{0}..{1}/{0}'.format(branch, remote)], cwd = app_path):
- print('Updates found for {0}'.format(app))
- if app=='frappe':
- # run bench update
- import shlex
- subprocess.check_output(shlex.split('bench update --no-backup'), cwd = '..')
- else:
- updated = False
- subprocess.check_output(['git', 'pull', '--rebase', remote, branch],
- cwd = app_path)
- # find all sites with that app
- for site in get_sites():
- frappe.init(site)
- if app in frappe.get_installed_apps():
- print('Updating {0}'.format(site))
- updated = True
- subprocess.check_output(['bench', '--site', site, 'clear-cache'], cwd = '..')
- if migrate:
- subprocess.check_output(['bench', '--site', site, 'migrate'], cwd = '..')
- frappe.destroy()
-
- if updated or restart:
- subprocess.check_output(['bench', 'restart'], cwd = '..')
- else:
- print('No Updates')
-
commands = [
build,
diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py
index bd0ea08cc7..05ece76c7f 100644
--- a/frappe/core/doctype/activity_log/test_activity_log.py
+++ b/frappe/core/doctype/activity_log/test_activity_log.py
@@ -65,12 +65,12 @@ class TestActivityLog(unittest.TestCase):
frappe.local.login_manager = LoginManager()
auth_log = self.get_auth_log()
- self.assertEquals(auth_log.status, 'Success')
+ self.assertEqual(auth_log.status, 'Success')
# test user logout log
frappe.local.login_manager.logout()
auth_log = self.get_auth_log(operation='Logout')
- self.assertEquals(auth_log.status, 'Success')
+ self.assertEqual(auth_log.status, 'Success')
# test invalid login
frappe.form_dict.update({ 'pwd': 'password' })
diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py
index 731cb85d7c..d3017055cf 100755
--- a/frappe/core/doctype/communication/email.py
+++ b/frappe/core/doctype/communication/email.py
@@ -272,22 +272,13 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None)
doc.attachments.append(a)
def set_incoming_outgoing_accounts(doc):
- doc.incoming_email_account = doc.outgoing_email_account = None
+ from frappe.email.doctype.email_account.email_account import EmailAccount
+ incoming_email_account = EmailAccount.find_incoming(
+ match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
+ doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None
- if not doc.incoming_email_account and doc.sender:
- doc.incoming_email_account = frappe.db.get_value("Email Account",
- {"email_id": doc.sender, "enable_incoming": 1}, "email_id")
-
- if not doc.incoming_email_account and doc.reference_doctype:
- doc.incoming_email_account = frappe.db.get_value("Email Account",
- {"append_to": doc.reference_doctype, }, "email_id")
-
- if not doc.incoming_email_account:
- doc.incoming_email_account = frappe.db.get_value("Email Account",
- {"default_incoming": 1, "enable_incoming": 1}, "email_id")
-
- doc.outgoing_email_account = frappe.email.smtp.get_outgoing_email_account(raise_exception_not_set=False,
- append_to=doc.doctype, sender=doc.sender)
+ doc.outgoing_email_account = EmailAccount.find_outgoing(
+ match_by_email=doc.sender, match_by_doctype=doc.reference_doctype)
if doc.sent_or_received == "Sent":
doc.db_set("email_account", doc.outgoing_email_account.name)
diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 388d9389f2..720fe1dda7 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -233,7 +233,7 @@ class Importer:
return updated_doc
else:
# throw if no changes
- frappe.throw("No changes to update")
+ frappe.throw(_("No changes to update"))
def get_eta(self, current, total, processing_time):
self.last_eta = getattr(self, "last_eta", 0)
@@ -319,7 +319,7 @@ class ImportFile:
self.warnings = []
self.file_doc = self.file_path = self.google_sheets_url = None
- if isinstance(file, frappe.string_types):
+ if isinstance(file, str):
if frappe.db.exists("File", {"file_url": file}):
self.file_doc = frappe.get_doc("File", {"file_url": file})
elif "docs.google.com/spreadsheets" in file:
@@ -626,7 +626,7 @@ class Row:
return
elif df.fieldtype in ["Date", "Datetime"]:
value = self.get_date(value, col)
- if isinstance(value, frappe.string_types):
+ if isinstance(value, str):
# value was not parsed as datetime object
self.warnings.append(
{
diff --git a/frappe/core/doctype/doctype/boilerplate/controller._py b/frappe/core/doctype/doctype/boilerplate/controller._py
index 583bd30908..6db99def55 100644
--- a/frappe/core/doctype/doctype/boilerplate/controller._py
+++ b/frappe/core/doctype/doctype/boilerplate/controller._py
@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
# Copyright (c) {year}, {app_publisher} and contributors
# For license information, please see license.txt
-from __future__ import unicode_literals
# import frappe
{base_class_import}
diff --git a/frappe/core/doctype/doctype/boilerplate/test_controller._py b/frappe/core/doctype/doctype/boilerplate/test_controller._py
index 8ed08ae15a..5f4150ce9b 100644
--- a/frappe/core/doctype/doctype/boilerplate/test_controller._py
+++ b/frappe/core/doctype/doctype/boilerplate/test_controller._py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
# Copyright (c) {year}, {app_publisher} and Contributors
# See license.txt
-from __future__ import unicode_literals
# import frappe
import unittest
diff --git a/frappe/core/doctype/report/boilerplate/controller.py b/frappe/core/doctype/report/boilerplate/controller.py
index 55c01e4f75..b8e9cb7467 100644
--- a/frappe/core/doctype/report/boilerplate/controller.py
+++ b/frappe/core/doctype/report/boilerplate/controller.py
@@ -1,7 +1,6 @@
# Copyright (c) 2013, {app_publisher} and contributors
# For license information, please see license.txt
-from __future__ import unicode_literals
# import frappe
def execute(filters=None):
diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py
index 9c76c839f3..d09799ca69 100644
--- a/frappe/core/doctype/report/test_report.py
+++ b/frappe/core/doctype/report/test_report.py
@@ -106,7 +106,7 @@ class TestReport(unittest.TestCase):
else:
report = frappe.get_doc('Report', 'Test Report')
- self.assertNotEquals(report.is_permitted(), True)
+ self.assertNotEqual(report.is_permitted(), True)
frappe.set_user('Administrator')
# test for the `_format` method if report data doesn't have sort_by parameter
diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py
index 2e9b832acc..47651fee72 100644
--- a/frappe/core/doctype/user_permission/test_user_permission.py
+++ b/frappe/core/doctype/user_permission/test_user_permission.py
@@ -46,7 +46,7 @@ class TestUserPermission(unittest.TestCase):
frappe.set_user('test_user_perm1@example.com')
doc = frappe.new_doc("Blog Post")
- self.assertEquals(doc.blog_category, 'general')
+ self.assertEqual(doc.blog_category, 'general')
frappe.set_user('Administrator')
def test_apply_to_all(self):
@@ -54,7 +54,7 @@ class TestUserPermission(unittest.TestCase):
user = create_user('test_bulk_creation_update@example.com')
param = get_params(user, 'User', user.name)
is_created = add_user_permissions(param)
- self.assertEquals(is_created, 1)
+ self.assertEqual(is_created, 1)
def test_for_apply_to_all_on_update_from_apply_all(self):
user = create_user('test_bulk_creation_update@example.com')
@@ -63,11 +63,11 @@ class TestUserPermission(unittest.TestCase):
# Initially create User Permission document with apply_to_all checked
is_created = add_user_permissions(param)
- self.assertEquals(is_created, 1)
+ self.assertEqual(is_created, 1)
is_created = add_user_permissions(param)
# User Permission should not be changed
- self.assertEquals(is_created, 0)
+ self.assertEqual(is_created, 0)
def test_for_applicable_on_update_from_apply_to_all(self):
''' Update User Permission from all to some applicable Doctypes'''
@@ -77,7 +77,7 @@ class TestUserPermission(unittest.TestCase):
# Initially create User Permission document with apply_to_all checked
is_created = add_user_permissions(get_params(user, 'User', user.name))
- self.assertEquals(is_created, 1)
+ self.assertEqual(is_created, 1)
is_created = add_user_permissions(param)
frappe.db.commit()
@@ -92,7 +92,7 @@ class TestUserPermission(unittest.TestCase):
# Check that User Permissions for applicable is created
self.assertIsNotNone(is_created_applicable_first)
self.assertIsNotNone(is_created_applicable_second)
- self.assertEquals(is_created, 1)
+ self.assertEqual(is_created, 1)
def test_for_apply_to_all_on_update_from_applicable(self):
''' Update User Permission from some to all applicable Doctypes'''
@@ -102,7 +102,7 @@ class TestUserPermission(unittest.TestCase):
# create User permissions that with applicable
is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"]))
- self.assertEquals(is_created, 1)
+ self.assertEqual(is_created, 1)
is_created = add_user_permissions(param)
is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user))
@@ -115,7 +115,7 @@ class TestUserPermission(unittest.TestCase):
# Check that all User Permission with applicable is removed
self.assertIsNone(removed_applicable_first)
self.assertIsNone(removed_applicable_second)
- self.assertEquals(is_created, 1)
+ self.assertEqual(is_created, 1)
def test_user_perm_for_nested_doctype(self):
"""Test if descendants' visibility is controlled for a nested DocType."""
@@ -183,7 +183,7 @@ class TestUserPermission(unittest.TestCase):
# User perm is created on ToDo but for doctype Assignment Rule only
# it should not have impact on Doc A
- self.assertEquals(new_doc.doc, "ToDo")
+ self.assertEqual(new_doc.doc, "ToDo")
frappe.set_user('Administrator')
remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo")
@@ -228,7 +228,7 @@ class TestUserPermission(unittest.TestCase):
# User perm is created on ToDo but for doctype Assignment Rule only
# it should not have impact on Doc A
- self.assertEquals(new_doc.doc, "ToDo")
+ self.assertEqual(new_doc.doc, "ToDo")
frappe.set_user('Administrator')
clear_session_defaults()
diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py
index fbc788f6bf..fec5019ca9 100644
--- a/frappe/core/doctype/user_permission/user_permission.py
+++ b/frappe/core/doctype/user_permission/user_permission.py
@@ -191,7 +191,7 @@ def clear_user_permissions(user, for_doctype):
def add_user_permissions(data):
''' Add and update the user permissions '''
frappe.only_for('System Manager')
- if isinstance(data, frappe.string_types):
+ if isinstance(data, str):
data = json.loads(data)
data = frappe._dict(data)
diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py
index 3126326636..fb49aa5da0 100644
--- a/frappe/custom/doctype/custom_field/custom_field.py
+++ b/frappe/custom/doctype/custom_field/custom_field.py
@@ -68,14 +68,15 @@ class CustomField(Document):
check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname)
def on_update(self):
- frappe.clear_cache(doctype=self.dt)
+ if not frappe.flags.in_setup_wizard:
+ frappe.clear_cache(doctype=self.dt)
if not self.flags.ignore_validate:
# validate field
from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype
validate_fields_for_doctype(self.dt)
# update the schema
- if not frappe.db.get_value('DocType', self.dt, 'issingle'):
+ if not frappe.db.get_value('DocType', self.dt, 'issingle') and not frappe.flags.in_setup_wizard:
frappe.db.updatedb(self.dt)
def on_trash(self):
@@ -144,6 +145,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
'''Add / update multiple custom fields
:param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`'''
+
+ if not ignore_validate and frappe.flags.in_setup_wizard:
+ ignore_validate = True
+
for doctype, fields in custom_fields.items():
if isinstance(fields, dict):
# only one field
@@ -163,6 +168,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True):
custom_field.update(df)
custom_field.save()
+ frappe.clear_cache(doctype=doctype)
+ frappe.db.updatedb(doctype)
+
+
@frappe.whitelist()
def add_custom_field(doctype, df):
diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py
index f5e0371c1f..75555a8205 100644
--- a/frappe/custom/doctype/customize_form/test_customize_form.py
+++ b/frappe/custom/doctype/customize_form/test_customize_form.py
@@ -47,64 +47,64 @@ class TestCustomizeForm(unittest.TestCase):
self.assertEqual(len(d.get("fields")), 0)
d = self.get_customize_form("Event")
- self.assertEquals(d.doc_type, "Event")
- self.assertEquals(len(d.get("fields")), 36)
+ self.assertEqual(d.doc_type, "Event")
+ self.assertEqual(len(d.get("fields")), 36)
d = self.get_customize_form("Event")
- self.assertEquals(d.doc_type, "Event")
+ self.assertEqual(d.doc_type, "Event")
self.assertEqual(len(d.get("fields")),
len(frappe.get_doc("DocType", d.doc_type).fields) + 1)
- self.assertEquals(d.get("fields")[-1].fieldname, "test_custom_field")
- self.assertEquals(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
+ self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field")
+ self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1)
return d
def test_save_customization_property(self):
d = self.get_customize_form("Event")
- self.assertEquals(frappe.db.get_value("Property Setter",
+ self.assertEqual(frappe.db.get_value("Property Setter",
{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
d.allow_copy = 1
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Property Setter",
+ self.assertEqual(frappe.db.get_value("Property Setter",
{"doc_type": "Event", "property": "allow_copy"}, "value"), '1')
d.allow_copy = 0
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Property Setter",
+ self.assertEqual(frappe.db.get_value("Property Setter",
{"doc_type": "Event", "property": "allow_copy"}, "value"), None)
def test_save_customization_field_property(self):
d = self.get_customize_form("Event")
- self.assertEquals(frappe.db.get_value("Property Setter",
+ self.assertEqual(frappe.db.get_value("Property Setter",
{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
repeat_this_event_field.reqd = 1
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Property Setter",
+ self.assertEqual(frappe.db.get_value("Property Setter",
{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1')
repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0]
repeat_this_event_field.reqd = 0
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Property Setter",
+ self.assertEqual(frappe.db.get_value("Property Setter",
{"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None)
def test_save_customization_custom_field_property(self):
d = self.get_customize_form("Event")
- self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+ self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0]
custom_field.reqd = 1
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
+ self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1)
custom_field = d.get("fields", {"is_custom_field": True})[0]
custom_field.reqd = 0
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
+ self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0)
def test_save_customization_new_field(self):
d = self.get_customize_form("Event")
@@ -115,14 +115,14 @@ class TestCustomizeForm(unittest.TestCase):
"is_custom_field": 1
})
d.run_method("save_customization")
- self.assertEquals(frappe.db.get_value("Custom Field",
+ self.assertEqual(frappe.db.get_value("Custom Field",
"Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data")
- self.assertEquals(frappe.db.get_value("Custom Field",
+ self.assertEqual(frappe.db.get_value("Custom Field",
"Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname)
frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form")
- self.assertEquals(frappe.db.get_value("Custom Field",
+ self.assertEqual(frappe.db.get_value("Custom Field",
"Event-test_add_custom_field_via_customize_form"), None)
@@ -142,7 +142,7 @@ class TestCustomizeForm(unittest.TestCase):
d.doc_type = "Event"
d.run_method('reset_to_defaults')
- self.assertEquals(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
+ self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0)
frappe.local.test_objects["Property Setter"] = []
make_test_records_for_doctype("Property Setter")
@@ -156,7 +156,7 @@ class TestCustomizeForm(unittest.TestCase):
d = self.get_customize_form("Event")
# don't allow for standard fields
- self.assertEquals(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
+ self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0)
# allow for custom field
self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1)
diff --git a/frappe/database/database.py b/frappe/database/database.py
index 58e5c8a46e..c9c1ec3909 100644
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@@ -858,7 +858,7 @@ class Database(object):
if not datetime:
return '0001-01-01 00:00:00.000000'
- if isinstance(datetime, frappe.string_types):
+ if isinstance(datetime, str):
if ':' not in datetime:
datetime = datetime + ' 00:00:00.000000'
else:
diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py
index 7d1d92408c..879c8394d7 100644
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@@ -1,5 +1,3 @@
-import warnings
-
import pymysql
from pymysql.constants import ER, FIELD_TYPE
from pymysql.converters import conversions, escape_string
@@ -55,7 +53,6 @@ class MariaDBDatabase(Database):
}
def get_connection(self):
- warnings.filterwarnings('ignore', category=pymysql.Warning)
usessl = 0
if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key:
usessl = 1
diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py
index 4faea78551..6ac2767a71 100644
--- a/frappe/database/postgres/database.py
+++ b/frappe/database/postgres/database.py
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
import re
import frappe
import psycopg2
@@ -13,9 +11,9 @@ from frappe.database.postgres.schema import PostgresTable
# cast decimals as floats
DEC2FLOAT = psycopg2.extensions.new_type(
- psycopg2.extensions.DECIMAL.values,
- 'DEC2FLOAT',
- lambda value, curs: float(value) if value is not None else None)
+ psycopg2.extensions.DECIMAL.values,
+ 'DEC2FLOAT',
+ lambda value, curs: float(value) if value is not None else None)
psycopg2.extensions.register_type(DEC2FLOAT)
@@ -65,7 +63,6 @@ class PostgresDatabase(Database):
}
def get_connection(self):
- # warnings.filterwarnings('ignore', category=psycopg2.Warning)
conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format(
self.host, self.user, self.user, self.password, self.port
))
@@ -114,7 +111,7 @@ class PostgresDatabase(Database):
if not date:
return '0001-01-01'
- if not isinstance(date, frappe.string_types):
+ if not isinstance(date, str):
date = date.strftime('%Y-%m-%d')
return date
diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py
index 20551559fd..25af92f532 100644
--- a/frappe/desk/doctype/notification_log/notification_log.py
+++ b/frappe/desk/doctype/notification_log/notification_log.py
@@ -46,7 +46,7 @@ def enqueue_create_notification(users, doc):
doc = frappe._dict(doc)
- if isinstance(users, frappe.string_types):
+ if isinstance(users, str):
users = [user.strip() for user in users.split(',') if user.strip()]
users = list(set(users))
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py
index c38cf47626..1ac5279508 100755
--- a/frappe/desk/page/setup_wizard/setup_wizard.py
+++ b/frappe/desk/page/setup_wizard/setup_wizard.py
@@ -124,6 +124,7 @@ def handle_setup_exception(args):
frappe.db.rollback()
if args:
traceback = frappe.get_traceback()
+ print(traceback)
for hook in frappe.get_hooks("setup_wizard_exception"):
frappe.get_attr(hook)(traceback, args)
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 9589507ca6..befaf7b01f 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -377,10 +377,17 @@ def handle_duration_fieldtype_values(result, columns):
if fieldtype == "Duration":
for entry in range(0, len(result)):
- val_in_seconds = result[entry][i]
- if val_in_seconds:
- duration_val = format_duration(val_in_seconds)
- result[entry][i] = duration_val
+ row = result[entry]
+ if isinstance(row, dict):
+ val_in_seconds = row[col.fieldname]
+ if val_in_seconds:
+ duration_val = format_duration(val_in_seconds)
+ row[col.fieldname] = duration_val
+ else:
+ val_in_seconds = row[i]
+ if val_in_seconds:
+ duration_val = format_duration(val_in_seconds)
+ row[i] = duration_val
return result
diff --git a/frappe/email/doctype/document_follow/test_document_follow.py b/frappe/email/doctype/document_follow/test_document_follow.py
index 1ac2d19305..38aa870232 100644
--- a/frappe/email/doctype/document_follow/test_document_follow.py
+++ b/frappe/email/doctype/document_follow/test_document_follow.py
@@ -17,14 +17,14 @@ class TestDocumentFollow(unittest.TestCase):
document_follow.unfollow_document("Event", event_doc.name, user.name)
doc = document_follow.follow_document("Event", event_doc.name, user.name)
- self.assertEquals(doc.user, user.name)
+ self.assertEqual(doc.user, user.name)
document_follow.send_hourly_updates()
email_queue_entry_name = frappe.get_all("Email Queue", limit=1)[0].name
email_queue_entry_doc = frappe.get_doc("Email Queue", email_queue_entry_name)
- self.assertEquals((email_queue_entry_doc.recipients[0].recipient), user.name)
+ self.assertEqual((email_queue_entry_doc.recipients[0].recipient), user.name)
self.assertIn(event_doc.doctype, email_queue_entry_doc.message)
self.assertIn(event_doc.name, email_queue_entry_doc.message)
diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py
index 4869c5a9bf..36b662bb39 100755
--- a/frappe/email/doctype/email_account/email_account.py
+++ b/frappe/email/doctype/email_account/email_account.py
@@ -8,9 +8,14 @@ import re
import json
import socket
import time
-from frappe import _
+import functools
+
+import email.utils
+
+from frappe import _, are_emails_muted
from frappe.model.document import Document
-from frappe.utils import validate_email_address, cint, cstr, get_datetime, DATE_FORMAT, strip, comma_or, sanitize_html, add_days
+from frappe.utils import (validate_email_address, cint, cstr, get_datetime,
+ DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr)
from frappe.utils.user import is_system_user
from frappe.utils.jinja import render_template
from frappe.email.smtp import SMTPServer
@@ -21,17 +26,37 @@ from datetime import datetime, timedelta
from frappe.desk.form import assign_to
from frappe.utils.user import get_system_managers
from frappe.utils.background_jobs import enqueue, get_jobs
-from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
from frappe.utils.html_utils import clean_email_html
+from frappe.utils.error import raise_error_on_no_output
from frappe.email.utils import get_port
+OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account")
+
class SentEmailInInbox(Exception):
pass
-class InvalidEmailCredentials(frappe.ValidationError):
- pass
+def cache_email_account(cache_name):
+ def decorator_cache_email_account(func):
+ @functools.wraps(func)
+ def wrapper_cache_email_account(*args, **kwargs):
+ if not hasattr(frappe.local, cache_name):
+ setattr(frappe.local, cache_name, {})
+
+ cached_accounts = getattr(frappe.local, cache_name)
+ match_by = list(kwargs.values()) + ['default']
+ matched_accounts = list(filter(None, [cached_accounts.get(key) for key in match_by]))
+ if matched_accounts:
+ return matched_accounts[0]
+
+ matched_accounts = func(*args, **kwargs)
+ cached_accounts.update(matched_accounts or {})
+ return matched_accounts and list(matched_accounts.values())[0]
+ return wrapper_cache_email_account
+ return decorator_cache_email_account
class EmailAccount(Document):
+ DOCTYPE = 'Email Account'
+
def autoname(self):
"""Set name as `email_account_name` or make title from Email Address."""
if not self.email_account_name:
@@ -72,9 +97,8 @@ class EmailAccount(Document):
self.get_incoming_server()
self.no_failed = 0
-
if self.enable_outgoing:
- self.check_smtp()
+ self.validate_smtp_conn()
else:
if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication):
frappe.throw(_("Password is required or select Awaiting Password"))
@@ -90,6 +114,13 @@ class EmailAccount(Document):
if self.append_to not in valid_doctypes:
frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
+ def validate_smtp_conn(self):
+ if not self.smtp_server:
+ frappe.throw(_("SMTP Server is required"))
+
+ server = self.get_smtp_server()
+ return server.session
+
def before_save(self):
messages = []
as_list = 1
@@ -151,24 +182,6 @@ class EmailAccount(Document):
except Exception:
pass
- def check_smtp(self):
- """Checks SMTP settings."""
- if self.enable_outgoing:
- if not self.smtp_server:
- frappe.throw(_("{0} is required").format("SMTP Server"))
-
- server = SMTPServer(
- login = getattr(self, "login_id", None) or self.email_id,
- server=self.smtp_server,
- port=cint(self.smtp_port),
- use_tls=cint(self.use_tls),
- use_ssl=cint(self.use_ssl_for_outgoing)
- )
- if self.password and not self.no_smtp_authentication:
- server.password = self.get_password()
-
- server.sess
-
def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
"""Returns logged in POP3/IMAP connection object."""
if frappe.cache().get_value("workers:no-internet") == True:
@@ -231,7 +244,7 @@ class EmailAccount(Document):
return None
elif not in_receive and any(map(lambda t: t in message, auth_error_codes)):
- self.throw_invalid_credentials_exception()
+ SMTPServer.throw_invalid_credentials_exception()
else:
frappe.throw(cstr(e))
@@ -249,13 +262,142 @@ class EmailAccount(Document):
else:
raise
+ @property
+ def _password(self):
+ raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test)
+ return self.get_password(raise_exception=raise_exception)
+
+ @property
+ def default_sender(self):
+ return email.utils.formataddr((self.name, self.get("email_id")))
+
+ def is_exists_in_db(self):
+ """Some of the Email Accounts we create from configs and those doesn't exists in DB.
+ This is is to check the specific email account exists in DB or not.
+ """
+ return self.find_one_by_filters(name=self.name)
+
+ @classmethod
+ def from_record(cls, record):
+ email_account = frappe.new_doc(cls.DOCTYPE)
+ email_account.update(record)
+ return email_account
+
@classmethod
- def throw_invalid_credentials_exception(cls):
- frappe.throw(
- _("Incorrect email or password. Please check your login credentials."),
- exc=InvalidEmailCredentials,
- title=_("Invalid Credentials")
- )
+ def find(cls, name):
+ return frappe.get_doc(cls.DOCTYPE, name)
+
+ @classmethod
+ def find_one_by_filters(cls, **kwargs):
+ name = frappe.db.get_value(cls.DOCTYPE, kwargs)
+ return cls.find(name) if name else None
+
+ @classmethod
+ def find_from_config(cls):
+ config = cls.get_account_details_from_site_config()
+ return cls.from_record(config) if config else None
+
+ @classmethod
+ def create_dummy(cls):
+ return cls.from_record({"sender": "notifications@example.com"})
+
+ @classmethod
+ @raise_error_on_no_output(
+ keep_quiet = lambda: not cint(frappe.get_system_settings('setup_complete')),
+ error_message = OUTGOING_EMAIL_ACCOUNT_MISSING, error_type = frappe.OutgoingEmailError) # noqa
+ @cache_email_account('outgoing_email_account')
+ def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False):
+ """Find the outgoing Email account to use.
+
+ :param match_by_email: Find account using emailID
+ :param match_by_doctype: Find account by matching `Append To` doctype
+ :param _raise_error: This is used by raise_error_on_no_output decorator to raise error.
+ """
+ if match_by_email:
+ match_by_email = parse_addr(match_by_email)[1]
+ doc = cls.find_one_by_filters(enable_outgoing=1, email_id=match_by_email)
+ if doc:
+ return {match_by_email: doc}
+
+ if match_by_doctype:
+ doc = cls.find_one_by_filters(enable_outgoing=1, enable_incoming=1, append_to=match_by_doctype)
+ if doc:
+ return {match_by_doctype: doc}
+
+ doc = cls.find_default_outgoing()
+ if doc:
+ return {'default': doc}
+
+ @classmethod
+ def find_default_outgoing(cls):
+ """ Find default outgoing account.
+ """
+ doc = cls.find_one_by_filters(enable_outgoing=1, default_outgoing=1)
+ doc = doc or cls.find_from_config()
+ return doc or (are_emails_muted() and cls.create_dummy())
+
+ @classmethod
+ def find_incoming(cls, match_by_email=None, match_by_doctype=None):
+ """Find the incoming Email account to use.
+ :param match_by_email: Find account using emailID
+ :param match_by_doctype: Find account by matching `Append To` doctype
+ """
+ doc = cls.find_one_by_filters(enable_incoming=1, email_id=match_by_email)
+ if doc:
+ return doc
+
+ doc = cls.find_one_by_filters(enable_incoming=1, append_to=match_by_doctype)
+ if doc:
+ return doc
+
+ doc = cls.find_default_incoming()
+ return doc
+
+ @classmethod
+ def find_default_incoming(cls):
+ doc = cls.find_one_by_filters(enable_incoming=1, default_incoming=1)
+ return doc
+
+ @classmethod
+ def get_account_details_from_site_config(cls):
+ if not frappe.conf.get("mail_server"):
+ return {}
+
+ field_to_conf_name_map = {
+ 'smtp_server': {'conf_names': ('mail_server',)},
+ 'smtp_port': {'conf_names': ('mail_port',)},
+ 'use_tls': {'conf_names': ('use_tls', 'mail_login')},
+ 'login_id': {'conf_names': ('mail_login',)},
+ 'email_id': {'conf_names': ('auto_email_id', 'mail_login'), 'default': 'notifications@example.com'},
+ 'password': {'conf_names': ('mail_password',)},
+ 'always_use_account_email_id_as_sender':
+ {'conf_names': ('always_use_account_email_id_as_sender',), 'default': 0},
+ 'always_use_account_name_as_sender_name':
+ {'conf_names': ('always_use_account_name_as_sender_name',), 'default': 0},
+ 'name': {'conf_names': ('email_sender_name',), 'default': 'Frappe'},
+ 'from_site_config': {'default': True}
+ }
+
+ account_details = {}
+ for doc_field_name, d in field_to_conf_name_map.items():
+ conf_names, default = d.get('conf_names') or [], d.get('default')
+ value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)]
+ account_details[doc_field_name] = (value and value[0]) or default
+ return account_details
+
+ def sendmail_config(self):
+ return {
+ 'server': self.smtp_server,
+ 'port': cint(self.smtp_port),
+ 'login': getattr(self, "login_id", None) or self.email_id,
+ 'password': self._password,
+ 'use_ssl': cint(self.use_ssl_for_outgoing),
+ 'use_tls': cint(self.use_tls)
+ }
+
+ def get_smtp_server(self):
+ config = self.sendmail_config()
+ return SMTPServer(**config)
def handle_incoming_connect_error(self, description):
if test_internet():
@@ -642,6 +784,8 @@ class EmailAccount(Document):
def send_auto_reply(self, communication, email):
"""Send auto reply if set."""
+ from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
+
if self.enable_auto_reply:
set_incoming_outgoing_accounts(communication)
@@ -653,7 +797,7 @@ class EmailAccount(Document):
frappe.sendmail(recipients = [email.from_email],
sender = self.email_id,
reply_to = communication.incoming_email_account,
- subject = _("Re: ") + communication.subject,
+ subject = " ".join([_("Re:"), communication.subject]),
content = render_template(self.auto_reply_message or "", communication.as_dict()) or \
frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()),
reference_doctype = communication.reference_doctype,
diff --git a/frappe/email/doctype/email_domain/test_records.json b/frappe/email/doctype/email_domain/test_records.json
index 32bc66e150..a6ccc99f06 100644
--- a/frappe/email/doctype/email_domain/test_records.json
+++ b/frappe/email/doctype/email_domain/test_records.json
@@ -10,7 +10,8 @@
"incoming_port": "993",
"attachment_limit": "1",
"smtp_server": "smtp.test.com",
- "smtp_port": "587"
+ "smtp_port": "587",
+ "password": "password"
},
{
"doctype": "Email Account",
@@ -25,6 +26,7 @@
"incoming_port": "143",
"attachment_limit": "1",
"smtp_server": "smtp.test.com",
- "smtp_port": "587"
+ "smtp_port": "587",
+ "password": "password"
}
]
diff --git a/frappe/email/doctype/email_queue/email_queue.json b/frappe/email/doctype/email_queue/email_queue.json
index 4529ea8211..f251786c90 100644
--- a/frappe/email/doctype/email_queue/email_queue.json
+++ b/frappe/email/doctype/email_queue/email_queue.json
@@ -24,7 +24,8 @@
"unsubscribe_method",
"expose_recipients",
"attachments",
- "retry"
+ "retry",
+ "email_account"
],
"fields": [
{
@@ -139,13 +140,19 @@
"fieldtype": "Int",
"label": "Retry",
"read_only": 1
+ },
+ {
+ "fieldname": "email_account",
+ "fieldtype": "Link",
+ "label": "Email Account",
+ "options": "Email Account"
}
],
"icon": "fa fa-envelope",
"idx": 1,
"in_create": 1,
"links": [],
- "modified": "2020-07-17 15:58:15.369419",
+ "modified": "2021-04-29 06:33:25.191729",
"modified_by": "Administrator",
"module": "Email",
"name": "Email Queue",
diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index 267fbdfe9c..076dfc5417 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -2,15 +2,26 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt
-from __future__ import unicode_literals
+import traceback
+import json
+
+from rq.timeouts import JobTimeoutException
+import smtplib
+import quopri
+from email.parser import Parser
+
import frappe
-from frappe import _
+from frappe import _, safe_encode, task
from frappe.model.document import Document
-from frappe.email.queue import send_one
-from frappe.utils import now_datetime
-
+from frappe.email.queue import get_unsubcribed_url
+from frappe.email.email_body import add_attachment
+from frappe.utils import cint
+from email.policy import SMTPUTF8
+MAX_RETRY_COUNT = 3
class EmailQueue(Document):
+ DOCTYPE = 'Email Queue'
+
def set_recipients(self, recipients):
self.set("recipients", [])
for r in recipients:
@@ -30,6 +41,241 @@ class EmailQueue(Document):
duplicate.set_recipients(recipients)
return duplicate
+ @classmethod
+ def find(cls, name):
+ return frappe.get_doc(cls.DOCTYPE, name)
+
+ def update_db(self, commit=False, **kwargs):
+ frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+ if commit:
+ frappe.db.commit()
+
+ def update_status(self, status, commit=False, **kwargs):
+ self.update_db(status = status, commit = commit, **kwargs)
+ if self.communication:
+ communication_doc = frappe.get_doc('Communication', self.communication)
+ communication_doc.set_delivery_status(commit=commit)
+
+ @property
+ def cc(self):
+ return (self.show_as_cc and self.show_as_cc.split(",")) or []
+
+ @property
+ def to(self):
+ return [r.recipient for r in self.recipients if r.recipient not in self.cc]
+
+ @property
+ def attachments_list(self):
+ return json.loads(self.attachments) if self.attachments else []
+
+ def get_email_account(self):
+ from frappe.email.doctype.email_account.email_account import EmailAccount
+
+ if self.email_account:
+ return frappe.get_doc('Email Account', self.email_account)
+
+ return EmailAccount.find_outgoing(
+ match_by_email = self.sender, match_by_doctype = self.reference_doctype)
+
+ def is_to_be_sent(self):
+ return self.status in ['Not Sent','Partially Sent']
+
+ def can_send_now(self):
+ hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1)
+ if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue:
+ return False
+
+ return True
+
+ def send(self, is_background_task=False):
+ """ Send emails to recipients.
+ """
+ if not self.can_send_now():
+ frappe.db.rollback()
+ return
+
+ with SendMailContext(self, is_background_task) as ctx:
+ message = None
+ for recipient in self.recipients:
+ if not recipient.is_mail_to_be_sent():
+ continue
+
+ message = ctx.build_message(recipient.recipient)
+ if not frappe.flags.in_test:
+ ctx.smtp_session.sendmail(recipient.recipient, self.sender, message)
+ ctx.add_to_sent_list(recipient)
+
+ if frappe.flags.in_test:
+ frappe.flags.sent_mail = message
+ return
+
+ if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to:
+ ctx.email_account_doc.append_email_to_sent_folder(message)
+
+
+@task(queue = 'short')
+def send_mail(email_queue_name, is_background_task=False):
+ """This is equalent to EmqilQueue.send.
+
+ This provides a way to make sending mail as a background job.
+ """
+ record = EmailQueue.find(email_queue_name)
+ record.send(is_background_task=is_background_task)
+
+class SendMailContext:
+ def __init__(self, queue_doc: Document, is_background_task: bool = False):
+ self.queue_doc = queue_doc
+ self.is_background_task = is_background_task
+ self.email_account_doc = queue_doc.get_email_account()
+ self.smtp_server = self.email_account_doc.get_smtp_server()
+ self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()]
+
+ def __enter__(self):
+ self.queue_doc.update_status(status='Sending', commit=True)
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ exceptions = [
+ smtplib.SMTPServerDisconnected,
+ smtplib.SMTPAuthenticationError,
+ smtplib.SMTPRecipientsRefused,
+ smtplib.SMTPConnectError,
+ smtplib.SMTPHeloError,
+ JobTimeoutException
+ ]
+
+ self.smtp_server.quit()
+ self.log_exception(exc_type, exc_val, exc_tb)
+
+ if exc_type in exceptions:
+ email_status = (self.sent_to and 'Partially Sent') or 'Not Sent'
+ self.queue_doc.update_status(status = email_status, commit = True)
+ elif exc_type:
+ if self.queue_doc.retry < MAX_RETRY_COUNT:
+ update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1}
+ else:
+ update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'}
+ self.queue_doc.update_status(**update_fields, commit = True)
+ else:
+ email_status = self.is_mail_sent_to_all() and 'Sent'
+ email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent'
+ self.queue_doc.update_status(status = email_status, commit = True)
+
+ def log_exception(self, exc_type, exc_val, exc_tb):
+ if exc_type:
+ traceback_string = "".join(traceback.format_tb(exc_tb))
+ traceback_string += f"\n Queue Name: {self.queue_doc.name}"
+
+ if self.is_background_task:
+ frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string)
+ else:
+ frappe.log_error(message = traceback_string)
+
+ @property
+ def smtp_session(self):
+ if frappe.flags.in_test:
+ return
+ return self.smtp_server.session
+
+ def add_to_sent_list(self, recipient):
+ # Update recipient status
+ recipient.update_db(status='Sent', commit=True)
+ self.sent_to.append(recipient.recipient)
+
+ def is_mail_sent_to_all(self):
+ return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients])
+
+ def get_message_object(self, message):
+ return Parser(policy=SMTPUTF8).parsestr(message)
+
+ def message_placeholder(self, placeholder_key):
+ map = {
+ 'tracker': '',
+ 'unsubscribe_url': '',
+ 'cc': '',
+ 'recipient': '',
+ }
+ return map.get(placeholder_key)
+
+ def build_message(self, recipient_email):
+ """Build message specific to the recipient.
+ """
+ message = self.queue_doc.message
+ if not message:
+ return ""
+
+ message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str())
+ message = message.replace(self.message_placeholder('unsubscribe_url'),
+ self.get_unsubscribe_str(recipient_email))
+ message = message.replace(self.message_placeholder('cc'), self.get_receivers_str())
+ message = message.replace(self.message_placeholder('recipient'),
+ self.get_receipient_str(recipient_email))
+ message = self.include_attachments(message)
+ return message
+
+ def get_tracker_str(self):
+ tracker_url_html = \
+ '
'
+
+ message = ''
+ if frappe.conf.use_ssl and self.queue_doc.track_email_status:
+ message = quopri.encodestring(
+ tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
+ ).decode()
+ return message
+
+ def get_unsubscribe_str(self, recipient_email):
+ unsubscribe_url = ''
+ if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype:
+ doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name
+ unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email,
+ self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param)
+
+ return quopri.encodestring(unsubscribe_url.encode()).decode()
+
+ def get_receivers_str(self):
+ message = ''
+ if self.queue_doc.expose_recipients == "footer":
+ to_str = ', '.join(self.queue_doc.to)
+ cc_str = ', '.join(self.queue_doc.cc)
+ message = f"This email was sent to {to_str}"
+ message = message + f" and copied to {cc_str}" if cc_str else message
+ return message
+
+ def get_receipient_str(self, recipient_email):
+ message = ''
+ if self.queue_doc.expose_recipients != "header":
+ message = recipient_email
+ return message
+
+ def include_attachments(self, message):
+ message_obj = self.get_message_object(message)
+ attachments = self.queue_doc.attachments_list
+
+ for attachment in attachments:
+ if attachment.get('fcontent'):
+ continue
+
+ fid = attachment.get("fid")
+ if fid:
+ _file = frappe.get_doc("File", fid)
+ fcontent = _file.get_content()
+ attachment.update({
+ 'fname': _file.file_name,
+ 'fcontent': fcontent,
+ 'parent': message_obj
+ })
+ attachment.pop("fid", None)
+ add_attachment(**attachment)
+
+ elif attachment.get("print_format_attachment") == 1:
+ attachment.pop("print_format_attachment", None)
+ print_format_file = frappe.attach_print(**attachment)
+ print_format_file.update({"parent": message_obj})
+ add_attachment(**print_format_file)
+
+ return safe_encode(message_obj.as_string())
+
@frappe.whitelist()
def retry_sending(name):
doc = frappe.get_doc("Email Queue", name)
@@ -42,7 +288,9 @@ def retry_sending(name):
@frappe.whitelist()
def send_now(name):
- send_one(name, now=True)
+ record = EmailQueue.find(name)
+ if record:
+ record.send()
def on_doctype_update():
"""Add index in `tabCommunication` for `(reference_doctype, reference_name)`"""
diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
index 42956a1180..3f07ec58f3 100644
--- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
+++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
@@ -7,4 +7,16 @@ import frappe
from frappe.model.document import Document
class EmailQueueRecipient(Document):
- pass
+ DOCTYPE = 'Email Queue Recipient'
+
+ def is_mail_to_be_sent(self):
+ return self.status == 'Not Sent'
+
+ def is_main_sent(self):
+ return self.status == 'Sent'
+
+ def update_db(self, commit=False, **kwargs):
+ frappe.db.set_value(self.DOCTYPE, self.name, kwargs)
+ if commit:
+ frappe.db.commit()
+
diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json
index c1c877efd4..8b6900a3c9 100644
--- a/frappe/email/doctype/notification/notification.json
+++ b/frappe/email/doctype/notification/notification.json
@@ -102,7 +102,8 @@
"default": "0",
"fieldname": "is_standard",
"fieldtype": "Check",
- "label": "Is Standard"
+ "label": "Is Standard",
+ "no_copy": 1
},
{
"depends_on": "is_standard",
@@ -281,7 +282,7 @@
"icon": "fa fa-envelope",
"index_web_pages_for_search": 1,
"links": [],
- "modified": "2020-11-24 14:25:43.245677",
+ "modified": "2021-05-04 11:17:11.882314",
"modified_by": "Administrator",
"module": "Email",
"name": "Notification",
diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py
index 1190a6f00b..3b03c42b95 100755
--- a/frappe/email/email_body.py
+++ b/frappe/email/email_body.py
@@ -4,7 +4,7 @@
from __future__ import unicode_literals
import frappe, re, os
from frappe.utils.pdf import get_pdf
-from frappe.email.smtp import get_outgoing_email_account
+from frappe.email.doctype.email_account.email_account import EmailAccount
from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint,
split_emails, to_markdown, markdown, random_string, parse_addr)
import email.utils
@@ -75,7 +75,8 @@ class EMail:
self.bcc = bcc or []
self.html_set = False
- self.email_account = email_account or get_outgoing_email_account(sender=sender)
+ self.email_account = email_account or \
+ EmailAccount.find_outgoing(match_by_email=sender, _raise_error=True)
def set_html(self, message, text_content = None, footer=None, print_html=None,
formatted=None, inline_images=None, header=None):
@@ -249,8 +250,8 @@ class EMail:
def get_formatted_html(subject, message, footer=None, print_html=None,
email_account=None, header=None, unsubscribe_link=None, sender=None, with_container=False):
- if not email_account:
- email_account = get_outgoing_email_account(False, sender=sender)
+
+ email_account = email_account or EmailAccount.find_outgoing(match_by_email=sender)
signature = None
if "" not in message:
diff --git a/frappe/email/queue.py b/frappe/email/queue.py
index 2aff04edc9..52c91baf1c 100755
--- a/frappe/email/queue.py
+++ b/frappe/email/queue.py
@@ -7,7 +7,8 @@ import sys
from six.moves import html_parser as HTMLParser
import smtplib, quopri, json
from frappe import msgprint, _, safe_decode, safe_encode, enqueue
-from frappe.email.smtp import SMTPServer, get_outgoing_email_account
+from frappe.email.smtp import SMTPServer
+from frappe.email.doctype.email_account.email_account import EmailAccount
from frappe.email.email_body import get_email, get_formatted_html, add_attachment
from frappe.utils.verified_command import get_signed_params, verify_request
from html2text import html2text
@@ -73,7 +74,9 @@ def send(recipients=None, sender=None, subject=None, message=None, text_content=
if isinstance(send_after, int):
send_after = add_days(nowdate(), send_after)
- email_account = get_outgoing_email_account(True, append_to=reference_doctype, sender=sender)
+ email_account = EmailAccount.find_outgoing(
+ match_by_doctype=reference_doctype, match_by_email=sender, _raise_error=True)
+
if not sender or sender == "Administrator":
sender = email_account.default_sender
@@ -170,19 +173,19 @@ def add(recipients, sender, subject, **kwargs):
if not email_queue:
email_queue = get_email_queue([r], sender, subject, **kwargs)
if kwargs.get('now'):
- send_one(email_queue.name, now=True)
+ email_queue.send()
else:
duplicate = email_queue.get_duplicate([r])
duplicate.insert(ignore_permissions=True)
if kwargs.get('now'):
- send_one(duplicate.name, now=True)
+ duplicate.send()
frappe.db.commit()
else:
email_queue = get_email_queue(recipients, sender, subject, **kwargs)
if kwargs.get('now'):
- send_one(email_queue.name, now=True)
+ email_queue.send()
def get_email_queue(recipients, sender, subject, **kwargs):
'''Make Email Queue object'''
@@ -234,6 +237,9 @@ def get_email_queue(recipients, sender, subject, **kwargs):
', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent')
recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', [])))
+ email_account = kwargs.get('email_account')
+ email_account_name = email_account and email_account.is_exists_in_db() and email_account.name
+
e.set_recipients(recipients)
e.reference_doctype = kwargs.get('reference_doctype')
e.reference_name = kwargs.get('reference_name')
@@ -245,8 +251,8 @@ def get_email_queue(recipients, sender, subject, **kwargs):
e.send_after = kwargs.get('send_after')
e.show_as_cc = ",".join(kwargs.get('cc', []))
e.show_as_bcc = ",".join(kwargs.get('bcc', []))
+ e.email_account = email_account_name or None
e.insert(ignore_permissions=True)
-
return e
def get_emails_sent_this_month():
@@ -328,44 +334,25 @@ def return_unsubscribed_page(email, doctype, name):
indicator_color='green')
def flush(from_test=False):
- """flush email queue, every time: called from scheduler"""
- # additional check
-
- auto_commit = not from_test
+ """flush email queue, every time: called from scheduler
+ """
+ from frappe.email.doctype.email_queue.email_queue import send_mail
+ # To avoid running jobs inside unit tests
if frappe.are_emails_muted():
msgprint(_("Emails are muted"))
from_test = True
- smtpserver_dict = frappe._dict()
-
- for email in get_queue():
-
- if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
- break
-
- if email.name:
- smtpserver = smtpserver_dict.get(email.sender)
- if not smtpserver:
- smtpserver = SMTPServer()
- smtpserver_dict[email.sender] = smtpserver
+ if cint(frappe.defaults.get_defaults().get("hold_queue"))==1:
+ return
- if from_test:
- send_one(email.name, smtpserver, auto_commit)
- else:
- send_one_args = {
- 'email': email.name,
- 'smtpserver': smtpserver,
- 'auto_commit': auto_commit,
- }
- enqueue(
- method = 'frappe.email.queue.send_one',
- queue = 'short',
- **send_one_args
- )
+ for row in get_queue():
+ try:
+ func = send_mail if from_test else send_mail.enqueue
+ is_background_task = not from_test
+ func(email_queue_name = row.name, is_background_task = is_background_task)
+ except Exception:
+ frappe.log_error()
- # NOTE: removing commit here because we pass auto_commit
- # finally:
- # frappe.db.commit()
def get_queue():
return frappe.db.sql('''select
name, sender
@@ -378,213 +365,6 @@ def get_queue():
by priority desc, creation asc
limit 500''', { 'now': now_datetime() }, as_dict=True)
-
-def send_one(email, smtpserver=None, auto_commit=True, now=False):
- '''Send Email Queue with given smtpserver'''
-
- email = frappe.db.sql('''select
- name, status, communication, message, sender, reference_doctype,
- reference_name, unsubscribe_param, unsubscribe_method, expose_recipients,
- show_as_cc, add_unsubscribe_link, attachments, retry
- from
- `tabEmail Queue`
- where
- name=%s
- for update''', email, as_dict=True)
-
- if len(email):
- email = email[0]
- else:
- return
-
- recipients_list = frappe.db.sql('''select name, recipient, status from
- `tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1)
-
- if frappe.are_emails_muted():
- frappe.msgprint(_("Emails are muted"))
- return
-
- if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 :
- return
-
- if email.status not in ('Not Sent','Partially Sent') :
- # rollback to release lock and return
- frappe.db.rollback()
- return
-
- frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
- (now_datetime(), email.name), auto_commit=auto_commit)
-
- if email.communication:
- frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
- email_sent_to_any_recipient = None
-
- try:
- message = None
-
- if not frappe.flags.in_test:
- if not smtpserver:
- smtpserver = SMTPServer()
-
- # to avoid always using default email account for outgoing
- if getattr(frappe.local, "outgoing_email_account", None):
- frappe.local.outgoing_email_account = {}
-
- smtpserver.setup_email_account(email.reference_doctype, sender=email.sender)
-
- for recipient in recipients_list:
- if recipient.status != "Not Sent":
- continue
-
- message = prepare_message(email, recipient.recipient, recipients_list)
- if not frappe.flags.in_test:
- smtpserver.sess.sendmail(email.sender, recipient.recipient, message)
-
- recipient.status = "Sent"
- frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
- (now_datetime(), recipient.name), auto_commit=auto_commit)
-
- email_sent_to_any_recipient = any("Sent" == s.status for s in recipients_list)
-
- #if all are sent set status
- if email_sent_to_any_recipient:
- frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
- (now_datetime(), email.name), auto_commit=auto_commit)
- else:
- frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
- where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
- if frappe.flags.in_test:
- frappe.flags.sent_mail = message
- return
- if email.communication:
- frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
- if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient:
- smtpserver.email_account.append_email_to_sent_folder(message)
-
- except (smtplib.SMTPServerDisconnected,
- smtplib.SMTPConnectError,
- smtplib.SMTPHeloError,
- smtplib.SMTPAuthenticationError,
- smtplib.SMTPRecipientsRefused,
- JobTimeoutException):
-
- # bad connection/timeout, retry later
-
- if email_sent_to_any_recipient:
- frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
- (now_datetime(), email.name), auto_commit=auto_commit)
- else:
- frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
- (now_datetime(), email.name), auto_commit=auto_commit)
-
- if email.communication:
- frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
- # no need to attempt further
- return
-
- except Exception as e:
- frappe.db.rollback()
-
- if email.retry < 3:
- frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""",
- (now_datetime(), email.name), auto_commit=auto_commit)
- else:
- if email_sent_to_any_recipient:
- frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
- (text_type(e), email.name), auto_commit=auto_commit)
- else:
- frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
- where name=%s""", (text_type(e), email.name), auto_commit=auto_commit)
-
- if email.communication:
- frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
-
- if now:
- print(frappe.get_traceback())
- raise e
-
- else:
- # log to Error Log
- frappe.log_error('frappe.email.queue.flush')
-
-def prepare_message(email, recipient, recipients_list):
- message = email.message
- if not message:
- return ""
-
- # Parse "Email Account" from "Email Sender"
- email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender)
- if frappe.conf.use_ssl and email_account.track_email_status:
- # Using SSL => Publically available domain => Email Read Reciept Possible
- message = message.replace("", quopri.encodestring(''.format(frappe.local.site, email.communication).encode()).decode())
- else:
- # No SSL => No Email Read Reciept
- message = message.replace("", quopri.encodestring("".encode()).decode())
-
- if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
- unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
- email.unsubscribe_method, email.unsubscribe_params)
- message = message.replace("", quopri.encodestring(unsubscribe_url.encode()).decode())
-
- if email.expose_recipients == "header":
- pass
- else:
- if email.expose_recipients == "footer":
- if isinstance(email.show_as_cc, string_types):
- email.show_as_cc = email.show_as_cc.split(",")
- email_sent_to = [r.recipient for r in recipients_list]
- email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
- email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
-
- if email_sent_cc:
- email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
- else:
- email_sent_message = _("This email was sent to {0}").format(email_sent_to)
- message = message.replace("", quopri.encodestring(email_sent_message.encode()).decode())
-
- message = message.replace("", recipient)
-
- message = (message and message.encode('utf8')) or ''
- message = safe_decode(message)
-
- if PY3:
- from email.policy import SMTPUTF8
- message = Parser(policy=SMTPUTF8).parsestr(message)
- else:
- message = Parser().parsestr(message)
-
- if email.attachments:
- # On-demand attachments
-
- attachments = json.loads(email.attachments)
-
- for attachment in attachments:
- if attachment.get('fcontent'):
- continue
-
- fid = attachment.get("fid")
- if fid:
- _file = frappe.get_doc("File", fid)
- fcontent = _file.get_content()
- attachment.update({
- 'fname': _file.file_name,
- 'fcontent': fcontent,
- 'parent': message
- })
- attachment.pop("fid", None)
- add_attachment(**attachment)
-
- elif attachment.get("print_format_attachment") == 1:
- attachment.pop("print_format_attachment", None)
- print_format_file = frappe.attach_print(**attachment)
- print_format_file.update({"parent": message})
- add_attachment(**print_format_file)
-
- return safe_encode(message.as_string())
-
def clear_outbox(days=None):
"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
Note: Used separate query to avoid deadlock
diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index 9ba81fa146..3acb76af23 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -9,11 +9,24 @@ import _socket, sys
from frappe import _
from frappe.utils import cint, cstr, parse_addr
+CONNECTION_FAILED = _('Could not connect to outgoing email server')
+AUTH_ERROR_TITLE = _("Invalid Credentials")
+AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.")
+SOCKET_ERROR_TITLE = _("Incorrect Configuration")
+SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port")
+SEND_MAIL_FAILED = _("Unable to send emails at this time")
+EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
+
+class InvalidEmailCredentials(frappe.ValidationError):
+ pass
+
def send(email, append_to=None, retry=1):
"""Deprecated: Send the message or add it to Outbox Email"""
def _send(retry):
+ from frappe.email.doctype.email_account.email_account import EmailAccount
try:
- smtpserver = SMTPServer(append_to=append_to)
+ email_account = EmailAccount.find_outgoing(match_by_doctype=append_to)
+ smtpserver = email_account.get_smtp_server()
# validate is called in as_string
email_body = email.as_string()
@@ -34,224 +47,80 @@ def send(email, append_to=None, retry=1):
_send(retry)
-def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None):
- """Returns outgoing email account based on `append_to` or the default
- outgoing account. If default outgoing account is not found, it will
- try getting settings from `site_config.json`."""
-
- sender_email_id = None
- _email_account = None
-
- if sender:
- sender_email_id = parse_addr(sender)[1]
-
- if not getattr(frappe.local, "outgoing_email_account", None):
- frappe.local.outgoing_email_account = {}
-
- if not (frappe.local.outgoing_email_account.get(append_to)
- or frappe.local.outgoing_email_account.get(sender_email_id)
- or frappe.local.outgoing_email_account.get("default")):
- email_account = None
-
- if sender_email_id:
- # check if the sender has an email account with enable_outgoing
- email_account = _get_email_account({"enable_outgoing": 1,
- "email_id": sender_email_id})
-
- if not email_account and append_to:
- # append_to is only valid when enable_incoming is checked
- email_accounts = frappe.db.get_values("Email Account", {
- "enable_outgoing": 1,
- "enable_incoming": 1,
- "append_to": append_to,
- }, cache=True)
-
- if email_accounts:
- _email_account = email_accounts[0]
-
- else:
- email_account = _get_email_account({
- "enable_outgoing": 1,
- "enable_incoming": 1,
- "append_to": append_to
- })
-
- if not email_account:
- # sender don't have the outging email account
- sender_email_id = None
- email_account = get_default_outgoing_email_account(raise_exception_not_set=raise_exception_not_set)
-
- if not email_account and _email_account:
- # if default email account is not configured then setup first email account based on append to
- email_account = _email_account
-
- if not email_account and raise_exception_not_set and cint(frappe.db.get_single_value('System Settings', 'setup_complete')):
- frappe.throw(_("Please setup default Email Account from Setup > Email > Email Account"),
- frappe.OutgoingEmailError)
-
- if email_account:
- if email_account.enable_outgoing and not getattr(email_account, 'from_site_config', False):
- raise_exception = True
- if email_account.smtp_server in ['localhost','127.0.0.1'] or email_account.no_smtp_authentication:
- raise_exception = False
- email_account.password = email_account.get_password(raise_exception=raise_exception)
- email_account.default_sender = email.utils.formataddr((email_account.name, email_account.get("email_id")))
-
- frappe.local.outgoing_email_account[append_to or sender_email_id or "default"] = email_account
-
- return frappe.local.outgoing_email_account.get(append_to) \
- or frappe.local.outgoing_email_account.get(sender_email_id) \
- or frappe.local.outgoing_email_account.get("default")
-
-def get_default_outgoing_email_account(raise_exception_not_set=True):
- '''conf should be like:
- {
- "mail_server": "smtp.example.com",
- "mail_port": 587,
- "use_tls": 1,
- "mail_login": "emails@example.com",
- "mail_password": "Super.Secret.Password",
- "auto_email_id": "emails@example.com",
- "email_sender_name": "Example Notifications",
- "always_use_account_email_id_as_sender": 0,
- "always_use_account_name_as_sender_name": 0
- }
- '''
- email_account = _get_email_account({"enable_outgoing": 1, "default_outgoing": 1})
- if email_account:
- email_account.password = email_account.get_password(raise_exception=False)
-
- if not email_account and frappe.conf.get("mail_server"):
- # from site_config.json
- email_account = frappe.new_doc("Email Account")
- email_account.update({
- "smtp_server": frappe.conf.get("mail_server"),
- "smtp_port": frappe.conf.get("mail_port"),
-
- # legacy: use_ssl was used in site_config instead of use_tls, but meant the same thing
- "use_tls": cint(frappe.conf.get("use_tls") or 0) or cint(frappe.conf.get("use_ssl") or 0),
- "login_id": frappe.conf.get("mail_login"),
- "email_id": frappe.conf.get("auto_email_id") or frappe.conf.get("mail_login") or 'notifications@example.com',
- "password": frappe.conf.get("mail_password"),
- "always_use_account_email_id_as_sender": frappe.conf.get("always_use_account_email_id_as_sender", 0),
- "always_use_account_name_as_sender_name": frappe.conf.get("always_use_account_name_as_sender_name", 0)
- })
- email_account.from_site_config = True
- email_account.name = frappe.conf.get("email_sender_name") or "Frappe"
-
- if not email_account and not raise_exception_not_set:
- return None
-
- if frappe.are_emails_muted():
- # create a stub
- email_account = frappe.new_doc("Email Account")
- email_account.update({
- "email_id": "notifications@example.com"
- })
-
- return email_account
-
-def _get_email_account(filters):
- name = frappe.db.get_value("Email Account", filters)
- return frappe.get_doc("Email Account", name) if name else None
-
class SMTPServer:
- def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None):
- # get defaults from mail settings
-
- self._sess = None
- self.email_account = None
- self.server = None
- self.append_emails_to_sent_folder = None
+ def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None):
+ self.login = login
+ self.password = password
+ self._server = server
+ self._port = port
+ self.use_tls = use_tls
+ self.use_ssl = use_ssl
+ self._session = None
+
+ if not self.server:
+ frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError)
- if server:
- self.server = server
- self.port = port
- self.use_tls = cint(use_tls)
- self.use_ssl = cint(use_ssl)
- self.login = login
- self.password = password
+ @property
+ def port(self):
+ port = self._port or (self.use_ssl and 465) or (self.use_tls and 587)
+ return cint(port)
- else:
- self.setup_email_account(append_to)
+ @property
+ def server(self):
+ return cstr(self._server or "")
- def setup_email_account(self, append_to=None, sender=None):
- self.email_account = get_outgoing_email_account(raise_exception_not_set=False, append_to=append_to, sender=sender)
- if self.email_account:
- self.server = self.email_account.smtp_server
- self.login = (getattr(self.email_account, "login_id", None) or self.email_account.email_id)
- if not self.email_account.no_smtp_authentication:
- if self.email_account.ascii_encode_password:
- self.password = frappe.safe_encode(self.email_account.password, 'ascii')
- else:
- self.password = self.email_account.password
- else:
- self.password = None
- self.port = self.email_account.smtp_port
- self.use_tls = self.email_account.use_tls
- self.sender = self.email_account.email_id
- self.use_ssl = self.email_account.use_ssl_for_outgoing
- self.append_emails_to_sent_folder = self.email_account.append_emails_to_sent_folder
- self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender"))
- self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name"))
+ def secure_session(self, conn):
+ """Secure the connection incase of TLS.
+ """
+ if self.use_tls:
+ conn.ehlo()
+ conn.starttls()
+ conn.ehlo()
@property
- def sess(self):
- """get session"""
- if self._sess:
- return self._sess
+ def session(self):
+ if self.is_session_active():
+ return self._session
- # check if email server specified
- if not getattr(self, 'server'):
- err_msg = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account')
- frappe.msgprint(err_msg)
- raise frappe.OutgoingEmailError(err_msg)
+ SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP
try:
- if self.use_ssl:
- if not self.port:
- self.port = 465
-
- self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port))
- else:
- if self.use_tls and not self.port:
- self.port = 587
-
- self._sess = smtplib.SMTP(cstr(self.server or ""),
- cint(self.port) or None)
-
- if not self._sess:
- err_msg = _('Could not connect to outgoing email server')
- frappe.msgprint(err_msg)
- raise frappe.OutgoingEmailError(err_msg)
-
- if self.use_tls:
- self._sess.ehlo()
- self._sess.starttls()
- self._sess.ehlo()
+ self._session = SMTP(self.server, self.port)
+ if not self._session:
+ frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError)
+ self.secure_session(self._session)
if self.login and self.password:
- ret = self._sess.login(str(self.login or ""), str(self.password or ""))
+ res = self._session.login(str(self.login or ""), str(self.password or ""))
# check if logged correctly
- if ret[0]!=235:
- frappe.msgprint(ret[1])
- raise frappe.OutgoingEmailError(ret[1])
+ if res[0]!=235:
+ frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError)
- return self._sess
+ return self._session
except smtplib.SMTPAuthenticationError as e:
- from frappe.email.doctype.email_account.email_account import EmailAccount
- EmailAccount.throw_invalid_credentials_exception()
+ self.throw_invalid_credentials_exception()
except _socket.error as e:
# Invalid mail server -- due to refusing connection
- frappe.throw(
- _("Invalid Outgoing Mail Server or Port"),
- exc=frappe.ValidationError,
- title=_("Incorrect Configuration")
- )
+ frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE)
except smtplib.SMTPException:
- frappe.msgprint(_('Unable to send emails at this time'))
+ frappe.msgprint(SEND_MAIL_FAILED)
raise
+
+ def is_session_active(self):
+ if self._session:
+ try:
+ return self._session.noop()[0] == 250
+ except Exception:
+ return False
+
+ def quit(self):
+ if self.is_session_active():
+ self._session.quit()
+
+ @classmethod
+ def throw_invalid_credentials_exception(cls):
+ frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials)
diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py
index 3fcabb9495..33668cddba 100644
--- a/frappe/email/test_email_body.py
+++ b/frappe/email/test_email_body.py
@@ -7,10 +7,10 @@ from frappe import safe_decode
from frappe.email.receive import Email
from frappe.email.email_body import (replace_filename_with_cid,
get_email, inline_style_in_html, get_header)
-from frappe.email.queue import prepare_message, get_email_queue
+from frappe.email.queue import get_email_queue
+from frappe.email.doctype.email_queue.email_queue import SendMailContext
from six import PY3
-
class TestEmailBody(unittest.TestCase):
def setUp(self):
email_html = '''
@@ -57,7 +57,8 @@ This is the text version of this email
content='' + uni_chr1 + 'abcd' + uni_chr2 + '
',
formatted='' + uni_chr1 + 'abcd' + uni_chr2 + '
',
text_content='whatever')
- result = prepare_message(email=email, recipient='test@test.com', recipients_list=[])
+ mail_ctx = SendMailContext(queue_doc = email)
+ result = mail_ctx.build_message(recipient_email = 'test@test.com')
self.assertTrue(b"=EA=80=80abcd=DE=B4
" in result)
def test_prepare_message_returns_cr_lf(self):
@@ -68,8 +69,10 @@ This is the text version of this email
content='\n this is a test of newlines\n' + '
',
formatted='\n this is a test of newlines\n' + '
',
text_content='whatever')
- result = safe_decode(prepare_message(email=email,
- recipient='test@test.com', recipients_list=[]))
+
+ mail_ctx = SendMailContext(queue_doc = email)
+ result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com'))
+
if PY3:
self.assertTrue(result.count('\n') == result.count("\r"))
else:
diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py
index 0b11c559a2..58e4fdd8a6 100644
--- a/frappe/email/test_smtp.py
+++ b/frappe/email/test_smtp.py
@@ -4,7 +4,7 @@
import unittest
import frappe
from frappe.email.smtp import SMTPServer
-from frappe.email.smtp import get_outgoing_email_account
+from frappe.email.doctype.email_account.email_account import EmailAccount
class TestSMTP(unittest.TestCase):
def test_smtp_ssl_session(self):
@@ -33,13 +33,13 @@ class TestSMTP(unittest.TestCase):
frappe.local.outgoing_email_account = {}
# lowest preference given to email account with default incoming enabled
- create_email_account(email_id="default_outgoing_enabled@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1)
- self.assertEqual(get_outgoing_email_account().email_id, "default_outgoing_enabled@gmail.com")
+ create_email_account(email_id="default_outgoing_enabled@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1)
+ self.assertEqual(EmailAccount.find_outgoing().email_id, "default_outgoing_enabled@gmail.com")
frappe.local.outgoing_email_account = {}
# highest preference given to email account with append_to matching
- create_email_account(email_id="append_to@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
- self.assertEqual(get_outgoing_email_account(append_to="Blog Post").email_id, "append_to@gmail.com")
+ create_email_account(email_id="append_to@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post")
+ self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com")
# add back the mail_server
frappe.conf['mail_server'] = mail_server
@@ -75,4 +75,4 @@ def make_server(port, ssl, tls):
use_tls = tls
)
- server.sess
+ server.session
diff --git a/frappe/handler.py b/frappe/handler.py
index a38feb90fa..b622667e18 100755
--- a/frappe/handler.py
+++ b/frappe/handler.py
@@ -228,10 +228,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
is_whitelisted(fn)
is_valid_http_method(fn)
- try:
- fnargs = inspect.getargspec(method_obj)[0]
- except ValueError:
- fnargs = inspect.getfullargspec(method_obj).args
+ fnargs = inspect.getfullargspec(method_obj).args
if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"):
response = doc.run_method(method)
diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 1c863a1577..e0c3406c46 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -465,7 +465,7 @@ class DatabaseQuery(object):
elif f.operator.lower() in ('in', 'not in'):
values = f.value or ''
- if isinstance(values, frappe.string_types):
+ if isinstance(values, str):
values = values.split(",")
fallback = "''"
diff --git a/frappe/model/document.py b/frappe/model/document.py
index 4169919091..623916597e 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -1347,6 +1347,22 @@ class Document(BaseDocument):
from frappe.desk.doctype.tag.tag import DocTags
return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
+ def __repr__(self):
+ name = self.name or "unsaved"
+ doctype = self.__class__.__name__
+
+ docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
+ parent = f" parent={self.parent}" if self.parent else ""
+
+ return f"<{doctype}: {name}{docstatus}{parent}>"
+
+ def __str__(self):
+ name = self.name or "unsaved"
+ doctype = self.__class__.__name__
+
+ return f"{doctype}({name})"
+
+
def execute_action(doctype, name, action, **kwargs):
"""Execute an action on a document (called by background worker)"""
doc = frappe.get_doc(doctype, name)
diff --git a/frappe/model/meta.py b/frappe/model/meta.py
index 7f58c28397..66e8b08d92 100644
--- a/frappe/model/meta.py
+++ b/frappe/model/meta.py
@@ -118,7 +118,7 @@ class Meta(Document):
# non standard list object, skip
continue
- if (isinstance(value, (frappe.text_type, int, float, datetime, list, tuple))
+ if (isinstance(value, (str, int, float, datetime, list, tuple))
or (not no_nulls and value is None)):
out[key] = value
diff --git a/frappe/patches.txt b/frappe/patches.txt
index 60c3112f4a..e70be0a37b 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -336,3 +336,4 @@ frappe.patches.v13_0.remove_twilio_settings
frappe.patches.v12_0.rename_uploaded_files_with_proper_name
frappe.patches.v13_0.queryreport_columns
frappe.patches.v13_0.jinja_hook
+frappe.patches.v13_0.update_notification_channel_if_empty
diff --git a/frappe/patches/v13_0/update_notification_channel_if_empty.py b/frappe/patches/v13_0/update_notification_channel_if_empty.py
new file mode 100644
index 0000000000..2c2a40e81b
--- /dev/null
+++ b/frappe/patches/v13_0/update_notification_channel_if_empty.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+def execute():
+
+ frappe.reload_doc("Email", "doctype", "Notification")
+
+ notifications = frappe.get_all('Notification', {'is_standard': 1}, {'name', 'channel'})
+ for notification in notifications:
+ if not notification.channel:
+ frappe.db.set_value("Notification", notification.name, "channel", "Email", update_modified=False)
+ frappe.db.commit()
diff --git a/frappe/public/js/frappe/dom.js b/frappe/public/js/frappe/dom.js
index db9407ed53..2769e9061d 100644
--- a/frappe/public/js/frappe/dom.js
+++ b/frappe/public/js/frappe/dom.js
@@ -319,7 +319,7 @@ frappe.get_data_pill = (label, target_id=null, remove_action=null, image=null) =
frappe.get_modal = function(title, content) {
return $(`