Merge branch 'develop' into hide-notification-on-click

cypress/integration/control_link.js

@@ -58,6 +58,23 @@ context('Control Link', () => {
 		cy.get('.frappe-control[data-fieldname=link] input').should('have.value', '');
 	});
 
+	it("should be possible set empty value explicitly", () => {
+		get_dialog_with_link().as("dialog");
+
+		cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link");
+
+		cy.get(".frappe-control[data-fieldname=link] input")
+			.type("  ", { delay: 100 })
+			.blur();
+		cy.wait("@validate_link");
+		cy.get(".frappe-control[data-fieldname=link] input").should("have.value", "");
+		cy.window()
+			.its("cur_dialog")
+			.then((dialog) => {
+				expect(dialog.get_value("link")).to.equal('');
+			});
+	});
+
 	it('should route to form on arrow click', () => {
 		get_dialog_with_link().as('dialog');
 
@@ -78,7 +95,7 @@ context('Control Link', () => {
 		});
 	});
 
-	it('should fetch valid value', () => {
+	it('should update dependant fields (via fetch_from)', () => {
 		cy.get('@todos').then(todos => {
 			cy.visit(`/app/todo/${todos[0]}`);
 			cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link');
@@ -89,7 +106,67 @@ context('Control Link', () => {
 			cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should(
 				'contain', 'Administrator'
 			);
+
+			cy.window()
+				.its("cur_frm.doc.assigned_by")
+				.should("eq", "Administrator");
+
+			// invalid input
+			cy.get('@input').clear().type('invalid input', {delay: 100}).blur();
+			cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should(
+				'contain', ''
+			);
+
+			cy.window()
+				.its("cur_frm.doc.assigned_by")
+				.should("eq", null);
+
+			// set valid value again
+			cy.get('@input').clear().type('Administrator', {delay: 100}).blur();
+			cy.wait('@validate_link');
+
+			cy.window()
+				.its("cur_frm.doc.assigned_by")
+				.should("eq", "Administrator");
+
+			// clear input
+			cy.get('@input').clear().blur();
+			cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should(
+				'contain', ''
+			);
+
+			cy.window()
+				.its("cur_frm.doc.assigned_by")
+				.should("eq", "");
 		});
 	});
 
+	it("should set default values", () => {
+		cy.insert_doc("Property Setter", {
+			"doctype_or_field": "DocField",
+			"doc_type": "ToDo",
+			"field_name": "assigned_by",
+			"property": "default",
+			"property_type": "Text",
+			"value": "Administrator"
+		}, true);
+		cy.reload();
+		cy.new_form("ToDo");
+		cy.fill_field("description", "new", "Text Editor");
+		cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form");
+		cy.findByRole("button", {name: "Save"}).click();
+		cy.wait("@save_form");
+		cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should(
+			"contain", "Administrator"
+		);
+		// if user clears default value explicitly, system should not reset default again
+		cy.get_field("assigned_by").clear().blur();
+		cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form");
+		cy.findByRole("button", {name: "Save"}).click();
+		cy.wait("@save_form");
+		cy.get_field("assigned_by").should("have.value", "");
+		cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should(
+			"contain", ""
+		);
+	});
 });

cypress/support/commands.js

@@ -110,34 +110,6 @@ Cypress.Commands.add('get_doc', (doctype, name) => {
 		});
 });
 
-Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => {
-	return cy
-		.window()
-		.its('frappe.csrf_token')
-		.then(csrf_token => {
-			return cy
-				.request({
-					method: 'POST',
-					url: `/api/resource/${doctype}`,
-					body: args,
-					headers: {
-						Accept: 'application/json',
-						'Content-Type': 'application/json',
-						'X-Frappe-CSRF-Token': csrf_token
-					},
-					failOnStatusCode: !ignore_duplicate
-				})
-				.then(res => {
-					let status_codes = [200];
-					if (ignore_duplicate) {
-						status_codes.push(409);
-					}
-					expect(res.status).to.be.oneOf(status_codes);
-					return res.body;
-				});
-		});
-});
-
 Cypress.Commands.add('remove_doc', (doctype, name) => {
 	return cy
 		.window()

frappe/__init__.py

@@ -143,6 +143,8 @@ lang = local("lang")
 # This if block is never executed when running the code. It is only used for
 # telling static code analyzer where to find dynamically defined attributes.
 if typing.TYPE_CHECKING:
+	from frappe.utils.redis_wrapper import RedisWrapper
+
 	from frappe.database.mariadb.database import MariaDBDatabase
 	from frappe.database.postgres.database import PostgresDatabase
 	from frappe.query_builder.builder import MariaDB, Postgres
@@ -150,6 +152,7 @@ if typing.TYPE_CHECKING:
 	db: typing.Union[MariaDBDatabase, PostgresDatabase]
 	qb: typing.Union[MariaDB, Postgres]
 
+
 # end: static analysis hack
 
 def init(site, sites_path=None, new_site=False):
@@ -311,9 +314,8 @@ def destroy():
 
 	release_local(local)
 
-# memcache
 redis_server = None
-def cache():
+def cache() -> "RedisWrapper":
 	"""Returns redis connection."""
 	global redis_server
 	if not redis_server:

frappe/api.py

@@ -94,7 +94,8 @@ def handle():
 						"data": doc.save().as_dict()
 					})
 
-					if doc.parenttype and doc.parent:
+					# check for child table doctype
+					if doc.get("parenttype"):
 						frappe.get_doc(doc.parenttype, doc.parent).save()
 
 					frappe.db.commit()

frappe/app.py

@@ -192,12 +192,7 @@ def make_form_dict(request):
 	if not isinstance(args, dict):
 		frappe.throw(_("Invalid request arguments"))
 
-	try:
-		frappe.local.form_dict = frappe._dict({
-			k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items()
-		})
-	except IndexError:
-		frappe.local.form_dict = frappe._dict(args)
+	frappe.local.form_dict = frappe._dict(args)
 
 	if "_" in frappe.local.form_dict:
 		# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict

frappe/boot.py

@@ -7,6 +7,7 @@ bootstrap client session
 import frappe
 import frappe.defaults
 import frappe.desk.desk_page
+from frappe.desk.doctype.route_history.route_history import frequently_visited_links
 from frappe.desk.form.load import get_meta_bundle
 from frappe.utils.change_log import get_versions
 from frappe.translate import get_lang_dict
@@ -15,7 +16,6 @@ from frappe.social.doctype.energy_point_settings.energy_point_settings import is
 from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled
 from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points
 from frappe.model.base_document import get_controller
-from frappe.social.doctype.post.post import frequently_visited_links
 from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo
 from frappe.utils import get_time_zone, add_user_info
 

frappe/client.py

@@ -99,7 +99,6 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
 	if not filters:
 		filters = None
 
-
 	if frappe.get_meta(doctype).issingle:
 		value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug)
 	else:
@@ -129,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None):
 	:param fieldname: fieldname string or JSON / dict with key value pair
 	:param value: value if fieldname is JSON / dict'''
 
-	if fieldname!="idx" and fieldname in frappe.model.default_fields:
+	if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
 		frappe.throw(_("Cannot edit standard fields"))
 
 	if not value:
@@ -142,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None):
 	else:
 		values = {fieldname: value}
 
-	doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
-	if doc and doc.parent and doc.parenttype:
+	# check for child table doctype
+	if not frappe.get_meta(doctype).istable:
+		doc = frappe.get_doc(doctype, name)
+		doc.update(values)
+	else:
+		doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
 		doc = frappe.get_doc(doc.parenttype, doc.parent)
 		child = doc.getone({"doctype": doctype, "name": name})
 		child.update(values)
-	else:
-		doc = frappe.get_doc(doctype, name)
-		doc.update(values)
 
 	doc.save()
 
@@ -163,10 +163,10 @@ def insert(doc=None):
 	if isinstance(doc, str):
 		doc = json.loads(doc)
 
-	if doc.get("parent") and doc.get("parenttype"):
+	if doc.get("parenttype"):
 		# inserting a child record
-		parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
-		parent.append(doc.get("parentfield"), doc)
+		parent = frappe.get_doc(doc.parenttype, doc.parent)
+		parent.append(doc.parentfield, doc)
 		parent.save()
 		return parent.as_dict()
 	else:
@@ -187,10 +187,10 @@ def insert_many(docs=None):
 		frappe.throw(_('Only 200 inserts allowed in one request'))
 
 	for doc in docs:
-		if doc.get("parent") and doc.get("parenttype"):
+		if doc.get("parenttype"):
 			# inserting a child record
-			parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
-			parent.append(doc.get("parentfield"), doc)
+			parent = frappe.get_doc(doc.parenttype, doc.parent)
+			parent.append(doc.parentfield, doc)
 			parent.save()
 			out.append(parent.name)
 		else:

frappe/commands/utils.py

@@ -623,6 +623,7 @@ def transform_database(context, table, engine, row_format, failfast):
 @click.command('run-tests')
 @click.option('--app', help="For App")
 @click.option('--doctype', help="For DocType")
+@click.option('--case', help="Select particular TestCase")
 @click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt")
 @click.option('--test', multiple=True, help="Specific test")
 @click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests")
@@ -636,7 +637,7 @@ def transform_database(context, table, engine, row_format, failfast):
 @pass_context
 def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False,
 		coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None,
-		skip_test_records=False, skip_before_tests=False, failfast=False):
+		skip_test_records=False, skip_before_tests=False, failfast=False, case=None):
 
 	with CodeCoverage(coverage, app):
 		import frappe.test_runner
@@ -658,7 +659,7 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 
 		ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
 			force=context.force, profile=profile, junit_xml_output=junit_xml_output,
-			ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast)
+			ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast, case=case)
 
 		if len(ret.failures) == 0 and len(ret.errors) == 0:
 			ret = 0

frappe/core/doctype/communication/email.py

@@ -1,29 +1,32 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import frappe
 import json
-from email.utils import formataddr
-from frappe.core.utils import get_parent_doc
-from frappe.utils import (get_url, get_formatted_email, cint, list_to_str,
-	validate_email_address, split_emails, parse_addr, get_datetime)
-from frappe.email.email_body import get_message_id
+from typing import TYPE_CHECKING, Dict
+
+import frappe
 import frappe.email.smtp
-import time
 from frappe import _
-from frappe.utils.background_jobs import enqueue
+from frappe.email.email_body import get_message_id
+from frappe.utils import (cint, get_datetime, get_formatted_email,
+	list_to_str, split_emails, validate_email_address)
+
+if TYPE_CHECKING:
+	from frappe.core.doctype.communication.communication import Communication
+
 
 OUTGOING_EMAIL_ACCOUNT_MISSING = _("""
 	Unable to send mail because of a missing email account.
 	Please setup default Email Account from Setup > Email > Email Account
 """)
 
+
 @frappe.whitelist()
 def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
 	sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False,
 	print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None,
 	flags=None, read_receipt=None, print_letterhead=True, email_template=None, communication_type=None,
-	ignore_permissions=False):
+	ignore_permissions=False) -> Dict[str, str]:
 	"""Make a new communication.
 
 	:param doctype: Reference DocType.
@@ -56,7 +59,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 	cc = list_to_str(cc) if isinstance(cc, list) else cc
 	bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc
 
-	comm = frappe.get_doc({
+	comm: "Communication" = frappe.get_doc({
 		"doctype":"Communication",
 		"subject": subject,
 		"content": content,
@@ -73,16 +76,13 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 		"message_id":get_message_id().strip(" <>"),
 		"read_receipt":read_receipt,
 		"has_attachment": 1 if attachments else 0,
-		"communication_type": communication_type
+		"communication_type": communication_type,
 	}).insert(ignore_permissions=True)
 
-	comm.save(ignore_permissions=True)
-
-	if isinstance(attachments, str):
-		attachments = json.loads(attachments)
-
 	# if not committed, delayed task doesn't find the communication
 	if attachments:
+		if isinstance(attachments, str):
+			attachments = json.loads(attachments)
 		add_attachments(comm.name, attachments)
 
 	if cint(send_email):
@@ -93,12 +93,13 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 			send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead)
 
 	emails_not_sent_to = comm.exclude_emails_list(include_sender=send_me_a_copy)
+
 	return {
 		"name": comm.name,
-		"emails_not_sent_to": ", ".join(emails_not_sent_to or [])
+		"emails_not_sent_to": ", ".join(emails_not_sent_to)
 	}
 
-def validate_email(doc):
+def validate_email(doc: "Communication") -> None:
 	"""Validate Email Addresses of Recipients and CC"""
 	if not (doc.communication_type=="Communication" and doc.communication_medium == "Email") or doc.flags.in_receive:
 		return
@@ -114,8 +115,6 @@ def validate_email(doc):
 	for email in split_emails(doc.bcc):
 		validate_email_address(email, throw=True)
 
-	# validate sender
-
 def set_incoming_outgoing_accounts(doc):
 	from frappe.email.doctype.email_account.email_account import EmailAccount
 	incoming_email_account = EmailAccount.find_incoming(

frappe/core/doctype/communication/mixins.py

@@ -1,3 +1,4 @@
+from typing import List
 import frappe
 from frappe import _
 from frappe.core.utils import get_parent_doc
@@ -194,14 +195,18 @@ class CommunicationEmailMixin:
 			return _("Leave this conversation")
 		return ''
 
-	def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False):
+	def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False) -> List:
 		"""List of mail id's excluded while sending mail.
 		"""
 		all_ids = self.get_all_email_addresses(exclude_displayname=True)
-		final_ids = self.mail_recipients(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
-			self.mail_bcc(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
-			self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender)
-		return set(all_ids) - set(final_ids)
+
+		final_ids = (
+			self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)
+			+ self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation)
+			+ self.mail_cc(is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender)
+		)
+
+		return list(set(all_ids) - set(final_ids))
 
 	def get_assignees(self):
 		"""Get owners of the reference document.

frappe/core/doctype/data_import/data_import.js

@@ -44,6 +44,7 @@ frappe.ui.form.on('Data Import', {
 			}
 			frm.dashboard.show_progress(__('Import Progress'), percent, message);
 			frm.page.set_indicator(__('In Progress'), 'orange');
+			frm.trigger('update_primary_action');
 
 			// hide progress when complete
 			if (data.current === data.total) {
@@ -80,7 +81,10 @@ frappe.ui.form.on('Data Import', {
 		frm.trigger('show_import_log');
 		frm.trigger('show_import_warnings');
 		frm.trigger('toggle_submit_after_import');
-		frm.trigger('show_import_status');
+
+		if (frm.doc.status != 'Pending')
+			frm.trigger('show_import_status');
+
 		frm.trigger('show_report_error_button');
 
 		if (frm.doc.status === 'Partial Success') {
@@ -128,40 +132,49 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	show_import_status(frm) {
-		let import_log = JSON.parse(frm.doc.import_log || '[]');
-		let successful_records = import_log.filter(log => log.success);
-		let failed_records = import_log.filter(log => !log.success);
-		if (successful_records.length === 0) return;
-
-		let message;
-		if (failed_records.length === 0) {
-			let message_args = [successful_records.length];
-			if (frm.doc.import_type === 'Insert New Records') {
-				message =
-					successful_records.length > 1
-						? __('Successfully imported {0} records.', message_args)
-						: __('Successfully imported {0} record.', message_args);
-			} else {
-				message =
-					successful_records.length > 1
-						? __('Successfully updated {0} records.', message_args)
-						: __('Successfully updated {0} record.', message_args);
-			}
-		} else {
-			let message_args = [successful_records.length, import_log.length];
-			if (frm.doc.import_type === 'Insert New Records') {
-				message =
-					successful_records.length > 1
-						? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
-						: __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
-			} else {
-				message =
-					successful_records.length > 1
-						? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
-						: __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
+		frappe.call({
+			'method': 'frappe.core.doctype.data_import.data_import.get_import_status',
+			'args': {
+				'data_import_name': frm.doc.name
+			},
+			'callback': function(r) {
+				let successful_records = cint(r.message.success);
+				let failed_records = cint(r.message.failed);
+				let total_records = cint(r.message.total_records);
+
+				if (!total_records) return;
+
+				let message;
+				if (failed_records === 0) {
+					let message_args = [successful_records];
+					if (frm.doc.import_type === 'Insert New Records') {
+						message =
+							successful_records > 1
+								? __('Successfully imported {0} records.', message_args)
+								: __('Successfully imported {0} record.', message_args);
+					} else {
+						message =
+							successful_records > 1
+								? __('Successfully updated {0} records.', message_args)
+								: __('Successfully updated {0} record.', message_args);
+					}
+				} else {
+					let message_args = [successful_records, total_records];
+					if (frm.doc.import_type === 'Insert New Records') {
+						message =
+							successful_records > 1
+								? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
+								: __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
+					} else {
+						message =
+							successful_records > 1
+								? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
+								: __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
+					}
+				}
+				frm.dashboard.set_headline(message);
 			}
-		}
-		frm.dashboard.set_headline(message);
+		});
 	},
 
 	show_report_error_button(frm) {
@@ -275,7 +288,7 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	show_import_preview(frm, preview_data) {
-		let import_log = JSON.parse(frm.doc.import_log || '[]');
+		let import_log = preview_data.import_log;
 
 		if (
 			frm.import_preview &&
@@ -316,6 +329,15 @@ frappe.ui.form.on('Data Import', {
 		);
 	},
 
+	export_import_log(frm) {
+		open_url_post(
+			'/api/method/frappe.core.doctype.data_import.data_import.download_import_log',
+			{
+				data_import_name: frm.doc.name
+			}
+		);
+	},
+
 	show_import_warnings(frm, preview_data) {
 		let columns = preview_data.columns;
 		let warnings = JSON.parse(frm.doc.template_warnings || '[]');
@@ -391,92 +413,131 @@ frappe.ui.form.on('Data Import', {
 		frm.trigger('show_import_log');
 	},
 
-	show_import_log(frm) {
-		let import_log = JSON.parse(frm.doc.import_log || '[]');
-		let logs = import_log;
-		frm.toggle_display('import_log', false);
-		frm.toggle_display('import_log_section', logs.length > 0);
+	render_import_log(frm) {
+		frappe.call({
+			'method': 'frappe.client.get_list',
+			'args': {
+				'doctype': 'Data Import Log',
+				'filters': {
+					'data_import': frm.doc.name
+				},
+				'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'],
+				'limit_page_length': 5000,
+				'order_by': 'log_index'
+			},
+			callback: function(r) {
+				let logs = r.message;
+
+				if (logs.length === 0) return;
+
+				frm.toggle_display('import_log_section', true);
+
+				let rows = logs
+					.map(log => {
+						let html = '';
+						if (log.success) {
+							if (frm.doc.import_type === 'Insert New Records') {
+								html = __('Successfully imported {0}', [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`
+								]);
+							} else {
+								html = __('Successfully updated {0}', [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`
+								]);
+							}
+						} else {
+							let messages = (JSON.parse(log.messages || '[]'))
+								.map(JSON.parse)
+								.map(m => {
+									let title = m.title ? `<strong>${m.title}</strong>` : '';
+									let message = m.message ? `<div>${m.message}</div>` : '';
+									return title + message;
+								})
+								.join('');
+							let id = frappe.dom.get_unique_id();
+							html = `${messages}
+								<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
+									${__('Show Traceback')}
+								</button>
+								<div class="collapse" id="${id}" style="margin-top: 15px;">
+									<div class="well">
+										<pre>${log.exception}</pre>
+									</div>
+								</div>`;
+						}
+						let indicator_color = log.success ? 'green' : 'red';
+						let title = log.success ? __('Success') : __('Failure');
 
-		if (logs.length === 0) {
-			frm.get_field('import_log_preview').$wrapper.empty();
-			return;
-		}
+						if (frm.doc.show_failed_logs && log.success) {
+							return '';
+						}
 
-		let rows = logs
-			.map(log => {
-				let html = '';
-				if (log.success) {
-					if (frm.doc.import_type === 'Insert New Records') {
-						html = __('Successfully imported {0}', [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`
-						]);
-					} else {
-						html = __('Successfully updated {0}', [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`
-						]);
-					}
-				} else {
-					let messages = log.messages
-						.map(JSON.parse)
-						.map(m => {
-							let title = m.title ? `<strong>${m.title}</strong>` : '';
-							let message = m.message ? `<div>${m.message}</div>` : '';
-							return title + message;
-						})
-						.join('');
-					let id = frappe.dom.get_unique_id();
-					html = `${messages}
-						<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
-							${__('Show Traceback')}
-						</button>
-						<div class="collapse" id="${id}" style="margin-top: 15px;">
-							<div class="well">
-								<pre>${log.exception}</pre>
-							</div>
-						</div>`;
-				}
-				let indicator_color = log.success ? 'green' : 'red';
-				let title = log.success ? __('Success') : __('Failure');
+						return `<tr>
+							<td>${JSON.parse(log.row_indexes).join(', ')}</td>
+							<td>
+								<div class="indicator ${indicator_color}">${title}</div>
+							</td>
+							<td>
+								${html}
+							</td>
+						</tr>`;
+					})
+					.join('');
 
-				if (frm.doc.show_failed_logs && log.success) {
-					return '';
+				if (!rows && frm.doc.show_failed_logs) {
+					rows = `<tr><td class="text-center text-muted" colspan=3>
+						${__('No failed logs')}
+					</td></tr>`;
 				}
 
-				return `<tr>
-					<td>${log.row_indexes.join(', ')}</td>
-					<td>
-						<div class="indicator ${indicator_color}">${title}</div>
-					</td>
-					<td>
-						${html}
-					</td>
-				</tr>`;
-			})
-			.join('');
+				frm.get_field('import_log_preview').$wrapper.html(`
+					<table class="table table-bordered">
+						<tr class="text-muted">
+							<th width="10%">${__('Row Number')}</th>
+							<th width="10%">${__('Status')}</th>
+							<th width="80%">${__('Message')}</th>
+						</tr>
+						${rows}
+					</table>
+				`);
+			}
+		});
+	},
+
+	show_import_log(frm) {
+		frm.toggle_display('import_log_section', false);
 
-		if (!rows && frm.doc.show_failed_logs) {
-			rows = `<tr><td class="text-center text-muted" colspan=3>
-				${__('No failed logs')}
-			</td></tr>`;
+		if (frm.import_in_progress) {
+			return;
 		}
 
-		frm.get_field('import_log_preview').$wrapper.html(`
-			<table class="table table-bordered">
-				<tr class="text-muted">
-					<th width="10%">${__('Row Number')}</th>
-					<th width="10%">${__('Status')}</th>
-					<th width="80%">${__('Message')}</th>
-				</tr>
-				${rows}
-			</table>
-		`);
+		frappe.call({
+			'method': 'frappe.client.get_count',
+			'args': {
+				'doctype': 'Data Import Log',
+				'filters': {
+					'data_import': frm.doc.name
+				}
+			},
+			'callback': function(r) {
+				let count = r.message;
+				if (count < 5000) {
+					frm.trigger('render_import_log');
+				} else {
+					frm.toggle_display('import_log_section', false);
+					frm.add_custom_button(__('Export Import Log'), () =>
+						frm.trigger('export_import_log')
+					);
+				}
+			}
+		});
 	},
 });

frappe/core/doctype/data_import/data_import.json

@@ -1,194 +1,197 @@
 {
- "actions": [],
- "autoname": "format:{reference_doctype} Import on {creation}",
- "beta": 1,
- "creation": "2019-08-04 14:16:08.318714",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "reference_doctype",
-  "import_type",
-  "download_template",
-  "import_file",
-  "html_5",
-  "google_sheets_url",
-  "refresh_google_sheet",
-  "column_break_5",
-  "status",
-  "submit_after_import",
-  "mute_emails",
-  "template_options",
-  "import_warnings_section",
-  "template_warnings",
-  "import_warnings",
-  "section_import_preview",
-  "import_preview",
-  "import_log_section",
-  "import_log",
-  "show_failed_logs",
-  "import_log_preview"
- ],
- "fields": [
-  {
-   "fieldname": "reference_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "reqd": 1,
-   "set_only_once": 1
-  },
-  {
-   "fieldname": "import_type",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Import Type",
-   "options": "\nInsert New Records\nUpdate Existing Records",
-   "reqd": 1,
-   "set_only_once": 1
-  },
-  {
-   "depends_on": "eval:!doc.__islocal",
-   "fieldname": "import_file",
-   "fieldtype": "Attach",
-   "in_list_view": 1,
-   "label": "Import File",
-   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-  },
-  {
-   "fieldname": "import_preview",
-   "fieldtype": "HTML",
-   "label": "Import Preview"
-  },
-  {
-   "fieldname": "section_import_preview",
-   "fieldtype": "Section Break",
-   "label": "Preview"
-  },
-  {
-   "fieldname": "column_break_5",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "template_options",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Template Options",
-   "options": "JSON",
-   "read_only": 1
-  },
-  {
-   "fieldname": "import_log",
-   "fieldtype": "Code",
-   "label": "Import Log",
-   "options": "JSON"
-  },
-  {
-   "fieldname": "import_log_section",
-   "fieldtype": "Section Break",
-   "label": "Import Log"
-  },
-  {
-   "fieldname": "import_log_preview",
-   "fieldtype": "HTML",
-   "label": "Import Log Preview"
-  },
-  {
-   "default": "Pending",
-   "fieldname": "status",
-   "fieldtype": "Select",
-   "hidden": 1,
-   "label": "Status",
-   "options": "Pending\nSuccess\nPartial Success\nError",
-   "read_only": 1
-  },
-  {
-   "fieldname": "template_warnings",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Template Warnings",
-   "options": "JSON"
-  },
-  {
-   "default": "0",
-   "fieldname": "submit_after_import",
-   "fieldtype": "Check",
-   "label": "Submit After Import",
-   "set_only_once": 1
-  },
-  {
-   "fieldname": "import_warnings_section",
-   "fieldtype": "Section Break",
-   "label": "Import File Errors and Warnings"
-  },
-  {
-   "fieldname": "import_warnings",
-   "fieldtype": "HTML",
-   "label": "Import Warnings"
-  },
-  {
-   "depends_on": "eval:!doc.__islocal",
-   "fieldname": "download_template",
-   "fieldtype": "Button",
-   "label": "Download Template"
-  },
-  {
-   "default": "1",
-   "fieldname": "mute_emails",
-   "fieldtype": "Check",
-   "label": "Don't Send Emails",
-   "set_only_once": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "show_failed_logs",
-   "fieldtype": "Check",
-   "label": "Show Failed Logs"
-  },
-  {
-   "depends_on": "eval:!doc.__islocal && !doc.import_file",
-   "fieldname": "html_5",
-   "fieldtype": "HTML",
-   "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
-  },
-  {
-   "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
-   "description": "Must be a publicly accessible Google Sheets URL",
-   "fieldname": "google_sheets_url",
-   "fieldtype": "Data",
-   "label": "Import from Google Sheets",
-   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-  },
-  {
-   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
-   "fieldname": "refresh_google_sheet",
-   "fieldtype": "Button",
-   "label": "Refresh Google Sheet"
-  }
- ],
- "hide_toolbar": 1,
- "links": [],
- "modified": "2021-04-11 01:50:42.074623",
- "modified_by": "Administrator",
- "module": "Core",
- "name": "Data Import",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
+    "actions": [],
+    "autoname": "format:{reference_doctype} Import on {creation}",
+    "beta": 1,
+    "creation": "2019-08-04 14:16:08.318714",
+    "doctype": "DocType",
+    "editable_grid": 1,
+    "engine": "InnoDB",
+    "field_order": [
+        "reference_doctype",
+        "import_type",
+        "download_template",
+        "import_file",
+        "payload_count",
+        "html_5",
+        "google_sheets_url",
+        "refresh_google_sheet",
+        "column_break_5",
+        "status",
+        "submit_after_import",
+        "mute_emails",
+        "template_options",
+        "import_warnings_section",
+        "template_warnings",
+        "import_warnings",
+        "section_import_preview",
+        "import_preview",
+        "import_log_section",
+        "show_failed_logs",
+        "import_log_preview"
+    ],
+    "fields": [
+        {
+            "fieldname": "reference_doctype",
+            "fieldtype": "Link",
+            "in_list_view": 1,
+            "label": "Document Type",
+            "options": "DocType",
+            "reqd": 1,
+            "set_only_once": 1
+        },
+        {
+            "fieldname": "import_type",
+            "fieldtype": "Select",
+            "in_list_view": 1,
+            "label": "Import Type",
+            "options": "\nInsert New Records\nUpdate Existing Records",
+            "reqd": 1,
+            "set_only_once": 1
+        },
+        {
+            "depends_on": "eval:!doc.__islocal",
+            "fieldname": "import_file",
+            "fieldtype": "Attach",
+            "in_list_view": 1,
+            "label": "Import File",
+            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+        },
+        {
+            "fieldname": "import_preview",
+            "fieldtype": "HTML",
+            "label": "Import Preview"
+        },
+        {
+            "fieldname": "section_import_preview",
+            "fieldtype": "Section Break",
+            "label": "Preview"
+        },
+        {
+            "fieldname": "column_break_5",
+            "fieldtype": "Column Break"
+        },
+        {
+            "fieldname": "template_options",
+            "fieldtype": "Code",
+            "hidden": 1,
+            "label": "Template Options",
+            "options": "JSON",
+            "read_only": 1
+        },
+        {
+            "fieldname": "import_log_section",
+            "fieldtype": "Section Break",
+            "label": "Import Log"
+        },
+        {
+            "fieldname": "import_log_preview",
+            "fieldtype": "HTML",
+            "label": "Import Log Preview"
+        },
+        {
+            "default": "Pending",
+            "fieldname": "status",
+            "fieldtype": "Select",
+            "hidden": 1,
+            "label": "Status",
+            "options": "Pending\nSuccess\nPartial Success\nError",
+            "read_only": 1
+        },
+        {
+            "fieldname": "template_warnings",
+            "fieldtype": "Code",
+            "hidden": 1,
+            "label": "Template Warnings",
+            "options": "JSON"
+        },
+        {
+            "default": "0",
+            "fieldname": "submit_after_import",
+            "fieldtype": "Check",
+            "label": "Submit After Import",
+            "set_only_once": 1
+        },
+        {
+            "fieldname": "import_warnings_section",
+            "fieldtype": "Section Break",
+            "label": "Import File Errors and Warnings"
+        },
+        {
+            "fieldname": "import_warnings",
+            "fieldtype": "HTML",
+            "label": "Import Warnings"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal",
+            "fieldname": "download_template",
+            "fieldtype": "Button",
+            "label": "Download Template"
+        },
+        {
+            "default": "1",
+            "fieldname": "mute_emails",
+            "fieldtype": "Check",
+            "label": "Don't Send Emails",
+            "set_only_once": 1
+        },
+        {
+            "default": "0",
+            "fieldname": "show_failed_logs",
+            "fieldtype": "Check",
+            "label": "Show Failed Logs"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal && !doc.import_file",
+            "fieldname": "html_5",
+            "fieldtype": "HTML",
+            "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
+            "description": "Must be a publicly accessible Google Sheets URL",
+            "fieldname": "google_sheets_url",
+            "fieldtype": "Data",
+            "label": "Import from Google Sheets",
+            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+        },
+        {
+            "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
+            "fieldname": "refresh_google_sheet",
+            "fieldtype": "Button",
+            "label": "Refresh Google Sheet"
+        },
+        {
+            "fieldname": "payload_count",
+            "fieldtype": "Int",
+            "hidden": 1,
+            "label": "Payload Count",
+            "read_only": 1
+        }
+    ],
+    "hide_toolbar": 1,
+    "links": [],
+    "modified": "2022-02-01 20:08:37.624914",
+    "modified_by": "Administrator",
+    "module": "Core",
+    "name": "Data Import",
+    "naming_rule": "Expression",
+    "owner": "Administrator",
+    "permissions": [
+        {
+            "create": 1,
+            "delete": 1,
+            "email": 1,
+            "export": 1,
+            "print": 1,
+            "read": 1,
+            "report": 1,
+            "role": "System Manager",
+            "share": 1,
+            "write": 1
+        }
+    ],
+    "sort_field": "modified",
+    "sort_order": "DESC",
+    "states": [],
+    "track_changes": 1
 }

frappe/core/doctype/data_import/data_import.py

@@ -27,6 +27,7 @@ class DataImport(Document):
 
 		self.validate_import_file()
 		self.validate_google_sheets_url()
+		self.set_payload_count()
 
 	def validate_import_file(self):
 		if self.import_file:
@@ -38,6 +39,12 @@ class DataImport(Document):
 			return
 		validate_google_sheets_url(self.google_sheets_url)
 
+	def set_payload_count(self):
+		if self.import_file:
+			i = self.get_importer()
+			payloads = i.import_file.get_payloads_for_import()
+			self.payload_count = len(payloads)
+
 	@frappe.whitelist()
 	def get_preview_from_template(self, import_file=None, google_sheets_url=None):
 		if import_file:
@@ -67,7 +74,7 @@ class DataImport(Document):
 			enqueue(
 				start_import,
 				queue="default",
-				timeout=6000,
+				timeout=10000,
 				event="data_import",
 				job_name=self.name,
 				data_import=self.name,
@@ -80,6 +87,9 @@ class DataImport(Document):
 	def export_errored_rows(self):
 		return self.get_importer().export_errored_rows()
 
+	def download_import_log(self):
+		return self.get_importer().export_import_log()
+
 	def get_importer(self):
 		return Importer(self.reference_doctype, data_import=self)
 
@@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N
 		import_file, google_sheets_url
 	)
 
-
 @frappe.whitelist()
 def form_start_import(data_import):
 	return frappe.get_doc("Data Import", data_import).start_import()
@@ -145,6 +154,30 @@ def download_errored_template(data_import_name):
 	data_import = frappe.get_doc("Data Import", data_import_name)
 	data_import.export_errored_rows()
 
+@frappe.whitelist()
+def download_import_log(data_import_name):
+	data_import = frappe.get_doc("Data Import", data_import_name)
+	data_import.download_import_log()
+
+@frappe.whitelist()
+def get_import_status(data_import_name):
+	import_status = {}
+
+	logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'],
+		filters={'data_import': data_import_name},
+		group_by='success')
+
+	total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count')
+
+	for log in logs:
+		if log.get('success'):
+			import_status['success'] = log.get('count')
+		else:
+			import_status['failed'] = log.get('count')
+
+	import_status['total_records'] = total_payload_count
+
+	return import_status
 
 def import_file(
 	doctype, file_path, import_type, submit_after_import=False, console=False

frappe/core/doctype/data_import/data_import_list.js

@@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = {
 			'Error': 'red'
 		};
 		let status = doc.status;
+
 		if (imports_in_progress.includes(doc.name)) {
 			status = 'In Progress';
 		}
 		if (status == 'Pending') {
 			status = 'Not Started';
 		}
+
 		return [__(status), colors[status], 'status,=,' + doc.status];
 	},
 	formatters: {

frappe/core/doctype/data_import/importer.py

@@ -47,7 +47,13 @@ class Importer:
 		)
 
 	def get_data_for_import_preview(self):
-		return self.import_file.get_data_for_import_preview()
+		out = self.import_file.get_data_for_import_preview()
+
+		out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index", limit=10)
+
+		return out
 
 	def before_import(self):
 		# set user lang for translations
@@ -58,7 +64,6 @@ class Importer:
 		frappe.flags.in_import = True
 		frappe.flags.mute_emails = self.data_import.mute_emails
 
-		self.data_import.db_set("status", "Pending")
 		self.data_import.db_set("template_warnings", "")
 
 	def import_data(self):
@@ -79,20 +84,25 @@ class Importer:
 			return
 
 		# setup import log
-		if self.data_import.import_log:
-			import_log = frappe.parse_json(self.data_import.import_log)
-		else:
-			import_log = []
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
 
-		# remove previous failures from import log
-		import_log = [log for log in import_log if log.get("success")]
+		log_index = 0
+
+		# Do not remove rows in case of retry after an error or pending data import
+		if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
+			# remove previous failures from import log only in case of retry after partial success
+			import_log = [log for log in import_log if log.get("success")]
 
 		# get successfully imported rows
 		imported_rows = []
 		for log in import_log:
 			log = frappe._dict(log)
-			if log.success:
-				imported_rows += log.row_indexes
+			if log.success or len(import_log) < self.data_import.payload_count:
+				imported_rows += json.loads(log.row_indexes)
+
+			log_index = log.log_index
 
 		# start import
 		total_payload_count = len(payloads)
@@ -146,25 +156,41 @@ class Importer:
 							},
 						)
 
-					import_log.append(
-						frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes)
-					)
+					create_import_log(self.data_import.name, log_index, {
+						'success': True,
+						'docname': doc.name,
+						'row_indexes': row_indexes
+					})
+
+					log_index += 1
+
+					if not self.data_import.status == "Partial Success":
+						self.data_import.db_set("status", "Partial Success")
+					
 					# commit after every successful import
 					frappe.db.commit()
 
 				except Exception:
-					import_log.append(
-						frappe._dict(
-							success=False,
-							exception=frappe.get_traceback(),
-							messages=frappe.local.message_log,
-							row_indexes=row_indexes,
-						)
-					)
+					messages = frappe.local.message_log
 					frappe.clear_messages()
+
 					# rollback if exception
 					frappe.db.rollback()
 
+					create_import_log(self.data_import.name, log_index, {
+						'success': False,
+						'exception': frappe.get_traceback(),
+						'messages': messages,
+						'row_indexes': row_indexes
+					})
+
+					log_index += 1
+
+		# Logs are db inserted directly so will have to be fetched again
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
+
 		# set status
 		failures = [log for log in import_log if not log.get("success")]
 		if len(failures) == total_payload_count:
@@ -178,7 +204,6 @@ class Importer:
 			self.print_import_log(import_log)
 		else:
 			self.data_import.db_set("status", status)
-			self.data_import.db_set("import_log", json.dumps(import_log))
 
 		self.after_import()
 
@@ -248,11 +273,14 @@ class Importer:
 		if not self.data_import:
 			return
 
-		import_log = frappe.parse_json(self.data_import.import_log or "[]")
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
+
 		failures = [log for log in import_log if not log.get("success")]
 		row_indexes = []
 		for f in failures:
-			row_indexes.extend(f.get("row_indexes", []))
+			row_indexes.extend(json.loads(f.get("row_indexes", [])))
 
 		# de duplicate
 		row_indexes = list(set(row_indexes))
@@ -264,6 +292,30 @@ class Importer:
 
 		build_csv_response(rows, _(self.doctype))
 
+	def export_import_log(self):
+		from frappe.utils.csvutils import build_csv_response
+
+		if not self.data_import:
+			return
+
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index")
+
+		header_row = ["Row Numbers", "Status", "Message", "Exception"]
+
+		rows = [header_row]
+
+		for log in import_log:
+			row_number = json.loads(log.get("row_indexes"))[0]
+			status = "Success" if log.get('success') else "Failure"
+			message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \
+				log.get("messages")
+			exception = frappe.utils.cstr(log.get("exception", ''))
+			rows += [[row_number, status, message, exception]]
+
+		build_csv_response(rows, self.doctype)
+
 	def print_import_log(self, import_log):
 		failed_records = [log for log in import_log if not log.success]
 		successful_records = [log for log in import_log if log.success]
@@ -566,7 +618,7 @@ class Row:
 			)
 
 		# remove standard fields and __islocal
-		for key in frappe.model.default_fields + ("__islocal",):
+		for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",):
 			doc.pop(key, None)
 
 		for col, value in zip(columns, values):
@@ -1172,3 +1224,17 @@ def df_as_json(df):
 
 def get_select_options(df):
 	return [d for d in (df.options or "").split("\n") if d]
+
+def create_import_log(data_import, log_index, log_details):
+	frappe.get_doc({
+		'doctype': 'Data Import Log',
+		'log_index': log_index,
+		'success': log_details.get('success'),
+		'data_import': data_import,
+		'row_indexes': json.dumps(log_details.get('row_indexes')),
+		'docname': log_details.get('docname'),
+		'messages': json.dumps(log_details.get('messages', '[]')),
+		'exception': log_details.get('exception')
+	}).db_insert()
+
+

frappe/core/doctype/data_import/test_importer.py

@@ -4,6 +4,7 @@
 import unittest
 import frappe
 from frappe.core.doctype.data_import.importer import Importer
+from frappe.tests.test_query_builder import db_type_is, run_only_if
 from frappe.utils import getdate, format_duration
 
 doctype_name = 'DocType for Import'
@@ -54,21 +55,27 @@ class TestImporter(unittest.TestCase):
 		self.assertEqual(len(preview.data), 4)
 		self.assertEqual(len(preview.columns), 16)
 
+	# ignored on postgres because myisam doesn't exist on pg
+	@run_only_if(db_type_is.MARIADB)
 	def test_data_import_without_mandatory_values(self):
 		import_file = get_import_file('sample_import_file_without_mandatory')
 		data_import = self.get_importer(doctype_name, import_file)
 		frappe.local.message_log = []
 		data_import.start_import()
 		data_import.reload()
-		import_log = frappe.parse_json(data_import.import_log)
-		self.assertEqual(import_log[0]['row_indexes'], [2,3])
+
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"],
+			filters={"data_import": data_import.name},
+			order_by="log_index")
+
+		self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3])
 		expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #1: Value missing for: Child Title"
-		self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error)
+		self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error)
 		expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #2: Value missing for: Child Title"
-		self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error)
+		self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error)
 
-		self.assertEqual(import_log[1]['row_indexes'], [4])
-		self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required")
+		self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4])
+		self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required")
 
 	def test_data_import_update(self):
 		existing_doc = frappe.get_doc(

frappe/core/doctype/data_import_log/data_import_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Import Log', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/data_import_log/data_import_log.json

@@ -0,0 +1,84 @@
+{
+ "actions": [],
+ "creation": "2021-12-25 16:12:20.205889",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "MyISAM",
+ "field_order": [
+  "data_import",
+  "row_indexes",
+  "success",
+  "docname",
+  "messages",
+  "exception",
+  "log_index"
+ ],
+ "fields": [
+  {
+   "fieldname": "data_import",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Data Import",
+   "options": "Data Import"
+  },
+  {
+   "fieldname": "docname",
+   "fieldtype": "Data",
+   "label": "Reference Name"
+  },
+  {
+   "fieldname": "exception",
+   "fieldtype": "Text",
+   "label": "Exception"
+  },
+  {
+   "fieldname": "row_indexes",
+   "fieldtype": "Code",
+   "label": "Row Indexes",
+   "options": "JSON"
+  },
+  {
+   "default": "0",
+   "fieldname": "success",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Success"
+  },
+  {
+   "fieldname": "log_index",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Log Index"
+  },
+  {
+   "fieldname": "messages",
+   "fieldtype": "Code",
+   "label": "Messages",
+   "options": "JSON"
+  }
+ ],
+ "in_create": 1,
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-12-29 11:19:19.646076",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Data Import Log",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC"
+}

frappe/core/doctype/data_import_log/data_import_log.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class DataImportLog(Document):
+	pass

frappe/core/doctype/data_import_log/test_data_import_log.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+import unittest
+
+class TestDataImportLog(unittest.TestCase):
+	pass

frappe/core/doctype/doctype/doctype.py

@@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache
 import frappe
 from frappe import _
 from frappe.utils import now, cint
-from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
+from frappe.model import (
+	no_value_fields, default_fields, table_fields, data_field_options, child_table_fields
+)
 from frappe.model.document import Document
 from frappe.model.base_document import get_controller
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
@@ -74,6 +76,7 @@ class DocType(Document):
 		self.make_amendable()
 		self.make_repeatable()
 		self.validate_nestedset()
+		self.validate_child_table()
 		self.validate_website()
 		self.ensure_minimum_max_attachment_limit()
 		validate_links_table_fieldnames(self)
@@ -689,6 +692,22 @@ class DocType(Document):
 		})
 		self.nsm_parent_field = parent_field_name
 
+	def validate_child_table(self):
+		if not self.get("istable") or self.is_new():
+			# if the doctype is not a child table then return
+			# if the doctype is a new doctype and also a child table then
+			# don't move forward as it will be handled via schema
+			return
+
+		self.add_child_table_fields()
+
+	def add_child_table_fields(self):
+		from frappe.database.schema import add_column
+
+		add_column(self.name, "parent", "Data")
+		add_column(self.name, "parenttype", "Data")
+		add_column(self.name, "parentfield", "Data")
+
 	def get_max_idx(self):
 		"""Returns the highest `idx`"""
 		max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""",
@@ -699,6 +718,13 @@ class DocType(Document):
 		if not name:
 			name = self.name
 
+		# a Doctype name is the tablename created in database
+		# `tab<Doctype Name>` the length of tablename is limited to 64 characters
+		max_length = frappe.db.MAX_COLUMN_LENGTH - 3
+		if len(name) > max_length:
+			# length(tab + <Doctype Name>) should be equal to 64 characters hence doctype should be 61 characters
+			frappe.throw(_("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError)
+
 		flags = {"flags": re.ASCII}
 
 		# a DocType name should not start or end with an empty space
@@ -1009,7 +1035,7 @@ def validate_fields(meta):
 				sort_fields = [d.split()[0] for d in meta.sort_field.split(',')]
 
 			for fieldname in sort_fields:
-				if not fieldname in fieldname_list + list(default_fields):
+				if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)):
 					frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname),
 						InvalidFieldNameError)
 

frappe/core/doctype/doctype/test_doctype.py

@@ -23,6 +23,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert)
 		self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert)
 		self.assertRaises(frappe.NameError, new_doctype("Some (DocType)").insert)
+		self.assertRaises(frappe.NameError, new_doctype("Some Doctype with a name whose length is more than 61 characters").insert)
 		for name in ("Some DocType", "Some_DocType"):
 			if frappe.db.exists("DocType", name):
 				frappe.delete_doc("DocType", name)
@@ -353,7 +354,6 @@ class TestDocType(unittest.TestCase):
 		dump_docs = json.dumps(docs.get('docs'))
 		cancel_all_linked_docs(dump_docs)
 		data_link_doc.cancel()
-		data_doc.name = '{}-CANC-0'.format(data_doc.name)
 		data_doc.load_from_db()
 		self.assertEqual(data_link_doc.docstatus, 2)
 		self.assertEqual(data_doc.docstatus, 2)
@@ -377,7 +377,7 @@ class TestDocType(unittest.TestCase):
 		for data in link_doc.get('permissions'):
 			data.submit = 1
 			data.cancel = 1
-		link_doc.insert(ignore_if_duplicate=True)
+		link_doc.insert()
 
 		#create first parent doctype
 		test_doc_1 = new_doctype('Test Doctype 1')
@@ -392,7 +392,7 @@ class TestDocType(unittest.TestCase):
 		for data in test_doc_1.get('permissions'):
 			data.submit = 1
 			data.cancel = 1
-		test_doc_1.insert(ignore_if_duplicate=True)
+		test_doc_1.insert()
 
 		#crete second parent doctype
 		doc = new_doctype('Test Doctype 2')
@@ -407,7 +407,7 @@ class TestDocType(unittest.TestCase):
 		for data in link_doc.get('permissions'):
 			data.submit = 1
 			data.cancel = 1
-		doc.insert(ignore_if_duplicate=True)
+		doc.insert()
 
 		# create doctype data
 		data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1')
@@ -438,7 +438,6 @@ class TestDocType(unittest.TestCase):
 		# checking that doc for Test Doctype 2 is not canceled
 		self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel)
 
-		data_doc_2.name = '{}-CANC-0'.format(data_doc_2.name)
 		data_doc.load_from_db()
 		data_doc_2.load_from_db()
 		self.assertEqual(data_link_doc_1.docstatus, 2)

frappe/core/doctype/file/file.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 """
@@ -7,7 +7,6 @@ record of files
 naming for same name files: file.gif, file-1.gif, file-2.gif etc
 """
 
-import base64
 import hashlib
 import imghdr
 import io
@@ -17,9 +16,10 @@ import os
 import re
 import shutil
 import zipfile
+from typing import TYPE_CHECKING, Tuple
 
 import requests
-import requests.exceptions
+from requests.exceptions import HTTPError, SSLError
 from PIL import Image, ImageFile, ImageOps
 from io import BytesIO
 from urllib.parse import quote, unquote
@@ -31,6 +31,11 @@ from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, g
 from frappe.utils.image import strip_exif_data, optimize_image
 from frappe.utils.file_manager import safe_b64decode
 
+if TYPE_CHECKING:
+	from PIL.ImageFile import ImageFile
+	from requests.models import Response
+
+
 class MaxFileSizeReachedError(frappe.ValidationError):
 	pass
 
@@ -276,7 +281,7 @@ class File(Document):
 					image, filename, extn = get_local_image(self.file_url)
 				else:
 					image, filename, extn = get_web_image(self.file_url)
-			except (requests.exceptions.HTTPError, requests.exceptions.SSLError, IOError, TypeError):
+			except (HTTPError, SSLError, IOError, TypeError):
 					return
 
 			size = width, height
@@ -572,12 +577,10 @@ class File(Document):
 
 	@staticmethod
 	def zip_files(files):
-		from six import string_types
-
 		zip_file = io.BytesIO()
 		zf = zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED)
 		for _file in files:
-			if isinstance(_file, string_types):
+			if isinstance(_file, str):
 				_file = frappe.get_doc("File", _file)
 			if not isinstance(_file, File):
 				continue
@@ -650,9 +653,17 @@ def setup_folder_path(filename, new_parent):
 		from frappe.model.rename_doc import rename_doc
 		rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True)
 
-def get_extension(filename, extn, content):
+def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str:
 	mimetype = None
 
+	if response:
+		content_type = response.headers.get("Content-Type")
+
+		if content_type:
+			_extn = mimetypes.guess_extension(content_type)
+			if _extn:
+				return _extn[1:]
+
 	if extn:
 		# remove '?' char and parameters from extn if present
 		if '?' in extn:
@@ -695,14 +706,14 @@ def get_local_image(file_url):
 
 	return image, filename, extn
 
-def get_web_image(file_url):
+def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]:
 	# download
 	file_url = frappe.utils.get_url(file_url)
 	r = requests.get(file_url, stream=True)
 	try:
 		r.raise_for_status()
-	except requests.exceptions.HTTPError as e:
-		if "404" in e.args[0]:
+	except HTTPError:
+		if r.status_code == 404:
 			frappe.msgprint(_("File '{0}' not found").format(file_url))
 		else:
 			frappe.msgprint(_("Unable to read file format for {0}").format(file_url))
@@ -721,7 +732,10 @@ def get_web_image(file_url):
 		filename = get_random_filename()
 		extn = None
 
-	extn = get_extension(filename, extn, r.content)
+	extn = get_extension(filename, extn, response=r)
+	if extn == "bin":
+		extn = get_extension(filename, extn, content=r.content) or "png"
+
 	filename = "/files/" + strip(unquote(filename))
 
 	return image, filename, extn
@@ -864,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False):
 		else:
 			filename = get_random_filename(content_type=mtype)
 
-		doctype = doc.parenttype if doc.parent else doc.doctype
-		name = doc.parent or doc.name
+		# attaching a file to a child table doc, attaches it to the parent doc
+		doctype = doc.parenttype if doc.get("parent") else doc.doctype
+		name = doc.get("parent") or doc.name
 
 		_file = frappe.get_doc({
 			"doctype": "File",

frappe/core/doctype/file/test_file.py

@@ -1,15 +1,14 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import base64
 import json
 import frappe
 import os
 import unittest
+
 from frappe import _
-from frappe.core.doctype.file.file import get_attached_images, move_file, get_files_in_folder, unzip_file
+from frappe.core.doctype.file.file import File, get_attached_images, move_file, get_files_in_folder, unzip_file
 from frappe.utils import get_files_path
-# test_records = frappe.get_test_records('File')
 
 test_content1 = 'Hello'
 test_content2 = 'Hello World'
@@ -24,8 +23,6 @@ def make_test_doc():
 
 
 class TestSimpleFile(unittest.TestCase):
-
-
 	def setUp(self):
 		self.attached_to_doctype, self.attached_to_docname = make_test_doc()
 		self.test_content = test_content1
@@ -38,21 +35,13 @@ class TestSimpleFile(unittest.TestCase):
 		_file.save()
 		self.saved_file_url = _file.file_url
 
-
 	def test_save(self):
 		_file = frappe.get_doc("File", {"file_url": self.saved_file_url})
 		content = _file.get_content()
 		self.assertEqual(content, self.test_content)
 
 
-	def tearDown(self):
-		# File gets deleted on rollback, so blank
-		pass
-
-
 class TestBase64File(unittest.TestCase):
-
-
 	def setUp(self):
 		self.attached_to_doctype, self.attached_to_docname = make_test_doc()
 		self.test_content = base64.b64encode(test_content1.encode('utf-8'))
@@ -66,18 +55,12 @@ class TestBase64File(unittest.TestCase):
 		_file.save()
 		self.saved_file_url = _file.file_url
 
-
 	def test_saved_content(self):
 		_file = frappe.get_doc("File", {"file_url": self.saved_file_url})
 		content = _file.get_content()
 		self.assertEqual(content, test_content1)
 
 
-	def tearDown(self):
-		# File gets deleted on rollback, so blank
-		pass
-
-
 class TestSameFileName(unittest.TestCase):
 	def test_saved_content(self):
 		self.attached_to_doctype, self.attached_to_docname = make_test_doc()
@@ -130,8 +113,6 @@ class TestSameFileName(unittest.TestCase):
 
 
 class TestSameContent(unittest.TestCase):
-
-
 	def setUp(self):
 		self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc()
 		self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc()
@@ -186,10 +167,6 @@ class TestSameContent(unittest.TestCase):
 		limit_property.delete()
 		frappe.clear_cache(doctype='ToDo')
 
-	def tearDown(self):
-		# File gets deleted on rollback, so blank
-		pass
-
 
 class TestFile(unittest.TestCase):
 	def setUp(self):
@@ -398,7 +375,7 @@ class TestFile(unittest.TestCase):
 
 	def test_make_thumbnail(self):
 		# test web image
-		test_file = frappe.get_doc({
+		test_file: File = frappe.get_doc({
 			"doctype": "File",
 			"file_name": 'logo',
 			"file_url": frappe.utils.get_url('/_test/assets/image.jpg'),
@@ -407,6 +384,16 @@ class TestFile(unittest.TestCase):
 		test_file.make_thumbnail()
 		self.assertEquals(test_file.thumbnail_url, '/files/image_small.jpg')
 
+		# test web image without extension
+		test_file = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'logo',
+			"file_url": frappe.utils.get_url('/_test/assets/image'),
+		}).insert(ignore_permissions=True)
+
+		test_file.make_thumbnail()
+		self.assertTrue(test_file.thumbnail_url.endswith("_small.jpeg"))
+
 		# test local image
 		test_file.db_set('thumbnail_url', None)
 		test_file.reload()

frappe/core/doctype/report/report.py

@@ -61,7 +61,7 @@ class Report(Document):
 					delete_permanently=True)
 
 	def get_columns(self):
-		return [d.as_dict(no_default_fields = True) for d in self.columns]
+		return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
 
 	@frappe.whitelist()
 	def set_doctype_roles(self):

frappe/core/doctype/server_script/server_script_utils.py

@@ -34,19 +34,7 @@ def run_server_script_for_doc_event(doc, event):
 	if scripts:
 		# run all scripts for this doctype + event
 		for script_name in scripts:
-			try:
-				frappe.get_doc('Server Script', script_name).execute_doc(doc)
-			except Exception as e:
-				message = frappe._('Error executing Server Script {0}. Open Browser Console to see traceback.').format(
-					frappe.utils.get_link_to_form('Server Script', script_name)
-				)
-				exception = type(e)
-				if getattr(frappe, 'request', None):
-					# all exceptions throw 500 which is internal server error
-					# however server script error is a user error
-					# so we should throw 417 which is expectation failed
-					exception.http_status_code = 417
-				frappe.throw(title=frappe._('Server Script Error'), msg=message, exc=exception)
+			frappe.get_doc('Server Script', script_name).execute_doc(doc)
 
 def get_server_script_map():
 	# fetch cached server script methods

frappe/core/doctype/server_script/test_server_script.py

@@ -139,3 +139,42 @@ class TestServerScript(unittest.TestCase):
 
 		server_script.disabled = 1
 		server_script.save()
+
+	def test_restricted_qb(self):
+		todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote")
+		todo.insert()
+
+		script = frappe.get_doc(
+			doctype='Server Script',
+			name='test_qb_restrictions',
+			script_type = 'API',
+			api_method = 'test_qb_restrictions',
+			allow_guest = 1,
+			# whitelisted update
+			script = f'''
+frappe.db.set_value("ToDo", "{todo.name}", "description", "safe")
+'''
+		)
+		script.insert()
+		script.execute_method()
+
+		todo.reload()
+		self.assertEqual(todo.description, "safe")
+
+		# unsafe update
+		script.script = f"""
+todo = frappe.qb.DocType("ToDo")
+frappe.qb.update(todo).set(todo.description, "unsafe").where(todo.name == "{todo.name}").run()
+"""
+		script.save()
+		self.assertRaises(frappe.PermissionError, script.execute_method)
+		todo.reload()
+		self.assertEqual(todo.description, "safe")
+
+		# safe select
+		script.script = f"""
+todo = frappe.qb.DocType("ToDo")
+frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run()
+"""
+		script.save()
+		script.execute_method()

frappe/core/doctype/user/test_user.py

@@ -355,7 +355,10 @@ class TestUser(unittest.TestCase):
 			test_user.reload()
 			self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/")
 			update_password(old_password, old_password=new_password)
-			self.assertEqual(json.loads(frappe.message_log[0]), {"message": "Password reset instructions have been sent to your email"})
+			self.assertEqual(
+				json.loads(frappe.message_log[0]).get("message"), 
+				"Password reset instructions have been sent to your email"
+			)
 		sendmail.assert_called_once()
 		self.assertEqual(sendmail.call_args[1]["recipients"], "test2@example.com")
 

frappe/core/doctype/user/user.py

@@ -756,7 +756,7 @@ def verify_password(password):
 @frappe.whitelist(allow_guest=True)
 def sign_up(email, full_name, redirect_to):
 	if is_signup_disabled():
-		frappe.throw(_('Sign Up is disabled'), title='Not Allowed')
+		frappe.throw(_("Sign Up is disabled"), title=_("Not Allowed"))
 
 	user = frappe.db.get("User", {"email": email})
 	if user:
@@ -810,8 +810,10 @@ def reset_password(user):
 		user.validate_reset_password()
 		user.reset_password(send_email=True)
 
-		return frappe.msgprint(_("Password reset instructions have been sent to your email"))
-
+		return frappe.msgprint(
+			msg=_("Password reset instructions have been sent to your email"),
+			title=_("Password Email Sent")
+		)
 	except frappe.DoesNotExistError:
 		frappe.local.response['http_status_code'] = 400
 		frappe.clear_messages()

frappe/core/doctype/user_permission/test_user_permission.py

@@ -3,6 +3,7 @@
 from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable
 from frappe.permissions import has_user_permission
 from frappe.core.doctype.doctype.test_doctype import new_doctype
+from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
 
 import frappe
 import unittest
@@ -31,6 +32,18 @@ class TestUserPermission(unittest.TestCase):
 		param = get_params(user, 'User', perm_user.name, is_default=1)
 		self.assertRaises(frappe.ValidationError, add_user_permissions, param)
 
+	def test_default_user_permission_corectness(self):
+		user = create_user('test_default_corectness_permission_1@example.com')
+		param = get_params(user, 'User', user.name, is_default=1, hide_descendants= 1)
+		add_user_permissions(param)
+		#create a duplicate entry with default
+		perm_user = create_user('test_default_corectness2@example.com')
+		test_blog = make_test_blog()
+		param = get_params(perm_user, 'Blog Post', test_blog.name, is_default=1, hide_descendants= 1)
+		add_user_permissions(param)
+		frappe.db.delete('User Permission', filters={'for_value': test_blog.name})
+		frappe.delete_doc('Blog Post', test_blog.name)
+
 	def test_default_user_permission(self):
 		frappe.set_user('Administrator')
 		user = create_user('test_user_perm1@example.com', 'Website Manager')

frappe/core/doctype/user_permission/user_permission.py

@@ -48,7 +48,6 @@ class UserPermission(Document):
 			}, or_filters={
 				'applicable_for': cstr(self.applicable_for),
 				'apply_to_all_doctypes': 1,
-				'hide_descendants': cstr(self.hide_descendants)
 			}, limit=1)
 		if overlap_exists:
 			ref_link = frappe.get_desk_link(self.doctype, overlap_exists[0].name)

frappe/core/doctype/user_type/test_user_type.py

@@ -1,8 +1,68 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2021, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-# import frappe
+import frappe
 import unittest
 
+from frappe.installer import update_site_config
+
 class TestUserType(unittest.TestCase):
-	pass
+	def setUp(self):
+		create_role()
+
+	def test_add_select_perm_doctypes(self):
+		user_type = create_user_type('Test User Type')
+
+		# select perms added for all link fields
+		doc = frappe.get_meta('Contact')
+		link_fields = doc.get_link_fields()
+		select_doctypes = frappe.get_all('User Select Document Type', {'parent': user_type.name}, pluck='document_type')
+
+		for entry in link_fields:
+			self.assertTrue(entry.options in select_doctypes)
+
+		# select perms added for all child table link fields
+		link_fields = []
+		for child_table in doc.get_table_fields():
+			child_doc = frappe.get_meta(child_table.options)
+			link_fields.extend(child_doc.get_link_fields())
+
+		for entry in link_fields:
+			self.assertTrue(entry.options in select_doctypes)
+
+	def tearDown(self):
+		frappe.db.rollback()
+
+
+def create_user_type(user_type):
+	if frappe.db.exists('User Type', user_type):
+		frappe.delete_doc('User Type', user_type)
+
+	user_type_limit = {frappe.scrub(user_type): 1}
+	update_site_config('user_type_doctype_limit', user_type_limit)
+
+	doc = frappe.get_doc({
+		'doctype': 'User Type',
+		'name': user_type,
+		'role': '_Test User Type',
+		'user_id_field': 'user',
+		'apply_user_permission_on': 'User'
+	})
+
+	doc.append('user_doctypes', {
+		'document_type': 'Contact',
+		'read': 1,
+		'write': 1
+	})
+
+	return doc.insert()
+
+
+def create_role():
+	if not frappe.db.exists('Role', '_Test User Type'):
+		frappe.get_doc({
+			'doctype': 'Role',
+			'role_name': '_Test User Type',
+			'desk_access': 1,
+			'is_custom': 1
+		}).insert()

frappe/core/doctype/user_type/user_type.py

@@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters
 		['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]]
 
 	doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters,
-		order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
+		order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
 
 	custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)],
 		['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']]

frappe/core/notifications.py

@@ -39,43 +39,3 @@ def get_todays_events(as_list=False):
 	today = nowdate()
 	events = get_events(today, today)
 	return events if as_list else len(events)
-
-def get_unseen_likes():
-	"""Returns count of unseen likes"""
-
-	comment_doctype = DocType("Comment")
-	return frappe.db.count(comment_doctype,
-		filters=(
-			(comment_doctype.comment_type == "Like")
-			& (comment_doctype.modified >= Now() - Interval(years=1))
-			& (comment_doctype.owner.notnull())
-			& (comment_doctype.owner != frappe.session.user)
-			& (comment_doctype.reference_owner == frappe.session.user)
-			& (comment_doctype.seen == 0)
-		)
-	)
-
-
-def get_unread_emails():
-	"returns count of unread emails for a user"
-
-	communication_doctype = DocType("Communication")
-	user_doctype = DocType("User")
-	distinct_email_accounts = (
-		frappe.qb.from_(user_doctype)
-		.select(user_doctype.email_account)
-		.where(user_doctype.parent == frappe.session.user)
-		.distinct()
-	)
-
-	return frappe.db.count(communication_doctype,
-		filters=(
-			(communication_doctype.communication_type == "Communication")
-			& (communication_doctype.communication_medium == "Email")
-			& (communication_doctype.sent_or_received == "Received")
-			& (communication_doctype.email_status.notin(["spam", "Trash"]))
-			& (communication_doctype.email_account.isin(distinct_email_accounts))
-			& (communication_doctype.modified >= Now() - Interval(years=1))
-			& (communication_doctype.seen == 0)
-		)
-	)

frappe/core/page/dashboard_view/dashboard_view.js

@@ -30,6 +30,7 @@ class Dashboard {
 
 	show() {
 		this.route = frappe.get_route();
+		this.set_breadcrumbs();
 		if (this.route.length > 1) {
 			// from route
 			this.show_dashboard(this.route.slice(-1)[0]);
@@ -75,6 +76,10 @@ class Dashboard {
 		frappe.last_dashboard = current_dashboard_name;
 	}
 
+	set_breadcrumbs() {
+		frappe.breadcrumbs.add("Desk", "Dashboard");
+	}
+
 	refresh() {
 		frappe.run_serially([
 			() => this.render_cards(),

frappe/database/database.py

@@ -10,19 +10,20 @@ import re
 import string
 from contextlib import contextmanager
 from time import time
-from typing import Dict, List, Union, Tuple
+from typing import Dict, List, Tuple, Union
+
+from pypika.terms import Criterion, NullValue, PseudoColumn
 
 import frappe
 import frappe.defaults
 import frappe.model.meta
 from frappe import _
-from frappe.utils import now, getdate, cast, get_datetime
 from frappe.model.utils.link_count import flush_local_link_count
 from frappe.query_builder.functions import Count
-from frappe.query_builder.functions import Min, Max, Avg, Sum
-from frappe.query_builder.utils import Column
+from frappe.query_builder.utils import DocType
+from frappe.utils import cast, get_datetime, getdate, now, sbool
+
 from .query import Query
-from pypika.terms import Criterion, PseudoColumn
 
 
 class Database(object):
@@ -36,9 +37,9 @@ class Database(object):
 
 	OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"]
 	DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"]
-	STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype')
-	DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent',
-		'parentfield', 'parenttype', 'idx']
+	STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by')
+	DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx']
+	CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield')
 	MAX_WRITES_PER_TRANSACTION = 200_000
 
 	class InvalidColumnName(frappe.ValidationError): pass
@@ -278,7 +279,9 @@ class Database(object):
 				if self.auto_commit_on_many_writes:
 					self.commit()
 				else:
-					frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError)
+					msg = "<br><br>" + _("Too many changes to database in single action.") + "<br>"
+					msg += _("The changes have been reverted.") + "<br>"
+					raise frappe.TooManyWritesError(msg)
 
 	def check_implicit_commit(self, query):
 		if self.transaction_writes and \
@@ -432,11 +435,9 @@ class Database(object):
 
 		else:
 			fields = fieldname
-			if fieldname!="*":
+			if fieldname != "*":
 				if isinstance(fieldname, str):
 					fields = [fieldname]
-				else:
-					fields = fieldname
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:
@@ -555,7 +556,21 @@ class Database(object):
 	def get_list(*args, **kwargs):
 		return frappe.get_list(*args, **kwargs)
 
-	def get_single_value(self, doctype, fieldname, cache=False):
+	def set_single_value(self, doctype, fieldname, value, *args, **kwargs):
+		"""Set field value of Single DocType.
+
+		:param doctype: DocType of the single object
+		:param fieldname: `fieldname` of the property
+		:param value: `value` of the property
+
+		Example:
+
+			# Update the `deny_multiple_sessions` field in System Settings DocType.
+			company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True)
+		"""
+		return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs)
+
+	def get_single_value(self, doctype, fieldname, cache=True):
 		"""Get property of Single DocType. Cache locally by default
 
 		:param doctype: DocType of the single object whose value is requested
@@ -570,7 +585,7 @@ class Database(object):
 		if not doctype in self.value_cache:
 			self.value_cache[doctype] = {}
 
-		if fieldname in self.value_cache[doctype]:
+		if cache and fieldname in self.value_cache[doctype]:
 			return self.value_cache[doctype][fieldname]
 
 		val = self.query.get_sql(
@@ -677,53 +692,55 @@ class Database(object):
 		:param debug: Print the query in the developer / js console.
 		:param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit.
 		"""
-		if not modified:
-			modified = now()
-		if not modified_by:
-			modified_by = frappe.session.user
+		is_single_doctype = not (dn and dt != dn)
+		to_update = field if isinstance(field, dict) else {field: val}
 
-		to_update = {}
 		if update_modified:
-			to_update = {"modified": modified, "modified_by": modified_by}
+			modified = modified or now()
+			modified_by = modified_by or frappe.session.user
+			to_update.update({"modified": modified, "modified_by": modified_by})
+
+		if is_single_doctype:
+			frappe.db.delete(
+				"Singles",
+				filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug
+			)
+
+			singles_data = ((dt, key, sbool(value)) for key, value in to_update.items())
+			query = (
+				frappe.qb.into("Singles")
+					.columns("doctype", "field", "value")
+					.insert(*singles_data)
+			).run(debug=debug)
+			frappe.clear_document_cache(dt, dt)
 
-		if isinstance(field, dict):
-			to_update.update(field)
 		else:
-			to_update.update({field: val})
+			table = DocType(dt)
 
-		if dn and dt!=dn:
-			# with table
-			set_values = []
-			for key in to_update:
-				set_values.append('`{0}`=%({0})s'.format(key))
+			if for_update:
+				docnames = tuple(
+					self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True)
+				) or (NullValue(),)
+				query = frappe.qb.update(table).where(table.name.isin(docnames))
 
-			for name in self.get_values(dt, dn, 'name', for_update=for_update, debug=debug):
-				values = dict(name=name[0])
-				values.update(to_update)
+				for docname in docnames:
+					frappe.clear_document_cache(dt, docname)
 
-				self.sql("""update `tab{0}`
-					set {1} where name=%(name)s""".format(dt, ', '.join(set_values)),
-					values, debug=debug)
+			else:
+				query = self.query.build_conditions(table=dt, filters=dn, update=True)
+				# TODO: Fix this; doesn't work rn - gavin@frappe.io
+				# frappe.cache().hdel_keys(dt, "document_cache")
+				# Workaround: clear all document caches
+				frappe.cache().delete_value('document_cache')
 
-				frappe.clear_document_cache(dt, values['name'])
-		else:
-			# for singles
-			keys = list(to_update)
-			self.sql('''
-				delete from `tabSingles`
-				where field in ({0}) and
-					doctype=%s'''.format(', '.join(['%s']*len(keys))),
-					list(keys) + [dt], debug=debug)
-			for key, value in to_update.items():
-				self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''',
-					(dt, key, value), debug=debug)
-
-			frappe.clear_document_cache(dt, dn)
+			for column, value in to_update.items():
+				query = query.set(column, value)
+
+			query.run(debug=debug)
 
 		if dt in self.value_cache:
 			del self.value_cache[dt]
 
-
 	@staticmethod
 	def set(doc, field, val):
 		"""Set value in document. **Avoid**"""

frappe/database/mariadb/framework_mariadb.sql

@@ -171,9 +171,6 @@ CREATE TABLE `tabDocType` (
   `modified_by` varchar(255) DEFAULT NULL,
   `owner` varchar(255) DEFAULT NULL,
   `docstatus` int(1) NOT NULL DEFAULT 0,
-  `parent` varchar(255) DEFAULT NULL,
-  `parentfield` varchar(255) DEFAULT NULL,
-  `parenttype` varchar(255) DEFAULT NULL,
   `idx` int(8) NOT NULL DEFAULT 0,
   `search_fields` varchar(255) DEFAULT NULL,
   `issingle` int(1) NOT NULL DEFAULT 0,
@@ -228,8 +225,7 @@ CREATE TABLE `tabDocType` (
   `subject_field` varchar(255) DEFAULT NULL,
   `sender_field` varchar(255) DEFAULT NULL,
   `migration_hash` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`name`),
-  KEY `parent` (`parent`)
+  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --

frappe/database/mariadb/schema.py

@@ -18,6 +18,17 @@ class MariaDBTable(DBTable):
 		if index_defs:
 			additional_definitions += ',\n'.join(index_defs) + ',\n'
 
+		# child table columns
+		if self.meta.get("istable") or 0:
+			additional_definitions += ',\n'.join(
+				(
+					f"parent varchar({varchar_len})",
+					f"parentfield varchar({varchar_len})",
+					f"parenttype varchar({varchar_len})",
+					"index parent(parent)"
+				)
+			) + ',\n'
+
 		# create table
 		query = f"""create table `{self.table_name}` (
 			name varchar({varchar_len}) not null primary key,
@@ -26,12 +37,8 @@ class MariaDBTable(DBTable):
 			modified_by varchar({varchar_len}),
 			owner varchar({varchar_len}),
 			docstatus int(1) not null default '0',
-			parent varchar({varchar_len}),
-			parentfield varchar({varchar_len}),
-			parenttype varchar({varchar_len}),
 			idx int(8) not null default '0',
 			{additional_definitions}
-			index parent(parent),
 			index modified(modified))
 			ENGINE={engine}
 			ROW_FORMAT=DYNAMIC

frappe/database/postgres/database.py

@@ -170,11 +170,11 @@ class PostgresDatabase(Database):
 
 	@staticmethod
 	def is_primary_key_violation(e):
-		return e.pgcode == '23505' and '_pkey' in cstr(e.args[0])
+		return getattr(e, "pgcode", None) == '23505' and '_pkey' in cstr(e.args[0])
 
 	@staticmethod
 	def is_unique_key_violation(e):
-		return e.pgcode == '23505' and '_key' in cstr(e.args[0])
+		return getattr(e, "pgcode", None) == '23505' and '_key' in cstr(e.args[0])
 
 	@staticmethod
 	def is_duplicate_fieldname(e):

frappe/database/postgres/framework_postgres.sql

@@ -176,9 +176,6 @@ CREATE TABLE "tabDocType" (
   "modified_by" varchar(255) DEFAULT NULL,
   "owner" varchar(255) DEFAULT NULL,
   "docstatus" smallint NOT NULL DEFAULT 0,
-  "parent" varchar(255) DEFAULT NULL,
-  "parentfield" varchar(255) DEFAULT NULL,
-  "parenttype" varchar(255) DEFAULT NULL,
   "idx" bigint NOT NULL DEFAULT 0,
   "search_fields" varchar(255) DEFAULT NULL,
   "issingle" smallint NOT NULL DEFAULT 0,

frappe/database/postgres/schema.py

@@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition
 
 class PostgresTable(DBTable):
 	def create(self):
-		add_text = ''
+		add_text = ""
 
 		# columns
 		column_defs = self.get_column_definitions()
-		if column_defs: add_text += ',\n'.join(column_defs)
+		if column_defs:
+			add_text += ",\n".join(column_defs)
+
+		# child table columns
+		if self.meta.get("istable") or 0:
+			if column_defs:
+				add_text += ",\n"
+
+			add_text += ",\n".join(
+				(
+					"parent varchar({varchar_len})",
+					"parentfield varchar({varchar_len})",
+					"parenttype varchar({varchar_len})"
+				)
+			)
 
 		# TODO: set docstatus length
 		# create table
-		frappe.db.sql("""create table `%s` (
+		frappe.db.sql(("""create table `%s` (
 			name varchar({varchar_len}) not null primary key,
 			creation timestamp(6),
 			modified timestamp(6),
 			modified_by varchar({varchar_len}),
 			owner varchar({varchar_len}),
 			docstatus smallint not null default '0',
-			parent varchar({varchar_len}),
-			parentfield varchar({varchar_len}),
-			parenttype varchar({varchar_len}),
 			idx bigint not null default '0',
-			%s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text))
+			%s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN))
 
 		self.create_indexes()
 		frappe.db.commit()

frappe/database/schema.py

@@ -106,6 +106,9 @@ class DBTable:
 
 		columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
 			frappe.db.STANDARD_VARCHAR_COLUMNS]
+		if self.meta.get("istable"):
+			columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
+				frappe.db.CHILD_TABLE_COLUMNS]
 		columns += self.columns.values()
 
 		for col in columns:
@@ -300,12 +303,13 @@ def validate_column_length(fieldname):
 def get_definition(fieldtype, precision=None, length=None):
 	d = frappe.db.type_map.get(fieldtype)
 
-	# convert int to long int if the length of the int is greater than 11
+	if not d:
+		return
+
 	if fieldtype == "Int" and length and length > 11:
+		# convert int to long int if the length of the int is greater than 11
 		d = frappe.db.type_map.get("Long Int")
 
-	if not d: return
-
 	coltype = d[0]
 	size = d[1] if d[1] else None
 
@@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None):
 		if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6:
 			size = '21,9'
 
-		if coltype == "varchar" and length:
-			size = length
+		if length:
+			if coltype == "varchar":
+				size = length
+			elif coltype == "int" and length < 11:
+				# allow setting custom length for int if length provided is less than 11
+				# NOTE: this will only be applicable for mariadb as frappe implements int
+				# in postgres as bigint (as seen in type_map)
+				size = length
 
 	if size is not None:
 		coltype = "{coltype}({size})".format(coltype=coltype, size=size)
 
 	return coltype
 
-def add_column(doctype, column_name, fieldtype, precision=None):
+def add_column(
+	doctype,
+	column_name,
+	fieldtype,
+	precision=None,
+	length=None,
+	default=None,
+	not_null=False
+):
 	if column_name in frappe.db.get_table_columns(doctype):
 		# already exists
 		return
 
 	frappe.db.commit()
-	frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype,
-		column_name, get_definition(fieldtype, precision)))
+
+	query = "alter table `tab%s` add column %s %s" % (
+		doctype,
+		column_name,
+		get_definition(fieldtype, precision, length)
+	)
+
+	if not_null:
+		query += " not null"
+	if default:
+		query += f" default '{default}'"
+
+	frappe.db.sql(query)

frappe/desk/doctype/bulk_update/bulk_update.py

@@ -42,13 +42,13 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None):
 		doc = frappe.get_doc(doctype, d)
 		try:
 			message = ''
-			if action == 'submit' and doc.docstatus==0:
+			if action == 'submit' and doc.docstatus.is_draft():
 				doc.submit()
 				message = _('Submiting {0}').format(doctype)
-			elif action == 'cancel' and doc.docstatus==1:
+			elif action == 'cancel' and doc.docstatus.is_submitted():
 				doc.cancel()
 				message = _('Cancelling {0}').format(doctype)
-			elif action == 'update' and doc.docstatus < 2:
+			elif action == 'update' and not doc.docstatus.is_cancelled():
 				doc.update(data)
 				doc.save()
 				message = _('Updating {0}').format(doctype)

frappe/desk/doctype/route_history/route_history.py

@@ -52,3 +52,9 @@ def deferred_insert(routes):
 	]
 
 	_deferred_insert("Route History", json.dumps(routes))
+
+@frappe.whitelist()
+def frequently_visited_links():
+	return frappe.get_all('Route History', fields=['route', 'count(name) as count'], filters={
+		'user': frappe.session.user
+	}, group_by="route", order_by="count desc", limit=5)

frappe/desk/doctype/tag/tag.py

@@ -148,8 +148,6 @@ def update_tags(doc, tags):
 			"doctype": "Tag Link",
 			"document_type": doc.doctype,
 			"document_name": doc.name,
-			"parenttype": doc.doctype,
-			"parent": doc.name,
 			"title": doc.get_title() or '',
 			"tag": tag
 		}).insert(ignore_permissions=True)

frappe/desk/form/linked_with.py

@@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 		else:
 			return results
 
-	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
-
 	for dt, link in linkinfo.items():
 		filters = []
 		link["doctype"] = dt
@@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 					ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"))
 
 				elif link.get("get_parent"):
-					if me and me.parent and me.parenttype == dt:
+					ret = None
+
+					# check for child table
+					if not frappe.get_meta(doctype).istable:
+						continue
+
+					me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
+					if me and me.parenttype == dt:
 						ret = frappe.get_all(doctype=dt, fields=fields,
 							filters=[[dt, "name", '=', me.parent]])
-					else:
-						ret = None
 
 				elif link.get("child_doctype"):
 					or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")]
@@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
 	ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled))
 	ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled))
 
-	filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
+	filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 	# find links of parents
 	links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters)
@@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
 
 def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 
-	filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]]
+	filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 
 	# find links of parents
 	links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1)
-	links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
+	links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
 
 	ret = {}
 
@@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 	ret = {}
 
-	filters=[['fieldtype','=', 'Dynamic Link']]
+	filters = [['fieldtype','=', 'Dynamic Link']]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 
 	# find dynamic links of parents
 	links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
-	links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
+	links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
 
 	for df in links:
 		if is_single(df.doctype): continue
 
-		# optimized to get both link exists and parenttype
-		possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype},
-		fields=['parenttype'], distinct=True)
+		is_child = frappe.get_meta(df.doctype).istable
+		possible_link = frappe.get_all(
+			df.doctype,
+			filters={df.doctype_fieldname: doctype},
+			fields=["parenttype"] if is_child else None,
+			distinct=True
+		)
 
 		if not possible_link: continue
 
-		for d in possible_link:
-			# is child
-			if d.parenttype:
+		if is_child:
+			for d in possible_link:
 				ret[d.parenttype] = {
 					"child_doctype": df.doctype,
 					"fieldname": [df.fieldname],
 					"doctype_fieldname": df.doctype_fieldname
 				}
-			else:
-				ret[df.doctype] = {
-					"fieldname": [df.fieldname],
-					"doctype_fieldname": df.doctype_fieldname
-				}
+		else:
+			ret[df.doctype] = {
+				"fieldname": [df.fieldname],
+				"doctype_fieldname": df.doctype_fieldname
+			}
 
 	return ret

frappe/desk/form/load.py

@@ -91,8 +91,8 @@ def get_docinfo(doc=None, doctype=None, name=None):
 			raise frappe.PermissionError
 
 	all_communications = _get_communications(doc.doctype, doc.name)
-	automated_messages = filter(lambda x: x['communication_type'] == 'Automated Message', all_communications)
-	communications_except_auto_messages = filter(lambda x: x['communication_type'] != 'Automated Message', all_communications)
+	automated_messages = [msg for msg in all_communications if msg['communication_type'] == 'Automated Message']
+	communications_except_auto_messages = [msg for msg in all_communications if msg['communication_type'] != 'Automated Message']
 
 	docinfo = frappe._dict(user_info = {})
 
@@ -119,6 +119,7 @@ def get_docinfo(doc=None, doctype=None, name=None):
 	update_user_info(docinfo)
 
 	frappe.response["docinfo"] = docinfo
+	return docinfo
 
 def add_comments(doc, docinfo):
 	# divide comments into separate lists

frappe/desk/reportview.py

@@ -6,7 +6,7 @@
 import frappe, json
 import frappe.permissions
 from frappe.model.db_query import DatabaseQuery
-from frappe.model import default_fields, optional_fields
+from frappe.model import default_fields, optional_fields, child_table_fields
 from frappe import _
 from io import StringIO
 from frappe.core.doctype.access_log.access_log import make_access_log
@@ -156,7 +156,7 @@ def raise_invalid_field(fieldname):
 def is_standard(fieldname):
 	if '.' in fieldname:
 		parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
-	return fieldname in default_fields or fieldname in optional_fields
+	return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields
 
 def extract_fieldname(field):
 	for text in (',', '/*', '#'):
@@ -319,7 +319,7 @@ def export_query():
 	if add_totals_row:
 		ret = append_totals_row(ret)
 
-	data = [['Sr'] + get_labels(db_query.fields, doctype)]
+	data = [[_('Sr')] + get_labels(db_query.fields, doctype)]
 	for i, row in enumerate(ret):
 		data.append([i+1] + list(row))
 
@@ -378,7 +378,8 @@ def get_labels(fields, doctype):
 	for key in fields:
 		key = key.split(" as ")[0]
 
-		if key.startswith(('count(', 'sum(', 'avg(')): continue
+		if key.startswith(('count(', 'sum(', 'avg(')):
+			continue
 
 		if "." in key:
 			parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
@@ -386,10 +387,16 @@ def get_labels(fields, doctype):
 			parenttype = doctype
 			fieldname = fieldname.strip("`")
 
-		df = frappe.get_meta(parenttype).get_field(fieldname)
-		label = df.label if df else fieldname.title()
-		if label in labels:
-			label = doctype + ": " + label
+		if parenttype == doctype and fieldname == "name":
+			label = _("ID", context="Label of name column in report")
+		else:
+			df = frappe.get_meta(parenttype).get_field(fieldname)
+			label = _(df.label if df else fieldname.title())
+			if parenttype != doctype:
+				# If the column is from a child table, append the child doctype.
+				# For example, "Item Code (Sales Invoice Item)".
+				label += f" ({ _(parenttype) })"
+
 		labels.append(label)
 
 	return labels

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -252,7 +252,7 @@ def make_links(columns, data):
 				if col.options and row.get(col.options):
 					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
 			elif col.fieldtype == "Currency":
-				doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None
+				doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None
 				# Pass the Document to get the currency based on docfield option
 				row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc)
 	return columns, data

frappe/email/doctype/email_account/email_account.py

@@ -1,5 +1,6 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
+
 import email.utils
 import functools
 import imaplib
@@ -7,6 +8,7 @@ import socket
 import time
 from datetime import datetime, timedelta
 from poplib import error_proto
+from typing import List
 
 import frappe
 from frappe import _, are_emails_muted, safe_encode
@@ -82,9 +84,6 @@ class EmailAccount(Document):
 		if frappe.local.flags.in_patch or frappe.local.flags.in_test:
 			return
 
-		#if self.enable_incoming and not self.append_to:
-		#	frappe.throw(_("Append To is mandatory for incoming mails"))
-
 		if (not self.awaiting_password and not frappe.local.flags.in_install
 			and not frappe.local.flags.in_patch):
 			if self.password or self.smtp_server in ('127.0.0.1', 'localhost'):
@@ -422,10 +421,10 @@ class EmailAccount(Document):
 	def get_failed_attempts_count(self):
 		return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name)))
 
-	def receive(self, test_mails=None):
+	def receive(self):
 		"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
 		exceptions = []
-		inbound_mails = self.get_inbound_mails(test_mails=test_mails)
+		inbound_mails = self.get_inbound_mails()
 		for mail in inbound_mails:
 			try:
 				communication = mail.process()
@@ -442,7 +441,7 @@ class EmailAccount(Document):
 				frappe.db.rollback()
 			except Exception:
 				frappe.db.rollback()
-				frappe.log_error('email_account.receive')
+				frappe.log_error(title="EmailAccount.receive")
 				if self.use_imap:
 					self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback())
 				exceptions.append(frappe.get_traceback())
@@ -458,20 +457,19 @@ class EmailAccount(Document):
 		if exceptions:
 			raise Exception(frappe.as_json(exceptions))
 
-	def get_inbound_mails(self, test_mails=None):
+	def get_inbound_mails(self) -> List[InboundMail]:
 		"""retrive and return inbound mails.
 
 		"""
 		mails = []
 
-		def process_mail(messages):
+		def process_mail(messages, append_to=None):
 			for index, message in enumerate(messages.get("latest_messages", [])):
 				uid = messages['uid_list'][index] if messages.get('uid_list') else None
-				seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0
-				mails.append(InboundMail(message, self, uid, seen_status))
-
-		if frappe.local.flags.in_test:
-			return [InboundMail(msg, self) for msg in test_mails or []]
+				seen_status = messages.get('seen_status', {}).get(uid)
+				if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN":
+					# only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN'
+					mails.append(InboundMail(message, self, uid, seen_status, append_to))
 
 		if not self.enable_incoming:
 			return []
@@ -482,10 +480,10 @@ class EmailAccount(Document):
 			if self.use_imap:
 				# process all given imap folder
 				for folder in self.imap_folder:
-					email_server.select_imap_folder(folder.folder_name)
-					email_server.settings['uid_validity'] = folder.uidvalidity
-					messages = email_server.get_messages(folder=folder.folder_name) or {}
-					process_mail(messages)
+					if email_server.select_imap_folder(folder.folder_name):
+						email_server.settings['uid_validity'] = folder.uidvalidity
+						messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {}
+						process_mail(messages, folder.append_to)
 			else:
 				# process the pop3 account
 				messages = email_server.get_messages() or {}
@@ -495,7 +493,6 @@ class EmailAccount(Document):
 		except Exception:
 			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
 			return []
-
 		return mails
 
 	def handle_bad_emails(self, uid, raw, reason):
@@ -625,7 +622,6 @@ class EmailAccount(Document):
 			if frappe.db.exists("Email Account", {"enable_automatic_linking": 1, "name": ('!=', self.name)}):
 				frappe.throw(_("Automatic Linking can be activated only for one Email Account."))
 
-
 	def append_email_to_sent_folder(self, message):
 		email_server = None
 		try:
@@ -643,7 +639,8 @@ class EmailAccount(Document):
 				message = safe_encode(message)
 				email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message)
 			except Exception:
-				frappe.log_error()
+				frappe.log_error(title="EmailAccount.append_email_to_sent_folder")
+
 
 @frappe.whitelist()
 def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None):

frappe/email/doctype/email_account/test_email_account.py

@@ -14,11 +14,11 @@ from frappe.core.doctype.communication.email import make
 from frappe.desk.form.load import get_attachments
 from frappe.email.doctype.email_account.email_account import notify_unreplied
 
+from unittest.mock import patch
+
 make_test_records("User")
 make_test_records("Email Account")
 
-
-
 class TestEmailAccount(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
@@ -45,10 +45,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_incoming(self):
 		cleanup("test_sender@example.com")
 
-		test_mails = [self.get_test_mail('incoming-1.raw')]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-1.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("test_receiver@example.com" in comm.recipients)
@@ -72,11 +83,21 @@ class TestEmailAccount(unittest.TestCase):
 		existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'})
 		frappe.delete_doc("File", existing_file.name)
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-2.raw"), "r") as testfile:
-			test_mails = [testfile.read()]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-2.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("test_receiver@example.com" in comm.recipients)
@@ -93,11 +114,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_incoming_attached_email_from_outlook_plain_text_only(self):
 		cleanup("test_sender@example.com")
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-3.raw"), "r") as f:
-			test_mails = [f.read()]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-3.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content)
@@ -106,11 +137,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_incoming_attached_email_from_outlook_layers(self):
 		cleanup("test_sender@example.com")
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "incoming-4.raw"), "r") as f:
-			test_mails = [f.read()]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-4.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content)
@@ -151,11 +192,23 @@ class TestEmailAccount(unittest.TestCase):
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f:
 			raw = f.read()
 			raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id"))
-			test_mails = [raw]
 
 		# parse reply
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					raw
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
+
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		sent = frappe.get_doc("Communication", sent_name)
 
@@ -173,8 +226,20 @@ class TestEmailAccount(unittest.TestCase):
 			test_mails.append(f.read())
 
 		# parse reply
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': test_mails,
+				'seen_status': {
+					2: 'UNSEEN',
+					3: 'UNSEEN'
+				},
+				'uid_list': [2, 3]
+			}
+		}
+
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
 			fields=["name", "reference_doctype", "reference_name"])
@@ -197,11 +262,22 @@ class TestEmailAccount(unittest.TestCase):
 
 		# get test mail with message-id as in-reply-to
 		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f:
-			test_mails = [f.read().replace('{{ message_id }}', last_mail.message_id)]
+			messages = {
+				# append_to = ToDo
+				'"INBOX"': {
+					'latest_messages': [
+						f.read().replace('{{ message_id }}', last_mail.message_id)
+					],
+					'seen_status': {
+						2: 'UNSEEN'
+					},
+					'uid_list': [2]
+				}
+			}
 
 		# pull the mail
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
 			fields=["name", "reference_doctype", "reference_name"])
@@ -213,10 +289,21 @@ class TestEmailAccount(unittest.TestCase):
 	def test_auto_reply(self):
 		cleanup("test_sender@example.com")
 
-		test_mails = [self.get_test_mail('incoming-1.raw')]
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					self.get_test_mail('incoming-1.raw')
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
-		email_account.receive(test_mails=test_mails)
+		TestEmailAccount.mocked_email_receive(email_account, messages)
 
 		comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"})
 		self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype,
@@ -246,6 +333,91 @@ class TestEmailAccount(unittest.TestCase):
 		with self.assertRaises(Exception):
 			email_account.validate()
 
+	def test_append_to(self):
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		mail_content = self.get_test_mail(fname="incoming-2.raw")
+
+		inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo')
+		communication = inbound_mail.process()
+		# the append_to for the email is set to ToDO in "_Test Email Account 1"
+		self.assertEqual(communication.reference_doctype, 'ToDo')
+		self.assertTrue(communication.reference_name)
+		self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
+
+	def test_append_to_with_imap_folders(self):
+		mail_content_1 = self.get_test_mail(fname="incoming-1.raw")
+		mail_content_2 = self.get_test_mail(fname="incoming-2.raw")
+		mail_content_3 = self.get_test_mail(fname="incoming-3.raw")
+
+		messages = {
+			# append_to = ToDo
+			'"INBOX"': {
+				'latest_messages': [
+					mail_content_1,
+					mail_content_2
+				],
+				'seen_status': {
+					0: 'UNSEEN',
+					1: 'UNSEEN'
+				},
+				'uid_list': [0,1]
+			},
+			# append_to = Communication
+			'"Test Folder"': {
+				'latest_messages': [
+					mail_content_3
+				],
+				'seen_status': {
+					2: 'UNSEEN'
+				},
+				'uid_list': [2]
+			}
+		}
+
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		mails = TestEmailAccount.mocked_get_inbound_mails(email_account, messages)
+		self.assertEqual(len(mails), 3)
+
+		inbox_mails = 0
+		test_folder_mails = 0
+
+		for mail in mails:
+			communication = mail.process()
+			if mail.append_to == 'ToDo':
+				inbox_mails += 1
+				self.assertEqual(communication.reference_doctype, 'ToDo')
+				self.assertTrue(communication.reference_name)
+				self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
+			else:
+				test_folder_mails += 1
+				self.assertEqual(communication.reference_doctype, None)
+
+		self.assertEqual(inbox_mails, 2)
+		self.assertEqual(test_folder_mails, 1)
+
+	@patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True)
+	@patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None)
+	def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None):
+		from frappe.email.receive import EmailServer
+
+		def get_mocked_messages(**kwargs):
+			return messages.get(kwargs["folder"], {})
+
+		with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages):
+			mails = email_account.get_inbound_mails()
+
+		return mails
+
+	@patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True)
+	@patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None)
+	def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None):
+		def get_mocked_messages(**kwargs):
+			return messages.get(kwargs["folder"], {})
+
+		from frappe.email.receive import EmailServer
+		with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages):
+			email_account.receive()
+
 class TestInboundMail(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
@@ -313,11 +485,11 @@ class TestInboundMail(unittest.TestCase):
 
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
 		inbound_mail = InboundMail(mail_content, email_account, 12345, 1)
-		new_communiction = inbound_mail.process()
+		new_communication = inbound_mail.process()
 
 		# Make sure that uid is changed to new uid
-		self.assertEqual(new_communiction.uid, 12345)
-		self.assertEqual(communication.name, new_communiction.name)
+		self.assertEqual(new_communication.uid, 12345)
+		self.assertEqual(communication.name, new_communication.name)
 
 	def test_find_parent_email_queue(self):
 		"""If the mail is reply to the already sent mail, there will be a email queue record.

frappe/email/doctype/email_account/test_records.json

@@ -20,7 +20,7 @@
 		"pop3_server": "pop.test.example.com",
 		"no_remaining":"0",
 		"append_to": "ToDo",
-		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}],
+		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}],
 		"track_email_status": 1
 	},
 	{

frappe/email/doctype/email_queue/email_queue.py

@@ -11,7 +11,6 @@ import quopri
 from email.parser import Parser
 from email.policy import SMTPUTF8
 from html2text import html2text
-from six.moves import html_parser as HTMLParser
 
 import frappe
 from frappe import _, safe_encode, task
@@ -20,6 +19,7 @@ from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message
 from frappe.email.email_body import add_attachment, get_formatted_html, get_email
 from frappe.utils import cint, split_emails, add_days, nowdate, cstr, get_hook_method
 from frappe.email.doctype.email_account.email_account import EmailAccount
+from frappe.query_builder.utils import DocType
 
 
 MAX_RETRY_COUNT = 3
@@ -444,7 +444,7 @@ class QueueBuilder:
 
 		try:
 			text_content = html2text(self._message)
-		except HTMLParser.HTMLParseError:
+		except Exception:
 			text_content = "See html attachment"
 		return text_content + unsubscribe_text_message
 
@@ -477,18 +477,27 @@ class QueueBuilder:
 
 		all_ids = list(set(self.recipients + self.cc))
 
-		EmailUnsubscribe = frappe.qb.DocType("Email Unsubscribe")
-
-		unsubscribed = (frappe.qb.from_(EmailUnsubscribe)
-						.select(EmailUnsubscribe.email)
-						.where(EmailUnsubscribe.email.isin(all_ids) & 	
-									(
-										(
-											(EmailUnsubscribe.reference_doctype == self.reference_doctype) & (EmailUnsubscribe.reference_name == self.reference_name)
-										) | EmailUnsubscribe.global_unsubscribe == 1
-									)
-								).distinct()
-						).run(pluck=True)
+		EmailUnsubscribe = DocType("Email Unsubscribe")
+
+		if len(all_ids) > 0:
+			unsubscribed = (
+				frappe.qb.from_(EmailUnsubscribe).select(
+					EmailUnsubscribe.email
+				).where(
+					EmailUnsubscribe.email.isin(all_ids)
+					& (
+						(
+							(EmailUnsubscribe.reference_doctype == self.reference_doctype)
+							& (EmailUnsubscribe.reference_name == self.reference_name)
+						) | (
+							EmailUnsubscribe.global_unsubscribe == 1
+						)
+					)
+				).distinct()
+			).run(pluck=True)
+		else:
+			unsubscribed = None
+
 		self._unsubscribed_user_emails = unsubscribed or []
 		return self._unsubscribed_user_emails
 

frappe/email/doctype/notification/notification.py

@@ -137,7 +137,7 @@ def get_context(context):
 
 		if self.set_property_after_alert:
 			allow_update = True
-			if doc.docstatus == 1 and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit:
+			if doc.docstatus.is_submitted() and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit:
 				allow_update = False
 			try:
 				if allow_update and not doc.flags.in_notification_update:

frappe/email/receive.py

@@ -108,7 +108,8 @@ class EmailServer:
 				raise
 
 	def select_imap_folder(self, folder):
-		self.imap.select(folder)
+		res = self.imap.select(f'"{folder}"')
+		return res[0] == 'OK'	# The folder exsits TODO: handle other resoponses too
 
 	def logout(self):
 		if cint(self.settings.use_imap):
@@ -582,10 +583,11 @@ class Email:
 class InboundMail(Email):
 	"""Class representation of incoming mail along with mail handlers.
 	"""
-	def __init__(self, content, email_account, uid=None, seen_status=None):
+	def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None):
 		super().__init__(content)
 		self.email_account = email_account
 		self.uid = uid or -1
+		self.append_to = append_to
 		self.seen_status = seen_status or 0
 
 		# System documents related to this mail
@@ -623,15 +625,18 @@ class InboundMail(Email):
 		if self.parent_communication():
 			data['in_reply_to'] = self.parent_communication().name
 
+		append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to
+
 		if self.reference_document():
 			data['reference_doctype'] = self.reference_document().doctype
 			data['reference_name'] = self.reference_document().name
-		elif self.email_account.append_to and self.email_account.append_to != 'Communication':
-			reference_doc = self._create_reference_document(self.email_account.append_to)
-			if reference_doc:
-				data['reference_doctype'] = reference_doc.doctype
-				data['reference_name'] = reference_doc.name
-				data['is_first'] = True
+		else: 
+			if append_to and append_to != 'Communication':
+				reference_doc = self._create_reference_document(append_to)
+				if reference_doc:
+					data['reference_doctype'] = reference_doc.doctype
+					data['reference_name'] = reference_doc.name
+			data['is_first'] = True
 
 		if self.is_notification():
 			# Disable notifications for notification.

frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py

@@ -5,7 +5,7 @@ import frappe
 import json
 from frappe import _
 from frappe.model.document import Document
-from frappe.model import default_fields
+from frappe.model import default_fields, child_table_fields
 
 class DocumentTypeMapping(Document):
 	def validate(self):
@@ -14,7 +14,7 @@ class DocumentTypeMapping(Document):
 	def validate_inner_mapping(self):
 		meta = frappe.get_meta(self.local_doctype)
 		for field_map in self.field_mapping:
-			if field_map.local_fieldname not in default_fields:
+			if field_map.local_fieldname not in (default_fields + child_table_fields):
 				field = meta.get_field(field_map.local_fieldname)
 				if not field:
 					frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx))

frappe/exceptions.py

@@ -103,6 +103,7 @@ class DocumentAlreadyRestored(ValidationError): pass
 class AttachmentLimitReached(ValidationError): pass
 class QueryTimeoutError(Exception): pass
 class QueryDeadlockError(Exception): pass
+class TooManyWritesError(Exception): pass
 # OAuth exceptions
 class InvalidAuthorizationHeader(CSRFTokenError): pass
 class InvalidAuthorizationPrefix(CSRFTokenError): pass

frappe/model/__init__.py

@@ -90,11 +90,14 @@ default_fields = (
 	'creation',
 	'modified',
 	'modified_by',
+	'docstatus',
+	'idx'
+)
+
+child_table_fields = (
 	'parent',
 	'parentfield',
-	'parenttype',
-	'idx',
-	'docstatus'
+	'parenttype'
 )
 
 optional_fields = (

frappe/model/base_document.py

@@ -1,9 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+
 import frappe
 import datetime
 from frappe import _
-from frappe.model import default_fields, table_fields
+from frappe.model import default_fields, table_fields, child_table_fields
 from frappe.model.naming import set_new_name
 from frappe.model.utils.link_count import notify_link_count
 from frappe.modules import load_doctype_module
@@ -11,6 +12,7 @@ from frappe.model import display_fieldtypes
 from frappe.utils import (cint, flt, now, cstr, strip_html,
 	sanitize_html, sanitize_email, cast_fieldtype)
 from frappe.utils.html_utils import unescape_html
+from frappe.model.docstatus import DocStatus
 
 max_positive_value = {
 	'smallint': 2 ** 15,
@@ -20,6 +22,7 @@ max_positive_value = {
 
 DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link')
 
+
 def get_controller(doctype):
 	"""Returns the **class** object of the given DocType.
 	For `custom` type, returns `frappe.model.document.Document`.
@@ -101,6 +104,10 @@ class BaseDocument(object):
 				"balance": 42000
 			})
 		"""
+
+		# QUESTION: why do we need the 1st for loop?
+		# we're essentially setting the values in d, in the 2nd for loop (?)
+
 		# first set default field values of base document
 		for key in default_fields:
 			if key in d:
@@ -205,7 +212,10 @@ class BaseDocument(object):
 			raise ValueError
 
 	def remove(self, doc):
-		self.get(doc.parentfield).remove(doc)
+		# Usage: from the parent doc, pass the child table doc
+		# to remove that child doc from the child table, thus removing it from the parent doc
+		if doc.get("parentfield"):
+			self.get(doc.parentfield).remove(doc)
 
 	def _init_child(self, value, key):
 		if not self.doctype:
@@ -224,7 +234,7 @@ class BaseDocument(object):
 		value.parentfield = key
 
 		if value.docstatus is None:
-			value.docstatus = 0
+			value.docstatus = DocStatus.draft()
 
 		if not getattr(value, "idx", None):
 			value.idx = len(self.get(key) or []) + 1
@@ -282,8 +292,11 @@ class BaseDocument(object):
 			if key not in self.__dict__:
 				self.__dict__[key] = None
 
-			if key in ("idx", "docstatus") and self.__dict__[key] is None:
-				self.__dict__[key] = 0
+			if self.__dict__[key] is None:
+				if key == "docstatus":
+					self.docstatus = DocStatus.draft()
+				elif key == "idx":
+					self.__dict__[key] = 0
 
 		for key in self.get_valid_columns():
 			if key not in self.__dict__:
@@ -304,12 +317,27 @@ class BaseDocument(object):
 	def is_new(self):
 		return self.get("__islocal")
 
-	def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False):
+	@property
+	def docstatus(self):
+		return DocStatus(self.get("docstatus"))
+
+	@docstatus.setter
+	def docstatus(self, value):
+		self.__dict__["docstatus"] = DocStatus(cint(value))
+
+	def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False):
 		doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str)
 		doc["doctype"] = self.doctype
 		for df in self.meta.get_table_fields():
 			children = self.get(df.fieldname) or []
-			doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children]
+			doc[df.fieldname] = [
+				d.as_dict(
+					convert_dates_to_str=convert_dates_to_str,
+					no_nulls=no_nulls,
+					no_default_fields=no_default_fields,
+					no_child_table_fields=no_child_table_fields
+				) for d in children
+			]
 
 		if no_nulls:
 			for k in list(doc):
@@ -321,6 +349,11 @@ class BaseDocument(object):
 				if k in default_fields:
 					del doc[k]
 
+		if no_child_table_fields:
+			for k in list(doc):
+				if k in child_table_fields:
+					del doc[k]
+
 		for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"):
 			if self.get(key):
 				doc[key] = self.get(key)
@@ -492,7 +525,7 @@ class BaseDocument(object):
 					self.set(df.fieldname, flt(self.get(df.fieldname)))
 
 		if self.docstatus is not None:
-			self.docstatus = cint(self.docstatus)
+			self.docstatus = DocStatus(cint(self.docstatus))
 
 	def _get_missing_mandatory_fields(self):
 		"""Get mandatory fields that do not have any values"""
@@ -500,12 +533,12 @@ class BaseDocument(object):
 			if df.fieldtype in table_fields:
 				return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label))
 
-			elif self.parentfield:
+			# check if parentfield exists (only applicable for child table doctype)
+			elif self.get("parentfield"):
 				return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)),
 					_("Row"), self.idx, _("Value missing for"), _(df.label))
 
-			else:
-				return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
+			return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
 
 		missing = []
 
@@ -524,10 +557,11 @@ class BaseDocument(object):
 	def get_invalid_links(self, is_submittable=False):
 		"""Returns list of invalid links and also updates fetch values if not set"""
 		def get_msg(df, docname):
-			if self.parentfield:
+			# check if parentfield exists (only applicable for child table doctype)
+			if self.get("parentfield"):
 				return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname)
-			else:
-				return "{}: {}".format(_(df.label), docname)
+
+			return "{}: {}".format(_(df.label), docname)
 
 		invalid_links = []
 		cancelled_links = []
@@ -581,7 +615,7 @@ class BaseDocument(object):
 					setattr(self, df.fieldname, values.name)
 
 					for _df in fields_to_fetch:
-						if self.is_new() or self.docstatus != 1 or _df.allow_on_submit:
+						if self.is_new() or not self.docstatus.is_submitted() or _df.allow_on_submit:
 							self.set_fetch_from_value(doctype, _df, values)
 
 					notify_link_count(doctype, docname)
@@ -591,7 +625,7 @@ class BaseDocument(object):
 
 					elif (df.fieldname != "amended_from"
 						and (is_submittable or self.meta.is_submittable) and frappe.get_meta(doctype).is_submittable
-						and cint(frappe.db.get_value(doctype, docname, "docstatus"))==2):
+						and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled()):
 
 						cancelled_links.append((df.fieldname, docname, get_msg(df, docname)))
 
@@ -601,11 +635,8 @@ class BaseDocument(object):
 		fetch_from_fieldname = df.fetch_from.split('.')[-1]
 		value = values[fetch_from_fieldname]
 		if df.fieldtype in ['Small Text', 'Text', 'Data']:
-			if fetch_from_fieldname in default_fields:
-				from frappe.model.meta import get_default_df
-				fetch_from_df = get_default_df(fetch_from_fieldname)
-			else:
-				fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname)
+			from frappe.model.meta import get_default_df
+			fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname)
 
 			if not fetch_from_df:
 				frappe.throw(
@@ -740,9 +771,9 @@ class BaseDocument(object):
 
 
 	def throw_length_exceeded_error(self, df, max_length, value):
-		if self.parentfield and self.idx:
+		# check if parentfield exists (only applicable for child table doctype)
+		if self.get("parentfield"):
 			reference = _("{0}, Row {1}").format(_(self.doctype), self.idx)
-
 		else:
 			reference = "{0} {1}".format(_(self.doctype), self.name)
 
@@ -805,8 +836,8 @@ class BaseDocument(object):
 				or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code")
 
 				# cancelled and submit but not update after submit should be ignored
-				or self.docstatus==2
-				or (self.docstatus==1 and not df.get("allow_on_submit"))):
+				or self.docstatus.is_cancelled()
+				or (self.docstatus.is_submitted() and not df.get("allow_on_submit"))):
 				continue
 
 			else:
@@ -853,7 +884,7 @@ class BaseDocument(object):
 		:param parentfield: If fieldname is in child table."""
 		from frappe.model.meta import get_field_precision
 
-		if parentfield and not isinstance(parentfield, str):
+		if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"):
 			parentfield = parentfield.parentfield
 
 		cache_key = parentfield or "main"
@@ -880,7 +911,7 @@ class BaseDocument(object):
 		from frappe.utils.formatters import format_value
 
 		df = self.meta.get_field(fieldname)
-		if not df and fieldname in default_fields:
+		if not df:
 			from frappe.model.meta import get_default_df
 			df = get_default_df(fieldname)
 

frappe/model/delete_doc.py

@@ -212,7 +212,7 @@ def check_permission_and_not_submitted(doc):
 			.format(doc.doctype, doc.name), raise_exception=frappe.PermissionError)
 
 	# check if submitted
-	if doc.docstatus == 1:
+	if doc.docstatus.is_submitted():
 		frappe.msgprint(_("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format(_(doc.doctype), doc.name, "<a href='https://docs.erpnext.com//docs/user/manual/en/setting-up/articles/delete-submitted-document' target='_blank'>", "</a>"),
 			raise_exception=True)
 
@@ -222,32 +222,35 @@ def check_if_doc_is_linked(doc, method="Delete"):
 	"""
 	from frappe.model.rename_doc import get_link_fields
 	link_fields = get_link_fields(doc.doctype)
-	link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields]
+	ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
+
+	for lf in link_fields:
+		link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle']
 
-	for link_dt, link_field, issingle in link_fields:
 		if not issingle:
-			for item in frappe.db.get_values(link_dt, {link_field:doc.name},
-				["name", "parent", "parenttype", "docstatus"], as_dict=True):
-				linked_doctype = item.parenttype if item.parent else link_dt
+			fields = ["name", "docstatus"]
+			if frappe.get_meta(link_dt).istable:
+				fields.extend(["parent", "parenttype"])
 
-				ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
+			for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True):
+				# available only in child table cases
+				item_parent = getattr(item, "parent", None)
+				linked_doctype = item.parenttype if item_parent else link_dt
 
 				if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'):
 					# don't check for communication and todo!
 					continue
 
-				if not item:
-					continue
-				elif method != "Delete"  and (method != "Cancel" or item.docstatus != 1):
+				if method != "Delete" and (method != "Cancel" or item.docstatus != 1):
 					# don't raise exception if not
 					# linked to a non-cancelled doc when deleting or to a submitted doc when cancelling
 					continue
-				elif link_dt == doc.doctype and (item.parent or item.name) == doc.name:
+				elif link_dt == doc.doctype and (item_parent or item.name) == doc.name:
 					# don't raise exception if not
 					# linked to same item or doc having same name as the item
 					continue
 				else:
-					reference_docname = item.parent or item.name
+					reference_docname = item_parent or item.name
 					raise_link_exists_exception(doc, linked_doctype, reference_docname)
 
 		else:

frappe/model/docstatus.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
+# License: MIT. See LICENSE
+
+
+class DocStatus(int):
+	def is_draft(self):
+		return self == self.draft()
+
+	def is_submitted(self):
+		return self == self.submitted()
+
+	def is_cancelled(self):
+		return self == self.cancelled()
+
+	@classmethod
+	def draft(cls):
+		return cls(0)
+
+	@classmethod
+	def submitted(cls):
+		return cls(1)
+
+	@classmethod
+	def cancelled(cls):
+		return cls(2)

frappe/model/document.py

@@ -1,13 +1,16 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import frappe
+import hashlib
+import json
 import time
+from werkzeug.exceptions import NotFound
+
+import frappe
 from frappe import _, msgprint, is_whitelisted
 from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff
 from frappe.model.base_document import BaseDocument, get_controller
-from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc
-from werkzeug.exceptions import NotFound, Forbidden
-import hashlib, json
+from frappe.model.naming import set_new_name
+from frappe.model.docstatus import DocStatus
 from frappe.model import optional_fields, table_fields
 from frappe.model.workflow import validate_workflow
 from frappe.model.workflow import set_workflow_state_on_action
@@ -17,6 +20,7 @@ from frappe.desk.form.document_follow import follow_document
 from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event
 from frappe.utils.data import get_absolute_url
 
+
 # once_only validation
 # methods
 
@@ -307,9 +311,6 @@ class Document(BaseDocument):
 
 		self.check_permission("write", "save")
 
-		if self.docstatus == 2:
-			self._rename_doc_on_cancel()
-
 		self.set_user_and_timestamp()
 		self.set_docstatus()
 		self.check_if_latest()
@@ -474,7 +475,7 @@ class Document(BaseDocument):
 
 		# We'd probably want the creation and owner to be set via API
 		# or Data import at some point, that'd have to be handled here
-		if self.is_new():
+		if self.is_new() and not (frappe.flags.in_patch or frappe.flags.in_migrate):
 			self.creation = self.modified
 			self.owner = self.modified_by
 
@@ -490,7 +491,7 @@ class Document(BaseDocument):
 
 	def set_docstatus(self):
 		if self.docstatus is None:
-			self.docstatus=0
+			self.docstatus = DocStatus.draft()
 
 		for d in self.get_all_children():
 			d.docstatus = self.docstatus
@@ -526,7 +527,7 @@ class Document(BaseDocument):
 
 	def _validate_non_negative(self):
 		def get_msg(df):
-			if self.parentfield:
+			if self.get("parentfield"):
 				return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)),
 					_("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label)))
 			else:
@@ -720,6 +721,7 @@ class Document(BaseDocument):
 			else:
 				tmp = frappe.db.sql("""select modified, docstatus from `tab{0}`
 					where name = %s for update""".format(self.doctype), self.name, as_dict=True)
+
 				if not tmp:
 					frappe.throw(_("Record does not exist"))
 				else:
@@ -740,7 +742,7 @@ class Document(BaseDocument):
 		else:
 			self.check_docstatus_transition(0)
 
-	def check_docstatus_transition(self, docstatus):
+	def check_docstatus_transition(self, to_docstatus):
 		"""Ensures valid `docstatus` transition.
 		Valid transitions are (number in brackets is `docstatus`):
 
@@ -751,31 +753,32 @@ class Document(BaseDocument):
 
 		"""
 		if not self.docstatus:
-			self.docstatus = 0
-		if docstatus==0:
-			if self.docstatus==0:
+			self.docstatus = DocStatus.draft()
+
+		if to_docstatus == DocStatus.draft():
+			if self.docstatus.is_draft():
 				self._action = "save"
-			elif self.docstatus==1:
+			elif self.docstatus.is_submitted():
 				self._action = "submit"
 				self.check_permission("submit")
-			elif self.docstatus==2:
+			elif self.docstatus.is_cancelled():
 				raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)"))
 			else:
 				raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus)
 
-		elif docstatus==1:
-			if self.docstatus==1:
+		elif to_docstatus == DocStatus.submitted():
+			if self.docstatus.is_submitted():
 				self._action = "update_after_submit"
 				self.check_permission("submit")
-			elif self.docstatus==2:
+			elif self.docstatus.is_cancelled():
 				self._action = "cancel"
 				self.check_permission("cancel")
-			elif self.docstatus==0:
+			elif self.docstatus.is_draft():
 				raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)"))
 			else:
 				raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus)
 
-		elif docstatus==2:
+		elif to_docstatus == DocStatus.cancelled():
 			raise frappe.ValidationError(_("Cannot edit cancelled document"))
 
 	def set_parent_in_children(self):
@@ -929,14 +932,14 @@ class Document(BaseDocument):
 	@whitelist.__func__
 	def _submit(self):
 		"""Submit the document. Sets `docstatus` = 1, then saves."""
-		self.docstatus = 1
+		self.docstatus = DocStatus.submitted()
 		return self.save()
 
 	@whitelist.__func__
 	def _cancel(self):
 		"""Cancel the document. Sets `docstatus` = 2, then saves.
 		"""
-		self.docstatus = 2
+		self.docstatus = DocStatus.cancelled()
 		return self.save()
 
 	@whitelist.__func__
@@ -954,7 +957,7 @@ class Document(BaseDocument):
 		frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags)
 
 	def run_before_save_methods(self):
-		"""Run standard methods before  `INSERT` or `UPDATE`. Standard Methods are:
+		"""Run standard methods before	`INSERT` or `UPDATE`. Standard Methods are:
 
 		- `validate`, `before_save` for **Save**.
 		- `validate`, `before_submit` for **Submit**.
@@ -1199,7 +1202,7 @@ class Document(BaseDocument):
 		if not frappe.compare(val1, condition, val2):
 			label = doc.meta.get_label(fieldname)
 			condition_str = error_condition_map.get(condition, condition)
-			if doc.parentfield:
+			if doc.get("parentfield"):
 				msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2)
 			else:
 				msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2)
@@ -1223,7 +1226,7 @@ class Document(BaseDocument):
 				doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]}))
 
 		for fieldname in fieldnames:
-			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield)))
+			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield"))))
 
 	def get_url(self):
 		"""Returns Desk URL for this document."""
@@ -1371,19 +1374,16 @@ class Document(BaseDocument):
 		from frappe.desk.doctype.tag.tag import DocTags
 		return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
 
-	def _rename_doc_on_cancel(self):
-		new_name = gen_new_name_for_cancelled_doc(self)
-		frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False)
-		self.name = new_name
-
 	def __repr__(self):
 		name = self.name or "unsaved"
 		doctype = self.__class__.__name__
 
 		docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
-		parent = f" parent={self.parent}" if self.parent else ""
+		repr_str = f"<{doctype}: {name}{docstatus}"
 
-		return f"<{doctype}: {name}{docstatus}{parent}>"
+		if not hasattr(self, "parent"):
+			return repr_str + ">"
+		return f"{repr_str} parent={self.parent}>"
 
 	def __str__(self):
 		name = self.name or "unsaved"

frappe/model/mapper.py

@@ -4,7 +4,7 @@ import json
 
 import frappe
 from frappe import _
-from frappe.model import default_fields, table_fields
+from frappe.model import default_fields, table_fields, child_table_fields
 from frappe.utils import cstr
 
 
@@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent):
 	no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
 		+ [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
 		+ list(default_fields)
+		+ list(child_table_fields)
 		+ list(table_map.get("field_no_map", [])))
 
 	for df in target_doc.meta.get("fields"):

frappe/model/meta.py

@@ -18,7 +18,7 @@ from datetime import datetime
 import click
 import frappe, json, os
 from frappe.utils import cstr, cint, cast
-from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields
+from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields
 from frappe.model.document import Document
 from frappe.model.base_document import BaseDocument
 from frappe.modules import load_doctype_module
@@ -191,6 +191,8 @@ class Meta(Document):
 			else:
 				self._valid_columns = self.default_fields + \
 					[df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes]
+				if self.istable:
+					self._valid_columns += list(child_table_fields)
 
 		return self._valid_columns
 
@@ -520,7 +522,7 @@ class Meta(Document):
 		'''add `links` child table in standard link dashboard format'''
 		dashboard_links = []
 
-		if hasattr(self, 'links') and self.links:
+		if getattr(self, 'links', None):
 			dashboard_links.extend(self.links)
 
 		if not data.transactions:
@@ -625,9 +627,9 @@ def get_field_currency(df, doc=None):
 		frappe.local.field_currency = frappe._dict()
 
 	if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or
-		(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
+		(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
 
-		ref_docname = doc.parent or doc.name
+		ref_docname = doc.get("parent") or doc.name
 
 		if ":" in cstr(df.get("options")):
 			split_opts = df.get("options").split(":")
@@ -635,7 +637,7 @@ def get_field_currency(df, doc=None):
 				currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2])
 		else:
 			currency = doc.get(df.get("options"))
-			if doc.parent:
+			if doc.get("parenttype"):
 				if currency:
 					ref_docname = doc.name
 				else:
@@ -648,7 +650,7 @@ def get_field_currency(df, doc=None):
 				.setdefault(df.fieldname, currency)
 
 	return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \
-		(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
+		(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
 
 def get_field_precision(df, doc=None, currency=None):
 	"""get precision based on DocField options and fieldvalue in doc"""
@@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None):
 
 
 def get_default_df(fieldname):
-	if fieldname in default_fields:
+	if fieldname in (default_fields + child_table_fields):
 		if fieldname in ("creation", "modified"):
 			return frappe._dict(
 				fieldname = fieldname,
 				fieldtype = "Datetime"
 			)
 
-		else:
+		elif fieldname in ("idx", "docstatus"):
 			return frappe._dict(
 				fieldname = fieldname,
-				fieldtype = "Data"
+				fieldtype = "Int"
 			)
 
+		return frappe._dict(
+			fieldname = fieldname,
+			fieldtype = "Data"
+		)
+
+
 def trim_tables(doctype=None, dry_run=False, quiet=False):
 	"""
 	Removes database fields that don't exist in the doctype (json or custom field). This may be needed
@@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False):
 
 def trim_table(doctype, dry_run=True):
 	frappe.cache().hdel('table_columns', f"tab{doctype}")
-	ignore_fields = default_fields + optional_fields
+	ignore_fields = default_fields + optional_fields + child_table_fields
 	columns = frappe.db.get_table_columns(doctype)
 	fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value()
 	is_internal = lambda f: f not in ignore_fields and not f.startswith("_")

frappe/model/naming.py

@@ -1,14 +1,3 @@
-"""utilities to generate a document name based on various rules defined.
-
-NOTE:
-Till version 13, whenever a submittable document is amended it's name is set to orig_name-X,
-where X is a counter and it increments when amended again and so on.
-
-From Version 14, The naming pattern is changed in a way that amended documents will
-have the original name `orig_name` instead of `orig_name-X`. To make this happen
-the cancelled document naming pattern is changed to 'orig_name-CANC-X'.
-"""
-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
@@ -40,7 +29,7 @@ def set_new_name(doc):
 		doc.name = None
 
 	if getattr(doc, "amended_from", None):
-		doc.name = _get_amended_name(doc)
+		_set_amended_name(doc)
 		return
 
 	elif getattr(doc.meta, "issingle", False):
@@ -256,18 +245,6 @@ def revert_series_if_last(key, name, doc=None):
 			* prefix = #### and hashes = 2021 (hash doesn't exist)
 			* will search hash in key then accordingly get prefix = ""
 	"""
-	if hasattr(doc, 'amended_from'):
-		# Do not revert the series if the document is amended.
-		if doc.amended_from:
-			return
-
-		# Get document name by parsing incase of fist cancelled document
-		if doc.docstatus == 2 and not doc.amended_from:
-			if doc.name.endswith('-CANC'):
-				name, _ = NameParser.parse_docname(doc.name, sep='-CANC')
-			else:
-				name, _ = NameParser.parse_docname(doc.name, sep='-CANC-')
-
 	if ".#" in key:
 		prefix, hashes = key.rsplit(".", 1)
 		if "#" not in hashes:
@@ -356,9 +333,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
 	return value
 
 
-def _get_amended_name(doc):
-	name, _ = NameParser(doc).parse_amended_from()
-	return name
+def _set_amended_name(doc):
+	am_id = 1
+	am_prefix = doc.amended_from
+	if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"):
+		am_id = cint(doc.amended_from.split("-")[-1]) + 1
+		am_prefix = "-".join(doc.amended_from.split("-")[:-1])  # except the last hyphen
+
+	doc.name = am_prefix + "-" + str(am_id)
+	return doc.name
+
 
 def _field_autoname(autoname, doc, skip_slicing=None):
 	"""
@@ -399,83 +383,3 @@ def _format_autoname(autoname, doc):
 	name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value)
 
 	return name
-
-class NameParser:
-	"""Parse document name and return parts of it.
-
-	NOTE: It handles cancellend and amended doc parsing for now. It can be expanded.
-	"""
-	def __init__(self, doc):
-		self.doc = doc
-
-	def parse_amended_from(self):
-		"""
-		Cancelled document naming will be in one of these formats
-
-		* original_name-X-CANC - This is introduced to migrate old style naming to new style
-		* original_name-CANC - This is introduced to migrate old style naming to new style
-		* original_name-CANC-X - This is the new style naming
-
-		New style naming: In new style naming amended documents will have original name. That says,
-		when a document gets cancelled we need rename the document by adding `-CANC-X` to the end
-		so that amended documents can use the original name.
-
-		Old style naming: cancelled documents stay with original name and when amended, amended one
-		gets a new name as `original_name-X`. To bring new style naming we had to change the existing
-		cancelled document names and that is done by adding `-CANC` to cancelled documents through patch.
-		"""
-		if not getattr(self.doc, 'amended_from', None):
-			return (None, None)
-
-		# Handle old style cancelled documents (original_name-X-CANC, original_name-CANC)
-		if self.doc.amended_from.endswith('-CANC'):
-			name, _ = self.parse_docname(self.doc.amended_from, '-CANC')
-			amended_from_doc = frappe.get_all(
-				self.doc.doctype,
-				filters = {'name': self.doc.amended_from},
-				fields = ['amended_from'],
-				limit=1)
-
-			# Handle format original_name-X-CANC.
-			if amended_from_doc and amended_from_doc[0].amended_from:
-				return self.parse_docname(name, '-')
-			return name, None
-
-		# Handle new style cancelled documents
-		return self.parse_docname(self.doc.amended_from, '-CANC-')
-
-	@classmethod
-	def parse_docname(cls, name, sep='-'):
-		split_list = name.rsplit(sep, 1)
-
-		if len(split_list) == 1:
-			return (name, None)
-		return (split_list[0], split_list[1])
-
-def get_cancelled_doc_latest_counter(tname, docname):
-	"""Get the latest counter used for cancelled docs of given docname.
-	"""
-	name_prefix = f'{docname}-CANC-'
-
-	rows = frappe.db.sql("""
-		select
-			name
-		from `tab{tname}`
-		where
-			name like %(name_prefix)s and docstatus=2
-	""".format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1)
-
-	if not rows:
-		return -1
-	return max([int(row.name.replace(name_prefix, '') or -1) for row in rows])
-
-def gen_new_name_for_cancelled_doc(doc):
-	"""Generate a new name for cancelled document.
-	"""
-	if getattr(doc, "amended_from", None):
-		name, _ = NameParser(doc).parse_amended_from()
-	else:
-		name = doc.name
-
-	counter = get_cancelled_doc_latest_counter(doc.doctype, name)
-	return f'{name}-CANC-{counter+1}'

frappe/model/workflow.py

@@ -1,10 +1,11 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+import json
 
 import frappe
-from frappe.utils import cint
 from frappe import _
-import json
+from frappe.utils import cint
+from frappe.model.docstatus import DocStatus
 
 class WorkflowStateError(frappe.ValidationError): pass
 class WorkflowTransitionError(frappe.ValidationError): pass
@@ -102,13 +103,13 @@ def apply_workflow(doc, action):
 		doc.set(next_state.update_field, next_state.update_value)
 
 	new_docstatus = cint(next_state.doc_status)
-	if doc.docstatus == 0 and new_docstatus == 0:
+	if doc.docstatus.is_draft() and new_docstatus == DocStatus.draft():
 		doc.save()
-	elif doc.docstatus == 0 and new_docstatus == 1:
+	elif doc.docstatus.is_draft() and new_docstatus == DocStatus.submitted():
 		doc.submit()
-	elif doc.docstatus == 1 and new_docstatus == 1:
+	elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.submitted():
 		doc.save()
-	elif doc.docstatus == 1 and new_docstatus == 2:
+	elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.cancelled():
 		doc.cancel()
 	else:
 		frappe.throw(_('Illegal Document Status for {0}').format(next_state.state))
@@ -212,10 +213,10 @@ def bulk_workflow_approval(docnames, doctype, action):
 			frappe.db.commit()
 		except Exception as e:
 			if not frappe.message_log:
-				# Exception is  raised manually and not from msgprint or throw
+				# Exception is	raised manually and not from msgprint or throw
 				message = "{0}".format(e.__class__.__name__)
 				if e.args:
-					message +=  " : {0}".format(e.args[0])
+					message += " : {0}".format(e.args[0])
 				message_dict = {"docname": docname, "message": message}
 				failed_transactions[docname].append(message_dict)
 

frappe/modules/export_file.py

@@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export):
 
 	for df in doc.meta.get_table_fields():
 		for d in doc_export.get(df.fieldname):
-			for fieldname in frappe.model.default_fields:
+			for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
 				if fieldname in d:
 					del d[fieldname]
 

frappe/patches.txt

@@ -119,7 +119,6 @@ execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings')
 execute:frappe.delete_doc_if_exists('DocType', 'GSuite Templates')
 execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Account')
 execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings')
-frappe.patches.v12_0.remove_parent_and_parenttype_from_print_formats
 frappe.patches.v12_0.remove_example_email_thread_notify
 execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders()
 frappe.patches.v12_0.set_correct_url_in_files
@@ -175,6 +174,7 @@ execute:frappe.delete_doc_if_exists('Page', 'workspace')
 execute:frappe.delete_doc_if_exists('Page', 'dashboard', force=1)
 frappe.core.doctype.page.patches.drop_unused_pages
 execute:frappe.get_doc('Role', 'Guest').save() # remove desk access
+frappe.patches.v13_0.remove_chat
 frappe.patches.v13_0.rename_desk_page_to_workspace # 02.02.2021
 frappe.patches.v13_0.delete_package_publish_tool
 frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings
@@ -184,10 +184,10 @@ frappe.patches.v13_0.queryreport_columns
 frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
 frappe.patches.v13_0.set_first_day_of_the_week
-frappe.patches.v14_0.rename_cancelled_documents
 frappe.patches.v14_0.update_workspace2 # 20.09.2021
 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021
 frappe.patches.v14_0.transform_todo_schema
+frappe.patches.v14_0.remove_post_and_post_comment
 
 [post_model_sync]
 frappe.patches.v14_0.drop_data_import_legacy

frappe/patches/v12_0/remove_parent_and_parenttype_from_print_formats.py

@@ -1,14 +0,0 @@
-import frappe
-
-def execute():
-    frappe.db.sql("""
-        UPDATE
-            `tabPrint Format`
-        SET
-            `tabPrint Format`.`parent`='',
-            `tabPrint Format`.`parenttype`='',
-            `tabPrint Format`.parentfield=''
-        WHERE
-            `tabPrint Format`.parent != ''
-            OR `tabPrint Format`.parenttype != ''
-        """)

frappe/patches/v13_0/remove_chat.py

@@ -0,0 +1,17 @@
+import frappe
+import click
+
+def execute():
+	frappe.delete_doc_if_exists("DocType", "Chat Message")
+	frappe.delete_doc_if_exists("DocType", "Chat Message Attachment")
+	frappe.delete_doc_if_exists("DocType", "Chat Profile")
+	frappe.delete_doc_if_exists("DocType", "Chat Token")
+	frappe.delete_doc_if_exists("DocType", "Chat Room User")
+	frappe.delete_doc_if_exists("DocType", "Chat Room")
+	frappe.delete_doc_if_exists("Module Def", "Chat")
+
+	click.secho(
+		"Chat Module is moved to a separate app and is removed from Frappe in version-13.\n"
+		"Please install the app to continue using the chat feature: https://github.com/frappe/chat",
+		fg="yellow",
+	)

frappe/patches/v14_0/remove_post_and_post_comment.py

@@ -0,0 +1,5 @@
+import frappe
+
+def execute():
+	frappe.delete_doc_if_exists("DocType", "Post")
+	frappe.delete_doc_if_exists("DocType", "Post Comment")

frappe/patches/v14_0/rename_cancelled_documents.py

@@ -1,213 +0,0 @@
-import functools
-import traceback
-
-import frappe
-
-def execute():
-	"""Rename cancelled documents by adding a postfix.
-	"""
-	rename_cancelled_docs()
-
-def get_submittable_doctypes():
-	"""Returns list of submittable doctypes in the system.
-	"""
-	return frappe.db.get_all('DocType', filters={'is_submittable': 1}, pluck='name')
-
-def get_cancelled_doc_names(doctype):
-	"""Return names of cancelled document names those are in old format.
-	"""
-	docs = frappe.db.get_all(doctype, filters={'docstatus': 2}, pluck='name')
-	return [each for each in docs if not (each.endswith('-CANC') or ('-CANC-' in each))]
-
-@functools.lru_cache()
-def get_linked_doctypes():
-	"""Returns list of doctypes those are linked with given doctype using 'Link' fieldtype.
-	"""
-	filters=[['fieldtype','=', 'Link']]
-	links = frappe.get_all("DocField",
-		fields=["parent", "fieldname", "options as linked_to"],
-		filters=filters,
-		as_list=1)
-
-	links+= frappe.get_all("Custom Field",
-		fields=["dt as parent", "fieldname", "options as linked_to"],
-		filters=filters,
-		as_list=1)
-
-	links_by_doctype = {}
-	for doctype, fieldname, linked_to in links:
-		links_by_doctype.setdefault(linked_to, []).append((doctype, fieldname))
-	return links_by_doctype
-
-@functools.lru_cache()
-def get_single_doctypes():
-	return frappe.get_all("DocType", filters={'issingle': 1}, pluck='name')
-
-@functools.lru_cache()
-def get_dynamic_linked_doctypes():
-	filters=[['fieldtype','=', 'Dynamic Link']]
-
-	# find dynamic links of parents
-	links = frappe.get_all("DocField",
-		fields=["parent as doctype", "fieldname", "options as doctype_fieldname"],
-		filters=filters,
-		as_list=1)
-	links+= frappe.get_all("Custom Field",
-		fields=["dt as doctype", "fieldname", "options as doctype_fieldname"],
-		filters=filters,
-		as_list=1)
-	return links
-
-@functools.lru_cache()
-def get_child_tables():
-	"""
-	"""
-	filters =[['fieldtype', 'in', ('Table', 'Table MultiSelect')]]
-	links = frappe.get_all("DocField",
-		fields=["parent as doctype", "options as child_table"],
-		filters=filters,
-		as_list=1)
-
-	links+= frappe.get_all("Custom Field",
-		fields=["dt as doctype", "options as child_table"],
-		filters=filters,
-		as_list=1)
-
-	map = {}
-	for doctype, child_table in links:
-		map.setdefault(doctype, []).append(child_table)
-	return map
-
-def update_cancelled_document_names(doctype, cancelled_doc_names):
-	return frappe.db.sql("""
-		update
-			`tab{doctype}`
-		set
-			name=CONCAT(name, '-CANC')
-		where
-			docstatus=2
-			and
-			name in %(cancelled_doc_names)s;
-	""".format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names})
-
-def update_amended_field(doctype, cancelled_doc_names):
-	return frappe.db.sql("""
-		update
-			`tab{doctype}`
-		set
-			amended_from=CONCAT(amended_from, '-CANC')
-		where
-			amended_from in %(cancelled_doc_names)s;
-	""".format(doctype=doctype), {'cancelled_doc_names': cancelled_doc_names})
-
-def update_attachments(doctype, cancelled_doc_names):
-	frappe.db.sql("""
-		update
-			`tabFile`
-		set
-			attached_to_name=CONCAT(attached_to_name, '-CANC')
-		where
-			attached_to_doctype=%(dt)s and attached_to_name in %(cancelled_doc_names)s
-		""", {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
-
-def update_versions(doctype, cancelled_doc_names):
-	frappe.db.sql("""
-		UPDATE
-			`tabVersion`
-		SET
-			docname=CONCAT(docname, '-CANC')
-		WHERE
-			ref_doctype=%(dt)s AND docname in %(cancelled_doc_names)s
-		""", {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
-
-def update_linked_doctypes(doctype, cancelled_doc_names):
-	single_doctypes = get_single_doctypes()
-
-	for linked_dt, field in get_linked_doctypes().get(doctype, []):
-		if linked_dt not in single_doctypes:
-			frappe.db.sql("""
-				update
-					`tab{linked_dt}`
-				set
-					`{column}`=CONCAT(`{column}`, '-CANC')
-				where
-					`{column}` in %(cancelled_doc_names)s;
-			""".format(linked_dt=linked_dt, column=field),
-				{'cancelled_doc_names': cancelled_doc_names})
-		else:
-			doc = frappe.get_single(linked_dt)
-			if getattr(doc, field) in cancelled_doc_names:
-				setattr(doc, field, getattr(doc, field)+'-CANC')
-				doc.flags.ignore_mandatory=True
-				doc.flags.ignore_validate=True
-				doc.save(ignore_permissions=True)
-
-def update_dynamic_linked_doctypes(doctype, cancelled_doc_names):
-	single_doctypes = get_single_doctypes()
-
-	for linked_dt, fieldname, doctype_fieldname in get_dynamic_linked_doctypes():
-		if linked_dt not in single_doctypes:
-			frappe.db.sql("""
-				update
-					`tab{linked_dt}`
-				set
-					`{column}`=CONCAT(`{column}`, '-CANC')
-				where
-					`{column}` in %(cancelled_doc_names)s and {doctype_fieldname}=%(dt)s;
-			""".format(linked_dt=linked_dt, column=fieldname, doctype_fieldname=doctype_fieldname),
-				{'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
-		else:
-			doc = frappe.get_single(linked_dt)
-			if getattr(doc, doctype_fieldname) == doctype and getattr(doc, fieldname) in cancelled_doc_names:
-				setattr(doc, fieldname, getattr(doc, fieldname)+'-CANC')
-				doc.flags.ignore_mandatory=True
-				doc.flags.ignore_validate=True
-				doc.save(ignore_permissions=True)
-
-def update_child_tables(doctype, cancelled_doc_names):
-	child_tables = get_child_tables().get(doctype, [])
-	single_doctypes = get_single_doctypes()
-
-	for table in child_tables:
-		if table not in single_doctypes:
-			frappe.db.sql("""
-				update
-					`tab{table}`
-				set
-					parent=CONCAT(parent, '-CANC')
-				where
-					parenttype=%(dt)s and parent in %(cancelled_doc_names)s;
-			""".format(table=table), {'cancelled_doc_names': cancelled_doc_names, 'dt': doctype})
-		else:
-			doc = frappe.get_single(table)
-			if getattr(doc, 'parenttype')==doctype and getattr(doc, 'parent') in cancelled_doc_names:
-				setattr(doc, 'parent', getattr(doc, 'parent')+'-CANC')
-				doc.flags.ignore_mandatory=True
-				doc.flags.ignore_validate=True
-				doc.save(ignore_permissions=True)
-
-def rename_cancelled_docs():
-	submittable_doctypes = get_submittable_doctypes()
-
-	for dt in submittable_doctypes:
-		for retry in range(2):
-			try:
-				cancelled_doc_names = tuple(get_cancelled_doc_names(dt))
-				if not cancelled_doc_names:
-					break
-				update_cancelled_document_names(dt, cancelled_doc_names)
-				update_amended_field(dt, cancelled_doc_names)
-				update_child_tables(dt, cancelled_doc_names)
-				update_linked_doctypes(dt, cancelled_doc_names)
-				update_dynamic_linked_doctypes(dt, cancelled_doc_names)
-				update_attachments(dt, cancelled_doc_names)
-				update_versions(dt, cancelled_doc_names)
-				print(f"Renaming cancelled records of {dt} doctype")
-				frappe.db.commit()
-				break
-			except Exception:
-				if retry == 1:
-					print(f"Failed to rename the cancelled records of {dt} doctype, moving on!")
-					traceback.print_exc()
-				frappe.db.rollback()
-

frappe/public/js/frappe-web.bundle.js

@@ -2,6 +2,7 @@ import "./jquery-bootstrap";
 import "./frappe/class.js";
 import "./frappe/polyfill.js";
 import "./lib/md5.min.js";
+import "./lib/moment.js";
 import "./frappe/provide.js";
 import "./frappe/format.js";
 import "./frappe/utils/number_format.js";

frappe/public/js/frappe/data_import/import_preview.js

@@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
 	is_row_imported(row) {
 		let serial_no = row[0].content;
 		return this.import_log.find(log => {
-			return log.success && log.row_indexes.includes(serial_no);
+			return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no);
 		});
 	}
 };

frappe/public/js/frappe/file_uploader/FileUploader.vue

@@ -534,22 +534,21 @@ export default {
 			});
 		},
 		show_google_drive_picker() {
-			let dialog = cur_dialog;
-			dialog.hide();
+			this.close_dialog = true;
 			let google_drive = new GoogleDrivePicker({
-				pickerCallback: data => this.google_drive_callback(data, dialog),
+				pickerCallback: data => this.google_drive_callback(data),
 				...this.google_drive_settings
 			});
 			google_drive.loadPicker();
 		},
-		google_drive_callback(data, dialog) {
+		google_drive_callback(data) {
 			if (data.action == google.picker.Action.PICKED) {
 				this.upload_file({
 					file_url: data.docs[0].url,
 					file_name: data.docs[0].name
 				});
 			} else if (data.action == google.picker.Action.CANCEL) {
-				dialog.show();
+				cur_frm.attachments.new_attachment()
 			}
 		},
 		url_to_file(url, filename, mime_type) {

frappe/public/js/frappe/form/controls/link.js

@@ -374,10 +374,22 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 	}
 
 	set_custom_query(args) {
-		var set_nulls = function(obj) {
-			$.each(obj, function(key, value) {
-				if(value!==undefined) {
-					obj[key] = value;
+		const is_valid_value = (value, key) => {
+			if (value) return true;
+			// check if empty value is valid
+			if (this.frm) {
+				let field = frappe.meta.get_docfield(this.frm.doctype, key);
+				// empty value link fields is invalid
+				return !field || !["Link", "Dynamic Link"].includes(field.fieldtype);
+			} else {
+				return value !== undefined;
+			}
+		}
+
+		const set_nulls = (obj) => {
+			$.each(obj, (key, value) => {
+				if (!is_valid_value(value, key)) {
+					delete obj[key];
 				}
 			});
 			return obj;
@@ -458,7 +470,6 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 	validate_link_and_fetch(df, options, docname, value) {
 		if (!options) return;
 
-		let field_value = "";
 		const fetch_map = this.fetch_map;
 		const columns_to_fetch = Object.values(fetch_map);
 
@@ -467,16 +478,10 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 			return value;
 		}
 
-		return frappe.xcall("frappe.client.validate_link", {
-			doctype: options,
-			docname: value,
-			fields: columns_to_fetch,
-		}).then((response) => {
-			if (!docname || !columns_to_fetch.length) return response.name;
-
+		function update_dependant_fields(response) {
+			let field_value = "";
 			for (const [target_field, source_field] of Object.entries(fetch_map)) {
 				if (value) field_value = response[source_field];
-				
 				frappe.model.set_value(
 					df.parent,
 					docname,
@@ -485,9 +490,23 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat
 					df.fieldtype,
 				);
 			}
+		}
 
-			return response.name;
-		});
+		// to avoid unnecessary request
+		if (value) {
+			return frappe.xcall("frappe.client.validate_link", {
+				doctype: options,
+				docname: value,
+				fields: columns_to_fetch,
+			}).then((response) => {
+				if (!docname || !columns_to_fetch.length) return response.name;
+				update_dependant_fields(response);
+				return response.name;
+			});
+		} else {
+			update_dependant_fields({});
+			return value;
+		}
 	}
 
 	get fetch_map() {

frappe/public/js/frappe/form/form.js

@@ -860,36 +860,32 @@ frappe.ui.form.Form = class FrappeForm {
 	}
 
 	_cancel(btn, callback, on_error, skip_confirm) {
+		const me = this;
 		const cancel_doc = () => {
 			frappe.validated = true;
-			this.script_manager.trigger("before_cancel").then(() => {
+			me.script_manager.trigger("before_cancel").then(() => {
 				if (!frappe.validated) {
-					return this.handle_save_fail(btn, on_error);
+					return me.handle_save_fail(btn, on_error);
 				}
 
-				const original_name = this.docname;
-				const after_cancel = (r) => {
+				var after_cancel = function(r) {
 					if (r.exc) {
-						this.handle_save_fail(btn, on_error);
+						me.handle_save_fail(btn, on_error);
 					} else {
 						frappe.utils.play_sound("cancel");
+						me.refresh();
 						callback && callback();
-						this.script_manager.trigger("after_cancel");
-						frappe.run_serially([
-							() => this.rename_notify(this.doctype, original_name, r.docs[0].name),
-							() => frappe.router.clear_re_route(this.doctype, original_name),
-							() => this.refresh(),
-						]);
+						me.script_manager.trigger("after_cancel");
 					}
 				};
-				frappe.ui.form.save(this, "cancel", after_cancel, btn);
+				frappe.ui.form.save(me, "cancel", after_cancel, btn);
 			});
 		}
 
 		if (skip_confirm) {
 			cancel_doc();
 		} else {
-			frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error));
+			frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error));
 		}
 	};
 
@@ -911,7 +907,7 @@ frappe.ui.form.Form = class FrappeForm {
 			'docname': this.doc.name
 		}).then(is_amended => {
 			if (is_amended) {
-				frappe.throw(__('This document is already amended, you cannot amend it again'));
+				frappe.throw(__('This document is already amended, you cannot ammend it again'));
 			}
 			this.validate_form_action("Amend");
 			var me = this;

frappe/public/js/frappe/form/multi_select_dialog.js

@@ -150,8 +150,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
 		});
 	}
 
+	is_child_selection_enabled() {
+		return this.dialog.fields_dict['allow_child_item_selection'].get_value();
+	}
+
 	toggle_child_selection() {
-		if (this.dialog.fields_dict['allow_child_item_selection'].get_value()) {
+		if (this.is_child_selection_enabled()) {
 			this.show_child_results();
 		} else {
 			this.child_results = [];
@@ -289,7 +293,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
 			parent: this.dialog.get_field('filter_area').$wrapper,
 			doctype: this.doctype,
 			on_change: () => {
-				this.get_results();
+				if (this.is_child_selection_enabled()) {
+					this.show_child_results();
+				} else {
+					this.get_results();
+				}
 			}
 		});
 		// 'Apply Filter' breaks since the filers are not in a popover
@@ -325,7 +333,11 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
 
 		this.$parent.find('.input-with-feedback').on('change', () => {
 			frappe.flags.auto_scroll = false;
-			this.get_results();
+			if (this.is_child_selection_enabled()) {
+				this.show_child_results();
+			} else {
+				this.get_results();
+			}
 		});
 
 		this.$parent.find('[data-fieldtype="Data"]').on('input', () => {
@@ -333,8 +345,12 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog {
 			clearTimeout($this.data('timeout'));
 			$this.data('timeout', setTimeout(function () {
 				frappe.flags.auto_scroll = false;
-				me.empty_list();
-				me.get_results();
+				if (me.is_child_selection_enabled()) {
+					me.show_child_results();
+				} else {
+					me.empty_list();
+					me.get_results();
+				}
 			}, 300));
 		});
 	}

frappe/public/js/frappe/form/script_manager.js

@@ -192,7 +192,7 @@ frappe.ui.form.ScriptManager = class ScriptManager {
 		}
 
 		function setup_add_fetch(df) {
-			if ((['Data', 'Read Only', 'Text', 'Small Text', 'Currency', 'Check',
+			if ((['Data', 'Read Only', 'Text', 'Small Text', 'Currency', 'Check', 'Attach Image',
 				'Text Editor', 'Code', 'Link', 'Float', 'Int', 'Date', 'Select', 'Duration'].includes(df.fieldtype) || df.read_only==1)
 				&& df.fetch_from && df.fetch_from.indexOf(".")!=-1) {
 				var parts = df.fetch_from.split(".");

frappe/public/js/frappe/form/templates/form_sidebar.html

@@ -1,5 +1,5 @@
 <ul class="list-unstyled sidebar-menu user-actions hidden"></ul>
-<ul class="list-unstyled sidebar-menu sidebar-image-section hidden-xs hidden-sm hide">
+<ul class="list-unstyled sidebar-menu sidebar-image-section hide">
 	<li class="sidebar-image-wrapper">
 		<img class="sidebar-image">
 		<div class="sidebar-standard-image">

frappe/public/js/frappe/model/model.js

@@ -10,8 +10,7 @@ $.extend(frappe.model, {
 	layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'],
 
 	std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by',
-		'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus',
-		'parent', 'parenttype', 'parentfield', 'idx'],
+		'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus', 'idx'],
 
 	core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role',
 		'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',

frappe/public/js/frappe/router.js

@@ -250,12 +250,6 @@ frappe.router = {
 		}
 	},
 
-	clear_re_route(doctype, docname) {
-		delete frappe.re_route[
-			`${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}`
-		];
-	},
-
 	set_title(sub_path) {
 		if (frappe.route_titles[sub_path]) {
 			frappe.utils.set_title(frappe.route_titles[sub_path]);

frappe/public/js/frappe/views/breadcrumbs.js

@@ -70,6 +70,9 @@ frappe.breadcrumbs = {
 				this.set_form_breadcrumb(breadcrumbs, view);
 			} else if (breadcrumbs.doctype && view === 'list') {
 				this.set_list_breadcrumb(breadcrumbs);
+			} else if (breadcrumbs.doctype && view == 'dashboard-view') {
+				this.set_list_breadcrumb(breadcrumbs);
+				this.set_dashboard_breadcrumb(breadcrumbs);
 			}
 		}
 
@@ -164,6 +167,14 @@ frappe.breadcrumbs = {
 
 	},
 
+	set_dashboard_breadcrumb(breadcrumbs) {
+		const doctype = breadcrumbs.doctype;
+		const docname = frappe.get_route()[1];
+		let dashboard_route = `/app/${frappe.router.slug(doctype)}/${docname}`;
+		$(`<li><a href="${dashboard_route}">${__(docname)}</a></li>`)
+			.appendTo(this.$breadcrumbs);
+	},
+
 	setup_modules() {
 		if (!frappe.visible_modules) {
 			frappe.visible_modules = $.map(frappe.boot.allowed_workspaces, (m) => {

frappe/public/js/frappe/views/reports/report_view.js

@@ -866,7 +866,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 		}
 
 		doctype_fields = [{
-			label: __('ID'),
+			label: __('ID', null, 'Label of name column in report'),
 			fieldname: 'name',
 			fieldtype: 'Data',
 			reqd: 1

frappe/public/js/integrations/google_drive_picker.js

@@ -44,9 +44,16 @@ export default class GoogleDrivePicker {
 	}
 
 	handleAuthResult(authResult) {
+		let error_map = {
+			"popup_closed_by_user": __("Google Authentication was closed abruptly by the user")
+		};
+
 		if (authResult && !authResult.error) {
 			frappe.boot.user.google_drive_token = authResult.access_token;
 			this.createPicker();
+		} else {
+			let error = error_map[authResult.error] || __("Google Authentication Error");
+			frappe.throw(error);
 		}
 	}
 
@@ -58,20 +65,34 @@ export default class GoogleDrivePicker {
 	createPicker() {
 		// Create and render a Picker object for searching images.
 		if (this.pickerApiLoaded && frappe.boot.user.google_drive_token) {
-			var view = new google.picker.DocsView(google.picker.ViewId.DOCS)
+			this.view = new google.picker.DocsView(google.picker.ViewId.DOCS)
 				.setParent('root') // show the root folder by default
 				.setIncludeFolders(true); // also show folders, not just files
 
-			var picker = new google.picker.PickerBuilder()
+			this.picker = new google.picker.PickerBuilder()
 				.setAppId(this.appId)
 				.setDeveloperKey(this.developerKey)
 				.setOAuthToken(frappe.boot.user.google_drive_token)
-				.addView(view)
+				.addView(this.view)
 				.setLocale(frappe.boot.lang)
 				.setCallback(this.pickerCallback)
 				.build();
 
-			picker.setVisible(true);
+			this.picker.setVisible(true);
+			this.setupHide();
+		}
+	}
+
+	setupHide() {
+		let bg = $(".picker-dialog-bg");
+
+		for (let el of bg) {
+			el.onclick = () => {
+				this.picker.setVisible(false);
+				this.picker.Ob({
+					action: google.picker.Action.CANCEL
+				});
+			};
 		}
 	}
 }

frappe/public/js/lib/moment.js

@@ -0,0 +1,5 @@
+// This file is used to make sure that `moment` is bound to the window
+// before the bundle finishes loading, due to imports (datetime.js) in the bundle
+// that depend on `moment`.
+import momentTimezone from "moment-timezone/builds/moment-timezone-with-data.js";
+window.moment = momentTimezone;

frappe/public/js/libs.bundle.js

@@ -1,15 +1,12 @@
 import "./jquery-bootstrap";
 import Vue from "vue/dist/vue.esm.js";
-import moment from "moment/min/moment-with-locales.js";
-import momentTimezone from "moment-timezone/builds/moment-timezone-with-data.js";
+import "./lib/moment";
 import io from "socket.io-client/dist/socket.io.slim.js";
 import Sortable from "./lib/Sortable.min.js";
 // TODO: esbuild
 // Don't think jquery.hotkeys is being used anywhere. Will remove this after being sure.
 // import "./lib/jquery/jquery.hotkeys.js";
 
-window.moment = moment;
-window.moment = momentTimezone;
 window.Vue = Vue;
 window.Sortable = Sortable;
 window.io = io;

frappe/public/js/print_format_builder/print_format_builder.bundle.js

@@ -21,7 +21,7 @@ class PrintFormatBuilder {
 				this.$component.toggle_preview();
 			}
 		);
-		this.page.add_button(__("Reset Changes"), () =>
+		let $reset_changes_btn = this.page.add_button(__("Reset Changes"), () =>
 			this.$component.$store.reset_changes()
 		);
 		this.page.add_menu_item(__("Edit Print Format"), () => {
@@ -46,9 +46,11 @@ class PrintFormatBuilder {
 			if (value) {
 				this.page.set_indicator("Not Saved", "orange");
 				$toggle_preview_btn.hide();
+				$reset_changes_btn.show();
 			} else {
 				this.page.clear_indicator();
 				$toggle_preview_btn.show();
+				$reset_changes_btn.hide();
 			}
 		});
 		this.$component.$watch("show_preview", value => {

frappe/public/js/web_form.bundle.js

@@ -1,2 +1,3 @@
+import "./lib/moment.js";
 import "./frappe/utils/datetime.js";
 import "./frappe/web_form/webform_script.js";

frappe/public/scss/common/css_variables.scss

@@ -231,6 +231,10 @@
 
 	--highlight-shadow: 1px 1px 10px var(--blue-50), 0px 0px 4px var(--blue-600);
 
+	// code block
+	--code-block-bg: var(--gray-900);
+	--code-block-text: var(--gray-400);
+
 	// Border Sizes
 	--border-radius-sm: 4px;
 	--border-radius: 6px;
@@ -245,6 +249,7 @@
 	--checkbox-right-margin: var(--margin-xs);
 	--checkbox-size: 14px;
 	--checkbox-focus-shadow: 0 0 0 2px var(--gray-300);
+	--checkbox-gradient: linear-gradient(180deg, #4AC3F8 -124.51%, var(--primary) 100%);
 
 	--right-arrow-svg: url("data: image/svg+xml;utf8, <svg width='6' height='8' viewBox='0 0 6 8' fill='none' xmlns='http://www.w3.org/2000/svg'><path d='M1.25 7.5L4.75 4L1.25 0.5' stroke='%231F272E' stroke-linecap='round' stroke-linejoin='round'/></svg>");
 	--left-arrow-svg: url("data: image/svg+xml;utf8, <svg width='6' height='8' viewBox='0 0 6 8' fill='none' xmlns='http://www.w3.org/2000/svg'><path d='M7.5 9.5L4 6l3.5-3.5' stroke='%231F272E' stroke-linecap='round' stroke-linejoin='round'></path></svg>");

frappe/public/scss/common/global.scss

@@ -54,7 +54,7 @@ input[type="radio"] {
 	}
 
 	&:checked::before {
-		background-color: var(--blue-500);
+		background-color: var(--primary);
 		border-radius: 16px;
 		box-shadow: inset 0 0 0 2px white;
 	}
@@ -85,8 +85,8 @@ input[type="checkbox"] {
 	}
 
 	&:checked {
-		background-color: var(--blue-500);
-		background-image: $check-icon, linear-gradient(180deg, #4AC3F8 -124.51%, #2490EF 100%);
+		background-color: var(--primary);
+		background-image: $check-icon, var(--checkbox-gradient);
 		background-size: 57%, 100%;
 		box-shadow: none;
 		border: none;

frappe/public/scss/common/quill.scss

@@ -42,6 +42,7 @@
 		height: 300px;
 		border-bottom-left-radius: var(--border-radius);
 		border-bottom-right-radius: var(--border-radius);
+		resize: vertical;
 	}
 	.ql-stroke {
 		stroke: var(--icon-stroke);
@@ -85,10 +86,22 @@
 	margin-bottom: 8px;
 }
 
+.ql-code-block-container {
+	background-color: var(--code-block-bg);
+	color: var(--code-block-text);
+	padding: var(--padding-xs) var(--padding-sm) !important;
+	margin-bottom: var(--margin-xs) !important;
+	margin-top: var(--margin-xs)!important;
+	border-radius: var(--border-radius-sm);
+}
+
 .ql-bubble .ql-editor {
 	min-height: 100px;
 	max-height: 300px;
 	border-radius: var(--border-radius-sm);
+	.ql-code-block-container {
+		@extend .ql-code-block-container;
+	}
 }
 
 .ql-mention-list-container {

frappe/public/scss/desk/css_variables.scss

@@ -54,4 +54,6 @@ $input-height: 28px !default;
 	// skeleton
 	--skeleton-bg: var(--gray-100);
 
+	// progress bar
+	--progress-bar-bg: var(--primary);
 }

frappe/public/scss/desk/sidebar.scss

@@ -107,7 +107,7 @@ body[data-route^="Module"] .main-menu {
 		cursor: pointer;
 
 		.sidebar-image {
-			width: 100%;
+			width: min(100%, 170px);
 			height: auto;
 			max-height: 170px;
 			object-fit: cover;

frappe/public/scss/desk/variables.scss

@@ -118,6 +118,9 @@ $custom-control-label-color: var(--text-color);
 $custom-switch-indicator-size: 8px;
 $custom-control-indicator-border-width: 2px;
 
+// progress bar
+$progress-bar-bg: var(--progress-bar-bg);
+
 $navbar-nav-link-padding-x: 1rem !default;
 $navbar-padding-y: 1rem !default;
 $card-border-radius: 0.75rem !default;

frappe/query_builder/__init__.py

@@ -1,8 +1,21 @@
-from frappe.query_builder.terms import ParameterizedValueWrapper, ParameterizedFunction
-import pypika
+import pypika.terms
+from pypika import *
+from pypika import Field
+from pypika.utils import ignore_copy
+
+from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValueWrapper
+from frappe.query_builder.utils import (
+    Column,
+    DocType,
+    get_query_builder,
+    patch_query_aggregation,
+    patch_query_execute,
+)
 
 pypika.terms.ValueWrapper = ParameterizedValueWrapper
 pypika.terms.Function = ParameterizedFunction
 
-from pypika import *
-from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation
+# * Overrides the field() method and replaces it with the a `PseudoColumn` 'field' for consistency
+pypika.queries.Selectable.__getattr__ = ignore_copy(lambda table, x: Field(x, table=table))
+pypika.queries.Selectable.__getitem__ = ignore_copy(lambda table, x: Field(x, table=table))
+pypika.queries.Selectable.field = pypika.terms.PseudoColumn("field")

frappe/query_builder/builder.py

@@ -1,8 +1,12 @@
 from pypika import MySQLQuery, Order, PostgreSQLQuery, terms
-from pypika.queries import Schema, Table
-from frappe.utils import get_table_name
+from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
+from pypika.queries import QueryBuilder, Schema, Table
 from pypika.terms import Function
 
+from frappe.query_builder.terms import ParameterizedValueWrapper
+from frappe.utils import get_table_name
+
+
 class Base:
 	terms = terms
 	desc = Order.desc
@@ -19,13 +23,13 @@ class Base:
 		return Table(table_name, *args, **kwargs)
 
 	@classmethod
-	def into(cls, table, *args, **kwargs):
+	def into(cls, table, *args, **kwargs) -> QueryBuilder:
 		if isinstance(table, str):
 			table = cls.DocType(table)
 		return super().into(table, *args, **kwargs)
 
 	@classmethod
-	def update(cls, table, *args, **kwargs):
+	def update(cls, table, *args, **kwargs) -> QueryBuilder:
 		if isinstance(table, str):
 			table = cls.DocType(table)
 		return super().update(table, *args, **kwargs)
@@ -34,6 +38,10 @@ class Base:
 class MariaDB(Base, MySQLQuery):
 	Field = terms.Field
 
+	@classmethod
+	def _builder(cls, *args, **kwargs) -> "MySQLQueryBuilder":
+		return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs)
+
 	@classmethod
 	def from_(cls, table, *args, **kwargs):
 		if isinstance(table, str):
@@ -53,6 +61,10 @@ class Postgres(Base, PostgreSQLQuery):
 	# they are two different objects. The quick fix used here is to replace the
 	# Field names in the "Field" function.
 
+	@classmethod
+	def _builder(cls, *args, **kwargs) -> "PostgreSQLQueryBuilder":
+		return super()._builder(*args, wrapper_cls=ParameterizedValueWrapper, **kwargs)
+
 	@classmethod
 	def Field(cls, field_name, *args, **kwargs):
 		if field_name in cls.field_translation:

frappe/query_builder/terms.py

@@ -1,33 +1,77 @@
+from datetime import timedelta
 from typing import Any, Dict, Optional
+from frappe.utils.data import format_timedelta
 
 from pypika.terms import Function, ValueWrapper
 from pypika.utils import format_alias_sql
 
 
-class NamedParameterWrapper():
-	def __init__(self, parameters: Dict[str, Any]):
-		self.parameters = parameters
+class NamedParameterWrapper:
+	"""Utility class to hold parameter values and keys"""
 
-	def update_parameters(self, param_key: Any, param_value: Any, **kwargs):
+	def __init__(self) -> None:
+		self.parameters = {}
+
+	def get_sql(self, param_value: Any, **kwargs) -> str:
+		"""returns SQL for a parameter, while adding the real value in a dict
+
+		Args:
+				param_value (Any): Value of the parameter
+
+		Returns:
+				str: parameter used in the SQL query
+		"""
+		param_key = f"%(param{len(self.parameters) + 1})s"
 		self.parameters[param_key[2:-2]] = param_value
+		return param_key
 
-	def get_sql(self, **kwargs):
-		return f'%(param{len(self.parameters) + 1})s'
+	def get_parameters(self) -> Dict[str, Any]:
+		"""get dict with parameters and values
+
+		Returns:
+				Dict[str, Any]: parameter dict
+		"""
+		return self.parameters
 
 
 class ParameterizedValueWrapper(ValueWrapper):
-	def get_sql(self, quote_char: Optional[str] = None, secondary_quote_char: str = "'", param_wrapper= None, **kwargs: Any) -> str:
-		if param_wrapper is None:
-			sql = self.get_value_sql(quote_char=quote_char, secondary_quote_char=secondary_quote_char, **kwargs)
-			return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs)
+	"""
+	Class to monkey patch ValueWrapper
+
+	Adds functionality to parameterize queries when a `param wrapper` is passed in get_sql()
+	"""
+
+	def get_sql(
+		self,
+		quote_char: Optional[str] = None,
+		secondary_quote_char: str = "'",
+		param_wrapper: Optional[NamedParameterWrapper] = None,
+		**kwargs: Any,
+	) -> str:
+		if param_wrapper and isinstance(self.value, str):
+			# add quotes if it's a string value
+			value_sql = self.get_value_sql(quote_char=quote_char, **kwargs)
+			sql = param_wrapper.get_sql(param_value=value_sql, **kwargs)
 		else:
-			value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) if not isinstance(self.value,int) else self.value
-			param_sql = param_wrapper.get_sql(**kwargs)
-			param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs)
-		return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs)
+			# * BUG: pypika doesen't parse timedeltas
+			if isinstance(self.value, timedelta):
+				self.value = format_timedelta(self.value)
+			sql = self.get_value_sql(
+				quote_char=quote_char,
+				secondary_quote_char=secondary_quote_char,
+				param_wrapper=param_wrapper,
+				**kwargs,
+			)
+		return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs)
 
 
 class ParameterizedFunction(Function):
+	"""
+	Class to monkey patch pypika.terms.Functions
+
+	Only to pass `param_wrapper` in `get_function_sql`.
+	"""
+
 	def get_sql(self, **kwargs: Any) -> str:
 		with_alias = kwargs.pop("with_alias", False)
 		with_namespace = kwargs.pop("with_namespace", False)
@@ -35,15 +79,24 @@ class ParameterizedFunction(Function):
 		dialect = kwargs.pop("dialect", None)
 		param_wrapper = kwargs.pop("param_wrapper", None)
 
-		function_sql = self.get_function_sql(with_namespace=with_namespace, quote_char=quote_char, param_wrapper=param_wrapper, dialect=dialect)
+		function_sql = self.get_function_sql(
+			with_namespace=with_namespace,
+			quote_char=quote_char,
+			param_wrapper=param_wrapper,
+			dialect=dialect,
+		)
 
 		if self.schema is not None:
 			function_sql = "{schema}.{function}".format(
-				schema=self.schema.get_sql(quote_char=quote_char, dialect=dialect, **kwargs),
+				schema=self.schema.get_sql(
+					quote_char=quote_char, dialect=dialect, **kwargs
+				),
 				function=function_sql,
 			)
 
 		if with_alias:
-			return format_alias_sql(function_sql, self.alias, quote_char=quote_char, **kwargs)
+			return format_alias_sql(
+				function_sql, self.alias, quote_char=quote_char, **kwargs
+			)
 
 		return function_sql

frappe/query_builder/utils.py

@@ -1,16 +1,16 @@
 from enum import Enum
-from typing import Any, Callable, Dict, Union, get_type_hints
 from importlib import import_module
+from typing import Any, Callable, Dict, Union, get_type_hints
 
 from pypika import Query
 from pypika.queries import Column
+from pypika.terms import PseudoColumn
 
 import frappe
+from frappe.query_builder.terms import NamedParameterWrapper
 
 from .builder import MariaDB, Postgres
-from pypika.terms import PseudoColumn
 
-from frappe.query_builder.terms import NamedParameterWrapper
 
 class db_type_is(Enum):
 	MARIADB = "mariadb"
@@ -59,11 +59,29 @@ def patch_query_execute():
 		return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep
 
 	def prepare_query(query):
-		params = {}
-		query = query.get_sql(param_wrapper = NamedParameterWrapper(params))
+		import inspect
+
+		param_collector = NamedParameterWrapper()
+		query = query.get_sql(param_wrapper=param_collector)
 		if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"):
-			raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
-		return query, params
+			callstack = inspect.stack()
+			if len(callstack) >= 3 and ".py" in callstack[2].filename:
+				# ignore any query builder methods called from python files
+				# assumption is that those functions are whitelisted already.
+
+				# since query objects are patched everywhere any query.run()
+				# will have callstack like this:
+				# frame0: this function prepare_query()
+				# frame1: execute_query()
+				# frame2: frame that called `query.run()`
+				#
+				# if frame2 is server script it wont have a filename and hence
+				# it shouldn't be allowed.
+				# ps. stack() returns `"<unknown>"` as filename.
+				pass
+			else:
+				raise frappe.PermissionError('Only SELECT SQL allowed in scripting')
+		return query, param_collector.get_parameters()
 
 	query_class = get_attr(str(frappe.qb).split("'")[1])
 	builder_class = get_type_hints(query_class._builder).get('return')
@@ -78,7 +96,7 @@ def patch_query_execute():
 def patch_query_aggregation():
 	"""Patch aggregation functions to frappe.qb
 	"""
-	from frappe.query_builder.functions import _max, _min, _avg, _sum
+	from frappe.query_builder.functions import _avg, _max, _min, _sum
 
 	frappe.qb.max = _max
 	frappe.qb.min = _min