Merge branch 'develop' into cleanup-portal

.eslintrc

@@ -148,6 +148,7 @@
 		"context": true,
 		"before": true,
 		"beforeEach": true,
+		"after": true,
 		"qz": true,
 		"localforage": true,
 		"extend_cscript": true

.github/workflows/docs-checker.yml

@@ -12,7 +12,7 @@ jobs:
       - name: 'Setup Environment'
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.8
 
       - name: 'Clone repo'
         uses: actions/checkout@v2

cypress.json

@@ -9,5 +9,7 @@
   "retries": {
     "runMode": 2,
     "openMode": 2
-  }
+  },
+  "integrationFolder": ".",
+  "testFiles": ["cypress/integration/*.js", "**/ui_test_*.js"]
 }

cypress/integration/first_day_of_the_week.js

@@ -0,0 +1,45 @@
+context("First Day of the Week", () => {
+	before(() => {
+		cy.login();
+	});
+
+	beforeEach(() => {
+		cy.visit('/app/system-settings');
+		cy.findByText('Date and Number Format').click();
+	});
+
+	it("Date control starts with same day as selected in System Settings", () => {
+		cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings");
+		cy.fill_field('first_day_of_the_week', 'Tuesday', 'Select');
+		cy.findByRole('button', {name: 'Save'}).click();
+		cy.wait("@load_settings");
+		cy.dialog({
+			title: 'Date',
+			fields: [
+				{
+					label: 'Date',
+					fieldname: 'date',
+					fieldtype: 'Date'
+				}
+			]
+		});
+		cy.get_field('date').click();
+		cy.get('.datepicker--day-name').eq(0).should('have.text', 'Tu');
+	});
+
+	it("Calendar view starts with same day as selected in System Settings", () => {
+		cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings");
+		cy.fill_field('first_day_of_the_week', 'Monday', 'Select');
+		cy.findByRole('button', {name: 'Save'}).click();
+		cy.wait("@load_settings");
+		cy.visit("app/todo/view/calendar/default");
+		cy.get('.fc-day-header > span').eq(0).should('have.text', 'Mon');
+	});
+
+	after(() => {
+		cy.visit('/app/system-settings');
+		cy.findByText('Date and Number Format').click();
+		cy.fill_field('first_day_of_the_week', 'Sunday', 'Select');
+		cy.findByRole('button', {name: 'Save'}).click();
+	});
+});

cypress/integration/grid_keyboard_shortcut.js

@@ -1,48 +1,39 @@
 context('Grid Keyboard Shortcut', () => {
 	let total_count = 0;
-	beforeEach(() => {
-		cy.login();
-		cy.visit('/app/doctype/User');
-	});
 	before(() => {
 		cy.login();
-		cy.visit('/app/doctype/User');
-		return cy.window().its('frappe').then(frappe => {
-			frappe.db.count('DocField', {
-				filters: {
-					'parent': 'User', 'parentfield': 'fields', 'parenttype': 'DocType'
-				}
-			}).then((r) => {
-				total_count = r;
-			});
-		});
+	});
+	beforeEach(() => {
+		cy.reload();
+		cy.visit('/app/contact/new-contact-1');
+		cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click();
 	});
 	it('Insert new row at the end', () => {
 		cy.add_new_row_in_grid('{ctrl}{shift}{downarrow}', (cy, total_count) => {
-			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', `${total_count+1}`);
+			cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', `${total_count+1}`);
 		}, total_count);
 	});
 	it('Insert new row at the top', () => {
 		cy.add_new_row_in_grid('{ctrl}{shift}{uparrow}', (cy) => {
-			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '1');
+			cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2');
 		});
 	});
 	it('Insert new row below', () => {
 		cy.add_new_row_in_grid('{ctrl}{downarrow}', (cy) => {
-			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '2');
+			cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '1');
 		});
 	});
 	it('Insert new row above', () => {
 		cy.add_new_row_in_grid('{ctrl}{uparrow}', (cy) => {
-			cy.get('[data-name="new-docfield-1"]').should('have.attr', 'data-idx', '1');
+			cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2');
 		});
 	});
 });
 
 Cypress.Commands.add('add_new_row_in_grid', (shortcut_keys, callbackFn, total_count) => {
-	cy.get('.frappe-control[data-fieldname="fields"]').as('table');
-	cy.get('@table').find('.grid-body .col-xs-2').first().click();
-	cy.get('@table').find('.grid-body .col-xs-2')
+	cy.get('.frappe-control[data-fieldname="email_ids"]').as('table');
+	cy.get('@table').find('.grid-body [data-fieldname="email_id"]').first().click();
+	cy.get('@table').find('.grid-body [data-fieldname="email_id"]')
 		.first().type(shortcut_keys);
 
 	callbackFn(cy, total_count);

cypress/integration/web_form.js

@@ -0,0 +1,29 @@
+context('Web Form', () => {
+	before(() => {
+		cy.login();
+	});
+
+	it('Navigate and Submit a WebForm', () => {
+		cy.visit('/update-profile');
+		cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200);
+		cy.get('.web-form-actions .btn-primary').click();
+		cy.wait(500);
+		cy.get('.modal.show > .modal-dialog').should('be.visible');
+	});
+
+	it('Navigate and Submit a MultiStep WebForm', () => {
+		cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(() => {
+			cy.visit('/update-profile-duplicate');
+			cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200);
+			cy.get('.btn-next').should('be.visible');
+			cy.get('.web-form-footer .btn-primary').should('not.be.visible');
+			cy.get('.btn-next').click();
+			cy.get('.btn-previous').should('be.visible');
+			cy.get('.btn-next').should('not.be.visible');
+			cy.get('.web-form-footer .btn-primary').should('be.visible');
+			cy.get('.web-form-actions .btn-primary').click();
+			cy.wait(500);
+			cy.get('.modal.show > .modal-dialog').should('be.visible');
+		});
+	});
+});

cypress/support/commands.js

@@ -30,7 +30,7 @@ Cypress.Commands.add('login', (email, password) => {
 		email = 'Administrator';
 	}
 	if (!password) {
-		password = Cypress.config('adminPassword');
+		password = Cypress.env('adminPassword');
 	}
 	cy.request({
 		url: '/api/method/login',
@@ -161,7 +161,7 @@ Cypress.Commands.add('remove_doc', (doctype, name) => {
 
 Cypress.Commands.add('create_records', doc => {
 	return cy
-		.call('frappe.tests.ui_test_helpers.create_if_not_exists', {doc})
+		.call('frappe.tests.ui_test_helpers.create_if_not_exists', {doc: JSON.stringify(doc)})
 		.then(r => r.message);
 });
 
@@ -193,7 +193,8 @@ Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => {
 });
 
 Cypress.Commands.add('get_field', (fieldname, fieldtype = 'Data') => {
-	let selector = `[data-fieldname="${fieldname}"] input:visible`;
+	let field_element = fieldtype === 'Select' ? 'select': 'input';
+	let selector = `[data-fieldname="${fieldname}"] ${field_element}:visible`;
 
 	if (fieldtype === 'Text Editor') {
 		selector = `[data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]:visible`;

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -1,32 +1,47 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2019, Frappe Technologies and contributors
+# Copyright (c) 2022, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+from typing import Dict, Iterable, List
+
 import frappe
-from frappe.model.document import Document
-from frappe.desk.form import assign_to
-import frappe.cache_manager
 from frappe import _
+from frappe.cache_manager import clear_doctype_map, get_doctype_map
+from frappe.desk.form import assign_to
 from frappe.model import log_types
+from frappe.model.document import Document
 
-class AssignmentRule(Document):
 
+class AssignmentRule(Document):
 	def validate(self):
+		self.validate_document_types()
+		self.validate_assignment_days()
+
+	def clear_cache(self):
+		super().clear_cache()
+		clear_doctype_map(self.doctype, self.document_type)
+		clear_doctype_map(self.doctype, f"due_date_rules_for_{self.document_type}")
+
+	def validate_document_types(self):
+		if self.document_type == "ToDo":
+			frappe.throw(
+				_('Assignment Rule is not allowed on {0} document type').format(
+					frappe.bold("ToDo")
+				)
+			)
+
+	def validate_assignment_days(self):
 		assignment_days = self.get_assignment_days()
-		if not len(set(assignment_days)) == len(assignment_days):
-			repeated_days = get_repeated(assignment_days)
-			frappe.throw(_("Assignment Day {0} has been repeated.").format(frappe.bold(repeated_days)))
-		if self.document_type == 'ToDo':
-			frappe.throw(_('Assignment Rule is not allowed on {0} document type').format(frappe.bold("ToDo")))
 
-	def on_update(self):
-		clear_assignment_rule_cache(self)
-
-	def after_rename(self, old, new, merge):
-		clear_assignment_rule_cache(self)
+		if len(set(assignment_days)) != len(assignment_days):
+			repeated_days = get_repeated(assignment_days)
+			plural = "s" if len(repeated_days) > 1 else ""
 
-	def on_trash(self):
-		clear_assignment_rule_cache(self)
+			frappe.throw(
+				_("Assignment Day{0} {1} has been repeated.").format(
+					plural,
+					frappe.bold(", ".join(repeated_days))
+				)
+			)
 
 	def apply_unassign(self, doc, assignments):
 		if (self.unassign_condition and
@@ -35,7 +50,6 @@ class AssignmentRule(Document):
 
 		return False
 
-
 	def apply_assign(self, doc):
 		if self.safe_eval('assign_condition', doc):
 			return self.do_assignment(doc)
@@ -109,7 +123,7 @@ class AssignmentRule(Document):
 				user = d.user,
 				count = frappe.db.count('ToDo', dict(
 					reference_type = self.document_type,
-					owner = d.user,
+					allocated_to = d.user,
 					status = "Open"))
 			))
 
@@ -141,65 +155,68 @@ class AssignmentRule(Document):
 	def is_rule_not_applicable_today(self):
 		today = frappe.flags.assignment_day or frappe.utils.get_weekday()
 		assignment_days = self.get_assignment_days()
-		if assignment_days and not today in assignment_days:
-			return True
+		return assignment_days and today not in assignment_days
 
-		return False
 
-def get_assignments(doc):
+def get_assignments(doc) -> List[Dict]:
 	return frappe.get_all('ToDo', fields = ['name', 'assignment_rule'], filters = dict(
 		reference_type = doc.get('doctype'),
 		reference_name = doc.get('name'),
 		status = ('!=', 'Cancelled')
-	), limit = 5)
+	), limit=5)
+
 
 @frappe.whitelist()
 def bulk_apply(doctype, docnames):
-	import json
-	docnames = json.loads(docnames)
-
+	docnames = frappe.parse_json(docnames)
 	background = len(docnames) > 5
+
 	for name in docnames:
 		if background:
 			frappe.enqueue('frappe.automation.doctype.assignment_rule.assignment_rule.apply', doc=None, doctype=doctype, name=name)
 		else:
-			apply(None, doctype=doctype, name=name)
+			apply(doctype=doctype, name=name)
+
 
 def reopen_closed_assignment(doc):
-	todo_list = frappe.db.get_all('ToDo', filters = dict(
-		reference_type = doc.doctype,
-		reference_name = doc.name,
-		status = 'Closed'
-	))
-	if not todo_list:
-		return False
+	todo_list = frappe.get_all("ToDo", filters={
+		"reference_type": doc.doctype,
+		"reference_name": doc.name,
+		"status": "Closed",
+	}, pluck="name")
+
 	for todo in todo_list:
-		todo_doc = frappe.get_doc('ToDo', todo.name)
+		todo_doc = frappe.get_doc('ToDo', todo)
 		todo_doc.status = 'Open'
 		todo_doc.save(ignore_permissions=True)
-	return True
 
-def apply(doc, method=None, doctype=None, name=None):
-	if not doctype:
-		doctype = doc.doctype
+	return bool(todo_list)
 
-	if (frappe.flags.in_patch
+
+def apply(doc=None, method=None, doctype=None, name=None):
+	doctype = doctype or doc.doctype
+
+	skip_assignment_rules = (
+		frappe.flags.in_patch
 		or frappe.flags.in_install
 		or frappe.flags.in_setup_wizard
-		or doctype in log_types):
+		or doctype in log_types
+	)
+
+	if skip_assignment_rules:
 		return
 
 	if not doc and doctype and name:
 		doc = frappe.get_doc(doctype, name)
 
-	assignment_rules = frappe.cache_manager.get_doctype_map('Assignment Rule', doc.doctype, dict(
-		document_type = doc.doctype, disabled = 0), order_by = 'priority desc')
-
-	assignment_rule_docs = []
+	assignment_rules = get_doctype_map("Assignment Rule", doc.doctype, filters={
+		"document_type": doc.doctype, "disabled": 0
+	}, order_by="priority desc")
 
 	# multiple auto assigns
-	for d in assignment_rules:
-		assignment_rule_docs.append(frappe.get_cached_doc('Assignment Rule', d.get('name')))
+	assignment_rule_docs: List[AssignmentRule] = [
+		frappe.get_cached_doc("Assignment Rule", d.get('name')) for d in assignment_rules
+	]
 
 	if not assignment_rule_docs:
 		return
@@ -235,6 +252,7 @@ def apply(doc, method=None, doctype=None, name=None):
 
 	# apply close rule only if assignments exists
 	assignments = get_assignments(doc)
+
 	if assignments:
 		for assignment_rule in assignment_rule_docs:
 			if assignment_rule.is_rule_not_applicable_today():
@@ -242,38 +260,74 @@ def apply(doc, method=None, doctype=None, name=None):
 
 			if not new_apply:
 				# only reopen if close condition is not satisfied
-				if not assignment_rule.safe_eval('close_condition', doc):
-					reopen =  reopen_closed_assignment(doc)
-					if reopen:
+				to_close_todos = assignment_rule.safe_eval('close_condition', doc)
+
+				if to_close_todos:
+					# close todo status
+					todos_to_close = frappe.get_all("ToDo", filters={
+						"reference_type": doc.doctype,
+						"reference_name": doc.name,
+					}, pluck="name")
+
+					for todo in todos_to_close:
+						_todo = frappe.get_doc("ToDo", todo)
+						_todo.status = "Closed"
+						_todo.save()
+					break
+
+				else:
+					reopened = reopen_closed_assignment(doc)
+					if reopened:
 						break
+
+				# print(f"Rule:{assignment_rule}\nDoc: {doc}\nReOpened: {reopened}")
+
 			assignment_rule.close_assignments(doc)
 
+
 def update_due_date(doc, state=None):
-	# called from hook
-	if (frappe.flags.in_patch
-		or frappe.flags.in_install
-		or frappe.flags.in_migrate
+	"""Run on_update on every Document (via hooks.py)
+	"""
+	skip_document_update = (
+		frappe.flags.in_migrate
+		or frappe.flags.in_patch
 		or frappe.flags.in_import
-		or frappe.flags.in_setup_wizard):
+		or frappe.flags.in_setup_wizard
+		or frappe.flags.in_install
+	)
+
+	if skip_document_update:
 		return
-	assignment_rules = frappe.cache_manager.get_doctype_map('Assignment Rule', 'due_date_rules_for_' + doc.doctype, dict(
-		document_type = doc.doctype,
-		disabled = 0,
-		due_date_based_on = ['is', 'set']
-	))
+
+	assignment_rules = get_doctype_map(
+		doctype="Assignment Rule",
+		name=f"due_date_rules_for_{doc.doctype}",
+		filters={
+			"due_date_based_on": ["is", "set"],
+			"document_type": doc.doctype,
+			"disabled": 0,
+		}
+	)
+
 	for rule in assignment_rules:
-		rule_doc = frappe.get_cached_doc('Assignment Rule', rule.get('name'))
+		rule_doc = frappe.get_cached_doc("Assignment Rule", rule.get("name"))
 		due_date_field = rule_doc.due_date_based_on
-		if doc.meta.has_field(due_date_field) and \
-			doc.has_value_changed(due_date_field) and rule.get('name'):
-			assignment_todos = frappe.get_all('ToDo', {
-				'assignment_rule': rule.get('name'),
-				'status': 'Open',
-				'reference_type': doc.doctype,
-				'reference_name': doc.name
-			})
+		field_updated = (
+			doc.meta.has_field(due_date_field)
+			and doc.has_value_changed(due_date_field)
+			and rule.get("name")
+		)
+
+		if field_updated:
+			assignment_todos = frappe.get_all("ToDo", filters={
+				"assignment_rule": rule.get("name"),
+				"reference_type": doc.doctype,
+				"reference_name": doc.name,
+				"status": "Open",
+			}, pluck="name")
+
 			for todo in assignment_todos:
-				todo_doc = frappe.get_doc('ToDo', todo.name)
+				todo_doc = frappe.get_doc('ToDo', todo)
 				todo_doc.date = doc.get(due_date_field)
 				todo_doc.flags.updater_reference = {
 					'doctype': 'Assignment Rule',
@@ -282,20 +336,19 @@ def update_due_date(doc, state=None):
 				}
 				todo_doc.save(ignore_permissions=True)
 
-def get_assignment_rules():
-	return [d.document_type for d in frappe.db.get_all('Assignment Rule', fields=['document_type'], filters=dict(disabled = 0))]
 
-def get_repeated(values):
-	unique_list = []
-	diff = []
+def get_assignment_rules() -> List[str]:
+	return frappe.get_all("Assignment Rule", filters={"disabled": 0}, pluck="document_type")
+
+
+def get_repeated(values: Iterable) -> List:
+	unique = set()
+	repeated = set()
+
 	for value in values:
-		if value not in unique_list:
-			unique_list.append(str(value))
+		if value in unique:
+			repeated.add(value)
 		else:
-			if value not in diff:
-				diff.append(str(value))
-	return " ".join(diff)
+			unique.add(value)
 
-def clear_assignment_rule_cache(rule):
-	frappe.cache_manager.clear_doctype_map('Assignment Rule', rule.document_type)
-	frappe.cache_manager.clear_doctype_map('Assignment Rule', 'due_date_rules_for_' + rule.document_type)
+	return [str(x) for x in repeated]

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -1,12 +1,22 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2019, Frappe Technologies and Contributors
+# Copyright (c) 2021, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import frappe
+
 import unittest
-from frappe.utils import random_string
+
+import frappe
 from frappe.test_runner import make_test_records
+from frappe.utils import random_string
+
 
 class TestAutoAssign(unittest.TestCase):
+	@classmethod
+	def setUpClass(cls):
+		frappe.db.delete("Assignment Rule")
+
+	@classmethod
+	def tearDownClass(cls):
+		frappe.db.rollback()
+
 	def setUp(self):
 		make_test_records("User")
 		days = [
@@ -30,7 +40,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), 'test@example.com')
+		), 'allocated_to'), 'test@example.com')
 
 		note = make_note(dict(public=1))
 
@@ -39,7 +49,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), 'test1@example.com')
+		), 'allocated_to'), 'test1@example.com')
 
 		clear_assignments()
 
@@ -51,7 +61,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), 'test2@example.com')
+		), 'allocated_to'), 'test2@example.com')
 
 		# check loop back to first user
 		note = make_note(dict(public=1))
@@ -60,7 +70,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), 'test@example.com')
+		), 'allocated_to'), 'test@example.com')
 
 	def test_load_balancing(self):
 		self.assignment_rule.rule = 'Load Balancing'
@@ -71,11 +81,11 @@ class TestAutoAssign(unittest.TestCase):
 
 		# check if each user has 10 assignments (?)
 		for user in ('test@example.com', 'test1@example.com', 'test2@example.com'):
-			self.assertEqual(len(frappe.get_all('ToDo', dict(owner = user, reference_type = 'Note'))), 10)
+			self.assertEqual(len(frappe.get_all('ToDo', dict(allocated_to = user, reference_type = 'Note'))), 10)
 
 		# clear 5 assignments for first user
 		# can't do a limit in "delete" since postgres does not support it
-		for d in frappe.get_all('ToDo', dict(reference_type = 'Note', owner = 'test@example.com'), limit=5):
+		for d in frappe.get_all('ToDo', dict(reference_type = 'Note', allocated_to = 'test@example.com'), limit=5):
 			frappe.db.delete("ToDo", {"name": d.name})
 
 		# add 5 more assignments
@@ -84,7 +94,7 @@ class TestAutoAssign(unittest.TestCase):
 
 		# check if each user still has 10 assignments
 		for user in ('test@example.com', 'test1@example.com', 'test2@example.com'):
-			self.assertEqual(len(frappe.get_all('ToDo', dict(owner = user, reference_type = 'Note'))), 10)
+			self.assertEqual(len(frappe.get_all('ToDo', dict(allocated_to = user, reference_type = 'Note'))), 10)
 
 	def test_based_on_field(self):
 		self.assignment_rule.rule = 'Based on Field'
@@ -119,7 +129,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), None)
+		), 'allocated_to'), None)
 
 	def test_clear_assignment(self):
 		note = make_note(dict(public=1))
@@ -129,10 +139,10 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		))[0]
+		), limit=1)[0]
 
 		todo = frappe.get_doc('ToDo', todo['name'])
-		self.assertEqual(todo.owner, 'test@example.com')
+		self.assertEqual(todo.allocated_to, 'test@example.com')
 
 		# test auto unassign
 		note.public = 0
@@ -151,10 +161,10 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		))[0]
+		), limit=1)[0]
 
 		todo = frappe.get_doc('ToDo', todo['name'])
-		self.assertEqual(todo.owner, 'test@example.com')
+		self.assertEqual(todo.allocated_to, 'test@example.com')
 
 		note.content="Closed"
 		note.save()
@@ -164,7 +174,7 @@ class TestAutoAssign(unittest.TestCase):
 		# check if todo is closed
 		self.assertEqual(todo.status, 'Closed')
 		# check if closed todo retained assignment
-		self.assertEqual(todo.owner, 'test@example.com')
+		self.assertEqual(todo.allocated_to, 'test@example.com')
 
 	def check_multiple_rules(self):
 		note = make_note(dict(public=1, notify_on_login=1))
@@ -174,7 +184,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), 'test@example.com')
+		), 'allocated_to'), 'test@example.com')
 
 	def check_assignment_rule_scheduling(self):
 		frappe.db.delete("Assignment Rule")
@@ -192,7 +202,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), ['test@example.com', 'test1@example.com', 'test2@example.com'])
+		), 'allocated_to'), ['test@example.com', 'test1@example.com', 'test2@example.com'])
 
 		frappe.flags.assignment_day = "Friday"
 		note = make_note(dict(public=1))
@@ -201,7 +211,7 @@ class TestAutoAssign(unittest.TestCase):
 			reference_type = 'Note',
 			reference_name = note.name,
 			status = 'Open'
-		), 'owner'), ['test3@example.com'])
+		), 'allocated_to'), ['test3@example.com'])
 
 	def test_assignment_rule_condition(self):
 		frappe.db.delete("Assignment Rule")

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -96,7 +96,15 @@ class AutoRepeat(Document):
 		auto_repeat_days = self.get_auto_repeat_days()
 		if not len(set(auto_repeat_days)) == len(auto_repeat_days):
 			repeated_days = get_repeated(auto_repeat_days)
-			frappe.throw(_('Auto Repeat Day {0} has been repeated.').format(frappe.bold(repeated_days)))
+			plural = "s" if len(repeated_days) > 1 else ""
+
+			frappe.throw(
+				_("Auto Repeat Day{0} {1} has been repeated.").format(
+					plural,
+					frappe.bold(", ".join(repeated_days))
+				)
+			)
+
 
 	def update_auto_repeat_id(self):
 		#check if document is already on auto repeat

frappe/core/doctype/communication/communication.py

@@ -12,7 +12,7 @@ from frappe.core.utils import get_parent_doc
 from frappe.utils.bot import BotReply
 from frappe.utils import parse_addr, split_emails
 from frappe.core.doctype.comment.comment import update_comment_in_doc
-from email.utils import parseaddr
+from email.utils import getaddresses
 from urllib.parse import unquote
 from frappe.utils.user import is_system_user
 from frappe.contacts.doctype.contact.contact import get_contact_name
@@ -372,10 +372,9 @@ def get_contacts(email_strings, auto_create_contact=False):
 
 	for email_string in email_strings:
 		if email_string:
-			for email in email_string.split(","):
-				parsed_email = parseaddr(email)[1]
-				if parsed_email:
-					email_addrs.append(parsed_email)
+			result = getaddresses([email_string])
+			for email in result:
+				email_addrs.append(email[1])
 
 	contacts = []
 	for email in email_addrs:
@@ -488,10 +487,12 @@ def update_parent_document_on_communication(doc):
 def update_first_response_time(parent, communication):
 	if parent.meta.has_field("first_response_time") and not parent.get("first_response_time"):
 		if is_system_user(communication.sender):
-			first_responded_on = communication.creation
-			if parent.meta.has_field("first_responded_on") and communication.sent_or_received == "Sent":
-				parent.db_set("first_responded_on", first_responded_on)
-			parent.db_set("first_response_time", round(time_diff_in_seconds(first_responded_on, parent.creation), 2))
+			if communication.sent_or_received == "Sent":
+				first_responded_on = communication.creation
+				if parent.meta.has_field("first_responded_on"):
+					parent.db_set("first_responded_on", first_responded_on)
+				first_response_time = round(time_diff_in_seconds(first_responded_on, parent.creation), 2)
+				parent.db_set("first_response_time", first_response_time)
 
 def set_avg_response_time(parent, communication):
 	if parent.meta.has_field("avg_response_time") and communication.sent_or_received == "Sent":

frappe/core/doctype/doctype/doctype.py

@@ -1283,7 +1283,7 @@ def make_module_and_roles(doc, perm_fieldname="permissions"):
 		roles = [p.role for p in doc.get("permissions") or []] + default_roles
 
 		for role in list(set(roles)):
-			if not frappe.db.exists("Role", role):
+			if frappe.db.table_exists("Role", cached=False) and not frappe.db.exists("Role", role):
 				r = frappe.get_doc(dict(doctype= "Role", role_name=role, desk_access=1))
 				r.flags.ignore_mandatory = r.flags.ignore_permissions = True
 				r.insert()

frappe/core/doctype/doctype/test_doctype.py

@@ -15,6 +15,10 @@ from frappe.core.doctype.doctype.doctype import (UniqueFieldnameError,
 # test_records = frappe.get_test_records('DocType')
 
 class TestDocType(unittest.TestCase):
+
+	def tearDown(self):
+		frappe.db.rollback()
+
 	def test_validate_name(self):
 		self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert)
 		self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert)
@@ -42,6 +46,7 @@ class TestDocType(unittest.TestCase):
 
 		doc1.insert()
 		self.assertRaises(frappe.UniqueValidationError, doc2.insert)
+		frappe.db.rollback()
 
 		dt.fields[0].unique = 0
 		dt.save()

frappe/core/doctype/file/test_file.py

@@ -18,6 +18,7 @@ test_content2 = 'Hello World'
 def make_test_doc():
 	d = frappe.new_doc('ToDo')
 	d.description = 'Test'
+	d.assigned_by = frappe.session.user
 	d.save()
 	return d.doctype, d.name
 

frappe/core/doctype/module_def/module_def.json

@@ -10,7 +10,8 @@
   "custom",
   "package",
   "app_name",
-  "restrict_to_domain"
+  "restrict_to_domain",
+  "connections_tab"
  ],
  "fields": [
   {
@@ -50,6 +51,12 @@
    "fieldtype": "Link",
    "label": "Package",
    "options": "Package"
+  },
+  {
+   "fieldname": "connections_tab",
+   "fieldtype": "Tab Break",
+   "label": "Connections",
+   "show_dashboard": 1
   }
  ],
  "icon": "fa fa-sitemap",
@@ -116,7 +123,7 @@
    "link_fieldname": "module"
   }
  ],
- "modified": "2021-09-05 21:58:40.253909",
+ "modified": "2022-01-03 13:56:52.817954",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Module Def",
@@ -154,5 +161,6 @@
  "show_name_in_global_search": 1,
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }

frappe/core/doctype/system_settings/system_settings.js

@@ -10,6 +10,10 @@ frappe.ui.form.on("System Settings", {
 					frm.set_value(key, val);
 					frappe.sys_defaults[key] = val;
 				});
+				if (frm.re_setup_moment) {
+					frappe.app.setup_moment();
+					delete frm.re_setup_moment;
+				}
 			}
 		});
 	},
@@ -38,5 +42,8 @@ frappe.ui.form.on("System Settings", {
 			// Clear cache after saving to refresh the values of boot.
 			frappe.ui.toolbar.clear_cache();
 		}
-	}
+	},
+	first_day_of_the_week(frm) {
+		frm.re_setup_moment = true;
+	},
 });

frappe/core/doctype/system_settings/system_settings.json

@@ -17,10 +17,11 @@
   "date_and_number_format",
   "date_format",
   "time_format",
-  "column_break_7",
   "number_format",
+  "column_break_7",
   "float_precision",
   "currency_precision",
+  "first_day_of_the_week",
   "sec_backup_limit",
   "backup_limit",
   "encrypt_backup",
@@ -477,12 +478,19 @@
    "fieldname": "disable_system_update_notification",
    "fieldtype": "Check",
    "label": "Disable System Update Notification"
+  },
+  {
+   "default": "Sunday",
+   "fieldname": "first_day_of_the_week",
+   "fieldtype": "Select",
+   "label": "First Day of the Week",
+   "options": "Sunday\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2021-11-29 18:09:53.601629",
+ "modified": "2022-01-04 11:28:34.881192",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",
@@ -499,5 +507,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }

frappe/core/doctype/translation/translation.json

@@ -43,8 +43,7 @@
   {
    "fieldname": "context",
    "fieldtype": "Data",
-   "label": "Context",
-   "read_only": 1
+   "label": "Context"
   },
   {
    "default": "0",
@@ -83,7 +82,7 @@
   }
  ],
  "links": [],
- "modified": "2020-03-12 13:28:48.223409",
+ "modified": "2021-12-31 10:19:52.541055",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Translation",

frappe/core/doctype/user/test_user.py

@@ -70,7 +70,7 @@ class TestUser(unittest.TestCase):
 		delete_contact("_test@example.com")
 		delete_doc("User", "_test@example.com")
 
-		self.assertTrue(not frappe.db.sql("""select * from `tabToDo` where owner=%s""",
+		self.assertTrue(not frappe.db.sql("""select * from `tabToDo` where allocated_to=%s""",
 			("_test@example.com",)))
 
 		from frappe.core.doctype.role.test_role import test_records as role_records

frappe/core/doctype/user/user.json

@@ -10,15 +10,15 @@
   "enabled",
   "section_break_3",
   "email",
+  "first_name",
+  "middle_name",
   "last_name",
-  "language",
   "column_break0",
-  "first_name",
   "full_name",
-  "time_zone",
-  "column_break_11",
-  "middle_name",
   "username",
+  "column_break_11",
+  "language",
+  "time_zone",
   "send_welcome_email",
   "unsubscribed",
   "user_image",
@@ -660,7 +660,7 @@
   {
    "group": "Activity",
    "link_doctype": "ToDo",
-   "link_fieldname": "owner"
+   "link_fieldname": "allocated_to"
   },
   {
    "group": "Integrations",
@@ -669,7 +669,7 @@
   }
  ],
  "max_attachments": 5,
- "modified": "2021-11-17 17:17:16.098457",
+ "modified": "2022-01-03 11:53:25.250822",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User",
@@ -702,6 +702,7 @@
  "show_name_in_global_search": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "full_name",
  "track_changes": 1
-}
+}

frappe/core/doctype/user/user.py

@@ -363,7 +363,7 @@ class User(Document):
 			frappe.local.login_manager.logout(user=self.name)
 
 		# delete todos
-		frappe.db.delete("ToDo", {"owner": self.name})
+		frappe.db.delete("ToDo", {"allocated_to": self.name})
 		todo_table = DocType("ToDo")
 		(
 			frappe.qb.update(todo_table)

frappe/core/doctype/user_permission/user_permission.js

@@ -44,7 +44,7 @@ frappe.ui.form.on('User Permission', {
 
 	set_applicable_for_constraint: frm => {
 		frm.toggle_reqd('applicable_for', !frm.doc.apply_to_all_doctypes);
-		if (frm.doc.apply_to_all_doctypes) {
+		if (frm.doc.apply_to_all_doctypes && frm.doc.applicable_for) {
 			frm.set_value('applicable_for', null);
 		}
 	},

frappe/core/doctype/user_permission/user_permission.json

@@ -8,8 +8,8 @@
  "field_order": [
   "user",
   "allow",
-  "column_break_3",
   "for_value",
+  "column_break_3",
   "is_default",
   "advanced_control_section",
   "apply_to_all_doctypes",
@@ -37,10 +37,6 @@
    "options": "DocType",
    "reqd": 1
   },
-  {
-   "fieldname": "column_break_3",
-   "fieldtype": "Column Break"
-  },
   {
    "fieldname": "for_value",
    "fieldtype": "Dynamic Link",
@@ -87,10 +83,14 @@
    "fieldtype": "Check",
    "hidden": 1,
    "label": "Hide Descendants"
+  },
+  {
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break"
   }
  ],
  "links": [],
- "modified": "2021-01-21 18:14:10.839381",
+ "modified": "2022-01-03 11:25:03.726150",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User Permission",
@@ -111,6 +111,7 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "user",
  "track_changes": 1
 }

frappe/core/doctype/user_type/user_type.py

@@ -37,16 +37,14 @@ class UserType(Document):
 			return
 
 		modules = frappe.get_all("DocType",
-			fields=["module"],
 			filters={"name": ("in", [d.document_type for d in self.user_doctypes])},
 			distinct=True,
+			pluck="module",
 		)
 
-		self.set('user_type_modules', [])
-		for row in modules:
-			self.append('user_type_modules', {
-				'module': row.module
-			})
+		self.set("user_type_modules", [])
+		for module in modules:
+			self.append("user_type_modules", {"module": module})
 
 	def validate_document_type_limit(self):
 		limit = frappe.conf.get('user_type_doctype_limit', {}).get(frappe.scrub(self.name))

frappe/core/notifications.py

@@ -23,7 +23,7 @@ def get_things_todo(as_list=False):
 	data = frappe.get_list("ToDo",
 		fields=["name", "description"] if as_list else "count(*)",
 		filters=[["ToDo", "status", "=", "Open"]],
-		or_filters=[["ToDo", "owner", "=", frappe.session.user],
+		or_filters=[["ToDo", "allocated_to", "=", frappe.session.user],
 			["ToDo", "assigned_by", "=", frappe.session.user]],
 		as_list=True)
 

frappe/custom/doctype/customize_form/customize_form.py

@@ -516,6 +516,7 @@ docfield_properties = {
 	'options': 'Text',
 	'fetch_from': 'Small Text',
 	'fetch_if_empty': 'Check',
+	'show_dashboard': 'Check',
 	'permlevel': 'Int',
 	'width': 'Data',
 	'print_width': 'Data',

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -28,6 +28,7 @@
   "options",
   "fetch_from",
   "fetch_if_empty",
+  "show_dashboard",
   "permissions",
   "depends_on",
   "permlevel",
@@ -82,7 +83,7 @@
    "label": "Type",
    "oldfieldname": "fieldtype",
    "oldfieldtype": "Select",
-   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nTab Break",
+   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime",
    "reqd": 1,
    "search_index": 1
   },
@@ -422,18 +423,27 @@
    "fieldname": "non_negative",
    "fieldtype": "Check",
    "label": "Non Negative"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.fieldtype=='Tab Break'",
+   "fieldname": "show_dashboard",
+   "fieldtype": "Check",
+   "label": "Show Dashboard"
   }
  ],
  "idx": 1,
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-07-11 21:57:24.479749",
+ "modified": "2022-01-03 14:50:32.035768",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",
+ "naming_rule": "Random",
  "owner": "Administrator",
  "permissions": [],
  "sort_field": "modified",
- "sort_order": "ASC"
+ "sort_order": "ASC",
+ "states": []
 }

frappe/database/__init__.py

@@ -4,6 +4,8 @@
 # Database Module
 # --------------------
 
+from frappe.database.database import savepoint
+
 def setup_database(force, source_sql=None, verbose=None, no_mariadb_socket=False):
 	import frappe
 	if frappe.conf.db_type == 'postgres':

frappe/database/database.py

@@ -4,16 +4,18 @@
 # Database Module
 # --------------------
 
+import datetime
+import random
 import re
-import time
-from typing import Dict, List, Union
+import string
+from contextlib import contextmanager
+from time import time
+from typing import Dict, List, Union, Tuple
+
 import frappe
-import datetime
 import frappe.defaults
 import frappe.model.meta
-
 from frappe import _
-from time import time
 from frappe.utils import now, getdate, cast, get_datetime
 from frappe.model.utils.link_count import flush_local_link_count
 from frappe.query_builder.functions import Count
@@ -162,10 +164,7 @@ class Database(object):
 				frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2)))
 
 		except Exception as e:
-			if frappe.conf.db_type == 'postgres':
-				self.rollback()
-
-			elif self.is_syntax_error(e):
+			if self.is_syntax_error(e):
 				# only for mariadb
 				frappe.errprint('Syntax error in query:')
 				frappe.errprint(query)
@@ -176,6 +175,9 @@ class Database(object):
 			elif self.is_timedout(e):
 				raise frappe.QueryTimeoutError(e)
 
+			elif frappe.conf.db_type == 'postgres':
+				raise
+
 			if ignore_ddl and (self.is_missing_column(e) or self.is_missing_table(e) or self.cant_drop_field_or_key(e)):
 				pass
 			else:
@@ -265,9 +267,7 @@ class Database(object):
 		"""Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are
 		executed in one transaction. This is to ensure that writes are always flushed otherwise this
 		could cause the system to hang."""
-		if self.transaction_writes and \
-			query and query.strip().split()[0].lower() in ['start', 'alter', 'drop', 'create', "begin", "truncate"]:
-			raise Exception('This statement can cause implicit commit')
+		self.check_implicit_commit(query)
 
 		if query and query.strip().lower() in ('commit', 'rollback'):
 			self.transaction_writes = 0
@@ -280,6 +280,11 @@ class Database(object):
 				else:
 					frappe.throw(_("Too many writes in one request. Please send smaller requests"), frappe.ValidationError)
 
+	def check_implicit_commit(self, query):
+		if self.transaction_writes and \
+			query and query.strip().split()[0].lower() in ['start', 'alter', 'drop', 'create', "begin", "truncate"]:
+			raise Exception('This statement can cause implicit commit')
+
 	def fetch_as_dict(self, formatted=0, as_utf8=0):
 		"""Internal. Converts results to dict."""
 		result = self._cursor.fetchall()
@@ -699,6 +704,8 @@ class Database(object):
 				self.sql("""update `tab{0}`
 					set {1} where name=%(name)s""".format(dt, ', '.join(set_values)),
 					values, debug=debug)
+
+				frappe.clear_document_cache(dt, values['name'])
 		else:
 			# for singles
 			keys = list(to_update)
@@ -711,10 +718,11 @@ class Database(object):
 				self.sql('''insert into `tabSingles` (doctype, field, value) values (%s, %s, %s)''',
 					(dt, key, value), debug=debug)
 
+			frappe.clear_document_cache(dt, dn)
+
 		if dt in self.value_cache:
 			del self.value_cache[dt]
 
-		frappe.clear_document_cache(dt, dn)
 
 	@staticmethod
 	def set(doc, field, val):
@@ -811,6 +819,9 @@ class Database(object):
 			Avoid using savepoints when writing to filesystem."""
 		self.sql(f"savepoint {save_point}")
 
+	def release_savepoint(self, save_point):
+		self.sql(f"release savepoint {save_point}")
+
 	def rollback(self, *, save_point=None):
 		"""`ROLLBACK` current transaction. Optionally rollback to a known save_point."""
 		if save_point:
@@ -830,9 +841,9 @@ class Database(object):
 			'parent': dt
 		})
 
-	def table_exists(self, doctype):
+	def table_exists(self, doctype, cached=True):
 		"""Returns True if table for given doctype exists."""
-		return ("tab" + doctype) in self.get_tables()
+		return ("tab" + doctype) in self.get_tables(cached=cached)
 
 	def has_table(self, doctype):
 		return self.table_exists(doctype)
@@ -1097,3 +1108,28 @@ def enqueue_jobs_after_commit():
 			q.enqueue_call(execute_job, timeout=job.get("timeout"),
 							kwargs=job.get("queue_args"))
 		frappe.flags.enqueue_after_commit = []
+
+@contextmanager
+def savepoint(catch: Union[type, Tuple[type, ...]] = Exception):
+	""" Wrapper for wrapping blocks of DB operations in a savepoint.
+
+		as contextmanager:
+
+		for doc in docs:
+			with savepoint(catch=DuplicateError):
+				doc.insert()
+
+		as decorator (wraps FULL function call):
+
+		@savepoint(catch=DuplicateError)
+		def process_doc(doc):
+			doc.insert()
+	"""
+	try:
+		savepoint = ''.join(random.sample(string.ascii_lowercase, 10))
+		frappe.db.savepoint(savepoint)
+		yield # control back to calling function
+	except catch:
+		frappe.db.rollback(save_point=savepoint)
+	else:
+		frappe.db.release_savepoint(savepoint)

frappe/database/mariadb/framework_mariadb.sql

@@ -25,6 +25,7 @@ CREATE TABLE `tabDocField` (
   `oldfieldtype` varchar(255) DEFAULT NULL,
   `options` text,
   `search_index` int(1) NOT NULL DEFAULT 0,
+  `show_dashboard` int(1) NOT NULL DEFAULT 0,
   `hidden` int(1) NOT NULL DEFAULT 0,
   `set_only_once` int(1) NOT NULL DEFAULT 0,
   `allow_in_quick_entry` int(1) NOT NULL DEFAULT 0,

frappe/database/postgres/database.py

@@ -3,7 +3,7 @@ from typing import List, Tuple, Union
 
 import psycopg2
 import psycopg2.extensions
-from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
 from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION
 
 import frappe
@@ -69,14 +69,20 @@ class PostgresDatabase(Database):
 		conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format(
 			self.host, self.user, self.user, self.password, self.port
 		))
-		conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) # TODO: Remove this
+		conn.set_isolation_level(ISOLATION_LEVEL_REPEATABLE_READ)
 
 		return conn
 
 	def escape(self, s, percent=True):
-		"""Excape quotes and percent in given string."""
+		"""Escape quotes and percent in given string."""
 		if isinstance(s, bytes):
 			s = s.decode('utf-8')
+				
+		# MariaDB's driver treats None as an empty string
+		# So Postgres should do the same
+
+		if s is None: 
+			s = ''
 
 		if percent:
 			s = s.replace("%", "%%")
@@ -103,7 +109,7 @@ class PostgresDatabase(Database):
 
 		return super(PostgresDatabase, self).sql(*args, **kwargs)
 
-	def get_tables(self):
+	def get_tables(self, cached=True):
 		return [d[0] for d in self.sql("""select table_name
 			from information_schema.tables
 			where table_catalog='{0}'
@@ -138,6 +144,10 @@ class PostgresDatabase(Database):
 		# http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError
 		return isinstance(e, psycopg2.extensions.QueryCanceledError)
 
+	@staticmethod
+	def is_syntax_error(e):
+		return isinstance(e, psycopg2.errors.SyntaxError)
+
 	@staticmethod
 	def is_table_missing(e):
 		return getattr(e, 'pgcode', None) == '42P01'
@@ -255,8 +265,8 @@ class PostgresDatabase(Database):
 			key=key
 		)
 
-	def check_transaction_status(self, query):
-		pass
+	def check_implicit_commit(self, query):
+		pass # postgres can run DDL in transactions without implicit commits
 
 	def has_index(self, table_name, index_name):
 		return self.sql("""SELECT 1 FROM pg_indexes WHERE tablename='{table_name}'

frappe/database/postgres/framework_postgres.sql

@@ -27,6 +27,7 @@ CREATE TABLE "tabDocField" (
   "search_index" smallint NOT NULL DEFAULT 0,
   "hidden" smallint NOT NULL DEFAULT 0,
   "set_only_once" smallint NOT NULL DEFAULT 0,
+  "show_dashboard" smallint NOT NULL DEFAULT 0,
   "allow_in_quick_entry" smallint NOT NULL DEFAULT 0,
   "print_hide" smallint NOT NULL DEFAULT 0,
   "report_hide" smallint NOT NULL DEFAULT 0,

frappe/database/schema.py

@@ -206,6 +206,12 @@ class DbColumn:
 		if not current_def:
 			self.fieldname = validate_column_name(self.fieldname)
 			self.table.add_column.append(self)
+
+			if column_type not in ('text', 'longtext'):
+				if self.unique:
+					self.table.add_unique.append(self)
+				if self.set_index:
+					self.table.add_index.append(self)
 			return
 
 		# type

frappe/desk/doctype/bulk_update/bulk_update.py

@@ -7,6 +7,7 @@ from frappe.model.document import Document
 from frappe import _
 from frappe.utils import cint
 
+
 class BulkUpdate(Document):
 	pass
 
@@ -22,7 +23,7 @@ def update(doctype, field, value, condition='', limit=500):
 		frappe.throw(_('; not allowed in condition'))
 
 	docnames = frappe.db.sql_list(
-		'''select name from `tab{0}`{1} limit 0, {2}'''.format(doctype, condition, limit)
+		'''select name from `tab{0}`{1} limit {2} offset 0'''.format(doctype, condition, limit)
 	)
 	data = {}
 	data[field] = value

frappe/desk/doctype/dashboard/dashboard.py

@@ -1,23 +1,33 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2019, Frappe Technologies and contributors
+# Copyright (c) 2022, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-from frappe.model.document import Document
-from frappe.modules.export_file import export_to_files
-from frappe.config import get_modules_from_all_apps_for_user
+import json
+
 import frappe
 from frappe import _
-import json
+from frappe.config import get_modules_from_all_apps_for_user
+from frappe.model.document import Document
+from frappe.modules.export_file import export_to_files
+from frappe.query_builder import DocType
+
 
 class Dashboard(Document):
 	def on_update(self):
 		if self.is_default:
 			# make all other dashboards non-default
-			frappe.db.sql('''update
-				tabDashboard set is_default = 0 where name != %s''', self.name)
+			DashBoard = DocType("Dashboard")
+
+			frappe.qb.update(DashBoard).set(
+				DashBoard.is_default, 0
+			).where(
+				DashBoard.name != self.name
+			).run()
 
 		if frappe.conf.developer_mode and self.is_standard:
-			export_to_files(record_list=[['Dashboard', self.name, self.module + ' Dashboard']], record_module=self.module)
+			export_to_files(
+				record_list=[["Dashboard", self.name, f"{self.module} Dashboard"]],
+				record_module=self.module
+			)
 
 	def validate(self):
 		if not frappe.conf.developer_mode and self.is_standard:

frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py

@@ -8,6 +8,7 @@ from frappe.desk.doctype.dashboard_chart.dashboard_chart import get
 
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
+from unittest.mock import patch
 
 class TestDashboardChart(unittest.TestCase):
 	def test_period_ending(self):
@@ -15,8 +16,9 @@ class TestDashboardChart(unittest.TestCase):
 			getdate('2019-04-10'))
 
 		# week starts on monday
-		self.assertEqual(get_period_ending('2019-04-10', 'Weekly'),
-			getdate('2019-04-14'))
+		with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"):
+			self.assertEqual(get_period_ending('2019-04-10', 'Weekly'),
+				getdate('2019-04-14'))
 
 		self.assertEqual(get_period_ending('2019-04-10', 'Monthly'),
 			getdate('2019-04-30'))
@@ -200,13 +202,14 @@ class TestDashboardChart(unittest.TestCase):
 			timeseries = 1
 		)).insert()
 
-		result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1)
+		with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"):
+			result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1)
 
-		self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0])
-		self.assertEqual(
-			result.get('labels'),
-			['30-12-18', '06-01-19', '13-01-19', '20-01-19']
-		)
+			self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0])
+			self.assertEqual(
+				result.get('labels'),
+				['30-12-18', '06-01-19', '13-01-19', '20-01-19']
+			)
 
 		frappe.db.rollback()
 
@@ -231,13 +234,13 @@ class TestDashboardChart(unittest.TestCase):
 			timeseries = 1
 		)).insert()
 
-		result = get(chart_name='Test Average Dashboard Chart', refresh = 1)
-
-		self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 150.0, 266.6666666666667, 0.0])
-		self.assertEqual(
-			result.get('labels'),
-			['30-12-18', '06-01-19', '13-01-19', '20-01-19']
-		)
+		with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"):
+			result = get(chart_name='Test Average Dashboard Chart', refresh = 1)
+			self.assertEqual(
+				result.get('labels'),
+				['30-12-18', '06-01-19', '13-01-19', '20-01-19']
+			)
+			self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 150.0, 266.6666666666667, 0.0])
 
 		frappe.db.rollback()
 

frappe/desk/doctype/event/test_event.py

@@ -93,7 +93,7 @@ class TestEvent(unittest.TestCase):
 
 		# Remove an assignment
 		todo = frappe.get_doc("ToDo", {"reference_type": ev.doctype, "reference_name": ev.name,
-			"owner": self.test_user})
+			"allocated_to": self.test_user})
 		todo.status = "Cancelled"
 		todo.save()
 

frappe/desk/doctype/system_console/system_console.js

@@ -100,5 +100,5 @@ frappe.ui.form.on('System Console', {
 				</tr></thead>
 				<tbody>${rows}</thead>`);
 		});
-	}
+	},
 });

frappe/desk/doctype/todo/todo.json

@@ -13,7 +13,7 @@
   "column_break_2",
   "color",
   "date",
-  "owner",
+  "allocated_to",
   "description_section",
   "description",
   "section_break_6",
@@ -69,15 +69,6 @@
    "oldfieldname": "date",
    "oldfieldtype": "Date"
   },
-  {
-   "fieldname": "owner",
-   "fieldtype": "Link",
-   "ignore_user_permissions": 1,
-   "in_global_search": 1,
-   "in_standard_filter": 1,
-   "label": "Allocated To",
-   "options": "User"
-  },
   {
    "fieldname": "description_section",
    "fieldtype": "Section Break"
@@ -153,12 +144,21 @@
    "label": "Assignment Rule",
    "options": "Assignment Rule",
    "read_only": 1
+  },
+  {
+   "fieldname": "allocated_to",
+   "fieldtype": "Link",
+   "ignore_user_permissions": 1,
+   "in_global_search": 1,
+   "in_standard_filter": 1,
+   "label": "Allocated To",
+   "options": "User"
   }
  ],
  "icon": "fa fa-check",
  "idx": 2,
  "links": [],
- "modified": "2020-01-14 17:04:36.971002",
+ "modified": "2021-09-16 11:36:34.586898",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "ToDo",

frappe/desk/doctype/todo/todo.py

@@ -16,10 +16,10 @@ class ToDo(Document):
 		self._assignment = None
 		if self.is_new():
 
-			if self.assigned_by == self.owner:
+			if self.assigned_by == self.allocated_to:
 				assignment_message = frappe._("{0} self assigned this task: {1}").format(get_fullname(self.assigned_by), self.description)
 			else:
-				assignment_message = frappe._("{0} assigned {1}: {2}").format(get_fullname(self.assigned_by), get_fullname(self.owner), self.description)
+				assignment_message = frappe._("{0} assigned {1}: {2}").format(get_fullname(self.assigned_by), get_fullname(self.allocated_to), self.description)
 
 			self._assignment = {
 				"text": assignment_message,
@@ -29,12 +29,12 @@ class ToDo(Document):
 		else:
 			# NOTE the previous value is only available in validate method
 			if self.get_db_value("status") != self.status:
-				if self.owner == frappe.session.user:
+				if self.allocated_to == frappe.session.user:
 					removal_message = frappe._("{0} removed their assignment.").format(
 						get_fullname(frappe.session.user))
 				else:
 					removal_message = frappe._("Assignment of {0} removed by {1}").format(
-						get_fullname(self.owner), get_fullname(frappe.session.user))
+						get_fullname(self.allocated_to), get_fullname(frappe.session.user))
 
 				self._assignment = {
 					"text": removal_message,
@@ -69,15 +69,13 @@ class ToDo(Document):
 			return
 
 		try:
-			assignments = [d[0] for d in frappe.get_all("ToDo",
-				filters={
-					"reference_type": self.reference_type,
-					"reference_name": self.reference_name,
-					"status": ("!=", "Cancelled")
-				},
-				fields=["owner"], as_list=True)]
-
+			assignments = frappe.get_all("ToDo", filters={
+				"reference_type": self.reference_type,
+				"reference_name": self.reference_name,
+				"status": ("!=", "Cancelled")
+			}, pluck="allocated_to")
 			assignments.reverse()
+
 			frappe.db.set_value(self.reference_type, self.reference_name,
 				"_assign", json.dumps(assignments), update_modified=False)
 
@@ -98,8 +96,8 @@ class ToDo(Document):
 	def get_owners(cls, filters=None):
 		"""Returns list of owners after applying filters on todo's.
 		"""
-		rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['owner'])
-		return [parse_addr(row.owner)[1] for row in rows if row.owner]
+		rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['allocated_to'])
+		return [parse_addr(row.allocated_to)[1] for row in rows if row.allocated_to]
 
 # NOTE: todo is viewable if a user is an owner, or set as assigned_to value, or has any role that is allowed to access ToDo doctype.
 def on_doctype_update():
@@ -115,7 +113,7 @@ def get_permission_query_conditions(user):
 	if any(check in todo_roles for check in frappe.get_roles(user)):
 		return None
 	else:
-		return """(`tabToDo`.owner = {user} or `tabToDo`.assigned_by = {user})"""\
+		return """(`tabToDo`.allocated_to = {user} or `tabToDo`.assigned_by = {user})"""\
 			.format(user=frappe.db.escape(user))
 
 def has_permission(doc, ptype="read", user=None):
@@ -127,7 +125,7 @@ def has_permission(doc, ptype="read", user=None):
 	if any(check in todo_roles for check in frappe.get_roles(user)):
 		return True
 	else:
-		return doc.owner==user or doc.assigned_by==user
+		return doc.allocated_to==user or doc.assigned_by==user
 
 @frappe.whitelist()
 def new_todo(description):

frappe/desk/form/assign_to.py

@@ -19,11 +19,11 @@ def get(args=None):
 	if not args:
 		args = frappe.local.form_dict
 
-	return frappe.get_all('ToDo', fields=['owner', 'name'], filters=dict(
-		reference_type = args.get('doctype'),
-		reference_name = args.get('name'),
-		status = ('!=', 'Cancelled')
-	), limit=5)
+	return frappe.get_all("ToDo", fields=["allocated_to as owner", "name"], filters={
+		"reference_type": args.get("doctype"),
+		"reference_name": args.get("name"),
+		"status": ("!=", "Cancelled")
+	}, limit=5)
 
 @frappe.whitelist()
 def add(args=None):
@@ -48,7 +48,7 @@ def add(args=None):
 			"reference_type": args['doctype'],
 			"reference_name": args['name'],
 			"status": "Open",
-			"owner": assign_to
+			"allocated_to": assign_to
 		}
 
 		if frappe.get_all("ToDo", filters=filters):
@@ -61,7 +61,7 @@ def add(args=None):
 
 			d = frappe.get_doc({
 				"doctype": "ToDo",
-				"owner": assign_to,
+				"allocated_to": assign_to,
 				"reference_type": args['doctype'],
 				"reference_name": args['name'],
 				"description": args.get('description'),
@@ -87,7 +87,7 @@ def add(args=None):
 			follow_document(args['doctype'], args['name'], assign_to)
 
 			# notify
-			notify_assignment(d.assigned_by, d.owner, d.reference_type, d.reference_name, action='ASSIGN',
+			notify_assignment(d.assigned_by, d.allocated_to, d.reference_type, d.reference_name, action='ASSIGN',
 				description=args.get("description"))
 
 	if shared_with_users:
@@ -112,13 +112,13 @@ def add_multiple(args=None):
 		add(args)
 
 def close_all_assignments(doctype, name):
-	assignments = frappe.db.get_all('ToDo', fields=['owner'], filters =
+	assignments = frappe.db.get_all('ToDo', fields=['allocated_to'], filters =
 		dict(reference_type = doctype, reference_name = name, status=('!=', 'Cancelled')))
 	if not assignments:
 		return False
 
 	for assign_to in assignments:
-		set_status(doctype, name, assign_to.owner, status="Closed")
+		set_status(doctype, name, assign_to.allocated_to, status="Closed")
 
 	return True
 
@@ -130,13 +130,13 @@ def set_status(doctype, name, assign_to, status="Cancelled"):
 	"""remove from todo"""
 	try:
 		todo = frappe.db.get_value("ToDo", {"reference_type":doctype,
-		"reference_name":name, "owner":assign_to, "status": ('!=', status)})
+		"reference_name":name, "allocated_to":assign_to, "status": ('!=', status)})
 		if todo:
 			todo = frappe.get_doc("ToDo", todo)
 			todo.status = status
 			todo.save(ignore_permissions=True)
 
-			notify_assignment(todo.assigned_by, todo.owner, todo.reference_type, todo.reference_name)
+			notify_assignment(todo.assigned_by, todo.allocated_to, todo.reference_type, todo.reference_name)
 	except frappe.DoesNotExistError:
 		pass
 
@@ -150,25 +150,26 @@ def clear(doctype, name):
 	'''
 	Clears assignments, return False if not assigned.
 	'''
-	assignments = frappe.db.get_all('ToDo', fields=['owner'], filters =
+	assignments = frappe.db.get_all('ToDo', fields=['allocated_to'], filters =
 		dict(reference_type = doctype, reference_name = name))
 	if not assignments:
 		return False
 
 	for assign_to in assignments:
-		set_status(doctype, name, assign_to.owner, "Cancelled")
+		set_status(doctype, name, assign_to.allocated_to, "Cancelled")
 
 	return True
 
-def notify_assignment(assigned_by, owner, doc_type, doc_name, action='CLOSE',
+def notify_assignment(assigned_by, allocated_to, doc_type, doc_name, action='CLOSE',
 	description=None):
 	"""
 		Notify assignee that there is a change in assignment
 	"""
-	if not (assigned_by and owner and doc_type and doc_name): return
+	if not (assigned_by and allocated_to and doc_type and doc_name):
+		return
 
 	# return if self assigned or user disabled
-	if assigned_by == owner or not frappe.db.get_value('User', owner, 'enabled'):
+	if assigned_by == allocated_to or not frappe.db.get_value('User', allocated_to, 'enabled'):
 		return
 
 	# Search for email address in description -- i.e. assignee
@@ -194,7 +195,7 @@ def notify_assignment(assigned_by, owner, doc_type, doc_name, action='CLOSE',
 		'email_content': description_html
 	}
 
-	enqueue_create_notification(owner, notification_doc)
+	enqueue_create_notification(allocated_to, notification_doc)
 
 def format_message_for_assign_to(users):
 	return "<br><br>" + "<br>".join(users)

frappe/desk/form/load.py

@@ -253,7 +253,7 @@ def get_communication_data(doctype, name, start=0, limit=20, after=None, fields=
 
 def get_assignments(dt, dn):
 	cl = frappe.get_all("ToDo",
-		fields=['name', 'owner', 'description', 'status'],
+		fields=['name', 'allocated_to as owner', 'description', 'status'],
 		filters={
 			'reference_type': dt,
 			'reference_name': dn,

frappe/desk/listview.py

@@ -29,16 +29,16 @@ def get_group_by_count(doctype, current_filters, field):
 	subquery = frappe.get_all(doctype, filters=current_filters, run=False)
 	if field == 'assigned_to':
 		subquery_condition = ' and `tabToDo`.reference_name in ({subquery})'.format(subquery = subquery)
-		return frappe.db.sql("""select `tabToDo`.owner as name, count(*) as count
+		return frappe.db.sql("""select `tabToDo`.allocated_to as name, count(*) as count
 			from
 				`tabToDo`, `tabUser`
 			where
 				`tabToDo`.status!='Cancelled' and
-				`tabToDo`.owner = `tabUser`.name and
+				`tabToDo`.allocated_to = `tabUser`.name and
 				`tabUser`.user_type = 'System User'
 				{subquery_condition}
 			group by
-				`tabToDo`.owner
+				`tabToDo`.allocated_to
 			order by
 				count desc
 			limit 50""".format(subquery_condition = subquery_condition), as_dict=True)

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -388,7 +388,6 @@ def make_records(records, debug=False):
 
 	# LOG every success and failure
 	for record in records:
-
 		doctype = record.get("doctype")
 		condition = record.get('__condition')
 
@@ -405,6 +404,7 @@ def make_records(records, debug=False):
 
 		try:
 			doc.insert(ignore_permissions=True)
+			frappe.db.commit()
 
 		except frappe.DuplicateEntryError as e:
 			# print("Failed to insert duplicate {0} {1}".format(doctype, doc.name))
@@ -417,6 +417,7 @@ def make_records(records, debug=False):
 				raise
 
 		except Exception as e:
+			frappe.db.rollback()
 			exception = record.get('__exception')
 			if exception:
 				config = _dict(exception)

frappe/desk/treeview.py

@@ -4,6 +4,7 @@
 import frappe
 from frappe import _
 
+
 @frappe.whitelist()
 def get_all_nodes(doctype, label, parent, tree_method, **filters):
 	'''Recursively gets all data from tree nodes'''
@@ -40,8 +41,8 @@ def get_children(doctype, parent='', **filters):
 
 def _get_children(doctype, parent='', ignore_permissions=False):
 	parent_field = 'parent_' + doctype.lower().replace(' ', '_')
-	filters = [['ifnull(`{0}`,"")'.format(parent_field), '=', parent],
-		['docstatus', '<' ,'2']]
+	filters = [["ifnull(`{0}`,'')".format(parent_field), '=', parent],
+		['docstatus', '<' ,2]]
 
 	meta = frappe.get_meta(doctype)
 

frappe/email/doctype/email_queue/email_queue.py

@@ -475,28 +475,20 @@ class QueueBuilder:
 		if self._unsubscribed_user_emails is not None:
 			return self._unsubscribed_user_emails
 
-		all_ids = tuple(set(self.recipients + self.cc))
-
-		unsubscribed = frappe.db.sql_list('''
-			SELECT
-				distinct email
-			from
-				`tabEmail Unsubscribe`
-			where
-				email in %(all_ids)s
-				and (
-					(
-						reference_doctype = %(reference_doctype)s
-						and reference_name = %(reference_name)s
-					)
-					or global_unsubscribe = 1
-				)
-		''', {
-			'all_ids': all_ids,
-			'reference_doctype': self.reference_doctype,
-			'reference_name': self.reference_name,
-		})
-
+		all_ids = list(set(self.recipients + self.cc))
+
+		EmailUnsubscribe = frappe.qb.DocType("Email Unsubscribe")
+
+		unsubscribed = (frappe.qb.from_(EmailUnsubscribe)
+						.select(EmailUnsubscribe.email)
+						.where(EmailUnsubscribe.email.isin(all_ids) & 	
+									(
+										(
+											(EmailUnsubscribe.reference_doctype == self.reference_doctype) & (EmailUnsubscribe.reference_name == self.reference_name)
+										) | EmailUnsubscribe.global_unsubscribe == 1
+									)
+								).distinct()
+						).run(pluck=True)
 		self._unsubscribed_user_emails = unsubscribed or []
 		return self._unsubscribed_user_emails
 

frappe/email/doctype/email_template/email_template.json

@@ -12,7 +12,6 @@
   "use_html",
   "response_html",
   "response",
-  "owner",
   "section_break_4",
   "email_reply_help"
  ],
@@ -32,14 +31,6 @@
    "label": "Response",
    "mandatory_depends_on": "eval:!doc.use_html"
   },
-  {
-   "default": "user",
-   "fieldname": "owner",
-   "fieldtype": "Link",
-   "hidden": 1,
-   "label": "Owner",
-   "options": "User"
-  },
   {
    "fieldname": "section_break_4",
    "fieldtype": "Section Break"
@@ -66,7 +57,7 @@
  ],
  "icon": "fa fa-comment",
  "links": [],
- "modified": "2020-11-30 14:12:50.321633",
+ "modified": "2022-01-04 14:12:50.321633",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Template",

frappe/email/doctype/notification/notification.py

@@ -435,8 +435,8 @@ def get_context(doc):
 def get_assignees(doc):
 	assignees = []
 	assignees = frappe.get_all('ToDo', filters={'status': 'Open', 'reference_name': doc.name,
-		'reference_type': doc.doctype}, fields=['owner'])
+		'reference_type': doc.doctype}, fields=['allocated_to'])
 
-	recipients = [d.owner for d in assignees]
+	recipients = [d.allocated_to for d in assignees]
 
 	return recipients

frappe/email/receive.py

@@ -27,11 +27,7 @@ from frappe.utils.html_utils import clean_email_html
 
 # fix due to a python bug in poplib that limits it to 2048
 poplib._MAXLINE = 20480
-imaplib._MAXLINE = 20480
 
-# fix due to a python bug in poplib that limits it to 2048
-poplib._MAXLINE = 20480
-imaplib._MAXLINE = 20480
 
 
 class EmailSizeExceededError(frappe.ValidationError): pass

frappe/frappeclient.py

@@ -1,11 +1,13 @@
-import requests
-import json
-import frappe
-import base64
-
 '''
 FrappeClient is a library that helps you connect with other frappe systems
 '''
+import base64
+import json
+
+import requests
+
+import frappe
+
 
 class AuthError(Exception):
 	pass
@@ -46,7 +48,7 @@ class FrappeClient(object):
 
 	def _login(self, username, password):
 		'''Login/start a sesion. Called internally on init'''
-		r = self.session.post(self.url, data={
+		r = self.session.post(self.url, params={
 			'cmd': 'login',
 			'usr': username,
 			'pwd': password
@@ -289,14 +291,14 @@ class FrappeClient(object):
 	def get_api(self, method, params=None):
 		if params is None:
 			params = {}
-		res = self.session.get(self.url + "/api/method/" + method + "/",
+		res = self.session.get(f"{self.url}/api/method/{method}",
 			params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def post_api(self, method, params=None):
 		if params is None:
 			params = {}
-		res = self.session.post(self.url + "/api/method/" + method + "/",
+		res = self.session.post(f"{self.url}/api/method/{method}",
 			params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 

frappe/integrations/doctype/ldap_settings/ldap_settings.py

@@ -58,9 +58,9 @@ class LDAPSettings(Document):
 			import ssl
 
 			if self.require_trusted_certificate == 'Yes':
-				tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLSv1)
+				tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLS_CLIENT)
 			else:
-				tls_configuration = ldap3.Tls(validate=ssl.CERT_NONE, version=ssl.PROTOCOL_TLSv1)
+				tls_configuration = ldap3.Tls(validate=ssl.CERT_NONE, version=ssl.PROTOCOL_TLS_CLIENT)
 
 			if self.local_private_key_file:
 				tls_configuration.private_key_file = self.local_private_key_file

frappe/integrations/doctype/ldap_settings/test_ldap_settings.py

@@ -296,7 +296,7 @@ class LDAP_TestCase():
 
 					if local_doc['require_trusted_certificate'] == 'Yes':
 						tls_validate = ssl.CERT_REQUIRED
-						tls_version = ssl.PROTOCOL_TLSv1
+						tls_version = ssl.PROTOCOL_TLS_CLIENT
 						tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version)
 
 						self.assertTrue(kwargs['auto_bind'] == ldap3.AUTO_BIND_TLS_BEFORE_BIND,
@@ -304,7 +304,7 @@ class LDAP_TestCase():
 
 					else:
 						tls_validate = ssl.CERT_NONE
-						tls_version = ssl.PROTOCOL_TLSv1
+						tls_version = ssl.PROTOCOL_TLS_CLIENT
 						tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version)
 
 						self.assertTrue(kwargs['auto_bind'],

frappe/model/base_document.py

@@ -101,13 +101,10 @@ class BaseDocument(object):
 				"balance": 42000
 			})
 		"""
-		if "doctype" in d:
-			self.set("doctype", d.get("doctype"))
-
 		# first set default field values of base document
 		for key in default_fields:
 			if key in d:
-				self.set(key, d.get(key))
+				self.set(key, d[key])
 
 		for key, value in d.items():
 			self.set(key, value)
@@ -771,7 +768,9 @@ class BaseDocument(object):
 
 				else:
 					self_value = self.get_value(key)
-
+				# Postgres stores values as `datetime.time`, MariaDB as `timedelta`
+				if isinstance(self_value, datetime.timedelta) and isinstance(db_value, datetime.time):
+					db_value = datetime.timedelta(hours=db_value.hour, minutes=db_value.minute, seconds=db_value.second, microseconds=db_value.microsecond)
 				if self_value != db_value:
 					frappe.throw(_("Not allowed to change {0} after submission").format(df.label),
 						frappe.UpdateAfterSubmitError)
@@ -1011,15 +1010,12 @@ def _filter(data, filters, limit=None):
 			_filters[f] = fval
 
 	for d in data:
-		add = True
 		for f, fval in _filters.items():
 			if not frappe.compare(getattr(d, f, None), fval[0], fval[1]):
-				add = False
 				break
-
-		if add:
+		else:
 			out.append(d)
-			if limit and (len(out)-1)==limit:
+			if limit and len(out) >= limit:
 				break
 
 	return out

frappe/model/db_query.py

@@ -130,6 +130,11 @@ class DatabaseQuery(object):
 			args.fields = 'distinct ' + args.fields
 			args.order_by = '' # TODO: recheck for alternative
 
+		# Postgres requires any field that appears in the select clause to also
+		# appear in the order by and group by clause
+		if frappe.db.db_type == 'postgres' and args.order_by and args.group_by:
+			args = self.prepare_select_args(args)
+
 		query = """select %(fields)s
 			from %(tables)s
 			%(conditions)s
@@ -203,6 +208,19 @@ class DatabaseQuery(object):
 
 		return args
 
+	def prepare_select_args(self, args):
+		order_field = re.sub(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", "", args.order_by)
+
+		if order_field not in args.fields:
+			extracted_column = order_column = order_field.replace("`", "")
+			if "." in extracted_column:
+				extracted_column = extracted_column.split(".")[1]
+
+			args.fields += f", MAX({extracted_column}) as `{order_column}`"
+			args.order_by = args.order_by.replace(order_field, f"`{order_column}`")
+
+		return args
+
 	def parse_args(self):
 		"""Convert fields and filters from strings to list, dicts"""
 		if isinstance(self.fields, str):

frappe/model/delete_doc.py

@@ -117,7 +117,8 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa
 					doctype=doc.doctype, name=doc.name,
 					is_async=False if frappe.flags.in_test else True)
 
-
+		# clear cache for Document
+		doc.clear_cache()
 		# delete global search entry
 		delete_for_document(doc)
 		# delete tag link entry

frappe/model/document.py

@@ -396,6 +396,7 @@ class Document(BaseDocument):
 				"parenttype": self.doctype,
 				"parentfield": fieldname
 			})
+
 	def get_doc_before_save(self):
 		return getattr(self, '_doc_before_save', None)
 
@@ -468,9 +469,11 @@ class Document(BaseDocument):
 		self._original_modified = self.modified
 		self.modified = now()
 		self.modified_by = frappe.session.user
-		if not self.creation:
+
+		# We'd probably want the creation and owner to be set via API
+		# or Data import at some point, that'd have to be handled here
+		if self.is_new():
 			self.creation = self.modified
-		if not self.owner:
 			self.owner = self.modified_by
 
 		for d in self.get_all_children():
@@ -562,8 +565,12 @@ class Document(BaseDocument):
 					fail = value != original_value
 
 				if fail:
-					frappe.throw(_("Value cannot be changed for {0}").format(self.meta.get_label(field.fieldname)),
-						frappe.CannotChangeConstantError)
+					frappe.throw(
+						_("Value cannot be changed for {0}").format(
+							frappe.bold(self.meta.get_label(field.fieldname))
+						),
+						exc=frappe.CannotChangeConstantError
+					)
 
 		return False
 
@@ -1341,15 +1348,15 @@ class Document(BaseDocument):
 			), frappe.exceptions.InvalidDates)
 
 	def get_assigned_users(self):
-		assignments = frappe.get_all('ToDo',
-			fields=['owner'],
+		assigned_users = frappe.get_all('ToDo',
+			fields=['allocated_to'],
 			filters={
 				'reference_type': self.doctype,
 				'reference_name': self.name,
 				'status': ('!=', 'Cancelled'),
-			})
+			}, pluck='allocated_to')
 
-		users = set([assignment.owner for assignment in assignments])
+		users = set(assigned_users)
 		return users
 
 	def add_tag(self, tag):

frappe/model/meta.py

@@ -67,6 +67,10 @@ class Meta(Document):
 	_metaclass = True
 	default_fields = list(default_fields)[1:]
 	special_doctypes = ("DocField", "DocPerm", "DocType", "Module Def", 'DocType Action', 'DocType Link', 'DocType State')
+	standard_set_once_fields = [
+		frappe._dict(fieldname="creation", fieldtype="Datetime"),
+		frappe._dict(fieldname="owner", fieldtype="Data"),
+	]
 
 	def __init__(self, doctype):
 		self._fields = {}
@@ -154,6 +158,12 @@ class Meta(Document):
 		'''Return fields with `set_only_once` set'''
 		if not hasattr(self, "_set_only_once_fields"):
 			self._set_only_once_fields = self.get("fields", {"set_only_once": 1})
+			fieldnames = [d.fieldname for d in self._set_only_once_fields]
+
+			for df in self.standard_set_once_fields:
+				if df.fieldname not in fieldnames:
+					self._set_only_once_fields.append(df)
+
 		return self._set_only_once_fields
 
 	def get_table_fields(self):

frappe/model/rename_doc.py

@@ -110,6 +110,7 @@ def rename_doc(
 	if merge:
 		frappe.delete_doc(doctype, old)
 
+	new_doc.clear_cache()
 	frappe.clear_cache()
 	if rebuild_search:
 		frappe.enqueue('frappe.utils.global_search.rebuild_for_doctype', doctype=doctype)
@@ -292,7 +293,7 @@ def update_link_field_values(link_fields, old, new, doctype):
 			if parent == new and doctype == "DocType":
 				parent = old
 
-			frappe.db.set_value(parent, {docfield: old}, docfield, new)
+			frappe.db.set_value(parent, {docfield: old}, docfield, new, update_modified=False)
 
 		# update cached link_fields as per new
 		if doctype=='DocType' and field['parent'] == old:

frappe/patches.txt

@@ -182,6 +182,7 @@ frappe.patches.v13_0.queryreport_columns
 execute:frappe.reload_doc('core', 'doctype', 'doctype')
 frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
+frappe.patches.v13_0.set_first_day_of_the_week
 frappe.patches.v14_0.drop_data_import_legacy
 frappe.patches.v14_0.rename_cancelled_documents
 frappe.patches.v14_0.copy_mail_data #08.03.21
@@ -190,3 +191,4 @@ frappe.patches.v14_0.update_github_endpoints #08-11-2021
 frappe.patches.v14_0.remove_db_aggregation
 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021
 frappe.patches.v14_0.update_color_names_in_kanban_board_column
+frappe.patches.v14_0.transform_todo_schema

frappe/patches/v13_0/set_first_day_of_the_week.py

@@ -0,0 +1,7 @@
+import frappe
+
+def execute():
+	frappe.reload_doctype("System Settings")
+	# setting first_day_of_the_week value as "Monday" to avoid breaking change
+	# because before the configuration was introduced, system used to consider "Monday" as start of the week
+	frappe.db.set_value("System Settings", "System Settings", "first_day_of_the_week", "Monday")

frappe/patches/v14_0/transform_todo_schema.py

@@ -0,0 +1,12 @@
+import frappe
+from frappe.query_builder.utils import DocType
+
+
+def execute():
+	# Email Template & Help Article have owner field that doesn't have any additional functionality
+	# Only ToDo has to be updated.
+
+	ToDo = DocType("ToDo")
+	frappe.reload_doctype("ToDo", force=True)
+
+	frappe.qb.update(ToDo).set(ToDo.allocated_to, ToDo.owner).run()

frappe/public/js/desk.bundle.js

@@ -63,6 +63,7 @@ import "./frappe/utils/address_and_contact.js";
 import "./frappe/utils/preview_email.js";
 import "./frappe/utils/file_manager.js";
 import "./frappe/utils/diffview";
+import "./frappe/utils/datatable.js";
 
 import "./frappe/upload.js";
 import "./frappe/ui/tree.js";

frappe/public/js/frappe/data_import/import_preview.js

@@ -142,6 +142,8 @@ frappe.data_import.ImportPreview = class ImportPreview {
 			columns: this.columns,
 			layout: this.columns.length < 10 ? 'fluid' : 'fixed',
 			cellHeight: 35,
+			language: frappe.boot.lang,
+			translations: frappe.utils.datatable.get_translations(),
 			serialNoColumn: false,
 			checkboxColumn: false,
 			noDataMessage: __('No Data'),

frappe/public/js/frappe/desk.js

@@ -275,11 +275,7 @@ frappe.Application = class Application {
 			this.set_globals();
 			this.sync_pages();
 			frappe.router.setup();
-			moment.locale("en");
-			moment.user_utc_offset = moment().utcOffset();
-			if(frappe.boot.timezone_info) {
-				moment.tz.add(frappe.boot.timezone_info);
-			}
+			this.setup_moment();
 			if(frappe.boot.print_css) {
 				frappe.dom.set_style(frappe.boot.print_css, "print-style");
 			}
@@ -628,6 +624,19 @@ frappe.Application = class Application {
 			}
 		});
 	}
+
+	setup_moment() {
+		moment.updateLocale('en', {
+			week: {
+				dow: frappe.datetime.get_first_day_of_the_week_index(),
+			}
+		});
+		moment.locale("en");
+		moment.user_utc_offset = moment().utcOffset();
+		if (frappe.boot.timezone_info) {
+			moment.tz.add(frappe.boot.timezone_info);
+		}
+	}
 }
 
 frappe.get_module = function(m, default_module) {

frappe/public/js/frappe/form/controls/base_control.js

@@ -148,8 +148,9 @@ frappe.ui.form.Control = class BaseControl {
 			return this.doc[this.df.fieldname];
 		}
 	}
-	set_value(value) {
-		return this.validate_and_set_in_model(value);
+
+	set_value(value, force_set_value=false) {
+		return this.validate_and_set_in_model(value, null, force_set_value);
 	}
 	parse_validate_and_set_in_model(value, e) {
 		if(this.parse) {
@@ -157,12 +158,11 @@ frappe.ui.form.Control = class BaseControl {
 		}
 		return this.validate_and_set_in_model(value, e);
 	}
-	validate_and_set_in_model(value, e) {
-		var me = this;
-		let force_value_set = (this.doc && this.doc.__run_link_triggers);
-		let is_value_same = (this.get_model_value() === value);
+	validate_and_set_in_model(value, e, force_set_value=false) {
+		const me = this;
+		const is_value_same = (this.get_model_value() === value);
 
-		if (this.inside_change_event || (!force_value_set && is_value_same)) {
+		if (this.inside_change_event || (is_value_same && !force_set_value)) {
 			return Promise.resolve();
 		}
 

frappe/public/js/frappe/form/controls/date.js

@@ -10,14 +10,16 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat
 		this.set_t_for_today();
 	}
 	set_formatted_input(value) {
+		if (value === "Today") {
+			value = this.get_now_date();
+		}
+
 		super.set_formatted_input(value);
 		if (this.timepicker_only) return;
 		if (!this.datepicker) return;
 		if (!value) {
 			this.datepicker.clear();
 			return;
-		} else if (value === "Today") {
-			value = this.get_now_date();
 		}
 
 		let should_refresh = this.last_value && this.last_value !== value;
@@ -62,6 +64,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat
 			dateFormat: date_format,
 			startDate: this.get_start_date(),
 			keyboardNav: false,
+			firstDay: frappe.datetime.get_first_day_of_the_week_index(),
 			onSelect: () => {
 				this.$input.trigger('change');
 			},
@@ -77,7 +80,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat
 	}
 
 	get_start_date() {
-		return new Date(this.get_now_date());
+		return this.get_now_date();
 	}
 
 	set_datepicker() {
@@ -116,7 +119,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat
 		this.datepicker.update('position', position);
 	}
 	get_now_date() {
-		return frappe.datetime.convert_to_system_tz(frappe.datetime.now_date(true));
+		return frappe.datetime.convert_to_system_tz(frappe.datetime.now_date(true), false).toDate();
 	}
 	set_t_for_today() {
 		var me = this;

frappe/public/js/frappe/form/controls/multiselect_list.js

@@ -109,6 +109,7 @@ frappe.ui.form.ControlMultiSelectList = class ControlMultiSelectList extends fra
 		let value = decodeURIComponent($selectable_item.data().value);
 
 		if ($selectable_item.hasClass('selected')) {
+			this.values = this.values.slice();
 			this.values.push(value);
 		} else {
 			this.values = this.values.filter(val => val !== value);

frappe/public/js/frappe/form/form.js

@@ -215,7 +215,7 @@ frappe.ui.form.Form = class FrappeForm {
 
 		if (this.layout.tabs.length) {
 			this.layout.tabs.every(tab => {
-				if (tab.df.options === 'Dashboard') {
+				if (tab.df.show_dashboard) {
 					tab.wrapper.prepend(dashboard_parent);
 					dashboard_added = true;
 					return false;
@@ -983,7 +983,7 @@ frappe.ui.form.Form = class FrappeForm {
 			$.each(this.fields_dict, function(fieldname, field) {
 				if (field.df.fieldtype=="Link" && this.doc[fieldname]) {
 					// triggers add fetch, sets value in model and runs triggers
-					field.set_value(this.doc[fieldname]);
+					field.set_value(this.doc[fieldname], true);
 				}
 			});
 

frappe/public/js/frappe/form/grid_row.js

@@ -196,7 +196,7 @@ export default class GridRow {
 			// REDESIGN-TODO: Make translation contextual, this No is Number
 			var txt = (this.doc ? this.doc.idx : __("No."));
 			this.row_index = $(
-				`<div class="row-index sortable-handle col col-xs-1">
+				`<div class="row-index sortable-handle col">
 					${this.row_check_html}
 				<span class="hidden-xs">${txt}</span></div>`)
 				.appendTo(this.row)

frappe/public/js/frappe/list/list_view.js

@@ -1500,6 +1500,11 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 					read_only: 1,
 				},
 			],
+			primary_action_label: __("Copy to clipboard"),
+			primary_action: () => {
+				frappe.utils.copy_to_clipboard(this.get_share_url());
+				d.hide();
+			},
 		});
 		d.show();
 	}

frappe/public/js/frappe/utils/datatable.js

@@ -0,0 +1,22 @@
+frappe.provide("frappe.utils.datatable");
+
+frappe.utils.datatable.get_translations = function () {
+	let translations = {};
+	translations[frappe.boot.lang] = {
+		"Sort Ascending": __("Sort Ascending"),
+		"Sort Descending": __("Sort Descending"),
+		"Reset sorting": __("Reset sorting"),
+		"Remove column": __("Remove column"),
+		"No Data": __("No Data"),
+		"{count} cells copied": {
+			"1": __("{count} cell copied"),
+			"default": __("{count} cells copied")
+		},
+		"{count} rows selected": {
+			"1": __("{count} row selected"),
+			"default": __("{count} rows selected")
+		}
+	};
+
+	return translations;
+};

frappe/public/js/frappe/utils/datetime.js

@@ -254,6 +254,11 @@ $.extend(frappe.datetime, {
 		], true).isValid();
 	},
 
+	get_first_day_of_the_week_index() {
+		const first_day_of_the_week = frappe.sys_defaults.first_day_of_the_week || "Sunday";
+		return moment.weekdays().indexOf(first_day_of_the_week);
+	}
+
 });
 
 // Proxy for dateutil and get_today

frappe/public/js/frappe/views/reports/query_report.js

@@ -106,14 +106,17 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			return;
 		}
 
+		let route_options = {};
+		route_options = Object.assign(route_options, frappe.route_options);
+
 		if (this.report_name !== frappe.get_route()[1]) {
 			// different report
-			this.load_report();
+			this.load_report(route_options);
 		}
 		else if (frappe.has_route_options()) {
 			// filters passed through routes
 			// so refresh report again
-			this.refresh_report();
+			this.refresh_report(route_options);
 		} else {
 			// same report
 			// don't do anything to preserve state
@@ -121,7 +124,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		}
 	}
 
-	load_report() {
+	load_report(route_options) {
 		this.page.clear_inner_toolbar();
 		this.route = frappe.get_route();
 		this.page_name = frappe.get_route_str();
@@ -137,7 +140,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			() => this.get_report_settings(),
 			() => this.setup_progress_bar(),
 			() => this.setup_page_head(),
-			() => this.refresh_report(),
+			() => this.refresh_report(route_options),
 			() => this.add_chart_buttons_to_toolbar(true),
 			() => this.add_card_button_to_toolbar(true),
 		]);
@@ -343,13 +346,13 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		});
 	}
 
-	refresh_report() {
+	refresh_report(route_options) {
 		this.toggle_message(true);
 		this.toggle_report(false);
 
 		return frappe.run_serially([
 			() => this.setup_filters(),
-			() => this.set_route_filters(),
+			() => this.set_route_filters(route_options),
 			() => this.page.clear_custom_actions(),
 			() => this.report_settings.onload && this.report_settings.onload(this),
 			() => this.refresh()
@@ -525,15 +528,17 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		});
 	}
 
-	set_route_filters() {
-		if(frappe.route_options) {
-			const fields = Object.keys(frappe.route_options);
+	set_route_filters(route_options) {
+		if (!route_options) route_options = frappe.route_options;
+
+		if (route_options) {
+			const fields = Object.keys(route_options);
 
 			const filters_to_set = this.filters.filter(f => fields.includes(f.df.fieldname));
 
 			const promises = filters_to_set.map(f => {
 				return () => {
-					const value = frappe.route_options[f.df.fieldname];
+					const value = route_options[f.df.fieldname];
 					f.set_value(value);
 				};
 			});
@@ -844,6 +849,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				columns: columns,
 				data: data,
 				inlineFilters: true,
+				language: frappe.boot.lang,
+				translations: frappe.utils.datatable.get_translations(),
 				treeView: this.tree_report,
 				layout: 'fixed',
 				cellHeight: 33,

frappe/public/js/frappe/views/reports/report_view.js

@@ -284,6 +284,8 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 			columns: this.columns,
 			data: this.get_data(values),
 			getEditor: this.get_editing_object.bind(this),
+			language: frappe.boot.lang,
+			translations: frappe.utils.datatable.get_translations(),
 			checkboxColumn: true,
 			inlineFilters: true,
 			cellHeight: 35,

frappe/public/js/frappe/views/treeview.js

@@ -409,7 +409,9 @@ frappe.views.TreeView = class TreeView {
 			},
 		];
 
-		if (frappe.user.has_role('System Manager')) {
+		if (frappe.user.has_role('System Manager') &&
+			frappe.meta.has_field(me.doctype, "lft") &&
+			frappe.meta.has_field(me.doctype, "rgt")) {
 			this.menu_items.push(
 				{
 					label: __('Rebuild Tree'),

frappe/public/js/frappe/web_form/web_form.js

@@ -9,6 +9,7 @@ export default class WebForm extends frappe.ui.FieldGroup {
 		frappe.web_form = this;
 		frappe.web_form.events = {};
 		Object.assign(frappe.web_form.events, EventEmitterMixin);
+		this.current_section = 0;
 	}
 
 	prepare(web_form_doc, doc) {
@@ -19,12 +20,16 @@ export default class WebForm extends frappe.ui.FieldGroup {
 
 	make() {
 		super.make();
+		this.set_sections();
 		this.set_field_values();
+		this.setup_listeners();
 		if (this.introduction_text) this.set_form_description(this.introduction_text);
 		if (this.allow_print && !this.is_new) this.setup_print_button();
 		if (this.allow_delete && !this.is_new) this.setup_delete_button();
 		if (this.is_new) this.setup_cancel_button();
 		this.setup_primary_action();
+		this.setup_previous_next_button();
+		this.toggle_section();
 		$(".link-btn").remove();
 
 		// webform client script
@@ -40,6 +45,88 @@ export default class WebForm extends frappe.ui.FieldGroup {
 		};
 	}
 
+	setup_listeners() {
+		// Event listener for triggering Save/Next button for Multi Step Forms
+		// Do not use `on` event here since that can be used by user which will render this function useless
+		// setTimeout has 200ms delay so that all the base_control triggers for the fields have been run
+		let me = this;
+
+		if (!me.is_multi_step_form) {
+			return;
+		}
+
+		for (let field of $(".input-with-feedback")) {
+			$(field).change((e) => {
+				setTimeout(() => {
+					e.stopPropagation();
+					me.toggle_buttons();
+				}, 200);
+			});
+		}
+	}
+
+	set_sections() {
+		if (this.sections.length) return;
+
+		this.sections = $(`.form-section`);
+	}
+
+	setup_previous_next_button() {
+		let me = this;
+
+		if (!me.is_multi_step_form) {
+			return;
+		}
+
+		$('.web-form-footer').after(`
+			<div id="form-step-footer" class="pull-right">
+				<button class="btn btn-primary btn-previous btn-sm ml-2">${__("Previous")}</button>
+				<button class="btn btn-primary btn-next btn-sm ml-2">${__("Next")}</button>
+			</div>
+		`);
+
+		$('.btn-previous').on('click', function () {
+			let is_validated = me.validate_section();
+
+			if (!is_validated) return;
+
+			/**
+				The eslint utility cannot figure out if this is an infinite loop in backwards and
+				throws an error. Disabling for-direction just for this section.
+				for-direction doesnt throw an error if the values are hardcoded in the
+				reverse for-loop, but in this case its a dynamic loop.
+				https://eslint.org/docs/rules/for-direction
+			*/
+			/* eslint-disable for-direction */
+			for (let idx = me.current_section; idx < me.sections.length; idx--) {
+				let is_empty = me.is_previous_section_empty(idx);
+				me.current_section = me.current_section > 0 ? me.current_section - 1 : me.current_section;
+
+				if (!is_empty) {
+					break;
+				}
+			}
+			/* eslint-enable for-direction */
+			me.toggle_section();
+		});
+
+		$('.btn-next').on('click', function () {
+			let is_validated = me.validate_section();
+
+			if (!is_validated) return;
+
+			for (let idx = me.current_section; idx < me.sections.length; idx++) {
+				let is_empty = me.is_next_section_empty(idx);
+				me.current_section = me.current_section < me.sections.length ? me.current_section + 1 : me.current_section;
+
+				if (!is_empty) {
+					break;
+				}
+			}
+			me.toggle_section();
+		});
+	}
+
 	set_field_values() {
 		if (this.doc.name) this.set_values(this.doc);
 		else return;
@@ -104,6 +191,113 @@ export default class WebForm extends frappe.ui.FieldGroup {
 		);
 	}
 
+	validate_section() {
+		if (this.allow_incomplete) return true;
+
+		let fields = $(`.form-section:eq(${this.current_section}) .form-control`);
+		let errors = [];
+		let invalid_values = [];
+
+		for (let field of fields) {
+			let fieldname = $(field).attr("data-fieldname");
+			if (!fieldname) continue;
+
+			field = this.fields_dict[fieldname];
+
+			if (field.get_value) {
+				let value = field.get_value();
+				if (field.df.reqd && is_null(typeof value === 'string' ? strip_html(value) : value)) errors.push(__(field.df.label));
+
+				if (field.df.reqd && field.df.fieldtype === 'Text Editor' && is_null(strip_html(cstr(value)))) errors.push(__(field.df.label));
+
+				if (field.df.invalid) invalid_values.push(__(field.df.label));
+			}
+		}
+
+		let message = '';
+		if (invalid_values.length) {
+			message += __('Invalid values for fields:') + '<br><br><ul><li>' + invalid_values.join('<li>') + '</ul>';
+		}
+
+		if (errors.length) {
+			message += __('Mandatory fields required:') + '<br><br><ul><li>' + errors.join('<li>') + '</ul>';
+		}
+
+		if (invalid_values.length || errors.length) {
+			frappe.msgprint({
+				title: __('Error'),
+				message: message,
+				indicator: 'orange'
+			});
+		}
+
+		return !(errors.length || invalid_values.length);
+	}
+
+	toggle_section() {
+		if (!this.is_multi_step_form) return;
+
+		this.toggle_previous_button();
+		this.hide_sections();
+		this.show_section();
+		this.toggle_buttons();
+	}
+
+	toggle_buttons() {
+		for (let idx = this.current_section; idx < this.sections.length; idx++) {
+			if (this.is_next_section_empty(idx)) {
+				this.show_save_and_hide_next_button();
+			} else {
+				this.show_next_and_hide_save_button();
+				break;
+			}
+		}
+	}
+
+	is_next_section_empty(section) {
+		if (section + 1 > this.sections.length) return true;
+
+		let _section = $(`.form-section:eq(${section + 1})`);
+		let visible_controls = _section.find(".frappe-control:not(.hide-control)");
+
+		return !visible_controls.length ? true : false;
+	}
+
+	is_previous_section_empty(section) {
+		if (section - 1 > this.sections.length) return true;
+
+		let _section = $(`.form-section:eq(${section - 1})`);
+		let visible_controls = _section.find(".frappe-control:not(.hide-control)");
+
+		return !visible_controls.length ? true : false;
+	}
+
+	show_save_and_hide_next_button() {
+		$('.btn-next').hide();
+		$('.web-form-footer').show();
+	}
+
+	show_next_and_hide_save_button() {
+		$('.btn-next').show();
+		$('.web-form-footer').hide();
+	}
+
+	toggle_previous_button() {
+		this.current_section == 0 ? $('.btn-previous').hide() : $('.btn-previous').show();
+	}
+
+	show_section() {
+		$(`.form-section:eq(${this.current_section})`).show();
+	}
+
+	hide_sections() {
+		for (let idx=0; idx < this.sections.length; idx++) {
+			if (idx !== this.current_section) {
+				$(`.form-section:eq(${idx})`).hide();
+			}
+		}
+	}
+
 	save() {
 		let is_new = this.is_new;
 		if (this.validate && !this.validate()) {

frappe/public/scss/common/awesomeplete.scss

@@ -39,6 +39,7 @@
 			padding: var(--padding-sm);
 			color: var(--text-color);
 			border-radius: var(--border-radius);
+			white-space: unset;
 			@extend .ellipsis;
 			&:not(:last-child) {
 				margin-bottom: var(--margin-xs);

frappe/public/scss/desk/form.scss

@@ -54,7 +54,7 @@
 
 .form-section.card-section,
 .form-dashboard-section {
-	border-bottom: 1px solid var(--gray-200);
+	border-bottom: 1px solid var(--border-color);
 	padding: var(--padding-xs);
 }
 
@@ -316,12 +316,12 @@
 
 .form-tabs-list {
 	padding-left: var(--padding-xs);
-	border-bottom: 1px solid var(--gray-200);
+	border-bottom: 1px solid var(--border-color);
 
 	.form-tabs {
 		.nav-item {
 			.nav-link {
-				color: var(--gray-700);
+				color: var(--text-muted);
 				padding: var(--padding-md) 0;
 				margin: 0 var(--margin-md);
 

frappe/test_runner.py

@@ -335,7 +335,10 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False):
 		frappe.local.test_objects[doctype] += test_module._make_test_records(verbose)
 
 	elif hasattr(test_module, "test_records"):
-		frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force)
+		if doctype in frappe.local.test_objects:
+			frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force)
+		else: 
+			frappe.local.test_objects[doctype] = make_test_objects(doctype, test_module.test_records, verbose, force)
 
 	else:
 		test_records = frappe.get_test_records(doctype)

frappe/tests/test_db.py

@@ -12,6 +12,7 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.utils import random_string
 from frappe.utils.testutils import clear_custom_fields
 from frappe.query_builder import Field
+from frappe.database import savepoint
 
 from .test_query_builder import run_only_if, db_type_is
 from frappe.query_builder.functions import Concat_ws
@@ -267,6 +268,32 @@ class TestDB(unittest.TestCase):
 		for d in created_docs:
 			self.assertTrue(frappe.db.exists("ToDo", d))
 
+	def test_savepoints_wrapper(self):
+		frappe.db.rollback()
+
+		class SpecificExc(Exception):
+			pass
+
+		created_docs = []
+		failed_docs = []
+
+		for _ in range(5):
+			with savepoint(catch=SpecificExc):
+				doc_kept = frappe.get_doc(doctype="ToDo", description="nope").save()
+				created_docs.append(doc_kept.name)
+
+			with savepoint(catch=SpecificExc):
+				doc_gone = frappe.get_doc(doctype="ToDo", description="nope").save()
+				failed_docs.append(doc_gone.name)
+				raise SpecificExc
+
+		frappe.db.commit()
+
+		for d in failed_docs:
+			self.assertFalse(frappe.db.exists("ToDo", d))
+		for d in created_docs:
+			self.assertTrue(frappe.db.exists("ToDo", d))
+
 
 @run_only_if(db_type_is.MARIADB)
 class TestDDLCommandsMaria(unittest.TestCase):

frappe/tests/test_db_query.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import frappe, unittest
+import frappe
+import datetime
+import unittest
 
 from frappe.model.db_query import DatabaseQuery
 from frappe.desk.reportview import get_filters_cond
@@ -380,6 +382,22 @@ class TestReportview(unittest.TestCase):
 		owners = DatabaseQuery("DocType").execute(filters={"name": "DocType"}, pluck="owner")
 		self.assertEqual(owners, ["Administrator"])
 
+	def test_prepare_select_args(self):
+		# frappe.get_all inserts modified field into order_by clause
+		# test to make sure this is inserted into select field when postgres
+		doctypes = frappe.get_all("DocType",
+			filters={"docstatus": 0, "document_type": ("!=", "")},
+			group_by="document_type",
+			fields=["document_type", "sum(is_submittable) as is_submittable"],
+			limit=1,
+			as_list=True,
+		)
+		if frappe.conf.db_type == "mariadb":
+			self.assertTrue(len(doctypes[0]) == 2)
+		else:
+			self.assertTrue(len(doctypes[0]) == 3)
+			self.assertTrue(isinstance(doctypes[0][2], datetime.datetime))
+
 	def test_column_comparison(self):
 		"""Test DatabaseQuery.execute to test column comparison
 		"""

frappe/tests/test_document.py

@@ -252,3 +252,8 @@ class TestDocument(unittest.TestCase):
 			'currency': 100000
 		})
 		self.assertEquals(d.get_formatted('currency', currency='INR', format="#,###.##"), '₹ 100,000.00')
+
+	def test_limit_for_get(self):
+		doc = frappe.get_doc("DocType", "DocType")
+		# assuming DocType has more that 3 Data fields
+		self.assertEquals(len(doc.get("fields", filters={"fieldtype": "Data"}, limit=3)), 3)

frappe/tests/test_frappe_client.py

@@ -10,8 +10,9 @@ import requests
 import base64
 
 class TestFrappeClient(unittest.TestCase):
+	PASSWORD = "admin"
 	def test_insert_many(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Note", {"title": ("in", ('Sing','a','song','of','sixpence'))})
 		frappe.db.commit()
 
@@ -30,7 +31,7 @@ class TestFrappeClient(unittest.TestCase):
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'sixpence'}))
 
 	def test_create_doc(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Note", {"title": "test_create"})
 		frappe.db.commit()
 
@@ -39,13 +40,13 @@ class TestFrappeClient(unittest.TestCase):
 		self.assertTrue(frappe.db.get_value('Note', {'title': 'test_create'}))
 
 	def test_list_docs(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		doc_list = server.get_list("Note")
 
 		self.assertTrue(len(doc_list))
 
 	def test_get_doc(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Note", {"title": "get_this"})
 		frappe.db.commit()
 
@@ -56,7 +57,7 @@ class TestFrappeClient(unittest.TestCase):
 		self.assertTrue(doc)
 
 	def test_get_value(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Note", {"title": "get_value"})
 		frappe.db.commit()
 
@@ -74,14 +75,14 @@ class TestFrappeClient(unittest.TestCase):
 		self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"})
 
 	def test_get_single(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix')
 		self.assertEqual(server.get_value('Website Settings', 'title_prefix', 'Website Settings').get('title_prefix'), 'test-prefix')
 		self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix')
 		frappe.db.set_value('Website Settings', None, 'title_prefix', '')
 
 	def test_update_doc(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Note", {"title": ("in", ("Sing", "sing"))})
 		frappe.db.commit()
 
@@ -93,7 +94,7 @@ class TestFrappeClient(unittest.TestCase):
 		self.assertTrue(doc["title"] == changed_title)
 
 	def test_update_child_doc(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Contact", {"first_name": "George", "last_name": "Steevens"})
 		frappe.db.delete("Contact", {"first_name": "William", "last_name": "Shakespeare"})
 		frappe.db.delete("Communication", {"reference_doctype": "Event"})
@@ -130,7 +131,7 @@ class TestFrappeClient(unittest.TestCase):
 		self.assertTrue(frappe.db.exists("Communication Link", {"link_name": "William Shakespeare"}))
 
 	def test_delete_doc(self):
-		server = FrappeClient(get_url(), "Administrator", "admin", verify=False)
+		server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False)
 		frappe.db.delete("Note", {"title": "delete"})
 		frappe.db.commit()
 

frappe/tests/test_permissions.py

@@ -12,6 +12,7 @@ from frappe.core.page.permission_manager.permission_manager import update, reset
 from frappe.test_runner import make_test_records_for_doctype
 from frappe.core.doctype.user_permission.user_permission import clear_user_permissions
 from frappe.desk.form.load import getdoc
+from frappe.utils.data import now_datetime
 
 test_dependencies = ['Blogger', 'Blog Post', "User", "Contact", "Salutation"]
 
@@ -197,6 +198,32 @@ class TestPermissions(unittest.TestCase):
 		doc = frappe.get_doc("Blog Post", "-test-blog-post")
 		self.assertTrue(doc.has_permission("read"))
 
+	def test_set_standard_fields_manually(self):
+		# check that creation and owner cannot be set manually
+		from datetime import timedelta
+
+		fake_creation = now_datetime() + timedelta(days=-7)
+		fake_owner = frappe.db.get_value("User", {"name": ("!=", frappe.session.user)})
+
+		d = frappe.new_doc("ToDo")
+		d.description = "ToDo created via test_set_standard_fields_manually"
+		d.creation = fake_creation
+		d.owner = fake_owner
+		d.save()
+		self.assertNotEqual(d.creation, fake_creation)
+		self.assertNotEqual(d.owner, fake_owner)
+
+	def test_dont_change_standard_constants(self):
+		# check that Document.creation cannot be changed
+		user = frappe.get_doc("User", frappe.session.user)
+		user.creation = now_datetime()
+		self.assertRaises(frappe.CannotChangeConstantError, user.save)
+
+		# check that Document.owner cannot be changed
+		user.reload()
+		user.owner = frappe.db.get_value("User", {"name": ("!=", user.name)})
+		self.assertRaises(frappe.CannotChangeConstantError, user.save)
+
 	def test_set_only_once(self):
 		blog_post = frappe.get_meta("Blog Post")
 		doc = frappe.get_doc("Blog Post", "-test-blog-post-1")

frappe/tests/test_utils.py

@@ -15,6 +15,8 @@ import io
 from mimetypes import guess_type
 from datetime import datetime, timedelta, date
 
+from unittest.mock import patch
+
 class TestFilters(unittest.TestCase):
 	def test_simple_dict(self):
 		self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Open'}))
@@ -306,3 +308,24 @@ class TestDiffUtils(unittest.TestCase):
 		diff = get_version_diff(old_version, latest_version)
 		self.assertIn('-2;', diff)
 		self.assertIn('+42;', diff)
+
+class TestDateUtils(unittest.TestCase):
+	def test_first_day_of_week(self):
+		# Monday as start of the week
+		with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"):
+			self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-25"),
+				frappe.utils.getdate("2020-12-21"))
+			self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-20"),
+				frappe.utils.getdate("2020-12-14"))
+
+		# Sunday as start of the week
+		self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-25"),
+			frappe.utils.getdate("2020-12-20"))
+		self.assertEqual(frappe.utils.get_first_day_of_week("2020-12-21"),
+			frappe.utils.getdate("2020-12-20"))
+
+	def test_last_day_of_week(self):
+		self.assertEqual(frappe.utils.get_last_day_of_week("2020-12-24"),
+			frappe.utils.getdate("2020-12-26"))
+		self.assertEqual(frappe.utils.get_last_day_of_week("2020-12-28"),
+			frappe.utils.getdate("2021-01-02"))

frappe/tests/test_website.py

@@ -197,6 +197,7 @@ class TestWebsite(unittest.TestCase):
 		frappe.cache().delete_key('app_hooks')
 
 	def test_printview_page(self):
+		frappe.db.value_cache[('DocType', 'Language', 'name')] = (('Language',),)
 		content = get_response_content('/Language/ru')
 		self.assertIn('<div class="print-format">', content)
 		self.assertIn('<div>Language</div>', content)

frappe/tests/ui_test_helpers.py

@@ -244,3 +244,14 @@ def create_topic_and_reply(web_page):
 		})
 
 		reply.save()
+
+
+@frappe.whitelist()
+def update_webform_to_multistep():
+	doc = frappe.get_doc("Web Form", "edit-profile")
+	_doc = frappe.copy_doc(doc)
+	_doc.is_multi_step_form = 1
+	_doc.title = "update-profile-duplicate"
+	_doc.route = "update-profile-duplicate"
+	_doc.is_standard = False
+	_doc.save()

frappe/translations/de.csv

@@ -4700,3 +4700,7 @@ Value cannot be negative for {0}: {1},Der Wert kann für {0} nicht negativ sein:
 Negative Value,Negativer Wert,
 Authentication failed while receiving emails from Email Account: {0}.,Die Authentifizierung ist beim Empfang von E-Mails vom E-Mail-Konto fehlgeschlagen: {0}.,
 Message from server: {0},Nachricht vom Server: {0},
+Reset sorting,Sortierung zurücksetzen,
+Sort Ascending,Aufsteigend sortieren,
+Sort Descending,Absteigend sortieren,
+Remove column,Spalte entfernen,

frappe/utils/data.py

@@ -11,11 +11,26 @@ from code import compile_command
 from urllib.parse import quote, urljoin
 from frappe.desk.utils import slug
 from click import secho
+from enum import Enum
 
 DATE_FORMAT = "%Y-%m-%d"
 TIME_FORMAT = "%H:%M:%S.%f"
 DATETIME_FORMAT = DATE_FORMAT + " " + TIME_FORMAT
 
+class Weekday(Enum):
+	Sunday = 0
+	Monday = 1
+	Tuesday = 2
+	Wednesday = 3
+	Thursday = 4
+	Friday = 5
+	Saturday = 6
+
+def get_first_day_of_the_week():
+	return frappe.get_system_settings('first_day_of_the_week') or "Sunday"
+
+def get_start_of_week_index():
+	return Weekday[get_first_day_of_the_week()].value
 
 def is_invalid_date_string(date_string):
 	# dateutil parser does not agree with dates like "0001-01-01" or "0000-00-00"
@@ -98,6 +113,9 @@ def get_timedelta(time: Optional[str] = None) -> Optional[datetime.timedelta]:
 def to_timedelta(time_str):
 	from dateutil import parser
 
+	if isinstance(time_str, datetime.time):
+		time_str = str(time_str)
+
 	if isinstance(time_str, str):
 		t = parser.parse(time_str)
 		return datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond)
@@ -246,9 +264,22 @@ def get_quarter_start(dt, as_str=False):
 
 def get_first_day_of_week(dt, as_str=False):
 	dt = getdate(dt)
-	date = dt - datetime.timedelta(days=dt.weekday())
+	date = dt - datetime.timedelta(days=get_week_start_offset_days(dt))
 	return date.strftime(DATE_FORMAT) if as_str else date
 
+def get_week_start_offset_days(dt):
+	current_day_index = get_normalized_weekday_index(dt)
+	start_of_week_index = get_start_of_week_index()
+
+	if current_day_index >= start_of_week_index:
+		return current_day_index - start_of_week_index
+	else:
+		return 7 - (start_of_week_index - current_day_index)
+
+def get_normalized_weekday_index(dt):
+	# starts Sunday with 0
+	return (dt.weekday() + 1) % 7
+
 def get_year_start(dt, as_str=False):
 	dt = getdate(dt)
 	date = datetime.date(dt.year, 1, 1)

frappe/utils/install.py

@@ -5,11 +5,11 @@ import getpass
 from frappe.utils.password import update_password
 
 def before_install():
+	frappe.reload_doc("core", "doctype", "doctype_state")
 	frappe.reload_doc("core", "doctype", "docfield")
 	frappe.reload_doc("core", "doctype", "docperm")
 	frappe.reload_doc("core", "doctype", "doctype_action")
 	frappe.reload_doc("core", "doctype", "doctype_link")
-	frappe.reload_doc("core", "doctype", "doctype_state")
 	frappe.reload_doc("desk", "doctype", "form_tour_step")
 	frappe.reload_doc("desk", "doctype", "form_tour")
 	frappe.reload_doc("core", "doctype", "doctype")

frappe/utils/make_random.py

@@ -35,9 +35,13 @@ def get_random(doctype, filters=None, doc=False):
 		condition = " where " + " and ".join(condition)
 	else:
 		condition = ""
-
-	out = frappe.db.sql("""select name from `tab%s` %s
-		order by RAND() limit 0,1""" % (doctype, condition))
+	
+	out = frappe.db.multisql({
+		'mariadb': """select name from `tab%s` %s
+		order by RAND() limit 1 offset 0""" % (doctype, condition),
+		'postgres': """select name from `tab%s` %s
+		order by RANDOM() limit 1 offset 0""" % (doctype, condition)
+	})
 
 	out = out and out[0][0] or None
 

frappe/utils/nestedset.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 # Tree (Hierarchical) Nested Set Model (nsm)
@@ -109,7 +109,6 @@ def update_move_node(doc, parent_field):
 		new_parent = frappe.db.sql("""select lft, rgt from `tab%s`
 			where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0]
 
-
 		# set parent lft, rgt
 		frappe.db.sql("""update `tab{0}` set rgt = rgt + %s
 			where name = %s""".format(doc.doctype), (diff, parent))
@@ -134,6 +133,7 @@ def update_move_node(doc, parent_field):
 	frappe.db.sql("""update `tab{0}` set lft = -lft + %s, rgt = -rgt + %s
 		where lft < 0""".format(doc.doctype), (new_diff, new_diff))
 
+
 @frappe.whitelist()
 def rebuild_tree(doctype, parent_field):
 	"""
@@ -144,11 +144,15 @@ def rebuild_tree(doctype, parent_field):
 	if frappe.request and frappe.local.form_dict.cmd == 'rebuild_tree':
 		frappe.only_for('System Manager')
 
+	meta = frappe.get_meta(doctype)
+	if not meta.has_field("lft") or not meta.has_field("rgt"):
+		frappe.throw(_("Rebuilding of tree is not supported for {}").format(frappe.bold(doctype)),
+			title=_("Invalid Action"))
+
 	# get all roots
 	right = 1
 	table = DocType(doctype)
 	column = getattr(table, parent_field)
-
 	result = (
 		frappe.qb.from_(table)
 		.where(

frappe/utils/response.py

@@ -125,7 +125,7 @@ def json_handler(obj):
 	# serialize date
 	import collections.abc
 
-	if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime)):
+	if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime, datetime.time)):
 		return str(obj)
 
 	elif isinstance(obj, decimal.Decimal):

frappe/website/doctype/blog_post/blog_post.py

@@ -159,10 +159,10 @@ class BlogPost(WebsiteGenerator):
 		like_count = 0
 
 		if frappe.db.count('Feedback'):
-			like_count = frappe.db.count('Feedback', 
+			like_count = frappe.db.count('Feedback',
 				filters = dict(
-					reference_doctype = self.doctype, 
-					reference_name = self.name, 
+					reference_doctype = self.doctype,
+					reference_name = self.name,
 					like = True
 				)
 			)
@@ -183,7 +183,6 @@ def get_list_context(context=None):
 		get_list = get_blog_list,
 		no_breadcrumbs = True,
 		hide_filters = True,
-		children = get_children(),
 		# show_search = True,
 		title = _('Blog')
 	)
@@ -208,17 +207,34 @@ def get_list_context(context=None):
 	else:
 		list_context.parents = [{"name": _("Home"), "route": "/"}]
 
-	list_context.update(frappe.get_doc("Blog Settings").as_dict(no_default_fields=True))
+	blog_settings = frappe.get_doc("Blog Settings").as_dict(no_default_fields=True)
+	list_context.update(blog_settings)
+
+	if blog_settings.browse_by_category:
+		list_context.blog_categories = get_blog_categories()
 
 	return list_context
 
-def get_children():
-	return frappe.db.sql("""select route as name,
-		title from `tabBlog Category`
-		where published = 1
-		and exists (select name from `tabBlog Post`
-			where `tabBlog Post`.blog_category=`tabBlog Category`.name and published=1)
-		order by title asc""", as_dict=1)
+
+def get_blog_categories():
+	from pypika import Order
+	from pypika.terms import ExistsCriterion
+
+	post, category = frappe.qb.DocType("Blog Post"), frappe.qb.DocType("Blog Category")
+	return (
+		frappe.qb.from_(category)
+		.select(category.name, category.route, category.title)
+		.where(
+			(category.published == 1)
+			& ExistsCriterion(
+				frappe.qb.from_(post)
+				.select("name")
+				.where((post.published == 1) & (post.blog_category == category.name))
+			)
+		)
+		.orderby(category.title, order=Order.asc)
+		.run(as_dict=1)
+	)
 
 def clear_blog_cache():
 	for blog in frappe.db.sql_list("""select route from

frappe/website/doctype/blog_post/templates/blog_post_list.html

@@ -4,16 +4,34 @@
 
 {% block page_content %}
 
-{{ web_block("Hero",
-	values={
-		'title': blog_title or _("Blog"),
-		'subtitle': blog_introduction or '',
-	},
-	add_container=0,
-	add_top_padding=0,
-	add_bottom_padding=0,
-	css_class="py-5"
-) }}
+<div class="row py-8">
+	<div class="col-md-8">
+		<div class="hero">
+			<div class="hero-content">
+				<h1 class="hero-title">{{ blog_title or _('Blog') }}</h1>
+				<p class="hero-subtitle mb-0">{{ blog_introduction or '' }}</p>
+			</div>
+		</div>
+	</div>
+	<div class="col-md-4 align-self-end">
+		{%- if browse_by_category -%}
+		<label for="category-select" class="sr-only">{{ _("Browse by category") }}</label>
+		<select id="category-select" class="custom-select" onchange="window.location.pathname = this.value">
+			<option value="" {{ not frappe.form_dict.category and "selected" or "" }} disabled>
+				{{ _("Browse by category") }}
+			</option>
+			{%- if frappe.form_dict.category -%}
+			<option value="blog">{{ _("Show all blogs") }}</option>
+			{%- endif -%}
+			{%- for category in blog_categories -%}
+			<option value="{{ category.route }}" {{ frappe.form_dict.category == category.name and "selected" or "" }}>
+				{{ _(category.title) }}
+			</option>
+			{%- endfor -%}
+		</select>
+		{%- endif -%}
+	</div>
+</div>
 
 <div class="blog-list-content">
 	<div class="website-list" data-doctype="{{ doctype }}" data-txt="{{ txt or '[notxt]' | e }}">
@@ -34,5 +52,39 @@
 {% endblock %}
 
 {% block script %}
-<script>{% include "templates/includes/list/list.js" %}</script>
+<script>
+	frappe.ready(() => {
+		let result_wrapper = $(".website-list .result");
+		let next_start = {{ next_start or 0 }};
+
+		$(".website-list .btn-more").on("click", function() {
+			let $btn = $(this);
+			let args = $.extend(frappe.utils.get_query_params(), {
+				doctype: "Blog Post",
+				category: "{{ frappe.form_dict.category or '' }}",
+				limit_start: next_start,
+				pathname: location.pathname,
+			});
+			$btn.prop("disabled", true);
+			frappe.call('frappe.www.list.get', args)
+				.then(r => {
+					var data = r.message;
+					next_start = data.next_start;
+					$.each(data.result, function(i, d) {
+						$(d).appendTo(result_wrapper);
+					});
+					toggle_more(data.show_more);
+				})
+				.always(() => {
+					$btn.prop("disabled", false);
+				});
+		});
+
+		function toggle_more(show) {
+			if (!show) {
+				$(".website-list .more-block").addClass("hide");
+			}
+		}
+	});
+</script>
 {% endblock %}

frappe/website/doctype/blog_post/test_blog_post.py

@@ -58,15 +58,18 @@ class TestBlogPost(unittest.TestCase):
 		category_page_link = list(soup.find_all('a', href=re.compile(blog.blog_category)))[0]
 		category_page_url = category_page_link["href"]
 
+		cached_value = frappe.db.value_cache[('DocType', 'Blog Post', 'name')]
+		frappe.db.value_cache[('DocType', 'Blog Post', 'name')] = (('Blog Post',),)
+
 		# Visit the category page (by following the link found in above stage)
 		set_request(path=category_page_url)
 		category_page_response = get_response()
 		category_page_html = frappe.safe_decode(category_page_response.get_data())
-
 		# Category page should contain the blog post title
 		self.assertIn(blog.title, category_page_html)
 
 		# Cleanup
+		frappe.db.value_cache[('DocType', 'Blog Post', 'name')] = cached_value
 		frappe.delete_doc("Blog Post", blog.name)
 		frappe.delete_doc("Blog Category", blog.blog_category)
 

frappe/website/doctype/blog_post/ui_test_blog_post.js

@@ -0,0 +1,36 @@
+context('Blog Post', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app');
+	});
+
+	it('Blog Category dropdown works as expected', () => {
+		cy.create_records([
+			{
+				doctype: 'Blog Category',
+				title: 'Category 1',
+				published: 1
+			},
+			{
+				doctype: 'Blogger',
+				short_name: 'John',
+				full_name: 'John Doe'
+			},
+			{
+				doctype: 'Blog Post',
+				title: 'Test Blog Post',
+				content: 'Test Blog Post Content',
+				blog_category: 'category-1',
+				blogger: 'John',
+				published: 1
+			}
+		]);
+		cy.set_value('Blog Settings', 'Blog Settings', {browse_by_category: 1});
+		cy.visit('/blog');
+		cy.findByLabelText('Browse by category').select('Category 1');
+		cy.location('pathname').should('eq', '/blog/category-1');
+		cy.set_value('Blog Settings', 'Blog Settings', {browse_by_category: 0});
+		cy.visit('/blog');
+		cy.findByLabelText('Browse by category').should('not.exist');
+	});
+});

frappe/website/doctype/blog_settings/blog_settings.json

@@ -11,6 +11,7 @@
   "enable_social_sharing",
   "show_cta_in_blog",
   "allow_guest_to_comment",
+  "browse_by_category",
   "cta_section",
   "title",
   "subtitle",
@@ -110,14 +111,20 @@
    "default": "1",
    "fieldname": "allow_guest_to_comment",
    "fieldtype": "Check",
-   "label": "Allow guest to comment"
+   "label": "Allow Guest to comment"
+  },
+  {
+   "default": "0",
+   "fieldname": "browse_by_category",
+   "fieldtype": "Check",
+   "label": "Browse by category"
   }
  ],
  "icon": "fa fa-cog",
  "idx": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-10-28 20:44:44.143193",
+ "modified": "2021-12-20 13:40:32.312459",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Settings",
@@ -142,5 +149,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }

frappe/website/doctype/help_article/help_article.json

@@ -15,8 +15,7 @@
   "section_break_7",
   "content",
   "likes",
-  "route",
-  "owner"
+  "route"
  ],
  "fields": [
   {
@@ -79,13 +78,6 @@
    "fieldtype": "Data",
    "in_global_search": 1,
    "label": "Route"
-  },
-  {
-   "default": "user",
-   "fieldname": "owner",
-   "fieldtype": "Link",
-   "label": "Owner",
-   "options": "User"
   }
  ],
  "has_web_view": 1,
@@ -93,7 +85,7 @@
  "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2020-07-21 16:25:18.577325",
+ "modified": "2022-01-04 16:25:18.577325",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Help Article",