Merge branch 'develop' into mariadb-client-refactor

.github/helper/install.sh

@@ -52,12 +52,8 @@ bench -v setup requirements --dev
 
 if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi
 
-# install node-sass which is required for website theme test
-cd ./apps/frappe || exit
-yarn add node-sass@4.13.1
-cd ../..
-
 bench start &> bench_start.log &
 bench --site test_site reinstall --yes
+
 if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
 if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi

.github/workflows/patch-mariadb-tests.yml

@@ -57,7 +57,7 @@ jobs:
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 16
           check-latest: true
 
       - name: Add to Hosts

.github/workflows/publish-assets-develop.yml

@@ -15,7 +15,7 @@ jobs:
           path: 'frappe'
       - uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 16
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'

.github/workflows/publish-assets-releases.yml

@@ -16,9 +16,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'frappe'
+
       - uses: actions/setup-node@v3
         with:
-          python-version: '12.x'
+          node-version: 16
+
       - uses: actions/setup-python@v4
         with:
           python-version: '3.10'

.github/workflows/release.yml

@@ -16,10 +16,10 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
-      - name: Setup Node.js v14
+      - name: Setup Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 16
       - name: Setup dependencies
         run: |
           npm install @semantic-release/git @semantic-release/exec --no-save
@@ -31,4 +31,4 @@ jobs:
           GIT_AUTHOR_EMAIL: "developers@frappe.io"
           GIT_COMMITTER_NAME: "Frappe PR Bot"
           GIT_COMMITTER_EMAIL: "developers@frappe.io"
-        run: npx semantic-release 
+        run: npx semantic-release

.github/workflows/semantic-commits.yml

@@ -21,7 +21,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 16
           check-latest: true
 
       - name: Check commit titles

.github/workflows/server-mariadb-tests.yml

@@ -64,7 +64,7 @@ jobs:
       - uses: actions/setup-node@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
-          node-version: 14
+          node-version: 16
           check-latest: true
 
       - name: Add to Hosts

.github/workflows/server-postgres-tests.yml

@@ -67,7 +67,7 @@ jobs:
       - uses: actions/setup-node@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
-          node-version: '14'
+          node-version: '16'
           check-latest: true
 
       - name: Add to Hosts

.github/workflows/ui-tests.yml

@@ -63,7 +63,7 @@ jobs:
       - uses: actions/setup-node@v3
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         with:
-          node-version: 14
+          node-version: 16
           check-latest: true
 
       - name: Add to Hosts

cypress/integration/form.js

@@ -6,6 +6,7 @@ context('Form', () => {
 			return frappe.call("frappe.tests.ui_test_helpers.create_contact_records");
 		});
 	});
+
 	it('create a new form', () => {
 		cy.visit('/app/todo/new');
 		cy.get_field('description', 'Text Editor').type('this is a test todo', {force: true}).wait(200);
@@ -95,4 +96,63 @@ context('Form', () => {
 			})
 		})
 	});
+
+	it('let user undo/redo field value changes', { scrollBehavior: false }, () => {
+		const jump_to_field = (field_label) => {
+			cy.get("body")
+				.type("{esc}")  // lose focus if any
+				.type("{ctrl+j}")  // jump to field
+				.type(field_label)
+				.wait(500)
+				.type("{enter}")
+				.wait(200)
+				.type("{enter}")
+				.wait(500);
+		};
+
+		const type_value = (value) => {
+			cy.focused()
+				.clear()
+				.type(value)
+				.type("{esc}");
+		};
+
+		const undo = () => cy.get("body").type("{esc}").type("{ctrl+z}").wait(500);
+		const redo = () => cy.get("body").type("{esc}").type("{ctrl+y}").wait(500);
+
+		cy.new_form('User');
+
+		jump_to_field("Email");
+		type_value("admin@example.com");
+
+		jump_to_field("Username");
+		type_value("admin42");
+
+		jump_to_field("Birth Date");
+		type_value("12-31-01");
+
+		jump_to_field("Send Welcome Email");
+		cy.focused().uncheck()
+
+		// make a mistake
+		jump_to_field("Username");
+		type_value("admin24");
+
+		// undo behaviour
+		undo();
+		cy.get_field("username").should('have.value', 'admin42');
+
+		// redo behaviour
+		redo();
+		cy.get_field("username").should('have.value', 'admin24');
+
+		// undo everything & redo everything, ensure same values at the end
+		undo(); undo(); undo(); undo(); undo();
+		redo(); redo(); redo(); redo(); redo();
+
+		cy.get_field("username").should('have.value', 'admin24');
+		cy.get_field("email").should('have.value', 'admin@example.com');
+		cy.get_field("birth_date").should('have.value', '12-31-2001'); // parsed  value
+		cy.get_field("send_welcome_email").should('not.be.checked');
+	});
 });

frappe/boot.py

@@ -14,7 +14,7 @@ from frappe.email.inbox import get_email_accounts
 from frappe.model.base_document import get_controller
 from frappe.query_builder import DocType
 from frappe.query_builder.functions import Count
-from frappe.query_builder.terms import SubQuery
+from frappe.query_builder.terms import ParameterizedValueWrapper, SubQuery
 from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points
 from frappe.social.doctype.energy_point_settings.energy_point_settings import (
 	is_energy_point_enabled,
@@ -328,11 +328,11 @@ def get_unseen_notes():
 		frappe.qb.from_(note)
 		.select(note.name, note.title, note.content, note.notify_on_every_login)
 		.where(
-			(note.notify_on_every_login == 1)
+			(note.notify_on_login == 1)
 			& (note.expire_notification_on > frappe.utils.now())
 			& (
-				SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin(
-					[frappe.session.user]
+				ParameterizedValueWrapper(frappe.session.user).notin(
+					SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name))
 				)
 			)
 		)

frappe/core/doctype/doctype/doctype.py

@@ -17,7 +17,7 @@ from frappe.cache_manager import clear_controller_cache, clear_user_cache
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
 from frappe.database.schema import validate_column_length, validate_column_name
-from frappe.desk.notifications import delete_notification_count_for
+from frappe.desk.notifications import delete_notification_count_for, get_filters_for
 from frappe.desk.utils import validate_route_conflict
 from frappe.model import (
 	child_table_fields,
@@ -982,11 +982,7 @@ def validate_links_table_fieldnames(meta):
 
 	fieldnames = tuple(field.fieldname for field in meta.fields)
 	for index, link in enumerate(meta.links, 1):
-		if not frappe.get_meta(link.link_doctype).has_field(link.link_fieldname):
-			message = _("Document Links Row #{0}: Could not find field {1} in {2} DocType").format(
-				index, frappe.bold(link.link_fieldname), frappe.bold(link.link_doctype)
-			)
-			frappe.throw(message, InvalidFieldNameError, _("Invalid Fieldname"))
+		_test_connection_query(doctype=link.link_doctype, field=link.link_fieldname, idx=index)
 
 		if not link.is_child_table:
 			continue
@@ -1015,6 +1011,25 @@ def validate_links_table_fieldnames(meta):
 			frappe.throw(message, frappe.ValidationError, _("Invalid Table Fieldname"))
 
 
+def _test_connection_query(doctype, field, idx):
+	"""Make sure that connection can be queried.
+
+	This function executes query similar to one that would be executed for
+	finding count on dashboard and hence validates if fieldname/doctype are
+	correct.
+	"""
+	filters = get_filters_for(doctype) or {}
+	filters[field] = ""
+
+	try:
+		frappe.get_all(doctype, filters=filters, limit=1, distinct=True, ignore_ifnull=True)
+	except Exception as e:
+		frappe.clear_last_message()
+		msg = _("Document Links Row #{0}: Invalid doctype or fieldname.").format(idx)
+		msg += "<br>" + str(e)
+		frappe.throw(msg, InvalidFieldNameError)
+
+
 def validate_fields_for_doctype(doctype):
 	meta = frappe.get_meta(doctype, cached=False)
 	validate_links_table_fieldnames(meta)

frappe/core/doctype/doctype/test_doctype.py

@@ -543,7 +543,7 @@ class TestDocType(unittest.TestCase):
 
 		# check invalid doctype
 		doc.append("links", {"link_doctype": "User2", "link_fieldname": "first_name"})
-		self.assertRaises(frappe.DoesNotExistError, validate_links_table_fieldnames, doc)
+		self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc)
 		doc.links = []  # reset links table
 
 		# check invalid fieldname

frappe/core/doctype/system_settings/system_settings.py

@@ -44,12 +44,7 @@ class SystemSettings(Document):
 			frappe.flags.update_last_reset_password_date = True
 
 	def on_update(self):
-		for df in self.meta.get("fields"):
-			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
-				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
-
-		if self.language:
-			set_default_language(self.language)
+		self.set_defaults()
 
 		frappe.cache().delete_value("system_settings")
 		frappe.cache().delete_value("time_zone")
@@ -57,6 +52,14 @@ class SystemSettings(Document):
 		if frappe.flags.update_last_reset_password_date:
 			update_last_reset_password_date()
 
+	def set_defaults(self):
+		for df in self.meta.get("fields"):
+			if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
+				frappe.db.set_default(df.fieldname, self.get(df.fieldname))
+
+		if self.language:
+			set_default_language(self.language)
+
 
 def update_last_reset_password_date():
 	frappe.db.sql(

frappe/model/document.py

@@ -1093,7 +1093,7 @@ class Document(BaseDocument):
 
 		self.clear_cache()
 
-		if not hasattr(self.flags, "notify_update") or self.flags.notify_update:
+		if self.flags.get("notify_update", True):
 			self.notify_update()
 
 		update_global_search(self)
@@ -1147,7 +1147,7 @@ class Document(BaseDocument):
 		:param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
 		:param value: value of the property to be updated
 		:param update_modified: default True. updates the `modified` and `modified_by` properties
-		:param notify: default False. run doc.notify_updated() to send updates via socketio
+		:param notify: default False. run doc.notify_update() to send updates via socketio
 		:param commit: default False. run frappe.db.commit()
 		"""
 		if isinstance(fieldname, dict):

frappe/patches.txt

@@ -184,6 +184,7 @@ frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
 frappe.patches.v13_0.set_first_day_of_the_week
 frappe.patches.v13_0.encrypt_2fa_secrets
+frappe.patches.v13_0.reset_corrupt_defaults
 execute:frappe.reload_doc('custom', 'doctype', 'custom_field')
 frappe.patches.v14_0.update_workspace2 # 20.09.2021
 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021

frappe/patches/v13_0/encrypt_2fa_secrets.py

@@ -39,6 +39,7 @@ def execute():
 		.set(table.parent, PARENT_FOR_DEFAULTS)
 		.set(table.defvalue, defvalue_cases)
 		.where(table.parent == OLD_PARENT)
+		.where(table.defkey.like("%_otpsecret"))
 	).run()
 
 	clear_defaults_cache()

frappe/patches/v13_0/reset_corrupt_defaults.py

@@ -0,0 +1,33 @@
+import frappe
+from frappe.patches.v13_0.encrypt_2fa_secrets import DOCTYPE
+from frappe.patches.v13_0.encrypt_2fa_secrets import PARENT_FOR_DEFAULTS as TWOFACTOR_PARENT
+from frappe.utils import cint
+
+
+def execute():
+	"""
+	This patch is needed to fix parent incorrectly set as `__2fa` because of
+	https://github.com/frappe/frappe/commit/a822092211533ff17ff9b92dd86f6f868ed63e2e
+	"""
+
+	if not frappe.db.get_value(
+		DOCTYPE, {"parent": TWOFACTOR_PARENT, "defkey": ("not like", "%_otp%")}, "defkey"
+	):
+		return
+
+	# system settings
+	system_settings = frappe.get_single("System Settings")
+	system_settings.set_defaults()
+
+	# home page
+	frappe.db.set_default(
+		"desktop:home_page", "workspace" if cint(system_settings.setup_complete) else "setup-wizard"
+	)
+
+	# letter head
+	try:
+		letter_head = frappe.get_doc("Letter Head", {"is_default": 1})
+		letter_head.set_as_default()
+
+	except frappe.DoesNotExistError:
+		pass

frappe/public/js/frappe/form/controls/base_control.js

@@ -187,6 +187,15 @@ frappe.ui.form.Control = class BaseControl {
 			return Promise.resolve();
 		}
 
+		const old_value = this.get_model_value();
+		this.frm?.undo_manager?.record_change({
+			fieldname: me.df.fieldname,
+			old_value,
+			new_value: value,
+			doctype: this.doctype,
+			docname: this.docname,
+			is_child: Boolean(this.doc?.parenttype)
+		});
 		this.inside_change_event = true;
 		function set(value) {
 			me.inside_change_event = false;

frappe/public/js/frappe/form/form.js

@@ -13,6 +13,7 @@ import './script_helpers';
 import './sidebar/form_sidebar';
 import './footer/footer';
 import './form_tour';
+import { UndoManager } from './undo_manager';
 
 frappe.ui.form.Controller = class FormController {
 	constructor(opts) {
@@ -38,6 +39,7 @@ frappe.ui.form.Form = class FrappeForm {
 		this.fetch_dict = {};
 		this.parent = parent;
 		this.doctype_layout = frappe.get_doc('DocType Layout', doctype_layout_name);
+		this.undo_manager = new UndoManager({frm: this});
 		this.setup_meta(doctype);
 
 		this.beforeUnloadListener = (event) => {
@@ -143,6 +145,26 @@ frappe.ui.form.Form = class FrappeForm {
 			condition: () => !this.is_new()
 		});
 
+		// Undo and redo
+		frappe.ui.keys.add_shortcut({
+			shortcut: 'ctrl+z',
+			action: () => this.undo_manager.undo(),
+			page: this.page,
+			description: __('Undo last action'),
+		});
+		frappe.ui.keys.add_shortcut({
+			shortcut: 'shift+ctrl+z',
+			action: () => this.undo_manager.redo(),
+			page: this.page,
+			description: __('Redo last action'),
+		});
+		frappe.ui.keys.add_shortcut({
+			shortcut: 'ctrl+y',
+			action: () => this.undo_manager.redo(),
+			page: this.page,
+			description: __('Redo last action'),
+		});
+
 		let grid_shortcut_keys = [
 			{
 				'shortcut': 'Up Arrow',
@@ -357,6 +379,8 @@ frappe.ui.form.Form = class FrappeForm {
 
 		cur_frm = this;
 
+		this.undo_manager.erase_history();
+
 		if(this.docname) { // document to show
 			this.save_disabled = false;
 			// set the doc
@@ -1761,7 +1785,7 @@ frappe.ui.form.Form = class FrappeForm {
 		return sum;
 	}
 
-	scroll_to_field(fieldname) {
+	scroll_to_field(fieldname, focus=true) {
 		let field = this.get_field(fieldname);
 		if (!field) return;
 
@@ -1781,7 +1805,9 @@ frappe.ui.form.Form = class FrappeForm {
 		frappe.utils.scroll_to($el, true, 15);
 
 		// focus if text field
-		$el.find('input, select, textarea').focus();
+		if (focus) {
+			$el.find('input, select, textarea').focus();
+		}
 
 		// highlight control inside field
 		let control_element = $el.find('.form-control')

frappe/public/js/frappe/form/undo_manager.js

@@ -0,0 +1,81 @@
+export class UndoManager {
+	constructor({ frm }) {
+		this.frm = frm;
+		this.undo_stack = [];
+		this.redo_stack = [];
+	}
+	record_change({
+		fieldname,
+		old_value,
+		new_value,
+		doctype,
+		docname,
+		is_child,
+	}) {
+		if (old_value == new_value) {
+			return;
+		}
+
+		this.undo_stack.push({
+			fieldname,
+			old_value,
+			new_value,
+			doctype,
+			docname,
+			is_child,
+		});
+	}
+
+	erase_history() {
+		this.undo_stack = [];
+		this.redo_stack = [];
+	}
+
+	undo() {
+		const change = this.undo_stack.pop();
+		if (change) {
+			this._apply_change(change);
+			this._push_reverse_entry(change, this.redo_stack);
+		} else {
+			this._show_alert(__("Nothing left to undo"));
+		}
+	}
+
+	redo() {
+		const change = this.redo_stack.pop();
+		if (change) {
+			this._apply_change(change);
+			this._push_reverse_entry(change, this.undo_stack);
+		} else {
+			this._show_alert(__("Nothing left to redo"));
+		}
+	}
+
+	_push_reverse_entry(change, stack) {
+		stack.push({
+			...change,
+			new_value: change.old_value,
+			old_value: change.new_value,
+		});
+	}
+
+	_apply_change(change) {
+		if (change.is_child) {
+			frappe.model.set_value(
+				change.doctype,
+				change.docname,
+				change.fieldname,
+				change.old_value
+			);
+		} else {
+			this.frm.set_value(change.fieldname, change.old_value);
+			this.frm.scroll_to_field(change.fieldname, false);
+		}
+	}
+
+	_show_alert(msg) {
+		// reduce duration
+		// keyboard interactions shouldn't have long running annoying toasts
+		frappe.show_alert(msg, 3);
+	}
+}

frappe/tests/test_boot.py

@@ -0,0 +1,30 @@
+import unittest
+
+import frappe
+from frappe.boot import get_unseen_notes
+from frappe.desk.doctype.note.note import mark_as_seen
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestBootData(FrappeTestCase):
+	def test_get_unseen_notes(self):
+		frappe.db.delete("Note")
+		frappe.db.delete("Note Seen By")
+		note = frappe.get_doc(
+			{
+				"doctype": "Note",
+				"title": "Test Note",
+				"notify_on_login": 1,
+				"content": "Test Note 1",
+				"public": 1,
+			}
+		)
+		note.insert()
+
+		frappe.set_user("test@example.com")
+		unseen_notes = [d.title for d in get_unseen_notes()]
+		self.assertListEqual(unseen_notes, ["Test Note"])
+
+		mark_as_seen(note.name)
+		unseen_notes = [d.title for d in get_unseen_notes()]
+		self.assertListEqual(unseen_notes, [])

frappe/tests/test_document.py

@@ -3,7 +3,7 @@
 import unittest
 from contextlib import contextmanager
 from datetime import timedelta
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import frappe
 from frappe.app import make_form_dict
@@ -373,6 +373,19 @@ class TestDocument(unittest.TestCase):
 					except Exception as e:
 						self.fail(f"Invalid doc hook: {doctype}:{hook}\n{e}")
 
+	def test_realtime_notify(self):
+		todo = frappe.new_doc("ToDo")
+		todo.description = "this will trigger realtime update"
+		todo.notify_update = Mock()
+		todo.insert()
+		self.assertEqual(todo.notify_update.call_count, 1)
+
+		todo.reload()
+		todo.flags.notify_update = False
+		todo.description = "this won't trigger realtime update"
+		todo.save()
+		self.assertEqual(todo.notify_update.call_count, 1)
+
 
 class TestDocumentWebView(unittest.TestCase):
 	def get(self, path, user="Guest"):

frappe/website/doctype/blog_post/blog_post.js

@@ -2,21 +2,21 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Blog Post', {
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frappe.db.get_single_value('Blog Settings', 'show_cta_in_blog').then(value => {
 			frm.set_df_property("hide_cta", "hidden", !value);
 		});
 
 		generate_google_search_preview(frm);
 	},
-	title: function(frm) {
+	title: function (frm) {
 		generate_google_search_preview(frm);
 		frm.trigger('set_route');
 	},
-	meta_description: function(frm) {
+	meta_description: function (frm) {
 		generate_google_search_preview(frm);
 	},
-	blog_intro: function(frm) {
+	blog_intro: function (frm) {
 		generate_google_search_preview(frm);
 	},
 	blog_category(frm) {
@@ -36,8 +36,8 @@ function generate_google_search_preview(frm) {
 	if (!(frm.doc.meta_title || frm.doc.title)) return;
 	let google_preview = frm.get_field("google_preview");
 	let seo_title = (frm.doc.meta_title || frm.doc.title).slice(0, 60);
-	let seo_description =  (frm.doc.meta_description || frm.doc.blog_intro || "").slice(0, 160);
-	let date = frm.doc.published_on ? new frappe.datetime.datetime(frm.doc.published_on).moment.format('ll') + ' - ' : '';
+	let seo_description = (frm.doc.meta_description || frm.doc.blog_intro || "").slice(0, 160);
+	let date = frm.doc.published_on ? moment(frm.doc.published_on).format('ll') + '-' : '';
 	let route_array = frm.doc.route ? frm.doc.route.split('/') : [];
 	route_array.pop();
 
@@ -49,10 +49,10 @@ function generate_google_search_preview(frm) {
 					<span style="color: #5f6368;"> › ${route_array.join(' › ')}</span>
 				</cite>
 				<div style="font-size: 20px; line-height: 1.3; color: #1a0dab; padding-top: 4px; margin-bottom: 3px;">
-						${ seo_title }
+						${seo_title}
 				</div>
 				<p style="color: #545454; max-width: 48em; line-height: 1.58; font-size:14px;">
-					<span style="color: #70757a;">${ date }</span> ${ seo_description }
+					<span style="color: #70757a;">${date}</span> ${seo_description}
 				</p>
 			</div>
 	`);