Merge branch 'develop' into map-control

CODEOWNERS

@@ -6,9 +6,7 @@
 *                 @frappe/frappe-review-team
 templates/        @surajshetty3416
 www/              @surajshetty3416
-integrations/     @leela
 patches/          @surajshetty3416 @gavindsouza
-email/            @leela
 event_streaming/  @ruchamahabal
 data_import*      @netchampfaris
 core/             @surajshetty3416

cypress/integration/discussions.js

@@ -37,24 +37,24 @@ context('Discussions', () => {
 	};
 
 	const reply_through_comment_box = () => {
-		cy.get('.discussion-on-page:visible .comment-field')
+		cy.get('.discussion-form:visible .comment-field')
 			.type('This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.')
 			.should('have.value', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.');
 
-		cy.get('.discussion-on-page:visible .submit-discussion').click();
+		cy.get('.discussion-form:visible .submit-discussion').click();
 		cy.wait(3000);
 		cy.get('.discussion-on-page:visible').should('have.class', 'show');
-		cy.get('.discussion-on-page:visible').children(".reply-card").eq(1).children(".reply-text")
+		cy.get('.discussion-on-page:visible').children(".reply-card").eq(1).find(".reply-text")
 			.should('have.text', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.\n');
 	};
 
 	const cancel_and_clear_comment_box = () => {
-		cy.get('.discussion-on-page:visible .comment-field')
+		cy.get('.discussion-form:visible .comment-field')
 			.type('This is a discussion from the cypress ui tests.')
 			.should('have.value', 'This is a discussion from the cypress ui tests.');
 
-		cy.get('.discussion-on-page:visible .cancel-comment').click();
-		cy.get('.discussion-on-page:visible .comment-field').should('have.value', '');
+		cy.get('.discussion-form:visible .cancel-comment').click();
+		cy.get('.discussion-form:visible .comment-field').should('have.value', '');
 	};
 
 	const single_thread_discussion = () => {
@@ -62,13 +62,13 @@ context('Discussions', () => {
 		cy.get('.discussions-sidebar').should('have.length', 0);
 		cy.get('.reply').should('have.length', 0);
 
-		cy.get('.discussion-on-page .comment-field')
+		cy.get('.discussion-form:visible .comment-field')
 			.type('This comment is being made on a single thread discussion.')
 			.should('have.value', 'This comment is being made on a single thread discussion.');
 
-		cy.get('.discussion-on-page .submit-discussion').click();
+		cy.get('.discussion-form:visible .submit-discussion').click();
 		cy.wait(3000);
-		cy.get('.discussion-on-page').children(".reply-card").eq(-1).children(".reply-text")
+		cy.get('.discussion-on-page').children(".reply-card").eq(-1).find(".reply-text")
 			.should('have.text', 'This comment is being made on a single thread discussion.\n');
 	};
 

cypress/integration/form.js

@@ -29,10 +29,12 @@ context('Form', () => {
 		cy.get('.standard-filter-section [data-fieldname="name"] input').type('Test Form Contact 3').blur();
 		cy.click_listview_row_item(0);
 
+		cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist');
 		cy.get('.prev-doc').should('be.visible').click();
 		cy.get('.msgprint-dialog .modal-body').contains('No further records').should('be.visible');
 		cy.hide_dialog();
 
+		cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist');
 		cy.get('.next-doc').should('be.visible').click();
 		cy.get('.msgprint-dialog .modal-body').contains('No further records').should('be.visible');
 		cy.hide_dialog();

cypress/support/commands.js

@@ -200,16 +200,15 @@ Cypress.Commands.add('fill_table_field', (tablefieldname, row_idx, fieldname, va
 Cypress.Commands.add('get_table_field', (tablefieldname, row_idx, fieldname, fieldtype = 'Data') => {
 	let selector = `.frappe-control[data-fieldname="${tablefieldname}"]`;
 	selector += ` [data-idx="${row_idx}"]`;
-	selector += ` .form-in-grid`;
 
 	if (fieldtype === 'Text Editor') {
 		selector += ` [data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]`;
 	} else if (fieldtype === 'Code') {
 		selector += ` [data-fieldname="${fieldname}"] .ace_text-input`;
 	} else {
-		selector += ` .form-control[data-fieldname="${fieldname}"]`;
+		selector += ` [data-fieldname="${fieldname}"]`;
+		return cy.get(selector).find('.form-control:visible, .static-area:visible').first();
 	}
-
 	return cy.get(selector);
 });
 
@@ -290,6 +289,7 @@ Cypress.Commands.add('add_filter', () => {
 });
 
 Cypress.Commands.add('clear_filters', () => {
+	let has_filter = false;
 	cy.intercept({
 		method: 'POST',
 		url: 'api/method/frappe.model.utils.user_settings.save'
@@ -297,12 +297,17 @@ Cypress.Commands.add('clear_filters', () => {
 	cy.get('.filter-section .filter-button').click({force: true});
 	cy.wait(300);
 	cy.get('.filter-popover').should('exist');
+	cy.get('.filter-popover').then(popover => {
+		if (popover.find('input.input-with-feedback')[0].value != '') {
+			has_filter = true;
+		}
+	});
 	cy.get('.filter-popover').find('.clear-filters').click();
 	cy.get('.filter-section .filter-button').click();
 	cy.window().its('cur_list').then(cur_list => {
 		cur_list && cur_list.filter_area && cur_list.filter_area.clear();
+		has_filter && cy.wait('@filter-saved');
 	});
-	cy.wait('@filter-saved');
 });
 
 Cypress.Commands.add('click_modal_primary_button', (btn_name) => {

esbuild/esbuild.js

@@ -259,6 +259,7 @@ function get_build_options(files, outdir, plugins) {
 	return {
 		entryPoints: files,
 		entryNames: "[dir]/[name].[hash]",
+		target: ['es2017'],
 		outdir,
 		sourcemap: true,
 		bundle: true,

frappe/__init__.py

@@ -851,18 +851,25 @@ def set_value(doctype, docname, fieldname, value=None):
 	return frappe.client.set_value(doctype, docname, fieldname, value)
 
 def get_cached_doc(*args, **kwargs):
+	allow_dict = kwargs.pop("_allow_dict", False)
+
+	def _respond(doc, from_redis=False):
+		if not allow_dict and isinstance(doc, dict):
+			local.document_cache[key] = doc = get_doc(doc)
+
+		elif from_redis:
+			local.document_cache[key] = doc
+
+		return doc
+
 	if key := can_cache_doc(args):
 		# local cache
-		doc = local.document_cache.get(key)
-		if doc:
-			return doc
+		if doc := local.document_cache.get(key):
+			return _respond(doc)
 
 		# redis cache
-		doc = cache().hget('document_cache', key)
-		if doc:
-			doc = get_doc(doc)
-			local.document_cache[key] = doc
-			return doc
+		if doc := cache().hget('document_cache', key):
+			return _respond(doc, True)
 
 	# database
 	doc = get_doc(*args, **kwargs)
@@ -895,8 +902,13 @@ def clear_document_cache(doctype, name):
 		del local.document_cache[key]
 	cache().hdel('document_cache', key)
 
-def get_cached_value(doctype, name, fieldname, as_dict=False):
-	doc = get_cached_doc(doctype, name)
+def get_cached_value(doctype, name, fieldname="name", as_dict=False):
+	try:
+		doc = get_cached_doc(doctype, name, _allow_dict=True)
+	except DoesNotExistError:
+		clear_last_message()
+		return
+
 	if isinstance(fieldname, str):
 		if as_dict:
 			throw('Cannot make dict for single fieldname')

frappe/commands/site.py

@@ -780,9 +780,8 @@ def set_user_password(site, user, password, logout_all_sessions=False):
 @pass_context
 def set_last_active_for_user(context, user=None):
 	"Set users last active date to current datetime"
-
 	from frappe.core.doctype.user.user import get_system_users
-	from frappe.utils.user import set_last_active_to_now
+	from frappe.utils import now_datetime
 
 	site = get_site(context)
 
@@ -795,9 +794,10 @@ def set_last_active_for_user(context, user=None):
 			else:
 				return
 
-		set_last_active_to_now(user)
+		frappe.db.set_value("User", user, "last_active", now_datetime())
 		frappe.db.commit()
 
+
 @click.command('publish-realtime')
 @click.argument('event')
 @click.option('--message')

frappe/core/doctype/communication/communication.json

@@ -153,7 +153,7 @@
    "fieldname": "communication_type",
    "fieldtype": "Select",
    "label": "Communication Type",
-   "options": "Communication\nComment\nChat\nBot\nNotification\nFeedback\nAutomated Message",
+   "options": "Communication\nComment\nChat\nNotification\nFeedback\nAutomated Message",
    "read_only": 1,
    "reqd": 1
   },
@@ -164,7 +164,7 @@
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Comment Type",
-   "options": "\nComment\nLike\nInfo\nLabel\nWorkflow\nCreated\nSubmitted\nCancelled\nUpdated\nDeleted\nAssigned\nAssignment Completed\nAttachment\nAttachment Removed\nShared\nUnshared\nBot\nRelinked",
+   "options": "\nComment\nLike\nInfo\nLabel\nWorkflow\nCreated\nSubmitted\nCancelled\nUpdated\nDeleted\nAssigned\nAssignment Completed\nAttachment\nAttachment Removed\nShared\nUnshared\nRelinked",
    "read_only": 1
   },
   {
@@ -395,7 +395,7 @@
  "icon": "fa fa-comment",
  "idx": 1,
  "links": [],
- "modified": "2021-11-30 09:03:25.728637",
+ "modified": "2022-03-30 11:24:25.728637",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Communication",

frappe/core/doctype/communication/communication.py

@@ -10,7 +10,6 @@ from frappe.utils import validate_email_address, strip_html, cstr, time_diff_in_
 from frappe.core.doctype.communication.email import validate_email
 from frappe.core.doctype.communication.mixins import CommunicationEmailMixin
 from frappe.core.utils import get_parent_doc
-from frappe.utils.bot import BotReply
 from frappe.utils import parse_addr, split_emails
 from frappe.core.doctype.comment.comment import update_comment_in_doc
 from email.utils import getaddresses
@@ -105,7 +104,7 @@ class Communication(Document, CommunicationEmailMixin):
 		if self.communication_type == "Communication":
 			self.notify_change('add')
 
-		elif self.communication_type in ("Chat", "Notification", "Bot"):
+		elif self.communication_type in ("Chat", "Notification"):
 			if self.reference_name == frappe.session.user:
 				message = self.as_dict()
 				message['broadcast'] = True
@@ -160,7 +159,6 @@ class Communication(Document, CommunicationEmailMixin):
 
 		if self.comment_type != 'Updated':
 			update_parent_document_on_communication(self)
-			self.bot_reply()
 
 	def on_trash(self):
 		if self.communication_type == "Communication":
@@ -278,20 +276,6 @@ class Communication(Document, CommunicationEmailMixin):
 				if not self.sender_full_name:
 					self.sender_full_name = sender_email
 
-	def bot_reply(self):
-		if self.comment_type == 'Bot' and self.communication_type == 'Chat':
-			reply = BotReply().get_reply(self.content)
-			if reply:
-				frappe.get_doc({
-					"doctype": "Communication",
-					"comment_type": "Bot",
-					"communication_type": "Bot",
-					"content": cstr(reply),
-					"reference_doctype": self.reference_doctype,
-					"reference_name": self.reference_name
-				}).insert()
-				frappe.local.flags.commit = True
-
 	def set_delivery_status(self, commit=False):
 		'''Look into the status of Email Queue linked to this Communication and set the Delivery Status of this Communication'''
 		delivery_status = None

frappe/core/doctype/data_import/importer.py

@@ -244,6 +244,7 @@ class Importer:
 		existing_doc = frappe.get_doc(self.doctype, doc.get(id_field.fieldname))
 
 		updated_doc = frappe.get_doc(self.doctype, doc.get(id_field.fieldname))
+
 		updated_doc.update(doc)
 
 		if get_diff(existing_doc, updated_doc):

frappe/core/doctype/data_import/test_importer.py

@@ -92,11 +92,18 @@ class TestImporter(unittest.TestCase):
 
 		# update child table id in template date
 		i.import_file.raw_data[1][4] = existing_doc.table_field_1[0].name
-		i.import_file.raw_data[1][0] = existing_doc.name
+
+		# uppercase to check if autoname field isn't replaced in mariadb
+		if frappe.db.db_type == "mariadb":
+			i.import_file.raw_data[1][0] = existing_doc.name.upper()
+		else:
+			i.import_file.raw_data[1][0] = existing_doc.name
+
 		i.import_file.parse_data_from_template()
 		i.import_data()
 
 		updated_doc = frappe.get_doc(doctype_name, existing_doc.name)
+		self.assertEqual(existing_doc.title, updated_doc.title)
 		self.assertEqual(updated_doc.description, 'test description')
 		self.assertEqual(updated_doc.table_field_1[0].child_title, 'child title')
 		self.assertEqual(updated_doc.table_field_1[0].name, existing_doc.table_field_1[0].name)

frappe/core/page/background_jobs/background_jobs.css

@@ -1,43 +1,32 @@
-.list-jobs {
-	font-size: var(--text-base);
-}
 
-.table {
+.table-background-jobs {
 	margin-bottom: 0px;
 	margin-top: 0px;
+	font-size: var(--text-md);
+	table-layout: fixed;
 }
 
-thead {
-	background-color: var(--control-bg);
-	border-radius: var(--border-radius-sm);
+.table-background-jobs th {
+	font-weight: normal;
+	color: var(--text-muted);
 }
 
-thead > tr {
-	border-radius: var(--border-radius-sm);
+.table-background-jobs td {
+	color: var(--text-light);
 }
 
-thead > tr > th:first-child {
-	border-radius: var(--border-radius-sm) 0 0 var(--border-radius-sm);
-}
-thead > tr > th:last-child {
-	border-radius: 0 var(--border-radius-sm) var(--border-radius-sm) 0;
+.table-background-jobs th, .table-background-jobs td {
+	padding: var(--padding-sm) var(--padding-md);
 }
 
-.worker-name {
-	display: flex;
-	align-items: center;
+.table-background-jobs tbody tr:hover {
+	background-color: var(--highlight-color);
 }
 
 .job-name {
 	font-size: var(--text-md);
-	font-family: "Courier New", Courier, monospace;
-	/* background-color: var(--control-bg); */
-	/* padding: var(--padding-xs) var(--padding-sm); */
-	/* border-radius: var(--border-radius-md); */
-}
-
-.background-job-row:hover {
-	background-color: var(--bg-color);
+	font-family: var(--font-family-monospace);
+	word-break: break-word;
 }
 
 .no-background-jobs {
@@ -54,7 +43,5 @@ thead > tr > th:last-child {
 }
 
 .footer {
-	align-items: flex-end;
-	margin-top: var(--margin-md);
-	font-size: var(--text-base);
+	padding: var(--padding-md);
 }

frappe/core/page/background_jobs/background_jobs.html

@@ -1,51 +1,58 @@
-<div class="list-jobs">
-	{% if jobs.length %}
-	<table class="table table-borderless" style="table-layout: fixed;">
-		<thead>
-			<tr>
-				<th style="width: 20%">{{ __("Queue / Worker") }}</th>
-				<th>{{ __("Job") }}</th>
-				<th style="width: 15%">{{ __("Created") }}</th>
-			</tr>
-		</thead>
-		<tbody>
-			{% for j in jobs %}
-			<tr>
-				<td class="worker-name">
-					<span class="indicator-pill no-margin {{ j.color }}"></span>
-					<span class="ml-2">{{ j.queue.split(".").slice(-1)[0] }}</span>
-				</td>
-				<td style="overflow: auto;">
-					<div>
-						<span class="job-name">
-							{{ frappe.utils.encode_tags(j.job_name) }}
-						</span>
-					</div>
-					{% if j.exc_info %}
+{% if jobs.length %}
+<table class="table table-background-jobs">
+	<thead>
+		<tr>
+			<th style="width: 10%">{{ __("Queue") }}</th>
+			<th>{{ __("Job") }}</th>
+			<th style="width: 10%">{{ __("Status") }}</th>
+			<th style="width: 15%">{{ __("Created") }}</th>
+		</tr>
+	</thead>
+	<tbody>
+		{% for j in jobs %}
+		<tr>
+			<td class="worker-name">
+				{{ toTitle(j.queue.split(":").slice(-1)[0]) }}
+			</td>
+			<td>
+				<div>
+					<span class="job-name">
+						{{ frappe.utils.encode_tags(j.job_name) }}
+					</span>
+				</div>
+				{% if j.exc_info %}
+				<details>
+					<summary>{{ __("Exception") }}</summary>
 					<div class="exc_info">
 						<pre>{{ frappe.utils.encode_tags(j.exc_info) }}</pre>
 					</div>
-					{% endif %}
-				</td>
-				<td class="creation">{{ j.creation }}</td>
-			</tr>
-			{% endfor %}
-		</tbody>
-	</table>
-	{% else %}
-	<div class="no-background-jobs">
-		<img src="/assets/frappe/images/ui-states/list-empty-state.svg" alt="Empty State">
-		<p class="text-muted">{{ __("No pending or current jobs for this site") }}</p>
-	</div>
-	{% endif %}
-	<div class="footer row">
-		<div class="col-md-6 text-muted text-center text-md-left">{{ __("Last refreshed") }}
-			{{ frappe.datetime.now_datetime(true).toLocaleString() }}</div>
-		<div class="col-md-6 text-center text-md-right">
-			<span class="indicator-pill blue" class="mr-2">{{ __("Started") }}</span>
-			<span class="indicator-pill orange" class="mr-2">{{ __("Queued") }}</span>
-			<span class="indicator-pill red" class="mr-2">{{ __("Failed") }}</span>
-			<span class="indicator-pill green">{{ __("Finished") }}</span>
-		</div>
+				</details>
+				{% endif %}
+			</td>
+			<td>
+				<span class="indicator-pill {{ j.color }}">
+					{{ toTitle(j.status) }}
+				</span>
+			</td>
+			<td class="creation text-muted">
+				{{ frappe.datetime.prettyDate(j.creation) }}
+			</td>
+		</tr>
+		{% endfor %}
+	</tbody>
+</table>
+{% else %}
+<div class="no-background-jobs">
+	<img
+		src="/assets/frappe/images/ui-states/list-empty-state.svg"
+		alt="Empty State"
+	/>
+	<p class="text-muted">{{ __("No jobs found on this site") }}</p>
+</div>
+{% endif %}
+<div class="footer">
+	<div class="text-muted">
+		{{ __("Last refreshed") }}
+		{{ frappe.datetime.now_datetime(true).toLocaleString() }}
 	</div>
-</div>
+</div>

frappe/core/page/background_jobs/background_jobs.js

@@ -1,7 +1,7 @@
-frappe.pages["background_jobs"].on_page_load = (wrapper) => {
+frappe.pages["background_jobs"].on_page_load = wrapper => {
 	const background_job = new BackgroundJobs(wrapper);
 
-	$(wrapper).bind('show', () => {
+	$(wrapper).bind("show", () => {
 		background_job.show();
 	});
 
@@ -12,61 +12,135 @@ class BackgroundJobs {
 	constructor(wrapper) {
 		this.page = frappe.ui.make_app_page({
 			parent: wrapper,
-			title: __('Background Jobs'),
+			title: __("Background Jobs"),
 			single_column: true
 		});
 
-		this.called = false;
-		this.show_failed = false;
+		this.page.add_inner_button(__("Remove Failed Jobs"), () => {
+			frappe.confirm(
+				__("Are you sure you want to remove all failed jobs?"),
+				() => {
+					frappe
+						.call(
+							"frappe.core.page.background_jobs.background_jobs.remove_failed_jobs"
+						)
+						.then(() => this.refresh_jobs());
+				}
+			);
+		});
 
-		this.show_failed_button = this.page.add_inner_button(__("Show Failed Jobs"), () => {
-			this.show_failed = !this.show_failed;
-			if (this.show_failed_button) {
-				this.show_failed_button.text(
-					this.show_failed ? __("Hide Failed Jobs") : __("Show Failed Jobs")
-				);
+		this.page.main.addClass("frappe-card");
+		this.page.body.append('<div class="table-area"></div>');
+		this.$content = $(this.page.body).find(".table-area");
+
+		this.make_filters();
+		this.refresh_jobs = frappe.utils.throttle(
+			this.refresh_jobs.bind(this),
+			1000
+		);
+	}
+
+	make_filters() {
+		this.view = this.page.add_field({
+			label: __("View"),
+			fieldname: "view",
+			fieldtype: "Select",
+			options: ["Jobs", "Workers"],
+			default: "Jobs",
+			change: () => {
+				this.queue_timeout.toggle(this.view.get_value() === "Jobs");
+				this.job_status.toggle(this.view.get_value() === "Jobs");
 			}
 		});
-
-		// add a "Remove Failed Jobs button"
-		this.remove_failed_button = this.page.add_inner_button(__("Remove Failed Jobs"), () => {
-			frappe.call({
-				method: 'frappe.core.page.background_jobs.background_jobs.remove_failed_jobs',
-				callback: () => {
+		this.queue_timeout = this.page.add_field({
+			label: __("Queue"),
+			fieldname: "queue_timeout",
+			fieldtype: "Select",
+			options: [
+				{ label: "All Queues", value: "all" },
+				{ label: "Default", value: "default" },
+				{ label: "Short", value: "short" },
+				{ label: "Long", value: "long" }
+			],
+			default: "all"
+		});
+		this.job_status = this.page.add_field({
+			label: __("Job Status"),
+			fieldname: "job_status",
+			fieldtype: "Select",
+			options: [
+				{ label: "All Jobs", value: "all" },
+				{ label: "Queued", value: "queued" },
+				{ label: "Deferred", value: "deferred" },
+				{ label: "Started", value: "started" },
+				{ label: "Finished", value: "finished" },
+				{ label: "Failed", value: "failed" }
+			],
+			default: "all"
+		});
+		this.auto_refresh = this.page.add_field({
+			label: __("Auto Refresh"),
+			fieldname: "auto_refresh",
+			fieldtype: "Check",
+			default: 1,
+			change: () => {
+				if (this.auto_refresh.get_value()) {
 					this.refresh_jobs();
 				}
-			});
+			}
 		});
-
-		$(frappe.render_template('background_jobs_outer')).appendTo(this.page.body);
-		this.content = $(this.page.body).find('.table-area');
 	}
 
 	show() {
 		this.refresh_jobs();
+		this.update_scheduler_status();
+	}
+
+	update_scheduler_status() {
 		frappe.call({
-			method: 'frappe.core.page.background_jobs.background_jobs.get_scheduler_status',
-			callback: res => {
-				this.page.set_indicator(...res.message);
+			method:
+				"frappe.core.page.background_jobs.background_jobs.get_scheduler_status",
+			callback: r => {
+				let { status } = r.message;
+				if (status === "active") {
+					this.page.set_indicator(__("Scheduler: Active"), "green");
+				} else {
+					this.page.set_indicator(__("Scheduler: Inactive"), "red");
+				}
 			}
 		});
 	}
 
 	refresh_jobs() {
-		if (this.called) return;
-		this.called = true;
+		let view = this.view.get_value();
+		let args;
+		let { queue_timeout, job_status } = this.page.get_form_values();
+		if (view === "Jobs") {
+			args = { view, queue_timeout, job_status };
+		} else {
+			args = { view };
+		}
 
+		this.page.add_inner_message(__("Refreshing..."));
 		frappe.call({
-			method: 'frappe.core.page.background_jobs.background_jobs.get_info',
-			args: {
-				show_failed: this.show_failed
-			},
-			callback: (res) => {
-				this.called = false;
-				this.page.body.find('.list-jobs').remove();
-				$(frappe.render_template('background_jobs', { jobs: res.message || [] })).appendTo(this.content);
+			method: "frappe.core.page.background_jobs.background_jobs.get_info",
+			args,
+			callback: res => {
+				this.page.add_inner_message("");
+
+				let template =
+					view === "Jobs" ? "background_jobs" : "background_workers";
+				this.$content.html(
+					frappe.render_template(template, {
+						jobs: res.message || []
+					})
+				);
 
-				if (frappe.get_route()[0] === 'background_jobs') {
+				let auto_refresh = this.auto_refresh.get_value();
+				if (
+					frappe.get_route()[0] === "background_jobs" &&
+					auto_refresh
+				) {
 					setTimeout(() => this.refresh_jobs(), 2000);
 				}
 			}

frappe/core/page/background_jobs/background_jobs.py

@@ -8,8 +8,8 @@ from rq import Worker
 
 import frappe
 from frappe import _
-from frappe.utils import convert_utc_to_user_timezone, format_datetime
-from frappe.utils.background_jobs import get_redis_conn, get_queues
+from frappe.utils import convert_utc_to_user_timezone
+from frappe.utils.background_jobs import get_queues, get_workers
 from frappe.utils.scheduler import is_scheduler_inactive
 
 if TYPE_CHECKING:
@@ -24,16 +24,15 @@ JOB_COLORS = {
 
 
 @frappe.whitelist()
-def get_info(show_failed=False) -> List[Dict]:
-	if isinstance(show_failed, str):
-		show_failed = json.loads(show_failed)
-
-	conn = get_redis_conn()
-	queues = get_queues()
-	workers = Worker.all(conn)
+def get_info(view=None, queue_timeout=None, job_status=None) -> List[Dict]:
 	jobs = []
 
 	def add_job(job: 'Job', name: str) -> None:
+		if job_status != "all" and job.get_status() != job_status:
+			return
+		if queue_timeout != "all" and not name.endswith(f':{queue_timeout}'):
+			return
+
 		if job.kwargs.get('site') == frappe.local.site:
 			job_info = {
 				'job_name': job.kwargs.get('kwargs', {}).get('playbook_method')
@@ -41,7 +40,7 @@ def get_info(show_failed=False) -> List[Dict]:
 					or str(job.kwargs.get('job_name')),
 				'status': job.get_status(),
 				'queue': name,
-				'creation': format_datetime(convert_utc_to_user_timezone(job.created_at)),
+				'creation': convert_utc_to_user_timezone(job.created_at),
 				'color': JOB_COLORS[job.get_status()]
 			}
 
@@ -50,32 +49,31 @@ def get_info(show_failed=False) -> List[Dict]:
 
 			jobs.append(job_info)
 
-	# show worker jobs
-	for worker in workers:
-		job = worker.get_current_job()
-		if job:
-			add_job(job, worker.name)
-
-	for queue in queues:
-		# show active queued jobs
-		if queue.name != 'failed':
+	if view == 'Jobs':
+		queues = get_queues()
+		for queue in queues:
 			for job in queue.jobs:
 				add_job(job, queue.name)
 
-		# show failed jobs, if requested
-		if show_failed:
-			fail_registry = queue.failed_job_registry
-			for job_id in fail_registry.get_job_ids():
-				job = queue.fetch_job(job_id)
-				if job:
-					add_job(job, queue.name)
+	elif view == 'Workers':
+		workers = get_workers()
+		for worker in workers:
+			current_job = worker.get_current_job()
+			if current_job and current_job.kwargs.get('site') == frappe.local.site:
+				add_job(current_job, job.origin)
+			else:
+				jobs.append({
+					'queue': worker.name,
+					'job_name': 'idle',
+					'status': '',
+					'creation': ''
+				})
 
 	return jobs
 
 
 @frappe.whitelist()
 def remove_failed_jobs():
-	conn = get_redis_conn()
 	queues = get_queues()
 	for queue in queues:
 		fail_registry = queue.failed_job_registry
@@ -87,5 +85,5 @@ def remove_failed_jobs():
 @frappe.whitelist()
 def get_scheduler_status():
 	if is_scheduler_inactive():
-		return [_("Inactive"), "red"]
-	return [_("Active"), "green"]
+		return {'status': 'inactive'}
+	return {'status': 'active'}

frappe/core/page/background_jobs/background_jobs_outer.html

@@ -1,5 +0,0 @@
-<div class="frappe-card">
-	<div class="table-area">
-
-	</div>
-</div>

frappe/core/page/background_jobs/background_workers.html

@@ -0,0 +1,51 @@
+{% if jobs.length %}
+<table class="table table-background-jobs">
+    <thead>
+        <tr>
+            <th style="width: 40%">{{ __("Worker") }}</th>
+            <th>{{ __("Current Job") }}</th>
+            <th style="width: 10%">{{ __("Status") }}</th>
+            <th style="width: 15%">{{ __("Created") }}</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for j in jobs %}
+        <tr>
+            <td class="worker-name">
+                {{ j.queue }}
+            </td>
+            <td>
+                <div>
+                    <span class="job-name">
+                        {{ frappe.utils.encode_tags(j.job_name) }}
+                    </span>
+                </div>
+                {% if j.exc_info %}
+                <details>
+                    <summary>{{ __("Exception") }}</summary>
+                    <div class="exc_info">
+                        <pre>{{ frappe.utils.encode_tags(j.exc_info) }}</pre>
+                    </div>
+                </details>
+                {% endif %}
+            </td>
+            <td>
+                <span class="indicator-pill {{ j.color }}">{{ toTitle(j.status) }}</span>
+            </td>
+            <td class="creation text-muted">{{ frappe.datetime.prettyDate(j.creation) }}</td>
+        </tr>
+        {% endfor %}
+    </tbody>
+</table>
+{% else %}
+<div class="no-background-jobs">
+    <img src="/assets/frappe/images/ui-states/list-empty-state.svg" alt="Empty State">
+    <p class="text-muted">{{ __("No workers online on this site") }}</p>
+</div>
+{% endif %}
+<div class="footer">
+    <div class="text-muted">
+        {{ __("Last refreshed") }}
+        {{ frappe.datetime.now_datetime(true).toLocaleString() }}
+    </div>
+</div>

frappe/custom/doctype/custom_field/custom_field.py

@@ -33,29 +33,40 @@ class CustomField(Document):
 
 	def before_insert(self):
 		self.set_fieldname()
-		meta = frappe.get_meta(self.dt, cached=False)
-		fieldnames = [df.fieldname for df in meta.get("fields")]
-
-		if self.fieldname in fieldnames:
-			frappe.throw(_("A field with the name '{}' already exists in doctype {}.").format(self.fieldname, self.dt))
 
 	def validate(self):
+		# these imports have been added to avoid cyclical import, should fix in future
 		from frappe.custom.doctype.customize_form.customize_form import CustomizeForm
-
-		meta = frappe.get_meta(self.dt, cached=False)
-		fieldnames = [df.fieldname for df in meta.get("fields")]
-
-		if self.insert_after=='append':
-			self.insert_after = fieldnames[-1]
-
-		if self.insert_after and self.insert_after in fieldnames:
-			self.idx = fieldnames.index(self.insert_after) + 1
-
-		old_fieldtype = self.db_get('fieldtype')
-		is_fieldtype_changed = (not self.is_new()) and (old_fieldtype != self.fieldtype)
-
-		if not self.is_virtual and is_fieldtype_changed and not CustomizeForm.allow_fieldtype_change(old_fieldtype, self.fieldtype):
-			frappe.throw(_("Fieldtype cannot be changed from {0} to {1}").format(old_fieldtype, self.fieldtype))
+		from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts
+
+		# don't always get meta to improve performance
+		# setting idx is just an improvement, not a requirement
+		if self.is_new() or self.insert_after == "append":
+			meta = frappe.get_meta(self.dt, cached=False)
+			fieldnames = [df.fieldname for df in meta.get("fields")]
+
+			if self.is_new() and self.fieldname in fieldnames:
+				frappe.throw(
+					_("A field with the name {0} already exists in {1}")
+					.format(frappe.bold(self.fieldname), self.dt)
+				)
+
+			if self.insert_after == "append":
+				self.insert_after = fieldnames[-1]
+
+			if self.insert_after and self.insert_after in fieldnames:
+				self.idx = fieldnames.index(self.insert_after) + 1
+
+		if (
+			not self.is_virtual
+			and (doc_before_save := self.get_doc_before_save())
+			and (old_fieldtype := doc_before_save.fieldtype) != self.fieldtype
+			and not CustomizeForm.allow_fieldtype_change(old_fieldtype, self.fieldtype)
+		):
+			frappe.throw(
+				_("Fieldtype cannot be changed from {0} to {1}")
+				.format(old_fieldtype, self.fieldtype)
+			)
 
 		if not self.fieldname:
 			frappe.throw(_("Fieldname not set for Custom Field"))
@@ -63,13 +74,12 @@ class CustomField(Document):
 		if self.get('translatable', 0) and not supports_translation(self.fieldtype):
 			self.translatable = 0
 
-		if not self.flags.ignore_validate:
-			from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts
-			check_fieldname_conflicts(self)
+		check_fieldname_conflicts(self)
 
 	def on_update(self):
 		if not frappe.flags.in_setup_wizard:
 			frappe.clear_cache(doctype=self.dt)
+
 		if not self.flags.ignore_validate:
 			# validate field
 			from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype

frappe/custom/doctype/customize_form/customize_form.js

@@ -49,6 +49,14 @@ frappe.ui.form.on("Customize Form", {
 			if (grid_row.doc && grid_row.doc.fieldtype == "Section Break") {
 				$(grid_row.row).css({ "font-weight": "bold" });
 			}
+
+			grid_row.row.removeClass("highlight");
+
+			if (grid_row.doc.is_custom_field &&
+				!grid_row.row.hasClass('highlight') &&
+				!grid_row.doc.is_system_generated) {
+				grid_row.row.addClass("highlight");
+			}
 		});
 
 		$(frm.wrapper).on("grid-make-sortable", function(e, frm) {
@@ -84,17 +92,11 @@ frappe.ui.form.on("Customize Form", {
 	},
 
 	setup_sortable: function(frm) {
-		frm.page.body.find(".highlight").removeClass("highlight");
 		frm.doc.fields.forEach(function(f, i) {
-			var data_row = frm.page.body.find(
-				'[data-fieldname="fields"] [data-idx="' + f.idx + '"] .data-row'
-			);
-
-			if (f.is_custom_field) {
-				data_row.addClass("highlight");
-			} else {
+			if (!f.is_custom_field) {
 				f._sortable = false;
 			}
+
 			if (f.fieldtype == "Table") {
 				frm.add_custom_button(
 					f.options,

frappe/custom/doctype/customize_form/customize_form.py

@@ -67,7 +67,12 @@ class CustomizeForm(Document):
 			self.set(prop, meta.get(prop))
 
 		for d in meta.get("fields"):
-			new_d = {"fieldname": d.fieldname, "is_custom_field": d.get("is_custom_field"), "name": d.name}
+			new_d = {
+				"fieldname": d.fieldname,
+				"is_custom_field": d.get("is_custom_field"),
+				"is_system_generated": d.get("is_system_generated"),
+				"name": d.name
+			}
 			for prop in docfield_properties:
 				new_d[prop] = d.get(prop)
 			self.append("fields", new_d)

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -7,6 +7,7 @@
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
+  "is_system_generated",
   "label_and_type",
   "label",
   "fieldtype",
@@ -444,13 +445,21 @@
    "fieldname": "no_copy",
    "fieldtype": "Check",
    "label": "No Copy"
+  },
+  {
+   "default": "0",
+   "fieldname": "is_system_generated",
+   "fieldtype": "Check",
+   "hidden": 1,
+   "label": "Is System Generated",
+   "read_only": 1
   }
  ],
  "idx": 1,
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2022-02-25 16:01:12.616736",
+ "modified": "2022-03-31 12:05:11.799654",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",

frappe/database/database.py

@@ -115,6 +115,7 @@ class Database(object):
 				{"name": "a%", "owner":"test@example.com"})
 
 		"""
+		debug = debug or getattr(self, "debug", False)
 		query = str(query)
 		if not run:
 			return query
@@ -446,6 +447,7 @@ class Database(object):
 				pluck=pluck,
 				distinct=distinct,
 				limit=limit,
+				as_dict=as_dict,
 			)
 
 		else:
@@ -549,7 +551,7 @@ class Database(object):
 				return r and [[i[1] for i in r]] or []
 
 
-	def get_singles_dict(self, doctype, debug = False):
+	def get_singles_dict(self, doctype, debug=False, *, for_update=False):
 		"""Get Single DocType as dict.
 
 		:param doctype: DocType of the single object whose value is requested
@@ -560,10 +562,13 @@ class Database(object):
 			account_settings = frappe.db.get_singles_dict("Accounts Settings")
 		"""
 		result = self.query.get_sql(
-			"Singles", filters={"doctype": doctype}, fields=["field", "value"]
+			"Singles",
+			filters={"doctype": doctype},
+			fields=["field", "value"],
+			for_update=for_update,
 		).run()
-		dict_  = frappe._dict(result)
-		return dict_
+
+		return frappe._dict(result)
 
 	@staticmethod
 	def get_all(*args, **kwargs):
@@ -674,7 +679,20 @@ class Database(object):
 		)
 		return r
 
-	def _get_value_for_many_names(self, doctype, names, field, order_by, *, debug=False, run=True, pluck=False, distinct=False, limit=None):
+	def _get_value_for_many_names(
+		self,
+		doctype,
+		names,
+		field,
+		order_by,
+		*,
+		debug=False,
+		run=True,
+		pluck=False,
+		distinct=False,
+		limit=None,
+		as_dict=False
+	):
 		names = list(filter(None, names))
 		if names:
 			return self.get_all(
@@ -684,7 +702,7 @@ class Database(object):
 				order_by=order_by,
 				pluck=pluck,
 				debug=debug,
-				as_list=1,
+				as_list=not as_dict,
 				run=run,
 				distinct=distinct,
 				limit_page_length=limit

frappe/database/query.py

@@ -115,21 +115,23 @@ def change_orderby(order: str):
 
 
 OPERATOR_MAP = {
-					"+": operator.add,
-					"=": operator.eq,
-					"-": operator.sub,
-					"!=": operator.ne,
-					"<": operator.lt,
-					">": operator.gt,
-					"<=": operator.le,
-					">=": operator.ge,
-					"in": func_in,
-					"not in": func_not_in,
-					"like": like,
-					"not like": not_like,
-					"regex": func_regex,
-					"between": func_between
-				}
+	"+": operator.add,
+	"=": operator.eq,
+	"-": operator.sub,
+	"!=": operator.ne,
+	"<": operator.lt,
+	">": operator.gt,
+	"<=": operator.le,
+	"=<": operator.le,
+	">=": operator.ge,
+	"=>": operator.ge,
+	"in": func_in,
+	"not in": func_not_in,
+	"like": like,
+	"not like": not_like,
+	"regex": func_regex,
+	"between": func_between,
+}
 
 
 class Query:
@@ -211,8 +213,7 @@ class Query:
 					_operator = OPERATOR_MAP[f[1]]
 					conditions = conditions.where(_operator(Field(f[0]), f[2]))
 
-		conditions = self.add_conditions(conditions, **kwargs)
-		return conditions
+		return self.add_conditions(conditions, **kwargs)
 
 	def dict_query(self, table: str, filters: Dict[str, Union[str, int]] = None, **kwargs) -> frappe.qb:
 		"""Build conditions using the given dictionary filters
@@ -251,8 +252,7 @@ class Query:
 					field = getattr(_table, key)
 					conditions = conditions.where(field.isnull())
 
-		conditions = self.add_conditions(conditions, **kwargs)
-		return conditions
+		return self.add_conditions(conditions, **kwargs)
 
 	def build_conditions(
 		self,

frappe/email/doctype/email_queue/email_queue.py

@@ -220,9 +220,9 @@ class SendMailContext:
 
 	def message_placeholder(self, placeholder_key):
 		map = {
-			'tracker': '<!--email open check-->',
-			'unsubscribe_url': '<!--unsubscribe url-->',
-			'cc': '<!--cc message-->',
+			'tracker': '<!--email_open_check-->',
+			'unsubscribe_url': '<!--unsubscribe_url-->',
+			'cc': '<!--cc_message-->',
 			'recipient': '<!--recipient-->',
 		}
 		return map.get(placeholder_key)

frappe/email/queue.py

@@ -66,25 +66,25 @@ def get_emails_sent_today(email_account=None):
 
 def get_unsubscribe_message(unsubscribe_message, expose_recipients):
 	if unsubscribe_message:
-		unsubscribe_html = '''<a href="<!--unsubscribe url-->"
+		unsubscribe_html = '''<a href="<!--unsubscribe_url-->"
 			target="_blank">{0}</a>'''.format(unsubscribe_message)
 	else:
-		unsubscribe_link = '''<a href="<!--unsubscribe url-->"
+		unsubscribe_link = '''<a href="<!--unsubscribe_url-->"
 			target="_blank">{0}</a>'''.format(_('Unsubscribe'))
 		unsubscribe_html = _("{0} to stop receiving emails of this type").format(unsubscribe_link)
 
 	html = """<div class="email-unsubscribe">
-			<!--cc message-->
+			<!--cc_message-->
 			<div>
 				{0}
 			</div>
 		</div>""".format(unsubscribe_html)
 
 	if expose_recipients == "footer":
-		text = "\n<!--cc message-->"
+		text = "\n<!--cc_message-->"
 	else:
 		text = ""
-	text += "\n\n{unsubscribe_message}: <!--unsubscribe url-->\n".format(unsubscribe_message=unsubscribe_message)
+	text += "\n\n{unsubscribe_message}: <!--unsubscribe_url-->\n".format(unsubscribe_message=unsubscribe_message)
 
 	return frappe._dict({
 		"html": html,

frappe/hooks.py

@@ -282,14 +282,6 @@ sounds = [
 	# {"name": "chime", "src": "/assets/frappe/sounds/chime.mp3"},
 ]
 
-bot_parsers = [
-	'frappe.utils.bot.ShowNotificationBot',
-	'frappe.utils.bot.GetOpenListBot',
-	'frappe.utils.bot.ListBot',
-	'frappe.utils.bot.FindBot',
-	'frappe.utils.bot.CountBot'
-]
-
 setup_wizard_exception = [
 	"frappe.desk.page.setup_wizard.setup_wizard.email_setup_wizard_exception",
 	"frappe.desk.page.setup_wizard.setup_wizard.log_setup_wizard_exception"

frappe/installer.py

@@ -142,8 +142,10 @@ def find_org(org_repo: str) -> Tuple[str, str]:
 	import requests
 
 	for org in ["frappe", "erpnext"]:
-		res = requests.head(f"https://api.github.com/repos/{org}/{org_repo}")
-		if res.ok:
+		response = requests.head(f"https://api.github.com/repos/{org}/{org_repo}")
+		if response.status_code == 400:
+			response = requests.head(f"https://github.com/{org}/{org_repo}")
+		if response.ok:
 			return org, org_repo
 
 	raise InvalidRemoteException

frappe/model/base_document.py

@@ -132,32 +132,30 @@ class BaseDocument(object):
 	def get_db_value(self, key):
 		return frappe.db.get_value(self.doctype, self.name, key)
 
-	def get(self, key=None, filters=None, limit=None, default=None):
-		if key:
-			if isinstance(key, dict):
-				return _filter(self.get_all_children(), key, limit=limit)
-			if filters:
-				if isinstance(filters, dict):
-					value = _filter(self.__dict__.get(key, []), filters, limit=limit)
-				else:
-					default = filters
-					filters = None
-					value = self.__dict__.get(key, default)
+	def get(self, key, filters=None, limit=None, default=None):
+		if isinstance(key, dict):
+			return _filter(self.get_all_children(), key, limit=limit)
+
+		if filters:
+			if isinstance(filters, dict):
+				value = _filter(self.__dict__.get(key, []), filters, limit=limit)
 			else:
+				default = filters
+				filters = None
 				value = self.__dict__.get(key, default)
+		else:
+			value = self.__dict__.get(key, default)
 
-			if value is None and key in (
-				d.fieldname for d in self.meta.get_table_fields()
-			):
-				value = []
-				self.set(key, value)
+		if value is None and key in (
+			d.fieldname for d in self.meta.get_table_fields()
+		):
+			value = []
+			self.set(key, value)
 
-			if limit and isinstance(value, (list, tuple)) and len(value) > limit:
-				value = value[:limit]
+		if limit and isinstance(value, (list, tuple)) and len(value) > limit:
+			value = value[:limit]
 
-			return value
-		else:
-			return self.__dict__
+		return value
 
 	def getone(self, key, filters=None):
 		return self.get(key, filters=filters, limit=1)[0]
@@ -817,6 +815,13 @@ class BaseDocument(object):
 			elif language == "PythonExpression":
 				frappe.utils.validate_python_code(code_string, fieldname=field.label)
 
+	def _sync_autoname_field(self):
+		"""Keep autoname field in sync with `name`"""
+		autoname = self.meta.autoname or ""
+		_empty, _field_specifier, fieldname = autoname.partition("field:")
+
+		if fieldname and self.name and self.name != self.get("fieldname"):
+			self.set(fieldname, self.name)
 
 	def throw_length_exceeded_error(self, df, max_length, value):
 		# check if parentfield exists (only applicable for child table doctype)

frappe/model/document.py

@@ -88,35 +88,27 @@ class Document(BaseDocument):
 		If DocType name and document name are passed, the object will load
 		all values (including child documents) from the database.
 		"""
-		self.doctype = self.name = None
-		self._default_new_docs = {}
+		self.doctype = None
+		self.name = None
 		self.flags = frappe._dict()
 
-		if args and args[0] and isinstance(args[0], str):
-			# first arugment is doctype
-			if len(args)==1:
-				# single
-				self.doctype = self.name = args[0]
-			else:
+		if args and args[0]:
+			if isinstance(args[0], str):
+				# first arugment is doctype
 				self.doctype = args[0]
-				if isinstance(args[1], dict):
-					# filter
-					self.name = frappe.db.get_value(args[0], args[1], "name")
-					if self.name is None:
-						frappe.throw(_("{0} {1} not found").format(_(args[0]), args[1]),
-							frappe.DoesNotExistError)
-				else:
-					self.name = args[1]
 
-				if 'for_update' in kwargs:
-					self.flags.for_update = kwargs.get('for_update')
+				# doctype for singles, string value or filters for other documents
+				self.name = self.doctype if len(args) == 1 else args[1]
 
-			self.load_from_db()
-			return
+				# for_update is set in flags to avoid changing load_from_db signature
+				# since it is used in virtual doctypes and inherited in child classes
+				self.flags.for_update = kwargs.get("for_update")
+				self.load_from_db()
+				return
 
-		if args and args[0] and isinstance(args[0], dict):
-			# first argument is a dict
-			kwargs = args[0]
+			if isinstance(args[0], dict):
+				# first argument is a dict
+				kwargs = args[0]
 
 		if kwargs:
 			# init base document
@@ -133,17 +125,15 @@ class Document(BaseDocument):
 		frappe.whitelist()(fn)
 		return fn
 
-	def reload(self):
-		"""Reload document from database"""
-		self.load_from_db()
-
 	def load_from_db(self):
 		"""Load document and children from database and create properties
 		from fields"""
 		if not getattr(self, "_metaclass", False) and self.meta.issingle:
-			single_doc = frappe.db.get_singles_dict(self.doctype)
+			single_doc = frappe.db.get_singles_dict(
+				self.doctype, for_update=self.flags.for_update
+			)
 			if not single_doc:
-				single_doc = frappe.new_doc(self.doctype).as_dict()
+				single_doc = frappe.new_doc(self.doctype, as_dict=True)
 				single_doc["name"] = self.doctype
 				del single_doc["__islocal"]
 
@@ -177,6 +167,8 @@ class Document(BaseDocument):
 		if hasattr(self, "__setup__"):
 			self.__setup__()
 
+	reload = load_from_db
+
 	def get_latest(self):
 		if not getattr(self, "latest", None):
 			self.latest = frappe.get_doc(self.doctype, self.name)
@@ -500,6 +492,7 @@ class Document(BaseDocument):
 		self._validate_non_negative()
 		self._validate_length()
 		self._validate_code_fields()
+		self._sync_autoname_field()
 		self._extract_images_from_text_editor()
 		self._sanitize_content()
 		self._save_passwords()
@@ -848,16 +841,19 @@ class Document(BaseDocument):
 				frappe.CancelledLinkError)
 
 	def get_all_children(self, parenttype=None):
-		"""Returns all children documents from **Table** type field in a list."""
-		ret = []
-		for df in self.meta.get("fields", {"fieldtype": ['in', table_fields]}):
-			if parenttype:
-				if df.options==parenttype:
-					return self.get(df.fieldname)
+		"""Returns all children documents from **Table** type fields in a list."""
+
+		children = []
+
+		for df in self.meta.get_table_fields():
+			if parenttype and df.options != parenttype:
+				continue
+
 			value = self.get(df.fieldname)
 			if isinstance(value, list):
-				ret.extend(value)
-		return ret
+				children.extend(value)
+
+		return children
 
 	def run_method(self, method, *args, **kwargs):
 		"""run standard triggers, plus those in hooks"""
@@ -1375,11 +1371,9 @@ class Document(BaseDocument):
 		doctype = self.__class__.__name__
 
 		docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
-		repr_str = f"<{doctype}: {name}{docstatus}"
+		parent = f" parent={self.parent}" if getattr(self, "parent", None) else ""
 
-		if not hasattr(self, "parent"):
-			return repr_str + ">"
-		return f"{repr_str} parent={self.parent}>"
+		return f"<{doctype}: {name}{docstatus}{parent}>"
 
 	def __str__(self):
 		name = self.name or "unsaved"

frappe/printing/page/print/print.js

@@ -364,7 +364,11 @@ frappe.ui.form.PrintView = class {
 		let doc_letterhead = this.frm.doc.letter_head;
 
 		return frappe.db
-			.get_list('Letter Head', { fields: ['name', 'is_default'], limit: 0 })
+			.get_list('Letter Head', {
+				filters: {'disabled': 0},
+				fields: ['name', 'is_default'],
+				limit: 0
+			})
 			.then((letterheads) => {
 				letterheads.map((letterhead) => {
 					if (letterhead.is_default) default_letterhead = letterhead.name;

frappe/public/js/frappe/ui/page.js

@@ -850,9 +850,10 @@ frappe.ui.Page = class Page {
 	}
 	get_form_values() {
 		var values = {};
-		this.page_form.fields_dict.forEach(function(field, key) {
-			values[key] = field.get_value();
-		});
+		for (let fieldname in this.fields_dict) {
+			let field = this.fields_dict[fieldname];
+			values[fieldname] = field.get_value();
+		}
 		return values;
 	}
 	add_view(name, html) {

frappe/public/scss/desk/page.scss

@@ -53,6 +53,7 @@
 
 .custom-actions {
 	display: flex;
+	align-items: center;
 }
 
 .page-actions {

frappe/public/scss/website/index.scss

@@ -88,6 +88,20 @@
 	border-radius: $dropdown-border-radius;
 }
 
+.dropdown-item:active {
+	color: var(--fg-color);
+	text-decoration: none;
+	background-color: var(--gray-600);
+}
+
+.dropdown-item:active:hover {
+	color: var(--fg-color);
+}
+
+.dropdown-menu a:hover {
+	cursor: pointer;
+}
+
 .input-dark {
 	background-color: $dark;
 	border-color: darken($primary, 40%);

frappe/public/scss/website/page_builder.scss

@@ -479,6 +479,12 @@
 	align-items: center;
 }
 
+.collapsible-item-title {
+	font-weight: 600;
+	color: var(--text-color);
+	font-size: var(--text-2xl);
+}
+
 .collapsible-item a {
 	text-decoration: none;
 }
@@ -516,6 +522,7 @@
 	.section-description, .collapsible-items {
 		margin-left: auto;
 		margin-right: auto;
+		margin-top: 3rem;
 	}
 }
 

frappe/query_builder/functions.py

@@ -43,6 +43,10 @@ CombineDatetime = ImportMapper(
 	}
 )
 
+DateFormat = ImportMapper({
+	db_type_is.MARIADB: CustomFunction("DATE_FORMAT", ["date", "format"]),
+	db_type_is.POSTGRES: ToChar,
+})
 
 class Cast_(Function):
 	def __init__(self, value, as_type, alias=None):

frappe/query_builder/terms.py

@@ -1,10 +1,12 @@
 from datetime import timedelta
 from typing import Any, Dict, Optional
-from frappe.utils.data import format_timedelta
 
-from pypika.terms import Function, ValueWrapper
+from pypika.queries import QueryBuilder
+from pypika.terms import Criterion, Function, ValueWrapper
 from pypika.utils import format_alias_sql
 
+from frappe.utils.data import format_timedelta
+
 
 class NamedParameterWrapper:
 	"""Utility class to hold parameter values and keys"""
@@ -100,3 +102,12 @@ class ParameterizedFunction(Function):
 			)
 
 		return function_sql
+
+class subqry(Criterion):
+	def __init__(self, subq: QueryBuilder, alias: Optional[str] = None,) -> None:
+		super().__init__(alias)
+		self.subq = subq
+
+	def get_sql(self, **kwg: Any) -> str:
+		kwg["subquery"] = True
+		return self.subq.get_sql(**kwg)

frappe/templates/discussions/button.html

@@ -1,9 +1,6 @@
 {% if frappe.session.user != "Guest" and
 (condition is not defined or (condition is defined and condition )) %}
-<span class="btn btn-md btn-default reply">
+<span class="btn btn-md btn-secondary-dark reply">
 	{{ _(cta_title) }}
-	<!-- Below svg is not a part of the current design. Hence it is commented.
-		The comment will be removed after all design changes are implemented. -->
-	<!-- <svg class="icon icon-sm ml-1"><use href="#icon-add" style="stroke: var(--gray-700)"></use></svg> -->
 </span>
 {% endif %}

frappe/templates/discussions/comment_box.html

@@ -28,7 +28,7 @@
 		</div>
 
 		<a class="dark-links cancel-comment hide"> {{ _("Cancel") }} </a>
-		<div class="btn btn-md btn-default submit-discussion pull-right mb-1">
+		<div class="btn btn-sm btn-default submit-discussion pull-right mb-1">
 			{{ _("Post") }}
 		</div>
 	</div>

frappe/templates/discussions/discussions.js

@@ -4,8 +4,6 @@ frappe.ready(() => {
 
 	add_color_to_avatars();
 
-	expand_first_discussion();
-
 	$(".search-field").keyup((e) => {
 		search_topic(e);
 	});
@@ -14,11 +12,11 @@ frappe.ready(() => {
 		show_new_topic_modal(e);
 	});
 
-	$("#login-from-discussion").click((e) => {
+	$(".login-from-discussion").click((e) => {
 		login_from_discussion(e);
 	});
 
-	$(".sidebar-topic").click((e) => {
+	$(".sidebar-parent").click((e) => {
 		if ($(e.currentTarget).attr("aria-expanded") == "true") {
 			e.stopPropagation();
 		}
@@ -31,17 +29,6 @@ frappe.ready(() => {
 		}
 	});
 
-	$(document).on("input", ".discussion-on-page .comment-field", (e) => {
-		if ($(e.currentTarget).val()) {
-			$(e.currentTarget).css("height", "48px");
-			$(".cancel-comment").removeClass("hide").addClass("show");
-			$(e.currentTarget).css("height", $(e.currentTarget).prop("scrollHeight"));
-		} else {
-			$(".cancel-comment").removeClass("show").addClass("hide");
-			$(e.currentTarget).css("height", "48px");
-		}
-	});
-
 	$(document).on("click", ".submit-discussion", (e) => {
 		submit_discussion(e);
 	});
@@ -50,16 +37,26 @@ frappe.ready(() => {
 		clear_comment_box();
 	});
 
-	if ($(document).width() <= 550) {
-		$(document).on("click", ".sidebar-parent", () => {
-			hide_sidebar();
-		});
-	}
+	$(document).on("click", ".sidebar-parent", () => {
+		hide_sidebar();
+	});
 
-	$(document).on("click", ".back", (e) => {
+	$(document).on("click", ".back-button", (e) => {
 		back_to_sidebar(e);
 	});
 
+	$(document).on("click", ".dismiss-reply", (e) => {
+		dismiss_reply(e);
+	});
+
+	$(document).on("click", ".reply-card .dropdown-menu", (e) => {
+		perform_action(e);
+	});
+
+	$(document).on("input", ".discussion-on-page .comment-field", (e) => {
+		adjust_comment_box(e);
+	});
+
 });
 
 const show_new_topic_modal = (e) => {
@@ -79,10 +76,17 @@ const setup_socket_io = () => {
 		if (window.dev_server) {
 			frappe.boot.socketio_port = "9000";
 		}
+
 		frappe.socketio.init(9000);
 		frappe.socketio.socket.on("publish_message", (data) => {
 			publish_message(data);
 		});
+		frappe.socketio.socket.on("update_message", (data) => {
+			update_message(data);
+		});
+		frappe.socketio.socket.on("delete_message", (data) => {
+			delete_message(data);
+		});
 	});
 };
 
@@ -92,44 +96,47 @@ const publish_message = (data) => {
 	const topic = data.topic_info;
 	const single_thread = $(".is-single-thread").length;
 	const first_topic = !$(".reply-card").length;
-	const document_match_found = doctype == topic.reference_doctype && docname == topic.reference_docname;
+	const document_match_found = (doctype == topic.reference_doctype) && (docname == topic.reference_docname);
+
+	post_message_cleanup();
+	data.template = hide_actions_on_conditions(data.template, data.reply_owner);
+	data.template = style_avatar_frame(data.template);
+	data.sidebar = style_avatar_frame(data.sidebar);
+	data.new_topic_template = style_avatar_frame(data.new_topic_template);
 
 	if ($(`.discussion-on-page[data-topic=${topic.name}]`).length) {
-		post_message_cleanup();
-		data.template = style_avatar_frame(data.template);
-		$('<div class="card-divider-dark mb-8"></div>' + data.template)
-			.insertBefore(`.discussion-on-page[data-topic=${topic.name}] .discussion-form`);
+		$(data.template).insertBefore(`.discussion-on-page[data-topic=${topic.name}] .discussion-form`);
 
 	} else if (!first_topic && !single_thread && document_match_found) {
-		post_message_cleanup();
-		data.new_topic_template = style_avatar_frame(data.new_topic_template);
-
-		$(data.sidebar).insertAfter(`.discussions-sidebar .form-group`);
+		$(data.sidebar).insertBefore($(`.discussions-sidebar .sidebar-parent`).first());
 		$(`#discussion-group`).prepend(data.new_topic_template);
-
 		if (topic.owner == frappe.session.user) {
 			$(".discussion-on-page") && $(".discussion-on-page").collapse();
-			$(".sidebar-topic").first().click();
+			$(".sidebar-parent").first().click();
 		}
 
 	} else if (single_thread && document_match_found) {
-		post_message_cleanup();
-		data.template = style_avatar_frame(data.template);
 		$(data.template).insertBefore(`.discussion-form`);
 		$(".discussion-on-page").attr("data-topic", topic.name);
 
 	} else if (topic.owner == frappe.session.user && document_match_found) {
-		post_message_cleanup();
 		window.location.reload();
 	}
 
 	update_reply_count(topic.name);
 };
 
+const update_message = (data) => {
+	const reply_card = $(`[data-reply=${data.reply_name}]`);
+	reply_card.find(".reply-body").removeClass("hide");
+	reply_card.find(".reply-edit-card").addClass("hide");
+	reply_card.find(".reply-text").html(data.reply);
+	reply_card.find(".reply-actions").addClass("hide");
+};
+
 const post_message_cleanup = () => {
 	$(".topic-title").val("");
-	$(".comment-field").val("");
-	$(".discussion-on-page .comment-field").css("height", "48px");
+	$(".discussion-form .comment-field").val("");
 	$("#discussion-modal").modal("hide");
 	$("#no-discussions").addClass("hide");
 	$(".cancel-comment").addClass("hide");
@@ -141,15 +148,6 @@ const update_reply_count = (topic) => {
 	$(`[data-target='#t${topic}']`).find(".reply-count").text(reply_count);
 };
 
-const expand_first_discussion = () => {
-	if ($(document).width() > 550) {
-		$($(".discussions-parent .collapse")[0]).addClass("show");
-		$($(".discussions-sidebar [data-toggle='collapse']")[0]).attr("aria-expanded", true);
-	} else {
-		$("#discussion-group").addClass("hide");
-	}
-};
-
 const search_topic = (e) => {
 	let input = $(e.currentTarget).val();
 
@@ -160,7 +158,7 @@ const search_topic = (e) => {
 	}
 
 	topics.each((i, elem) => {
-		let topic_id = $(elem).parent().attr("data-target");
+		let topic_id = $(elem).closest(".sidebar-parent").attr("data-target");
 
 		/* Check match in replies */
 		let match_in_reply = false;
@@ -201,16 +199,20 @@ const submit_discussion = (e) => {
 	e.preventDefault();
 	e.stopImmediatePropagation();
 
+	const target = $(e.currentTarget);
+	const reply_name = target.closest(".reply-card").data("reply");
 	const title = $(".topic-title:visible").length ? $(".topic-title:visible").val().trim() : "";
-	const reply = $(".comment-field:visible").val().trim();
+	let reply = reply_name ? target.closest(".reply-card") : target.closest(".discussion-form");
+	reply = reply.find(".comment-field").val().trim();
 
 	if (reply) {
-		let doctype = $(e.currentTarget).closest(".discussions-parent").attr("data-doctype");
+		let doctype = target.closest(".discussions-parent").attr("data-doctype");
 		doctype = doctype ? decodeURIComponent(doctype) : doctype;
 
-		let docname = $(e.currentTarget).closest(".discussions-parent").attr("data-docname");
+		let docname = target.closest(".discussions-parent").attr("data-docname");
 		docname = docname ? decodeURIComponent(docname) : docname;
 
+
 		frappe.call({
 			method: "frappe.website.doctype.discussion_topic.discussion_topic.submit_discussion",
 			args: {
@@ -218,7 +220,8 @@ const submit_discussion = (e) => {
 				"docname": docname ? docname : "",
 				"reply": reply,
 				"title": title,
-				"topic_name": $(e.currentTarget).closest(".discussion-on-page").attr("data-topic")
+				"topic_name": target.closest(".discussion-on-page").attr("data-topic"),
+				"reply_name": reply_name
 			}
 		});
 	}
@@ -252,18 +255,64 @@ const style_avatar_frame = (template) => {
 };
 
 const clear_comment_box = () => {
-	$(".discussion-on-page .comment-field").val("");
+	$(".discussion-form .comment-field").val("");
 	$(".cancel-comment").removeClass("show").addClass("hide");
-	$(".discussion-on-page .comment-field").css("height", "48px");
 };
 
 const hide_sidebar = () => {
 	$(".discussions-sidebar").addClass("hide");
 	$("#discussion-group").removeClass("hide");
+	$(".search-field").addClass("hide");
+	$(".reply").addClass("hide");
 };
 
 const back_to_sidebar = () => {
 	$(".discussions-sidebar").removeClass("hide");
 	$("#discussion-group").addClass("hide");
 	$(".discussion-on-page").collapse("hide");
+	$(".search-field").removeClass("hide");
+	$(".reply").removeClass("hide");
+};
+
+const perform_action = (e) => {
+	const action = $(e.target).data().action;
+	const reply_card = $(e.target).closest(".reply-card");
+
+	if (action === "edit") {
+		reply_card.find(".reply-edit-card").removeClass("hide");
+		reply_card.find(".reply-body").addClass("hide");
+		reply_card.find(".reply-actions").removeClass("hide");
+	} else if (action === "delete") {
+		frappe.call({
+			method: "frappe.website.doctype.discussion_reply.discussion_reply.delete_message",
+			args: {
+				"reply_name": $(e.target).closest(".reply-card").data("reply")
+			}
+		});
+	}
+};
+
+const dismiss_reply = (e) => {
+	const reply_card = $(e.currentTarget).closest(".reply-card");
+	reply_card.find(".reply-edit-card").addClass("hide");
+	reply_card.find(".reply-body").removeClass("hide");
+	reply_card.find(".reply-actions").addClass("hide");
+};
+
+const adjust_comment_box = (e) => {
+	if ($(e.currentTarget).val()) {
+		$(".cancel-comment").removeClass("hide").addClass("show");
+	} else {
+		$(".cancel-comment").removeClass("show").addClass("hide");
+	}
+};
+
+const hide_actions_on_conditions = (template, owner) => {
+	let $template = $(template);
+	frappe.session.user != owner && $template.find(".dropdown").addClass("hide");
+	return $template.prop("outerHTML");
+};
+
+const delete_message = (data) => {
+	$(`[data-reply=${data.reply_name}]`).addClass("hide");
 };

frappe/templates/discussions/discussions_section.html

@@ -9,25 +9,31 @@
 
 	<div class="discussions-header">
 		<span class="discussion-heading">{{ _(title) }}</span>
+		{% if topics | length and not single_thread %}
+		{% include "frappe/templates/discussions/search.html" %}
+		{% endif %}
 		{% if topics and not single_thread %}
 		{% include "frappe/templates/discussions/button.html" %}
 		{% endif %}
 	</div>
 
-	<div class="card-style thread-card {% if topics | length and not single_thread %} discussions-card {% endif %}
-		{% if not topics | length %} empty-state {% endif %}">
+	<div class="">
 		{% if topics and not single_thread %}
 
-		<div class="discussions-sidebar">
-			{% include "frappe/templates/discussions/search.html" %}
+		<div class="discussions-sidebar card-style">
 
 			{% for topic in topics %}
 			{% set replies = frappe.get_all("Discussion Reply", {"topic": topic.name})%}
 			{% include "frappe/templates/discussions/sidebar.html" %}
+
+			{% if loop.index != topics | length %}
+			<div class="card-divider"></div>
+			{% endif %}
+
 			{% endfor %}
 		</div>
 
-		<div class="mr-2" id="discussion-group">
+		<div class="hide" id="discussion-group">
 			{% for topic in topics %}
 			{% include "frappe/templates/discussions/reply_section.html" %}
 			{% endfor %}
@@ -38,19 +44,25 @@
 		{% include "frappe/templates/discussions/reply_section.html" %}
 
 		{% else %}
-		<div class="no-discussions">
-			<img class="icon icon-xl" src="/assets/frappe/icons/timeless/message.svg">
-			<div class="discussion-heading mt-4 mb-0" style="color: inherit;"> {{ empty_state_title }} </div>
-			<div class="small mb-6"> {{ empty_state_subtitle }} </div>
-			{% if frappe.session.user == "Guest" %}
-			<div class="btn btn-default btn-md mt-3" id="login-from-discussion"> {{ _("Login") }} </div>
-			{% elif condition is defined and not condition %}
-			<div class="btn btn-default btn-md mt-3" id="login-from-discussion" data-redirect="{{ redirect_to }}">
-				{{ button_name }}
+		<div class="empty-state">
+			<div>
+			  <img class="icon icon-xl" src="/assets/frappe/icons/timeless/message.svg">
+			</div>
+			<div class="empty-state-text">
+			  <div class="empty-state-heading">{{ empty_state_title }}</div>
+			  <div class="course-meta">{{ empty_state_subtitle }}</div>
+			</div>
+			<div>
+				{% if frappe.session.user == "Guest" %}
+				<div class="btn btn-default btn-md login-from-discussion"> {{ _("Login") }} </div>
+				{% elif condition is defined and not condition %}
+				<div class="btn btn-default btn-md login-from-discussion" data-redirect="{{ redirect_to }}">
+					{{ button_name }}
+				</div>
+				{% else %}
+				{% include "frappe/templates/discussions/button.html" %}
+				{% endif %}
 			</div>
-			{% else %}
-			{% include "frappe/templates/discussions/button.html" %}
-			{% endif %}
 		</div>
 		{% endif %}
 	</div>

frappe/templates/discussions/reply_card.html

@@ -1,14 +1,50 @@
 {% from "frappe/templates/includes/avatar_macro.html" import avatar %}
-<div class="reply-card">
+<div class="reply-card" data-reply="{{ reply.name }}">
 	{% set member = frappe.db.get_value("User", reply.owner, ["name", "full_name", "username"], as_dict=True) %}
-	<div class="d-flex align-items-center small mb-2">
-		{% if loop.index == 1 or single_thread %}
+
+	<div class="reply-header">
 		{{ avatar(reply.owner) }}
-		{% endif %}
-		<a class="button-links {% if loop.index == 1 or single_thread %} ml-2 {% endif %}" {% if get_profile_url %} href="{{ get_profile_url(member.username) }}" {% endif %}>
+		<a class="button-links topic-author ml-4"
+			{% if get_profile_url %} href="{{ get_profile_url(member.username) }}" {% endif %}>
 			{{ member.full_name }}
 		</a>
-		<div class="ml-3 frappe-timestamp" data-timestamp="{{ reply.creation }}"> {{ frappe.utils.pretty_date(reply.creation) }} </div>
+		<div class="ml-2 frappe-timestamp small" data-timestamp="{{ reply.creation }}"> {{ frappe.utils.pretty_date(reply.creation) }} </div>
+		<div class="reply-actions hide">
+			<div class="submit-discussion mr-2"> {{ _("Post") }} </div>
+			<div class="dismiss-reply"> {{ _("Dismiss") }} </div>
+		</div>
+	</div>
+
+	<div class="reply-body">
+		{% if frappe.session.user == reply.owner %}
+		<div class="dropdown">
+			<svg class="icon icon-sm dropdown-toggle" type="button" id="dropdownMenuButton" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+				<use xlink:href="#icon-dot-horizontal"></use>
+			</svg>
+			<ul class="dropdown-menu" aria-labelledby="dropdownMenuButton">
+				<li>
+					<a class="dropdown-item small" data-action="edit"> {{ _("Edit") }} </a>
+				</li>
+				{% if index != 1 %}
+				<li>
+					<a class="dropdown-item small" data-action="delete"> {{ _("Delete") }} </a>
+				</li>
+				{% endif %}
+			</ul>
+		</div>
+		{% endif %}
+		<div class="reply-text">{{ frappe.utils.md_to_html(reply.reply) }}</div>
+	</div>
+
+	<div class="reply-edit-card hide">
+		<div class="form-group">
+			<div class="control-input-wrapper">
+				<div class="control-input">
+					<textarea type="text" autocomplete="off" class="input-with-feedback form-control comment-field"
+						data-fieldtype="Text" data-fieldname="feedback_comments" spellcheck="false">{{ reply.reply }}</textarea>
+				</div>
+			</div>
+		</div>
 	</div>
-	<div class="reply-text">{{ frappe.utils.md_to_html(reply.reply) }}</div>
 </div>
+

frappe/templates/discussions/reply_section.html

@@ -1,43 +1,52 @@
 {% if topic %}
 
 {% set replies = frappe.get_all("Discussion Reply", {"topic": topic.name},
-["reply", "owner", "creation"], order_by="creation")%}
+["reply", "owner", "creation", "name"], order_by="creation")%}
 
 {% endif %}
 
-<div class="collapse discussion-on-page" data-parent="#discussion-group"
+<div class=" {% if not single_thread %} collapse {% endif %} discussion-on-page card-style" data-parent="#discussion-group"
 	{% if topic %} id="t{{ topic.name }}" data-topic="{{ topic.name }}" {% endif %}>
 
-	{% if not single_thread %}
-	<div class="btn btn-md btn-default ellipsis back">
-		{{ _("Back") }}
-	</div>
-	{% endif %}
+	<div class="reply-section-header">
+		{% if not single_thread %}
+		<div class="back-button">
+			<svg class="icon icon-md mr-0">
+				<use class="" href="#icon-left"></use>
+			</svg>
+		</div>
+		{% endif %}
 
-	{% if topic and topic.title %}
-	<div class="discussion-heading p-0">{{ topic.title }}</div>
-	{% endif %}
+		{% if topic and topic.title %}
+		<div class="discussion-heading p-0">{{ topic.title }}</div>
+		{% endif %}
+	</div>
 
 	{% for reply in replies %}
+	{% set index = loop.index %}
 	{% include "frappe/templates/discussions/reply_card.html" %}
-
-	{% if loop.index != replies | length %}
-	<div class="card-divider-dark mb-8"></div>
-	{% endif %}
 	{% endfor %}
 
 	{% if frappe.session.user == "Guest" or (condition is defined and not condition) %}
-	<div class="d-flex flex-column align-items-center small">
-		{{ _("Want to join the discussion?") }}
-		{% if frappe.session.user == "Guest" %}
-		<div class="btn btn-default btn-md mt-3 mb-3" id="login-from-discussion">{{ _("Login") }}</div>
-		{% elif not condition %}
-		<div class="btn btn-default btn-md mt-3 mb-3" id="login-from-discussion" data-redirect="{{ redirect_to }}">{{ button_name }}
+	<div class="empty-state">
+		<div>
+		  <img class="icon icon-xl" src="/assets/frappe/icons/timeless/message.svg">
+		</div>
+		<div class="empty-state-text">
+		  <div class="empty-state-heading">{{ _("Want to discuss?") }}</div>
+		  <div class="course-meta">{{ _("Post it here, our mentors will help you out.") }}</div>
+		</div>
+		<div>
+			{% if frappe.session.user == "Guest" %}
+			<div class="btn btn-default btn-md login-from-discussion"> {{ _("Login") }} </div>
+			{% elif condition is defined and not condition %}
+			<div class="btn btn-default btn-md login-from-discussion" data-redirect="{{ redirect_to }}">
+				{{ button_name }}
+			</div>
+			{% endif %}
 		</div>
-		{% endif %}
 	</div>
 	{% else %}
 	{% include "frappe/templates/discussions/comment_box.html" %}
 	{% endif %}
-
 </div>

frappe/templates/discussions/search.html

@@ -1,9 +1,2 @@
-<div class="form-group">
-	<div class="control-input-wrapper">
-		<div class="control-input">
-			<input type="text" autocomplete="off" class="input-with-feedback form-control search-field"
-				data-fieldtype="Text" data-fieldname="feedback_comments" placeholder="Search {{ title }}"
-				spellcheck="false"></input>
-		</div>
-	</div>
-</div>
+<input type="text" autocomplete="off" class="search-field" data-fieldtype="Text"
+	data-fieldname="feedback_comments" placeholder="Search {{ title }}" spellcheck="false"></input>

frappe/templates/discussions/sidebar.html

@@ -1,19 +1,24 @@
-<div class="sidebar-parent">
-	<div class="sidebar-topic" data-target="#t{{ topic.name }}" data-toggle="collapse" aria-expanded="false">
+{% from "frappe/templates/includes/avatar_macro.html" import avatar %}
+{% set creator = frappe.db.get_value("User", topic.owner, ["name", "username", "full_name", "user_image"], as_dict=True) %}
+
+<div class="sidebar-parent" data-target="#t{{ topic.name }}" data-toggle="collapse" aria-expanded="false">
+	<div class="mr-4">
+		{{ avatar(creator.name, size="avatar-medium") }}
+	</div>
+	<div class="flex-grow-1">
 		<div class="discussion-topic-title">{{ topic.title }}</div>
-		<div class="sidebar-info">
-			{% set creator = frappe.get_doc("User", topic.owner) %}
-			<span class="reply-author ml-0">
-				{{ creator.full_name }}
-			</span>
-			<span class="small d-flex">
-				<span class="mr-2 d-flex align-items-center">
+		<div class="sidebar-topic">
+			<svg class="icon icon-md m-0 mr-2">
+				<use class="" href="#icon-reply"></use>
+			</svg>
+			<div class="topic-author">{{ creator.full_name }}</div>
+			<div class="ml-2 frappe-timestamp small" data-timestamp="{{ topic.creation }}"> {{ frappe.utils.pretty_date(topic.creation) }} </div>
+			<div class="ml-auto">
+				<span class="d-flex align-items-center">
 					<img class="mr-1" src="/assets/frappe/icons/timeless/message.svg">
 					<span class="reply-count">{{ replies | length }}</span>
 				</span>
-				<span> {{ frappe.utils.format_date(topic.creation, "dd MMM YYYY") }} </span>
-			</span>
+			</div>
 		</div>
 	</div>
-	<div class="card-divider"></div>
 </div>

frappe/templates/emails/email_footer.html

@@ -18,7 +18,7 @@
 	<!--unsubscribe link here-->
 
 	<div class="email-pixel">
-		<!--email open check-->
+		<!--email_open_check-->
 	</div>
 
 	<!-- default_mail_footer -->

frappe/templates/styles/discussion_style.css

@@ -1,25 +1,10 @@
-.thread-card {
-	flex-direction: column;
-	padding: 1rem;
-}
-
-.thread-card .form-control {
-	background-color: #FFFFFF;
-	font-size: inherit;
-	color: inherit;
-	padding: 0.75rem 1rem;
-	border-radius: 4px;
-	resize: none;
-}
-
 .modal .comment-field {
 	height: 300px;
 	resize: none;
 }
 
 .discussion-on-page .comment-field {
-	height: 48px;
-	box-shadow: inset 0px 0px 4px rgba(0, 0, 0, 0.2);
+	padding: 1rem;
 }
 
 .modal .cancel-comment {
@@ -31,61 +16,49 @@
 }
 
 .cancel-comment {
-	font-size: 0.75rem;
+	font-size: var(--text-sm);
 	margin-right: 0.5rem;
 	cursor: pointer;
 }
 
-.no-discussions {
-	width: 500px;
-	margin: 0 auto;
-	text-align: center;
-}
-
-.no-discussions .button {
-	margin: auto;
-}
-
 .discussions-header {
 	margin: 2.5rem 0 1.25rem;
+	display: flex;
+	align-items: center;
+}
+
+@media (max-width: 500px) {
+	.discussions-header {
+		flex-direction: column;
+		align-items: inherit;
+	}
 }
 
 .discussions-header .button {
 	float: right;
 }
 
-.discussions-parent .search-field {
-	background-color: #E2E6E9;
+.search-field {
 	background-image: url(/assets/frappe/icons/timeless/search.svg);
 	background-repeat: no-repeat;
 	text-indent: 1.5rem;
-	background-position: 1rem 0.7rem;
-	height: 36px;
-	font-size: 12px;
-	padding: 0.65rem 0.9rem;
-}
-
-.discussions-sidebar {
-	background-color: #F4F5F6;
-	padding: 0.75rem;
-	border-radius: 4px;
+	background-position: 1rem 0.65rem;
+	font-size: var(--text-md);
+	padding: 0.5rem 1rem;
+	border: 1px solid var(--dark-border-color);
+	border-radius: var(--border-radius-md);
+	margin-right: 0.5rem;
 }
 
-@media (max-width: 550px) {
-	.discussions-sidebar {
-		padding: 1rem;
+@media (max-width: 500px) {
+	.search-field {
+		margin: 0.75rem 0;
 	}
 }
 
 .sidebar-topic {
-	padding: 0.75rem;
-	margin: 0.75rem 0;
-	cursor: pointer;
-}
-
-.sidebar-topic[aria-expanded="true"] {
-	background: #FFFFFF;
-	border-radius: 4px;
+	display: flex;
+	align-items: center;
 }
 
 .comment-footer {
@@ -95,23 +68,46 @@
 }
 
 .reply-card {
-	margin-bottom: 2rem;
+	margin-bottom: 1.5rem;
 }
 
-.discussions-parent .collapsing {
-	transition: height 0s;
+.reply-card .dropdown {
+	float: right;
 }
 
 .discussion-topic-title {
-	color: var(--gray-900);
+	color: var(--text-color);
+	font-size: var(--text-lg);
+	font-weight: 600;
+	margin-bottom: 0.5rem;
 }
 
 .discussion-on-page .topic-title {
 	display: none;
 }
 
-.discussions-sidebar .sidebar-parent:last-child .card-divider {
-	display: none;
+.discussion-on-page {
+	flex-direction: column;
+	padding: 1.5rem;
+}
+
+.submit-discussion {
+	cursor: pointer;
+}
+
+.reply-body {
+	background: var(--bg-color);
+	padding: 1rem;
+	border-radius: var(--border-radius);
+	font-size: var(--text-md);
+	color: var(--text-color);
+}
+
+.reply-actions {
+	display: flex;
+	align-items: center;
+	font-size: var(--text-sm);
+	margin-left: auto;
 }
 
 .reply-text h1 {
@@ -130,6 +126,10 @@
 	font-size: 1rem;
 }
 
+.reply-text p {
+	margin-bottom: 0;
+}
+
 .sidebar-info {
 	margin-top: 0.5rem;
 	display: flex;
@@ -139,12 +139,11 @@
 
 .discussion-heading {
 	font-weight: 600;
-	font-size: 22px;
+	font-size: var(--text-3xl);
 	line-height: 146%;
 	letter-spacing: -0.0175em;
-	color: var(--gray-900);
-	margin-bottom: 1rem;
-	padding: 0 1rem;
+	color: var(--text-color);
+	flex-grow: 1;
 }
 
 .card-style {
@@ -152,7 +151,7 @@
 	background: white;
 	border-radius: 8px;
 	position: relative;
-	border: 1px solid var(--gray-200);
+	box-shadow: var(--shadow-sm);
 }
 
 .discussions-card {
@@ -179,48 +178,93 @@
 	}
 }
 
-@media (max-width: 550px) {
-	.back {
-		margin-top: 0.5rem;
-		margin-bottom: 1rem;
+.back-button {
+	margin-right: 1rem;
+	cursor: pointer;
+}
+
+.reply-author {
+	display: flex;
+	align-items: center;
+	margin: 0px 8px;
+	font-size: var(--text-sm);
+	line-height: 135%;
+	color: var(--text-color);
+}
+
+.discussions-header .btn {
+	float: right;
+}
+
+.empty-state {
+	background: var(--control-bg);
+	border-radius: var(--border-radius-lg);
+	padding: 2rem;
+	display: flex;
+	justify-content: space-between;
+	align-items: center;
+}
+
+.empty-state-text {
+	flex: 1;
+	margin-left: 1.25rem;
+}
+
+.empty-state-heading {
+	font-size: var(--text-xl);
+	color: var(--text-color);
+	font-weight: 600;
+}
+
+.sidebar-parent {
+	display: flex;
+	align-items: center;
+	padding: 1.25rem;
+	cursor: pointer;
+}
+
+@media (max-width: 500px) {
+	.sidebar-parent {
+		padding: 0.5rem;
 	}
 }
 
-@media (min-width: 550px) {
-	.back {
-		display: none;
+@media (max-width: 400px) {
+	.sidebar-parent {
+		font-size: var(--text-sm);
 	}
 }
 
-.reply-author {
-	margin: 0px 8px;
-	font-size: 12px;
-	line-height: 135%;
-	color: var(--gray-900);
+.topic-author {
+	color: var(--text-light);
+	font-weight: 500;
 }
 
-.card-divider {
-	border-top: 1px solid var(--gray-200);
-	margin-bottom: 1rem;
+.reply-section-header {
+	display: flex;
+	align-items: center;
+	margin-bottom: 2.5rem;
 }
 
-.card-divider-dark {
-	border-top: 1px solid var(--gray-300);
+.reply-header {
+	display: flex;
+	align-items: center;
 	margin-bottom: 1rem;
 }
 
-.empty-state {
-	background: var(--gray-200);
-	border: 1px dashed var(--gray-400);
-	box-sizing: border-box;
-	border-radius: 8px;
-	padding: 2.5rem;
+.dismiss-reply {
+	cursor: pointer;
 }
 
-.discussions-parent .btn-default {
-	color: var(--gray-700);
+.discussions-sidebar {
+	flex-direction: column;
 }
 
-.discussions-header .btn {
-	float: right;
+.card-divider {
+	border-top: 1px solid var(--dark-border-color);
+	margin-bottom: 0;
+}
+
+.reply-body .dropdown-menu {
+	min-width: 7rem;
 }

frappe/tests/test_api.py

@@ -83,17 +83,17 @@ class FrappeAPITestCase(unittest.TestCase):
 
 		return self._sid
 
-	def get(self, path: str, params: Optional[Dict] = None) -> TestResponse:
-		return make_request(target=self.TEST_CLIENT.get, args=(path, ), kwargs={"data": params})
+	def get(self, path: str, params: Optional[Dict] = None, **kwargs) -> TestResponse:
+		return make_request(target=self.TEST_CLIENT.get, args=(path, ), kwargs={"data": params, **kwargs})
 
-	def post(self, path, data) -> TestResponse:
-		return make_request(target=self.TEST_CLIENT.post, args=(path, ), kwargs={"data": data})
+	def post(self, path, data, **kwargs) -> TestResponse:
+		return make_request(target=self.TEST_CLIENT.post, args=(path, ), kwargs={"data": data, **kwargs})
 
-	def put(self, path, data) -> TestResponse:
-		return make_request(target=self.TEST_CLIENT.put, args=(path, ), kwargs={"data": data})
+	def put(self, path, data, **kwargs) -> TestResponse:
+		return make_request(target=self.TEST_CLIENT.put, args=(path, ), kwargs={"data": data, **kwargs})
 
-	def delete(self, path) -> TestResponse:
-		return make_request(target=self.TEST_CLIENT.delete, args=(path, ))
+	def delete(self, path, **kwargs) -> TestResponse:
+		return make_request(target=self.TEST_CLIENT.delete, args=(path, ), kwargs=kwargs)
 
 
 class TestResourceAPI(FrappeAPITestCase):

frappe/tests/test_background_jobs.py

@@ -28,6 +28,22 @@ class TestBackgroundJobs(unittest.TestCase):
 				fail_registry = queue.failed_job_registry
 				self.assertEqual(fail_registry.count, 0)
 
+	def test_enqueue_at_front(self):
+		kwargs = {
+			"method": "frappe.handler.ping",
+			"queue": "short",
+		}
+
+		# give worker something to work on first so that get_position doesn't return None
+		frappe.enqueue(**kwargs)
+
+		# test enqueue with at_front=True
+		low_priority_job = frappe.enqueue(**kwargs)
+		high_priority_job = frappe.enqueue(**kwargs, at_front=True)
+
+		# lesser is earlier
+		self.assertTrue(high_priority_job.get_position() < low_priority_job.get_position())
+
 
 def fail_function():
 	return 1 / 0

frappe/tests/test_email.py

@@ -27,7 +27,7 @@ class TestEmail(unittest.TestCase):
 		self.assertTrue('test@example.com' in queue_recipients)
 		self.assertTrue('test1@example.com' in queue_recipients)
 		self.assertEqual(len(queue_recipients), 2)
-		self.assertTrue('<!--unsubscribe url-->' in email_queue[0]['message'])
+		self.assertTrue('<!--unsubscribe_url-->' in email_queue[0]['message'])
 
 	def test_send_after(self):
 		self.test_email_queue(send_after=1)

frappe/tests/test_global_search.py

@@ -1,14 +1,15 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 import unittest
+
 import frappe
 
-from frappe.utils import global_search
-from frappe.test_runner import make_test_objects
+from frappe.custom.doctype.property_setter.property_setter import make_property_setter
 from frappe.desk.page.setup_wizard.install_fixtures import update_global_search_doctypes
+from frappe.utils import global_search, now_datetime
+from frappe.test_runner import make_test_objects
 
-import frappe.utils
 
 class TestGlobalSearch(unittest.TestCase):
 	def setUp(self):
@@ -17,7 +18,6 @@ class TestGlobalSearch(unittest.TestCase):
 		self.assertTrue('__global_search' in frappe.db.get_tables())
 		doctype = "Event"
 		global_search.reset()
-		from frappe.custom.doctype.property_setter.property_setter import make_property_setter
 		make_property_setter(doctype, "subject", "in_global_search", 1, "Int")
 		make_property_setter(doctype, "event_type", "in_global_search", 1, "Int")
 		make_property_setter(doctype, "roles", "in_global_search", 1, "Int")
@@ -42,12 +42,11 @@ class TestGlobalSearch(unittest.TestCase):
 				doctype='Event',
 				subject=text,
 				repeat_on='Monthly',
-				starts_on=frappe.utils.now_datetime())).insert()
+				starts_on=now_datetime())).insert()
 
 		global_search.sync_global_search()
 		frappe.db.commit()
 
-
 	def test_search(self):
 		self.insert_test_events()
 		results = global_search.search('awakens')
@@ -75,7 +74,6 @@ class TestGlobalSearch(unittest.TestCase):
 		results = global_search.search('Monthly')
 		self.assertEqual(len(results), 0)
 		doctype = "Event"
-		from frappe.custom.doctype.property_setter.property_setter import make_property_setter
 		make_property_setter(doctype, "repeat_on", "in_global_search", 1, "Int")
 		global_search.rebuild_for_doctype(doctype)
 		results = global_search.search('Monthly')
@@ -91,6 +89,7 @@ class TestGlobalSearch(unittest.TestCase):
 
 		frappe.delete_doc('Event', event_name)
 		global_search.sync_global_search()
+		frappe.db.commit()
 
 		results = global_search.search(test_subject)
 		self.assertTrue(all(r["name"] != event_name for r in results), msg="Deleted documents appearing in global search.")
@@ -111,7 +110,7 @@ class TestGlobalSearch(unittest.TestCase):
 			doc = frappe.get_doc({
 				'doctype':'Event',
 				'subject': text,
-				'starts_on': frappe.utils.now_datetime()
+				'starts_on': now_datetime()
 			})
 			doc.insert()
 
@@ -172,7 +171,7 @@ class TestGlobalSearch(unittest.TestCase):
 			doc = frappe.get_doc({
 				'doctype':'Event',
 				'subject': 'Lorem Ipsum',
-				'starts_on': frappe.utils.now_datetime(),
+				'starts_on': now_datetime(),
 				'description': case["data"]
 			})
 

frappe/tests/test_goal.py

@@ -1,33 +1,50 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import unittest
 import frappe
-
-from frappe.utils.goal import get_monthly_results, get_monthly_goal_graph_data
 from frappe.test_runner import make_test_objects
-import frappe.utils
+from frappe.utils import format_date, today
+from frappe.utils.goal import get_monthly_goal_graph_data, get_monthly_results
+from frappe.tests.utils import FrappeTestCase
+
 
-class TestGoal(unittest.TestCase):
+class TestGoal(FrappeTestCase):
 	def setUp(self):
-		make_test_objects('Event', reset=True)
+		make_test_objects("Event", reset=True)
 
 	def tearDown(self):
 		frappe.db.delete("Event")
-		# make_test_objects('Event', reset=True)
-		frappe.db.commit()
 
 	def test_get_monthly_results(self):
-		'''Test monthly aggregation values of a field'''
-		result_dict = get_monthly_results('Event', 'subject', 'creation', "event_type='Private'", 'count')
+		"""Test monthly aggregation values of a field"""
+		result_dict = get_monthly_results(
+			"Event",
+			"subject",
+			"creation",
+			filters={"event_type": "Private"},
+			aggregation="count",
+		)
 
-		from frappe.utils import today, formatdate
-		self.assertEqual(result_dict.get(formatdate(today(), "MM-yyyy")), 2)
+		self.assertEqual(result_dict.get(format_date(today(), "MM-yyyy")), 2)
 
 	def test_get_monthly_goal_graph_data(self):
-		'''Test for accurate values in graph data (based on test_get_monthly_results)'''
-		docname = frappe.get_list('Event', filters = {"subject": ["=", "_Test Event 1"]})[0]["name"]
-		frappe.db.set_value('Event', docname, 'description', 1)
-		data = get_monthly_goal_graph_data('Test', 'Event', docname, 'description', 'description', 'description',
-			'Event', '', 'description', 'creation', "starts_on = '2014-01-01'", 'count')
-		self.assertEqual(float(data['data']['datasets'][0]['values'][-1]), 1)
+		"""Test for accurate values in graph data (based on test_get_monthly_results)"""
+		docname = frappe.get_list("Event", filters={"subject": ["=", "_Test Event 1"]})[
+			0
+		]["name"]
+		frappe.db.set_value("Event", docname, "description", 1)
+		data = get_monthly_goal_graph_data(
+			"Test",
+			"Event",
+			docname,
+			"description",
+			"description",
+			"description",
+			"Event",
+			"",
+			"description",
+			"creation",
+			filters={"starts_on": "2014-01-01"},
+			aggregation="count",
+		)
+		self.assertEqual(float(data["data"]["datasets"][0]["values"][-1]), 1)

frappe/tests/test_naming.py

@@ -35,6 +35,17 @@ class TestNaming(unittest.TestCase):
 		title2 = append_number_if_name_exists('Note', 'Test', 'title', '_')
 		self.assertEqual(title2, 'Test_1')
 
+	def test_field_autoname_name_sync(self):
+
+		country = frappe.get_last_doc("Country")
+		original_name = country.name
+		country.country_name = "Not a country"
+		country.save()
+		country.reload()
+
+		self.assertEqual(country.name, original_name)
+		self.assertEqual(country.name, country.country_name)
+
 	def test_format_autoname(self):
 		'''
 		Test if braced params are replaced in format autoname

frappe/tests/ui_test_helpers.py

@@ -203,39 +203,40 @@ def create_data_for_discussions():
 
 def create_web_page(title, route, single_thread):
 	web_page = frappe.db.exists("Web Page", {"route": route})
-	if not web_page:
-		web_page = frappe.get_doc({
+	if web_page:
+		return web_page
+	web_page = frappe.get_doc({
 			"doctype": "Web Page",
 			"title": title,
 			"route": route,
 			"published": True
 		})
-		web_page.save()
-
-		web_page.append("page_blocks", {
-			"web_template": "Discussions",
-			"web_template_values": frappe.as_json({
-				"title": "Discussions",
-				"cta_title": "New Discussion",
-				"docname": web_page.name,
-				"single_thread": single_thread
-			})
+	web_page.save()
+
+	web_page.append("page_blocks", {
+		"web_template": "Discussions",
+		"web_template_values": frappe.as_json({
+			"title": "Discussions",
+			"cta_title": "New Discussion",
+			"docname": web_page.name,
+			"single_thread": single_thread
 		})
-		web_page.save()
+	})
+	web_page.save()
 
-	return web_page
+	return web_page.name
 
 def create_topic_and_reply(web_page):
 	topic = frappe.db.exists("Discussion Topic",{
 		"reference_doctype": "Web Page",
-		"reference_docname": web_page.name
+		"reference_docname": web_page
 	})
 
 	if not topic:
 		topic = frappe.get_doc({
 			"doctype": "Discussion Topic",
 			"reference_doctype": "Web Page",
-			"reference_docname": web_page.name,
+			"reference_docname": web_page,
 			"title": "Test Topic"
 		})
 		topic.save()
@@ -274,7 +275,6 @@ def update_child_table(name):
 
 		doc.save()
 
-
 @frappe.whitelist()
 def insert_doctype_with_child_table_record(name):
 	if frappe.db.get_all(name, {'title': 'Test Grid Search'}):

frappe/translations/fr.csv

@@ -246,7 +246,7 @@ Start Import,Démarrer l'import,
 State,Etat,
 Stopped,Arrêté,
 Subject,Sujet,
-Submit,Soumettre,
+Submit,Valider,
 Successful,Réussi,
 Summary,Résumé,
 Sunday,Dimanche,
@@ -4714,3 +4714,4 @@ Amend, Nouv. version
 Document has been submitted, Document validé
 Document has been cancelled, Document annulé
 Document is in draft state, Document au statut brouillon
+Copy to Clipboard,Copier vers le presse-papiers

frappe/utils/__init__.py

@@ -791,40 +791,27 @@ def get_build_version():
 		return frappe.utils.random_string(8)
 
 def get_assets_json():
-	if not hasattr(frappe.local, "assets_json"):
-		cache = frappe.cache()
+	def _get_assets():
+		# get merged assets.json and assets-rtl.json
+		assets = frappe.parse_json(frappe.read_file("assets/assets.json"))
 
-		# using .get instead of .get_value to avoid pickle.loads
-		try:
-			if not frappe.conf.developer_mode:
-				assets_json = cache.get("assets_json").decode('utf-8')
-			else:
-				assets_json = None
-		except (UnicodeDecodeError, AttributeError, ConnectionError):
-			assets_json = None
-
-		if not assets_json:
-			# get merged assets.json and assets-rtl.json
-			assets_dict = frappe.parse_json(
-				frappe.read_file("assets/assets.json")
+		if assets_rtl := frappe.read_file("assets/assets-rtl.json"):
+			assets.update(frappe.parse_json(assets_rtl))
+
+		return assets
+
+	if not hasattr(frappe.local, "assets_json"):
+		if not frappe.conf.developer_mode:
+			frappe.local.assets_json = frappe.cache().get_value(
+				"assets_json",
+				_get_assets,
+				shared=True,
 			)
 
-			assets_rtl = frappe.read_file("assets/assets-rtl.json")
-			if assets_rtl:
-				assets_dict.update(
-					frappe.parse_json(assets_rtl)
-				)
-			frappe.local.assets_json = frappe.as_json(assets_dict)
-			# save in cache
-			cache.set_value("assets_json", frappe.local.assets_json,
-				shared=True)
-
-			return assets_dict
 		else:
-			# from cache, decode and send
-			frappe.local.assets_json = frappe.safe_decode(assets_json)
+			frappe.local.assets_json = _get_assets()
 
-	return frappe.parse_json(frappe.local.assets_json)
+	return frappe.local.assets_json
 
 
 def get_bench_relative_path(file_path):

frappe/utils/background_jobs.py

@@ -40,8 +40,19 @@ def get_queues_timeout():
 
 redis_connection = None
 
-def enqueue(method, queue='default', timeout=None, event=None,
-	is_async=True, job_name=None, now=False, enqueue_after_commit=False, **kwargs):
+def enqueue(
+	method,
+	queue='default',
+	timeout=None,
+	event=None,
+	is_async=True,
+	job_name=None,
+	now=False,
+	enqueue_after_commit=False,
+	*,
+	at_front=False,
+	**kwargs
+):
 	'''
 		Enqueue method to be executed using a background worker
 
@@ -87,9 +98,8 @@ def enqueue(method, queue='default', timeout=None, event=None,
 			"queue_args":queue_args
 		})
 		return frappe.flags.enqueue_after_commit
-	else:
-		return q.enqueue_call(execute_job, timeout=timeout,
-			kwargs=queue_args)
+
+	return q.enqueue_call(execute_job, timeout=timeout, kwargs=queue_args, at_front=at_front)
 
 def enqueue_doc(doctype, name=None, method=None, queue='default', timeout=300,
 	now=False, **kwargs):
@@ -224,9 +234,12 @@ def get_queue_list(queue_list=None, build_queue_name=False):
 		queue_list = default_queue_list
 	return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list
 
-def get_workers(queue):
-	'''Returns a list of Worker objects tied to a queue object'''
-	return Worker.all(queue=queue)
+def get_workers(queue=None):
+	'''Returns a list of Worker objects tied to a queue object if queue is passed, else returns a list of all workers'''
+	if queue:
+		return Worker.all(queue=queue)
+	else:
+		return Worker.all(get_redis_conn())
 
 def get_running_jobs_in_queue(queue):
 	'''Returns a list of Jobs objects that are tied to a queue object and are currently running'''

frappe/utils/bot.py

@@ -1,224 +0,0 @@
-# Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-
-import frappe, re, frappe.utils
-from frappe.desk.notifications import get_notifications
-from frappe import _
-
-@frappe.whitelist()
-def get_bot_reply(question):
-	return BotReply().get_reply(question)
-
-class BotParser(object):
-	'''Base class for bot parser'''
-	def __init__(self, reply, query):
-		self.query = query
-		self.reply = reply
-		self.tables = reply.tables
-		self.doctype_names = reply.doctype_names
-
-	def has(self, *words):
-		'''return True if any of the words is present int the query'''
-		for word in words:
-			if re.search(r'\b{0}\b'.format(word), self.query):
-				return True
-
-	def startswith(self, *words):
-		'''return True if the query starts with any of the given words'''
-		for w in words:
-			if self.query.startswith(w):
-				return True
-
-	def strip_words(self, query, *words):
-		'''Remove the given words from the query'''
-		for word in words:
-			query = re.sub(r'\b{0}\b'.format(word), '', query)
-
-		return query.strip()
-
-	def format_list(self, data):
-		'''Format list as markdown'''
-		return _('I found these:') + ' ' + ', '.join(' [{title}](/app/Form/{doctype}/{name})'.format(
-			title = d.title or d.name,
-			doctype=self.get_doctype(),
-			name=d.name) for d in data)
-
-	def get_doctype(self):
-		'''returns the doctype name from self.tables'''
-		return self.doctype_names[self.tables[0]]
-
-class ShowNotificationBot(BotParser):
-	'''Show open notifications'''
-	def get_reply(self):
-		if self.has("whatsup", "what's up", "wassup", "whats up", 'notifications', 'open tasks'):
-			n = get_notifications()
-			open_items = sorted(n.get('open_count_doctype').items())
-
-			if open_items:
-				return ("Following items need your attention:\n\n"
-					+ "\n\n".join("{0} [{1}](/app/List/{1})".format(d[1], d[0])
-						for d in open_items if d[1] > 0))
-			else:
-				return 'Take it easy, nothing urgent needs your attention'
-
-class GetOpenListBot(BotParser):
-	'''Get list of open items'''
-	def get_reply(self):
-		if self.startswith('open', 'show open', 'list open', 'get open'):
-			if self.tables:
-				doctype = self.get_doctype()
-				from frappe.desk.notifications import get_notification_config
-				filters = get_notification_config().get('for_doctype').get(doctype, None)
-				if filters:
-					if isinstance(filters, dict):
-						data = frappe.get_list(doctype, filters=filters)
-					else:
-						data = [{'name':d[0], 'title':d[1]} for d in frappe.get_attr(filters)(as_list=True)]
-
-					return ", ".join('[{title}](/app/Form/{doctype}/{name})'.format(doctype=doctype,
-						name=d.get('name'), title=d.get('title') or d.get('name')) for d in data)
-				else:
-					return _("Can't identify open {0}. Try something else.").format(doctype)
-
-class ListBot(BotParser):
-	def get_reply(self):
-		if self.query.endswith(' ' + _('list')) and self.startswith(_('list')):
-			self.query = _('list') + ' ' + self.query.replace(' ' + _('list'), '')
-		if self.startswith(_('list'), _('show')):
-			like = None
-			if ' ' + _('like') + ' ' in self.query:
-				self.query, like = self.query.split(' ' + _('like') + ' ')
-
-			self.tables = self.reply.identify_tables(self.query.split(None, 1)[1])
-			if self.tables:
-				doctype = self.get_doctype()
-				meta = frappe.get_meta(doctype)
-				fields = ['name']
-				if meta.title_field:
-					fields.append('`{0}` as title'.format(meta.title_field))
-
-				filters = {}
-				if like:
-					filters={
-						meta.title_field or 'name': ('like', '%' + like + '%')
-					}
-				return self.format_list(frappe.get_list(self.get_doctype(), fields=fields, filters=filters))
-
-class CountBot(BotParser):
-	def get_reply(self):
-		if self.startswith('how many'):
-			self.tables = self.reply.identify_tables(self.query.split(None, 1)[1])
-			if self.tables:
-				return str(frappe.db.sql('select count(*) from `tab{0}`'.format(self.get_doctype()))[0][0])
-
-class FindBot(BotParser):
-	def get_reply(self):
-		if self.startswith('find', 'search'):
-			query = self.query.split(None, 1)[1]
-
-			if self.has('from'):
-				text, table = query.split('from')
-
-			if self.has('in'):
-				text, table = query.split('in')
-
-			if table:
-				text = text.strip()
-				self.tables = self.reply.identify_tables(table.strip())
-
-
-				if self.tables:
-					filters = {'name': ('like',  '%{0}%'.format(text))}
-					or_filters = None
-
-					title_field = frappe.get_meta(self.get_doctype()).title_field
-					if title_field and title_field!='name':
-						or_filters = {'title': ('like',  '%{0}%'.format(text))}
-
-					data = frappe.get_list(self.get_doctype(),
-						filters=filters, or_filters=or_filters)
-					if data:
-						return self.format_list(data)
-					else:
-						return _("Could not find {0} in {1}").format(text, self.get_doctype())
-
-				else:
-					self.out = _("Could not identify {0}").format(table)
-			else:
-				self.out = _("You can find things by asking 'find orange in customers'").format(table)
-
-class BotReply(object):
-	'''Build a reply for the bot by calling all parsers'''
-	def __init__(self):
-		self.tables = []
-
-	def get_reply(self, query):
-		self.query = query.lower()
-		self.setup()
-		self.pre_process()
-
-		# basic replies
-		if self.query.split()[0] in ("hello", "hi"):
-			return _("Hello {0}").format(frappe.utils.get_fullname())
-
-		if self.query == "help":
-			return help_text.format(frappe.utils.get_fullname())
-
-		# build using parsers
-		replies = []
-		for parser in frappe.get_hooks('bot_parsers'):
-			reply = None
-			try:
-				reply = frappe.get_attr(parser)(self, query).get_reply()
-			except frappe.PermissionError:
-				reply = _("Oops, you are not allowed to know that")
-
-			if reply:
-				replies.append(reply)
-
-		if replies:
-			return '\n\n'.join(replies)
-
-		if not reply:
-			return _("Don't know, ask 'help'")
-
-	def setup(self):
-		self.setup_tables()
-		self.identify_tables()
-
-	def pre_process(self):
-		if self.query.endswith("?"):
-			self.query = self.query[:-1]
-
-		if self.query in ("todo", "to do"):
-			self.query = "open todo"
-
-	def setup_tables(self):
-		tables = frappe.get_all("DocType", {"istable": 0})
-		self.all_tables = [d.name.lower() for d in tables]
-		self.doctype_names = {d.name.lower():d.name for d in tables}
-
-	def identify_tables(self, query=None):
-		if not query:
-			query = self.query
-		self.tables = []
-		for t in self.all_tables:
-			if t in query or t[:-1] in query:
-				self.tables.append(t)
-
-		return self.tables
-
-
-
-help_text = """Hello {0}, I am a K.I.S.S Bot, not AI, so be kind. I can try answering a few questions like,
-
-- "todo": list my todos
-- "show customers": list customers
-- "show customers like giant": list customer containing giant
-- "locate shirt": find where to find item "shirt"
-- "open issues": find open issues, try "open sales orders"
-- "how many users": count number of users
-- "find asian in sales orders": find sales orders where name or title has "asian"
-
-have fun!
-"""

frappe/utils/data.py

@@ -1645,18 +1645,21 @@ def validate_json_string(string: str) -> None:
 		raise frappe.ValidationError
 
 def get_user_info_for_avatar(user_id: str) -> Dict:
-	user_info = {
-		"email": user_id,
-		"image": "",
-		"name": user_id
-	}
 	try:
-		user_info["email"] = frappe.get_cached_value("User", user_id, "email")
-		user_info["name"] = frappe.get_cached_value("User", user_id, "full_name")
-		user_info["image"] = frappe.get_cached_value("User", user_id, "user_image")
-	except Exception:
-		frappe.local.message_log = []
-	return user_info
+		user = frappe.get_cached_doc("User", user_id)
+		return {
+			"email": user.email,
+			"image": user.user_image,
+			"name": user.full_name
+		}
+
+	except frappe.DoesNotExistError:
+		frappe.clear_last_message()
+		return {
+			"email": user_id,
+			"image": "",
+			"name": user_id
+		}
 
 
 def validate_python_code(string: str, fieldname=None, is_expression: bool = True) -> None:

frappe/utils/error.py

@@ -176,9 +176,13 @@ def collect_error_snapshots():
 
 def clear_old_snapshots():
 	"""Clear snapshots that are older than a month"""
+	from frappe.query_builder import DocType, Interval
+	from frappe.query_builder.functions import Now
 
-	frappe.db.sql("""delete from `tabError Snapshot`
-		where creation < (NOW() - INTERVAL '1' MONTH)""")
+	ErrorSnapshot = DocType("Error Snapshot")
+	frappe.db.delete(ErrorSnapshot, filters=(
+		ErrorSnapshot.creation < (Now() - Interval(months=1))
+	))
 
 	path = get_error_snapshot_path()
 	today = datetime.datetime.now()

frappe/utils/file_manager.py

@@ -6,6 +6,7 @@ import os, base64, re, json
 import hashlib
 import mimetypes
 import io
+from frappe.query_builder.utils import DocType
 from frappe.utils import get_hook_method, get_files_path, random_string, encode, cstr, call_hook_method, cint
 from frappe import _
 from frappe import conf
@@ -176,7 +177,7 @@ def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, d
 
 
 def get_file_data_from_hash(content_hash, is_private=0):
-	for name in frappe.db.sql_list("select name from `tabFile` where content_hash=%s and is_private=%s", (content_hash, is_private)):
+	for name in frappe.get_all("File", {"content_hash": content_hash, "is_private": is_private}, pluck="name"):
 		b = frappe.get_doc('File', name)
 		return {k: b.get(k) for k in frappe.get_hooks()['write_file_keys']}
 	return False
@@ -230,8 +231,7 @@ def write_file(content, fname, is_private=0):
 def remove_all(dt, dn, from_delete=False, delete_permanently=False):
 	"""remove all files in a transaction"""
 	try:
-		for fid in frappe.db.sql_list("""select name from `tabFile` where
-			attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)):
+		for fid in frappe.get_all("File", {"attached_to_doctype": dt, "attached_to_name": dn}, pluck="name"):
 			if from_delete:
 				# If deleting a doc, directly delete files
 				frappe.delete_doc("File", fid, ignore_permissions=True, delete_permanently=delete_permanently)
@@ -319,8 +319,10 @@ def get_file_path(file_name):
 	if '../' in file_name:
 		return
 
-	f = frappe.db.sql("""select file_url from `tabFile`
-		where name=%s or file_name=%s""", (file_name, file_name))
+	File = DocType("File")
+
+	f = frappe.qb.from_(File).where((File.name == file_name) | (File.file_name == file_name)).select(File.file_url).run()
+
 	if f:
 		file_name = f[0][0]
 
@@ -351,7 +353,7 @@ def get_file_name(fname, optional_suffix):
 	# convert to unicode
 	fname = cstr(fname)
 
-	n_records = frappe.db.sql("select name from `tabFile` where file_name=%s", fname)
+	n_records = frappe.get_all("File", {"file_name": fname}, pluck="name")
 	if len(n_records) > 0 or os.path.exists(encode(get_files_path(fname))):
 		f = fname.rsplit('.', 1)
 		if len(f) == 1:

frappe/utils/global_search.py

@@ -355,7 +355,9 @@ def sync_global_search():
 	:return:
 	"""
 	while frappe.cache().llen('global_search_queue') > 0:
-		value = json.loads(frappe.cache().lpop('global_search_queue').decode('utf-8'))
+		# rpop to follow FIFO
+		# Last one should override all previous contents of same document
+		value = json.loads(frappe.cache().rpop('global_search_queue').decode('utf-8'))
 		sync_value(value)
 
 def sync_value_in_queue(value):

frappe/utils/goal.py

@@ -1,157 +1,149 @@
-# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+from typing import Dict, Optional
+
 import frappe
 from frappe import _
+from frappe.query_builder.functions import DateFormat, Function
+from frappe.query_builder.utils import DocType
+from frappe.utils.data import add_to_date, cstr, flt, now_datetime
+from frappe.utils.formatters import format_value
+from contextlib import suppress
+
+def get_monthly_results(
+	goal_doctype: str,
+	goal_field: str,
+	date_col: str,
+	filters: Dict,
+	aggregation: str = "sum",
+) -> Dict:
+	"""Get monthly aggregation values for given field of doctype"""
+
+	Table = DocType(goal_doctype)
+	date_format = "%m-%Y" if frappe.db.db_type != "postgres" else "MM-YYYY"
+
+	return dict(
+		frappe.db.query.build_conditions(table=goal_doctype, filters=filters)
+		.select(
+			DateFormat(Table[date_col], date_format).as_("month_year"),
+			Function(aggregation, goal_field),
+		)
+		.groupby("month_year")
+		.run()
+	)
 
 
-def get_monthly_results(goal_doctype, goal_field, date_col, filter_str, aggregation = 'sum'):
-	'''Get monthly aggregation values for given field of doctype'''
-	# TODO: move to ORM?
-	if(frappe.db.db_type == 'postgres'):
-		month_year_format_query = '''to_char("{}", 'MM-YYYY')'''.format(date_col)
-	else:
-		month_year_format_query = 'date_format(`{}`, "%m-%Y")'.format(date_col)
-
-	conditions = ('where ' + filter_str) if filter_str else ''
-	results = frappe.db.sql('''SELECT {aggregation}(`{goal_field}`) AS {goal_field},
-		{month_year_format_query} AS month_year
-		FROM `{table_name}` {conditions}
-		GROUP BY month_year'''
-		.format(
-			aggregation=aggregation,
-			goal_field=goal_field,
-			month_year_format_query=month_year_format_query,
-			table_name="tab" + goal_doctype,
-			conditions=conditions
-		), as_dict=True)
-
-	month_to_value_dict = {}
-	for d in results:
-		month_to_value_dict[d['month_year']] = d[goal_field]
-
-	return month_to_value_dict
-
 @frappe.whitelist()
-def get_monthly_goal_graph_data(title, doctype, docname, goal_value_field, goal_total_field, goal_history_field,
-	goal_doctype, goal_doctype_link, goal_field, date_field, filter_str, aggregation="sum"):
-	'''
-		Get month-wise graph data for a doctype based on aggregation values of a field in the goal doctype
-
-		:param title: Graph title
-		:param doctype: doctype of graph doc
-		:param docname: of the doc to set the graph in
-		:param goal_value_field: goal field of doctype
-		:param goal_total_field: current month value field of doctype
-		:param goal_history_field: cached history field
-		:param goal_doctype: doctype the goal is based on
-		:param goal_doctype_link: doctype link field in goal_doctype
-		:param goal_field: field from which the goal is calculated
-		:param filter_str: where clause condition
-		:param aggregation: a value like 'count', 'sum', 'avg'
-
-		:return: dict of graph data
-	'''
-
-	from frappe.utils.formatters import format_value
-	import json
-
-	# should have atleast read perm
-	if not frappe.has_permission(goal_doctype):
-		return None
-
-	meta = frappe.get_meta(doctype)
+def get_monthly_goal_graph_data(
+	title: str,
+	doctype: str,
+	docname: str,
+	goal_value_field: str,
+	goal_total_field: str,
+	goal_history_field: str,
+	goal_doctype: str,
+	goal_doctype_link: str,
+	goal_field: str,
+	date_field: str,
+	filter_str: str = None,
+	aggregation: str = "sum",
+	filters: Optional[Dict] = None,
+) -> Dict:
+	"""
+	Get month-wise graph data for a doctype based on aggregation values of a field in the goal doctype
+
+	:param title: Graph title
+	:param doctype: doctype of graph doc
+	:param docname: of the doc to set the graph in
+	:param goal_value_field: goal field of doctype
+	:param goal_total_field: current month value field of doctype
+	:param goal_history_field: cached history field
+	:param goal_doctype: doctype the goal is based on
+	:param goal_doctype_link: doctype link field in goal_doctype
+	:param goal_field: field from which the goal is calculated
+	:param filter_str: [DEPRECATED] where clause condition. Use filters.
+	:param aggregation: a value like 'count', 'sum', 'avg'
+	:param filters: optional filters
+
+	:return: dict of graph data
+	"""
+	if isinstance(filter_str, str):
+		frappe.throw("String filters have been deprecated. Pass Dict filters instead.", exc=DeprecationWarning) # nosemgrep
+
 	doc = frappe.get_doc(doctype, docname)
+	doc.check_permission()
 
+	meta = doc.meta
 	goal = doc.get(goal_value_field)
-	formatted_goal = format_value(goal, meta.get_field(goal_value_field), doc)
+	today_date = now_datetime().date()
 
 	current_month_value = doc.get(goal_total_field)
-	formatted_value = format_value(current_month_value, meta.get_field(goal_total_field), doc)
+	current_month_year = today_date.strftime("%m-%Y")  # eg: "02-2022"
+	formatted_value = format_value(
+		current_month_value, meta.get_field(goal_total_field), doc
+	)
+	history = doc.get(goal_history_field)
 
-	from frappe.utils import today, getdate, formatdate, add_months
-	current_month_year = formatdate(today(), "MM-yyyy")
+	month_to_value_dict = None
+	if history and "{" in cstr(history):
+		with suppress(ValueError):
+			month_to_value_dict = frappe.parse_json(history)
 
-	history = doc.get(goal_history_field)
-	try:
-		month_to_value_dict = json.loads(history) if history and '{' in history else None
-	except ValueError:
-		month_to_value_dict = None
+	if month_to_value_dict is None: # nosemgrep
+		doc_filter = {}
+		with suppress(ValueError):
+			doc_filter = frappe.parse_json(filters or "{}")
+		if doctype != goal_doctype:
+			doc_filter[goal_doctype_link] = docname
 
-	if month_to_value_dict is None:
-		doc_filter = (goal_doctype_link + " = " + frappe.db.escape(docname)) if doctype != goal_doctype else ''
-		if filter_str:
-			doc_filter += ' and ' + filter_str if doc_filter else filter_str
-		month_to_value_dict = get_monthly_results(goal_doctype, goal_field, date_field, doc_filter, aggregation)
+		month_to_value_dict = get_monthly_results(
+			goal_doctype, goal_field, date_field, doc_filter, aggregation
+		)
 
 	month_to_value_dict[current_month_year] = current_month_value
 
-	months = []
-	months_formatted = []
-	values = []
+	month_labels = []
+	dataset_values = []
 	values_formatted = []
-	for i in range(0, 12):
-		date_value = add_months(today(), -i)
-		month_value = formatdate(date_value, "MM-yyyy")
-		month_word = getdate(date_value).strftime('%b %y')
-		month_year = getdate(date_value).strftime('%B') + ', ' + getdate(date_value).strftime('%Y')
-		months.insert(0, month_word)
-		months_formatted.insert(0, month_year)
-		if month_value in month_to_value_dict:
-			val = month_to_value_dict[month_value]
-		else:
-			val = 0
-		values.insert(0, val)
-		values_formatted.insert(0, format_value(val, meta.get_field(goal_total_field), doc))
-
-	y_markers = []
+	y_markers = {}
+
 	summary_values = [
-		{
-			'title': _("This month"),
-			'color': '#ffa00a',
-			'value': formatted_value
-		}
+		{"title": _("This month"), "color": "#ffa00a", "value": formatted_value},
 	]
 
-	if float(goal) > 0:
-		y_markers = [
-			{
-				'label': _("Goal"),
-				'lineType': "dashed",
-				'value': goal
-			},
-		]
+	if flt(goal) > 0:
+		formatted_goal = format_value(goal, meta.get_field(goal_value_field), doc)
 		summary_values += [
+			{"title": _("Goal"), "color": "#5e64ff", "value": formatted_goal},
 			{
-				'title': _("Goal"),
-				'color': '#5e64ff',
-				'value': formatted_goal
+				"title": _("Completed"),
+				"color": "#28a745",
+				"value": f"{int(round(flt(current_month_value) / flt(goal) * 100))}%",
 			},
-			{
-				'title': _("Completed"),
-				'color': '#28a745',
-				'value': str(int(round(float(current_month_value)/float(goal)*100))) + "%"
-			}
 		]
+		y_markers = {
+			"yMarkers": [{"label": _("Goal"), "lineType": "dashed", "value": flt(goal)}]
+		}
 
-	data = {
-		'title': title,
-		# 'subtitle':
-
-		'data': {
-			'datasets': [
-				{
-					'values': values,
-					'formatted': values_formatted
-				}
-			],
-			'labels': months,
+	for i in range(12):
+		date_value = add_to_date(today_date, months=-i, as_datetime=True)
+		month_word = date_value.strftime("%b %y")  # eg: "Feb 22"
+		month_labels.insert(0, month_word)
+
+		month_value = date_value.strftime("%m-%Y")  # eg: "02-2022"
+		val = month_to_value_dict.get(month_value, 0)
+		dataset_values.insert(0, val)
+		values_formatted.insert(
+			0, format_value(val, meta.get_field(goal_total_field), doc)
+		)
+
+	return {
+		"title": title,
+		"data": {
+			"datasets": [{"values": dataset_values, "formatted": values_formatted}],
+			"labels": month_labels,
+			**y_markers,
 		},
-
-		'summary': summary_values,
+		"summary": summary_values,
 	}
-
-	if y_markers:
-		data["data"]["yMarkers"] = y_markers
-
-	return data

frappe/utils/install.py

@@ -34,7 +34,7 @@ def after_install():
 	print_settings.save()
 
 	# all roles to admin
-	frappe.get_doc("User", "Administrator").add_roles(*frappe.db.sql_list("""select name from tabRole"""))
+	frappe.get_doc("User", "Administrator").add_roles(*frappe.get_all("Role", pluck="name"))
 
 	# update admin password
 	update_password("Administrator", get_admin_password())

frappe/utils/nestedset.py

@@ -16,6 +16,9 @@ import frappe
 from frappe import _
 from frappe.model.document import Document
 from frappe.query_builder import DocType, Order
+from frappe.query_builder.functions import Coalesce, Max
+from frappe.query_builder.utils import DocType
+
 
 class NestedSetRecursionError(frappe.ValidationError): pass
 class NestedSetMultipleRootsError(frappe.ValidationError): pass
@@ -51,87 +54,91 @@ def update_add_node(doc, parent, parent_field):
 	"""
 		insert a new node
 	"""
-
 	doctype = doc.doctype
 	name = doc.name
+	Table = DocType(doctype)
 
 	# get the last sibling of the parent
 	if parent:
-		left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s for update"
-			.format(doctype), parent)[0]
+		left, right = frappe.db.get_value(doctype, {"name": parent}, ["lft", "rgt"], for_update=True)
 		validate_loop(doc.doctype, doc.name, left, right)
 	else: # root
-		right = frappe.db.sql("""
-			SELECT COALESCE(MAX(rgt), 0) + 1 FROM `tab{0}`
-			WHERE COALESCE(`{1}`, '') = ''
-		""".format(doctype, parent_field))[0][0]
+		right = frappe.qb.from_(Table).select(
+			Coalesce(Max(Table.rgt), 0) + 1
+		).where(Coalesce(Table[parent_field], "") == "").run(pluck=True)[0]
+
 	right = right or 1
 
 	# update all on the right
-	frappe.db.sql("update `tab{0}` set rgt = rgt+2 where rgt >= %s"
-		.format(doctype), (right,))
-	frappe.db.sql("update `tab{0}` set lft = lft+2 where lft >= %s"
-		.format(doctype), (right,))
+	frappe.qb.update(Table).set(Table.rgt, Table.rgt + 2).where(Table.rgt >= right).run()
+	frappe.qb.update(Table).set(Table.lft, Table.lft + 2).where(Table.lft >= right).run()
 
-	# update index of new node
-	if frappe.db.sql("select * from `tab{0}` where lft=%s or rgt=%s".format(doctype), (right, right+1)):
-		frappe.msgprint(_("Nested set error. Please contact the Administrator."))
-		raise Exception
+	if frappe.qb.from_(Table).select("*").where((Table.lft == right) | (Table.rgt == right + 1)).run():
+		frappe.throw(_("Nested set error. Please contact the Administrator."))
 
-	frappe.db.sql("update `tab{0}` set lft=%s, rgt=%s where name=%s".format(doctype),
-		(right,right+1, name))
+	# update index of new node
+	frappe.qb.update(Table).set(Table.lft, right).set(Table.rgt, right + 1).where(Table.name == name).run()
 	return right
 
 
-def update_move_node(doc, parent_field):
-	parent = doc.get(parent_field)
+def update_move_node(doc: Document, parent_field: str):
+	parent: str = doc.get(parent_field)
+	Table = DocType(doc.doctype)
 
 	if parent:
-		new_parent = frappe.db.sql("""select lft, rgt from `tab{0}`
-			where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0]
+		new_parent = frappe.qb.from_(Table).select(
+			Table.lft, Table.rgt
+		).where(Table.name == parent).for_update().run(as_dict=True)[0]
 
 		validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt)
 
 	# move to dark side
-	frappe.db.sql("""update `tab{0}` set lft = -lft, rgt = -rgt
-		where lft >= %s and rgt <= %s""".format(doc.doctype), (doc.lft, doc.rgt))
+	frappe.qb.update(Table).set(Table.lft, - Table.lft).set(Table.rgt, - Table.rgt).where(
+		(Table.lft >= doc.lft) & (Table.rgt <= doc.rgt)
+	).run()
 
 	# shift left
 	diff = doc.rgt - doc.lft + 1
-	frappe.db.sql("""update `tab{0}` set lft = lft -%s, rgt = rgt - %s
-		where lft > %s""".format(doc.doctype), (diff, diff, doc.rgt))
+	frappe.qb.update(Table).set(Table.lft, Table.lft - diff).set(Table.rgt, Table.rgt - diff).where(
+		Table.lft > doc.rgt
+	).run()
 
 	# shift left rgts of ancestors whose only rgts must shift
-	frappe.db.sql("""update `tab{0}` set rgt = rgt - %s
-		where lft < %s and rgt > %s""".format(doc.doctype), (diff, doc.lft, doc.rgt))
+	frappe.qb.update(Table).set(Table.rgt, Table.rgt - diff).where(
+		(Table.lft < doc.lft) & (Table.rgt > doc.rgt)
+	).run()
 
 	if parent:
-		new_parent = frappe.db.sql("""select lft, rgt from `tab%s`
-			where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0]
+		# re-query value due to computation above
+		new_parent = frappe.qb.from_(Table).select(
+			Table.lft, Table.rgt
+		).where(Table.name == parent).for_update().run(as_dict=True)[0]
 
 		# set parent lft, rgt
-		frappe.db.sql("""update `tab{0}` set rgt = rgt + %s
-			where name = %s""".format(doc.doctype), (diff, parent))
+		frappe.qb.update(Table).set(Table.rgt, Table.rgt + diff).where(Table.name == parent).run()
 
 		# shift right at new parent
-		frappe.db.sql("""update `tab{0}` set lft = lft + %s, rgt = rgt + %s
-			where lft > %s""".format(doc.doctype), (diff, diff, new_parent.rgt))
+		frappe.qb.update(Table).set(Table.lft, Table.lft + diff).set(Table.rgt, Table.rgt + diff).where(
+			Table.lft > new_parent.rgt
+		).run()
 
 		# shift right rgts of ancestors whose only rgts must shift
-		frappe.db.sql("""update `tab{0}` set rgt = rgt + %s
-			where lft < %s and rgt > %s""".format(doc.doctype),
-			(diff, new_parent.lft, new_parent.rgt))
-
+		frappe.qb.update(Table).set(Table.rgt, Table.rgt + diff).where(
+			(Table.lft < new_parent.lft) & (Table.rgt > new_parent.rgt)
+		).run()
 
 		new_diff = new_parent.rgt - doc.lft
 	else:
 		# new root
-		max_rgt = frappe.db.sql("""select max(rgt) from `tab{0}`""".format(doc.doctype))[0][0]
+		max_rgt = frappe.qb.from_(Table).select(Max(Table.rgt)).run(pluck=True)[0]
 		new_diff = max_rgt + 1 - doc.lft
 
 	# bring back from dark side
-	frappe.db.sql("""update `tab{0}` set lft = -lft + %s, rgt = -rgt + %s
-		where lft < 0""".format(doc.doctype), (new_diff, new_diff))
+	frappe.qb.update(Table).set(
+		Table.lft, -Table.lft + new_diff
+	).set(
+		Table.rgt, -Table.rgt + new_diff
+	).where(Table.lft < 0).run()
 
 
 @frappe.whitelist()
@@ -197,10 +204,10 @@ def rebuild_node(doctype, parent, left, parent_field):
 
 def validate_loop(doctype, name, lft, rgt):
 	"""check if item not an ancestor (loop)"""
-	if name in frappe.db.sql_list("""select name from `tab{0}` where lft <= %s and rgt >= %s"""
-		 .format(doctype), (lft, rgt)):
+	if name in frappe.get_all(doctype, filters={"lft": ["<=", lft], "rgt": [">=", rgt]}, pluck="name"):
 		frappe.throw(_("Item cannot be added to its own descendents"), NestedSetRecursionError)
 
+
 class NestedSet(Document):
 	def __setup__(self):
 		if self.meta.get("nsm_parent_field"):
@@ -232,9 +239,7 @@ class NestedSet(Document):
 				raise
 
 	def validate_if_child_exists(self):
-		has_children = frappe.db.sql("""select count(name) from `tab{doctype}`
-			where `{nsm_parent_field}`=%s""".format(doctype=self.doctype, nsm_parent_field=self.nsm_parent_field),
-			(self.name,))[0][0]
+		has_children = frappe.db.count(self.doctype, filters={self.nsm_parent_field: self.name})
 		if has_children:
 			frappe.throw(_("Cannot delete {0} as it has child nodes").format(self.name), NestedSetChildExistsError)
 
@@ -251,8 +256,7 @@ class NestedSet(Document):
 			parent_field = self.nsm_parent_field
 
 		# set old_parent for children
-		frappe.db.sql("update `tab{0}` set old_parent=%s where {1}=%s"
-			.format(self.doctype, parent_field), (newdn, newdn))
+		frappe.db.set_value(self.doctype, {"old_parent": newdn}, {parent_field: newdn}, update_modified=False, for_update=False)
 
 		if merge:
 			rebuild_tree(self.doctype, parent_field)
@@ -269,8 +273,7 @@ class NestedSet(Document):
 
 	def validate_ledger(self, group_identifier="is_group"):
 		if hasattr(self, group_identifier) and not bool(self.get(group_identifier)):
-			if frappe.db.sql("""select name from `tab{0}` where {1}=%s and docstatus!=2"""
-				.format(self.doctype, self.nsm_parent_field), (self.name)):
+			if frappe.get_all(self.doctype, {self.nsm_parent_field: self.name, "docstatus": ("!=", 2)}):
 				frappe.throw(_("{0} {1} cannot be a leaf node as it has children").format(_(self.doctype), self.name))
 
 	def get_ancestors(self):
@@ -291,10 +294,20 @@ class NestedSet(Document):
 
 def get_root_of(doctype):
 	"""Get root element of a DocType with a tree structure"""
-	result = frappe.db.sql("""select t1.name from `tab{0}` t1 where
-		(select count(*) from `tab{1}` t2 where
-			t2.lft < t1.lft and t2.rgt > t1.rgt) = 0
-		and t1.rgt > t1.lft""".format(doctype, doctype))
+	from frappe.query_builder.functions import Count
+	from frappe.query_builder.terms import subqry
+
+	Table = DocType(doctype)
+	t1 = Table.as_("t1")
+	t2 = Table.as_("t2")
+
+	subq = frappe.qb.from_(t2).select(Count("*")).where(
+		(t2.lft < t1.lft) & (t2.rgt > t1.rgt)
+	)
+	result = frappe.qb.from_(t1).select(t1.name).where(
+		(subqry(subq) == 0) & (t1.rgt > t1.lft)
+	).run()
+
 	return result[0][0] if result else None
 
 def get_ancestors_of(doctype, name, order_by="lft desc", limit=None):

frappe/utils/redis_wrapper.py

@@ -138,6 +138,9 @@ class RedisWrapper(redis.Redis):
 	def lpop(self, key):
 		return super(RedisWrapper, self).lpop(self.make_key(key))
 
+	def rpop(self, key):
+		return super(RedisWrapper, self).rpop(self.make_key(key))
+
 	def llen(self, key):
 		return super(RedisWrapper, self).llen(self.make_key(key))
 

frappe/utils/reset_doc.py

@@ -1,151 +0,0 @@
-
-import frappe
-import json, os
-from frappe.modules import scrub, get_module_path, utils
-from frappe.custom.doctype.customize_form.customize_form import doctype_properties, docfield_properties
-from frappe.custom.doctype.property_setter.property_setter import make_property_setter
-from frappe.custom.doctype.custom_field.custom_field import create_custom_field
-from frappe.core.page.permission_manager.permission_manager import get_standard_permissions
-from frappe.permissions import setup_custom_perms
-from urllib.request import urlopen
-
-branch = 'develop'
-
-def reset_all():
-	for doctype in frappe.db.get_all('DocType', dict(custom=0)):
-		print(doctype.name)
-		reset_doc(doctype.name)
-
-def reset_doc(doctype):
-	'''
-		doctype = name of the DocType that you want to reset
-	'''
-	# fetch module name
-	module = frappe.db.get_value('DocType', doctype, 'module')
-	app = utils.get_module_app(module)
-
-	# get path for doctype's json and its equivalent git url
-	doc_path = os.path.join(get_module_path(module), 'doctype', scrub(doctype), scrub(doctype)+'.json')
-	try:
-		git_link = '/'.join(['https://raw.githubusercontent.com/frappe',\
-			app, branch, doc_path.split('apps/'+app)[1]])
-		original_file = urlopen(git_link).read()
-	except:
-		print('Did not find {0} in {1}'.format(doctype, app))
-		return
-
-	# load local and original json objects
-	local_doc = json.loads(open(doc_path, 'r').read())
-	original_doc = json.loads(original_file)
-
-	remove_duplicate_fields(doctype)
-	set_property_setter(doctype, local_doc, original_doc)
-	make_custom_fields(doctype, local_doc, original_doc)
-
-	with open(doc_path, 'w+') as f:
-		f.write(original_file)
-		f.close()
-
-	setup_perms_for(doctype)
-
-	frappe.db.commit()
-
-def remove_duplicate_fields(doctype):
-	for field in frappe.db.sql('''select fieldname, count(1) as cnt from tabDocField where parent=%s group by fieldname having cnt > 1''', doctype):
-		frappe.db.sql('delete from tabDocField where fieldname=%s and parent=%s limit 1', (field[0], doctype))
-		print('removed duplicate {0} in {1}'.format(field[0], doctype))
-
-def set_property_setter(doctype, local_doc, original_doc):
-	''' compare doctype_properties and docfield_properties and create property_setter '''
-
-	# doctype_properties reset
-	for dp in doctype_properties:
-		# make property_setter to mimic changes made in local json
-		if dp in local_doc and dp not in original_doc:
-			make_property_setter(doctype, '', dp, local_doc[dp], doctype_properties[dp], for_doctype=True)
-
-	local_fields = get_fields_dict(local_doc)
-	original_fields = get_fields_dict(original_doc)
-
-	# iterate through field and properties of each of those field
-	for docfield in original_fields:
-		for prop in original_fields[docfield]:
-			# skip fields that are not in local_fields
-			if docfield not in local_fields: continue
-
-			if prop in docfield_properties and prop in local_fields[docfield]\
-				and original_fields[docfield][prop] != local_fields[docfield][prop]:
-				# make property_setter equivalent of local changes
-				make_property_setter(doctype, docfield, prop, local_fields[docfield][prop],\
-					docfield_properties[prop])
-
-def make_custom_fields(doctype, local_doc, original_doc):
-	'''
-		check fields and create a custom field equivalent for non standard fields
-	'''
-	local_fields, original_fields = get_fields_dict(local_doc), get_fields_dict(original_doc)
-	local_fields = sorted(local_fields.items(), key=lambda x: x[1]['idx'])
-	doctype_doc = frappe.get_doc('DocType', doctype)
-
-	custom_docfield_properties, prev = get_custom_docfield_properties(), ""
-	for field, field_dict in local_fields:
-		df = {}
-		if field not in original_fields:
-			for prop in field_dict:
-				if prop in custom_docfield_properties:
-					df[prop] = field_dict[prop]
-
-			df['insert_after'] = prev if prev else ''
-
-			doctype_doc.fields = [d for d in doctype_doc.fields if d.fieldname != df['fieldname']]
-			doctype_doc.update_children()
-
-			create_custom_field(doctype, df)
-
-		# set current field as prev field for next field
-		prev = field
-
-def get_fields_dict(doc):
-	fields, idx = {}, 0
-	for field in doc['fields']:
-		field['idx'] = idx
-		fields[field.get('fieldname')] = field
-		idx += 1
-
-	return fields
-
-def get_custom_docfield_properties():
-	fields_meta = frappe.get_meta('Custom Field').fields
-	fields = {}
-	for d in fields_meta:
-		fields[d.fieldname] = d.fieldtype
-
-	return fields
-
-
-def setup_perms_for(doctype):
-	perms = frappe.get_all('DocPerm', fields='*', filters=dict(parent=doctype), order_by='idx asc')
-	# get default perms
-	try:
-		standard_perms = get_standard_permissions(doctype)
-	except (IOError, KeyError):
-		# no json file, doctype no longer exists!
-		return
-
-	same = True
-	if len(standard_perms) != len(perms):
-		same = False
-
-	else:
-		for i, p in enumerate(perms):
-			standard = standard_perms[i]
-			for fieldname in frappe.get_meta('DocPerm').get_fieldnames_with_value():
-				if p.get(fieldname) != standard.get(fieldname):
-					same = False
-					break
-
-			if not same:
-				break
-
-	if not same:
-		setup_custom_perms(doctype)

frappe/utils/user.py

@@ -1,15 +1,22 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import frappe, json
-from frappe import _dict
+from email.utils import formataddr
+from typing import Dict, List, Optional, TYPE_CHECKING
+
+import frappe
 import frappe.share
-from frappe.utils import cint
+from frappe import _dict
 from frappe.boot import get_allowed_reports
-from frappe.permissions import get_roles, get_valid_perms
 from frappe.core.doctype.domain_settings.domain_settings import get_active_modules
+from frappe.permissions import get_roles, get_valid_perms
 from frappe.query_builder import DocType
 from frappe.query_builder.functions import Concat_ws
+from frappe.query_builder import Order
+
+if TYPE_CHECKING:
+	from frappe.core.doctype.user.user import User
+
 
 class UserPermissions:
 	"""
@@ -64,14 +71,14 @@ class UserPermissions:
 
 	def build_doctype_map(self):
 		"""build map of special doctype properties"""
+		self.doctype_map = {}
 
 		active_domains = frappe.get_active_domains()
+		all_doctypes = frappe.get_all("DocType", fields=["name", "in_create", "module", "istable", "issingle", "read_only", "restrict_to_domain"])
 
-		self.doctype_map = {}
-		for r in frappe.db.sql("""select name, in_create, issingle, istable,
-			read_only, restrict_to_domain, module from tabDocType""", as_dict=1):
-			if (not r.restrict_to_domain) or (r.restrict_to_domain in active_domains):
-				self.doctype_map[r['name']] = r
+		for dt in all_doctypes:
+			if not dt.restrict_to_domain or (dt.restrict_to_domain in active_domains):
+				self.doctype_map[dt["name"]] = dt
 
 	def build_perm_map(self):
 		"""build map of permissions at level 0"""
@@ -150,10 +157,8 @@ class UserPermissions:
 		self.can_write += self.in_create
 		self.can_read += self.can_write
 
-		self.shared = frappe.db.sql_list("""select distinct share_doctype from `tabDocShare`
-			where `user`=%s and `read`=1""", self.name)
+		self.shared = frappe.get_all("DocShare", {"user": self.name, "read": 1}, distinct=True, pluck="share_doctype")
 		self.can_read = list(set(self.can_read + self.shared))
-
 		self.all_read += self.can_read
 
 		for dt in no_list_view_link:
@@ -161,11 +166,12 @@ class UserPermissions:
 				self.can_read.remove(dt)
 
 		if "System Manager" in self.get_roles():
-			docs = frappe.get_all("DocType", {'allow_import': 1})
-			self.can_import += [doc.name for doc in docs]
-
-			customizations = frappe.get_all("Property Setter", fields=['doc_type'], filters={'property': 'allow_import', 'value': "1"})
-			self.can_import += [custom.doc_type for custom in customizations]
+			self.can_import += frappe.get_all("DocType", {'allow_import': 1}, pluck="name")
+			self.can_import += frappe.get_all(
+				"Property Setter",
+				pluck="doc_type",
+				filters={"property": "allow_import", "value": "1"},
+			)
 
 		frappe.cache().hset("can_import", frappe.session.user, self.can_import)
 
@@ -186,10 +192,24 @@ class UserPermissions:
 		return self.can_read
 
 	def load_user(self):
-		d = frappe.db.sql("""select email, first_name, last_name, creation,
-			email_signature, user_type, desk_theme, language,
-			mute_sounds, send_me_a_copy, document_follow_notify
-			from tabUser where name = %s""", (self.name,), as_dict=1)[0]
+		d = frappe.db.get_value(
+			"User",
+			self.name,
+			[
+				"creation",
+				"desk_theme",
+				"document_follow_notify",
+				"email",
+				"email_signature",
+				"first_name",
+				"language",
+				"last_name",
+				"mute_sounds",
+				"send_me_a_copy",
+				"user_type",
+			],
+			as_dict=True,
+		)
 
 		if not self.can_read:
 			self.build_permissions()
@@ -209,142 +229,169 @@ class UserPermissions:
 	def get_all_reports(self):
 		return get_allowed_reports()
 
-def get_user_fullname(user):
+
+def get_user_fullname(user: str) -> str:
 	user_doctype = DocType("User")
-	fullname = frappe.get_value(
-		user_doctype,
-		filters={"name": user},
-		fieldname=Concat_ws(" ", user_doctype.first_name, user_doctype.last_name),
+	return (
+		frappe.get_value(
+			user_doctype,
+			filters={"name": user},
+			fieldname=Concat_ws(" ", user_doctype.first_name, user_doctype.last_name),
+		)
+		or ""
+	)
+
+
+def get_fullname_and_avatar(user: str) -> _dict:
+	first_name, last_name, avatar, name = frappe.db.get_value(
+		"User", user, ["first_name", "last_name", "user_image", "name"]
+	)
+	return _dict(
+		{
+			"fullname": " ".join(list(filter(None, [first_name, last_name]))),
+			"avatar": avatar,
+			"name": name,
+		}
 	)
-	return fullname or ''
-
-def get_fullname_and_avatar(user):
-	first_name, last_name, avatar, name = frappe.db.get_value("User",
-		user, ["first_name", "last_name", "user_image", "name"])
-	return _dict({
-		"fullname": " ".join(list(filter(None, [first_name, last_name]))),
-		"avatar": avatar,
-		"name": name
-	})
-
-def get_system_managers(only_name=False):
+
+
+def get_system_managers(only_name: bool = False) -> List[str]:
 	"""returns all system manager's user details"""
-	import email.utils
-	system_managers = frappe.db.sql("""SELECT DISTINCT `name`, `creation`,
-		CONCAT_WS(' ',
-			CASE WHEN `first_name`= '' THEN NULL ELSE `first_name` END,
-			CASE WHEN `last_name`= '' THEN NULL ELSE `last_name` END
-		) AS fullname
-		FROM `tabUser` AS p
-		WHERE `docstatus` < 2
-		AND `enabled` = 1
-		AND `name` NOT IN ({})
-		AND exists
-			(SELECT *
-				FROM `tabHas Role` AS ur
-				WHERE ur.parent = p.name
-				AND ur.role='System Manager')
-		ORDER BY `creation` DESC""".format(", ".join(["%s"]*len(frappe.STANDARD_USERS))),
-			frappe.STANDARD_USERS, as_dict=True)
+	HasRole = DocType("Has Role")
+	User = DocType("User")
+
+	if only_name:
+		fields = [User.name]
+	else:
+		fields = [User.full_name, User.name]
+
+	system_managers = (
+		frappe.qb.from_(User)
+		.join(HasRole)
+		.on((HasRole.parent == User.name))
+		.where(
+			(HasRole.parenttype == "User")
+			& (User.enabled == 1)
+			& (HasRole.role == "System Manager")
+			& (User.docstatus < 2)
+			& (User.name.notin(frappe.STANDARD_USERS))
+		)
+		.select(*fields)
+		.orderby(User.creation, order=Order.desc)
+		.run(as_dict=True)
+	)
 
 	if only_name:
 		return [p.name for p in system_managers]
 	else:
-		return [email.utils.formataddr((p.fullname, p.name)) for p in system_managers]
+		return [formataddr((p.full_name, p.name)) for p in system_managers]
+
 
-def add_role(user, role):
+def add_role(user: str, role: str) -> None:
 	frappe.get_doc("User", user).add_roles(role)
 
-def add_system_manager(email, first_name=None, last_name=None, send_welcome_email=False, password=None):
+
+def add_system_manager(
+	email: str,
+	first_name: Optional[str] = None,
+	last_name: Optional[str] = None,
+	send_welcome_email: bool = False,
+	password: str = None,
+) -> "User":
 	# add user
 	user = frappe.new_doc("User")
-	user.update({
-		"name": email,
-		"email": email,
-		"enabled": 1,
-		"first_name": first_name or email,
-		"last_name": last_name,
-		"user_type": "System User",
-		"send_welcome_email": 1 if send_welcome_email else 0
-	})
+	user.update(
+		{
+			"name": email,
+			"email": email,
+			"enabled": 1,
+			"first_name": first_name or email,
+			"last_name": last_name,
+			"user_type": "System User",
+			"send_welcome_email": 1 if send_welcome_email else 0,
+		}
+	)
 
 	user.insert()
 
 	# add roles
-	roles = frappe.get_all('Role',
-		fields=['name'],
-		filters={
-			'name': ['not in', ('Administrator', 'Guest', 'All')]
-		}
+	roles = frappe.get_all(
+		"Role",
+		fields=["name"],
+		filters={"name": ["not in", ("Administrator", "Guest", "All")]},
 	)
 	roles = [role.name for role in roles]
 	user.add_roles(*roles)
 
 	if password:
 		from frappe.utils.password import update_password
+
 		update_password(user=user.name, pwd=password)
+	return user
+
 
-def get_enabled_system_users():
-	# add more fields if required
-	return frappe.get_all('User',
-		fields=['email', 'language', 'name'],
+def get_enabled_system_users() -> List[Dict]:
+	return frappe.get_all(
+		"User",
+		fields=["email", "language", "name"],
 		filters={
-			'user_type': 'System User',
-			'enabled': 1,
-			'name': ['not in', ('Administrator', 'Guest')]
-		}
+			"user_type": "System User",
+			"enabled": 1,
+			"name": ["not in", ("Administrator", "Guest")],
+		},
 	)
 
-def is_website_user():
-	return frappe.db.get_value('User', frappe.session.user, 'user_type') == "Website User"
 
-def is_system_user(username):
-	return frappe.db.get_value("User", {"email": username, "enabled": 1, "user_type": "System User"})
+def is_website_user(username: Optional[str] = None) -> Optional[str]:
+	return (
+		frappe.db.get_value("User", username or frappe.session.user, "user_type")
+		== "Website User"
+	)
+
+
+def is_system_user(username: Optional[str] = None) -> Optional[str]:
+	return frappe.db.get_value(
+		"User",
+		{
+			"email": username or frappe.session.user,
+			"enabled": 1,
+			"user_type": "System User",
+		},
+	)
+
 
-def get_users():
+def get_users() -> List[Dict]:
 	from frappe.core.doctype.user.user import get_system_users
+
 	users = []
-	system_managers = frappe.utils.user.get_system_managers(only_name=True)
+	system_managers = get_system_managers(only_name=True)
+
 	for user in get_system_users():
-		users.append({
-			"full_name": frappe.utils.user.get_user_fullname(user),
-			"email": user,
-			"is_system_manager": 1 if (user in system_managers) else 0
-		})
+		users.append(
+			{
+				"full_name": get_user_fullname(user),
+				"email": user,
+				"is_system_manager": user in system_managers,
+			}
+		)
 
 	return users
 
-def set_last_active_to_now(user):
-	from frappe.utils import now_datetime
-	frappe.db.set_value("User", user, "last_active", now_datetime())
-
-
-def reset_simultaneous_sessions(user_limit):
-	for user in frappe.db.sql("""select name, simultaneous_sessions from tabUser
-		where name not in ('Administrator', 'Guest') and user_type = 'System User' and enabled=1
-		order by creation desc""", as_dict=1):
-		if user.simultaneous_sessions < user_limit:
-			user_limit = user_limit - user.simultaneous_sessions
-		else:
-			frappe.db.set_value("User", user.name, "simultaneous_sessions", 1)
-			user_limit = user_limit - 1
-
-def get_link_to_reset_password(user):
-	link = ''
-
-	if not cint(frappe.db.get_single_value('System Settings', 'setup_complete')):
-		user = frappe.get_doc("User", user)
-		link = user.reset_password(send_email=False)
-		frappe.db.commit()
-
-	return {
-		'link': link
-	}
-
-def get_users_with_role(role):
-	return [p[0] for p in frappe.db.sql("""SELECT DISTINCT `tabUser`.`name`
-		FROM `tabHas Role`, `tabUser`
-		WHERE `tabHas Role`.`role`=%s
-		AND `tabUser`.`name`!='Administrator'
-		AND `tabHas Role`.`parent`=`tabUser`.`name`
-		AND `tabUser`.`enabled`=1""", role)]
+
+def get_users_with_role(role: str) -> List[str]:
+	User = DocType("User")
+	HasRole = DocType("Has Role")
+
+	return (
+		frappe.qb.from_(HasRole)
+		.from_(User)
+		.where(
+			(HasRole.role == role)
+			& (User.name != "Administrator")
+			& (User.enabled == 1)
+			& (HasRole.parent == User.name)
+		)
+		.select(User.name)
+		.distinct()
+		.run(pluck=True)
+	)

frappe/website/doctype/discussion_reply/discussion_reply.py

@@ -1,12 +1,21 @@
-# Copyright (c) 2021, FOSS United and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # For license information, please see license.txt
 
 import frappe
 from frappe.model.document import Document
 
 class DiscussionReply(Document):
-	def after_insert(self):
 
+	def on_update(self):
+		frappe.publish_realtime(
+			event="update_message",
+			message = {
+				"reply": frappe.utils.md_to_html(self.reply),
+				"reply_name": self.name
+			},
+			after_commit=True)
+
+	def after_insert(self):
 		replies = frappe.db.count("Discussion Reply", {"topic": self.topic})
 		topic_info = frappe.get_all("Discussion Topic",
 			{"name": self.topic},
@@ -37,6 +46,19 @@ class DiscussionReply(Document):
 				"template": template,
 				"topic_info": topic_info[0],
 				"sidebar": sidebar,
-				"new_topic_template": new_topic_template
+				"new_topic_template": new_topic_template,
+				"reply_owner": self.owner
 			},
 			after_commit=True)
+
+	def after_delete(self):
+		frappe.publish_realtime(
+			event="delete_message",
+			message = {
+				"reply_name": self.name
+			},
+			after_commit=True)
+
+@frappe.whitelist()
+def delete_message(reply_name):
+	frappe.delete_doc("Discussion Reply", reply_name, ignore_permissions=True)

frappe/website/doctype/discussion_topic/discussion_topic.py

@@ -8,7 +8,14 @@ class DiscussionTopic(Document):
 	pass
 
 @frappe.whitelist()
-def submit_discussion(doctype, docname, reply, title, topic_name=None):
+def submit_discussion(doctype, docname, reply, title, topic_name=None, reply_name=None):
+
+	if reply_name:
+		doc = frappe.get_doc("Discussion Reply", reply_name)
+		doc.reply = reply
+		doc.save(ignore_permissions=True)
+		return
+
 	if topic_name:
 		save_message(reply, topic_name)
 		return topic_name

frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html

@@ -10,7 +10,7 @@
 			{%- set collapse_id = 'id-' + frappe.utils.generate_hash('Collapse', 12) -%}
 			<a class="collapsible-title" data-toggle="collapse" href="#{{ collapse_id }}" role="button"
 				aria-expanded="false" aria-controls="{{ collapse_id }}">
-				<h3>{{ item.title }}</h3>
+				<div class="collapsible-item-title">{{ item.title }}</div>
 				<svg class="collapsible-icon" width="24" height="24" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg">
 					<path class="vertical" d="M8 4V12" stroke="currentColor" stroke-width="1.5" stroke-miterlimit="10"
 						stroke-linecap="round"

frappe/website/web_template/section_with_features/section_with_features.html

@@ -17,12 +17,12 @@
 				<h3 class="feature-title">{{ feature.title }}</h3>
 				{%- endif -%}
 				{%- if feature.content -%}
-				<p class="feature-content">{{ feature.content }}</p>
+				<p class="feature-content">{{ frappe.utils.md_to_html(feature.content) }}</p>
 				{%- endif -%}
 			</div>
 			<div>
 				{%- if feature.url -%}
-				<a href="{{ feature.url }}" class="feature-url stretched-link">Learn more →</a>
+				<a href="{{ feature.url }}" class="feature-url stretched-link"> {{ _("Learn more") }} →</a>
 				{%- endif -%}
 			</div>
 		</div>

package.json

@@ -68,7 +68,7 @@
     "@frappe/esbuild-plugin-postcss2": "^0.1.3",
     "autoprefixer": "10",
     "chalk": "^2.3.2",
-    "esbuild": "^0.11.21",
+    "esbuild": "^0.14.29",
     "esbuild-vue": "^0.2.0",
     "fast-glob": "^3.2.5",
     "launch-editor": "^2.2.1",

yarn.lock

@@ -1382,6 +1382,91 @@ es-to-primitive@^1.2.1:
     is-date-object "^1.0.1"
     is-symbol "^1.0.2"
 
+esbuild-android-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-android-64/-/esbuild-android-64-0.14.29.tgz#c0960c84c9b832bade20831515e89d32549d4769"
+  integrity sha512-tJuaN33SVZyiHxRaVTo1pwW+rn3qetJX/SRuc/83rrKYtyZG0XfsQ1ao1nEudIt9w37ZSNXR236xEfm2C43sbw==
+
+esbuild-android-arm64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-android-arm64/-/esbuild-android-arm64-0.14.29.tgz#8eceb3abe5abde5489d6a5cbe6a7c1044f71115f"
+  integrity sha512-D74dCv6yYnMTlofVy1JKiLM5JdVSQd60/rQfJSDP9qvRAI0laPXIG/IXY1RG6jobmFMUfL38PbFnCqyI/6fPXg==
+
+esbuild-darwin-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-darwin-64/-/esbuild-darwin-64-0.14.29.tgz#26f3f14102310ecb8f2d9351c5b7a47a60d2cc8a"
+  integrity sha512-+CJaRvfTkzs9t+CjGa0Oa28WoXa7EeLutQhxus+fFcu0MHhsBhlmeWHac3Cc/Sf/xPi1b2ccDFfzGYJCfV0RrA==
+
+esbuild-darwin-arm64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.29.tgz#6d2d89dfd937992649239711ed5b86e51b13bd89"
+  integrity sha512-5Wgz/+zK+8X2ZW7vIbwoZ613Vfr4A8HmIs1XdzRmdC1kG0n5EG5fvKk/jUxhNlrYPx1gSY7XadQ3l4xAManPSw==
+
+esbuild-freebsd-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.29.tgz#2cb41a0765d0040f0838280a213c81bbe62d6394"
+  integrity sha512-VTfS7Bm9QA12JK1YXF8+WyYOfvD7WMpbArtDj6bGJ5Sy5xp01c/q70Arkn596aGcGj0TvQRplaaCIrfBG1Wdtg==
+
+esbuild-freebsd-arm64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.29.tgz#e1b79fbb63eaeff324cf05519efa7ff12ce4586a"
+  integrity sha512-WP5L4ejwLWWvd3Fo2J5mlXvG3zQHaw5N1KxFGnUc4+2ZFZknP0ST63i0IQhpJLgEJwnQpXv2uZlU1iWZjFqEIg==
+
+esbuild-linux-32@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-32/-/esbuild-linux-32-0.14.29.tgz#a4a5a0b165b15081bc3227986e10dd4943edb7d6"
+  integrity sha512-4myeOvFmQBWdI2U1dEBe2DCSpaZyjdQtmjUY11Zu2eQg4ynqLb8Y5mNjNU9UN063aVsCYYfbs8jbken/PjyidA==
+
+esbuild-linux-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-64/-/esbuild-linux-64-0.14.29.tgz#4c450088c84f8bfd22c51d116f59416864b85481"
+  integrity sha512-iaEuLhssReAKE7HMwxwFJFn7D/EXEs43fFy5CJeA4DGmU6JHh0qVJD2p/UP46DvUXLRKXsXw0i+kv5TdJ1w5pg==
+
+esbuild-linux-arm64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.29.tgz#d1a23993b26cb1f63f740329b2fc09218e498bd1"
+  integrity sha512-KYf7s8wDfUy+kjKymW3twyGT14OABjGHRkm9gPJ0z4BuvqljfOOUbq9qT3JYFnZJHOgkr29atT//hcdD0Pi7Mw==
+
+esbuild-linux-arm@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-arm/-/esbuild-linux-arm-0.14.29.tgz#a7e2fea558525eab812b1fe27d7a2659cd1bb723"
+  integrity sha512-OXa9D9QL1hwrAnYYAHt/cXAuSCmoSqYfTW/0CEY0LgJNyTxJKtqc5mlwjAZAvgyjmha0auS/sQ0bXfGf2wAokQ==
+
+esbuild-linux-mips64le@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.29.tgz#e708c527f0785574e400828cdbed3d9b17b5ddff"
+  integrity sha512-05jPtWQMsZ1aMGfHOvnR5KrTvigPbU35BtuItSSWLI2sJu5VrM8Pr9Owym4wPvA4153DFcOJ1EPN/2ujcDt54g==
+
+esbuild-linux-ppc64le@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.29.tgz#0137d1b38beae36a57176ef45e90740e734df502"
+  integrity sha512-FYhBqn4Ir9xG+f6B5VIQVbRuM4S6qwy29dDNYFPoxLRnwTEKToIYIUESN1qHyUmIbfO0YB4phG2JDV2JDN9Kgw==
+
+esbuild-linux-riscv64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.29.tgz#a2f73235347a58029dcacf0fb91c9eb8bebc8abb"
+  integrity sha512-eqZMqPehkb4nZcffnuOpXJQdGURGd6GXQ4ZsDHSWyIUaA+V4FpMBe+5zMPtXRD2N4BtyzVvnBko6K8IWWr36ew==
+
+esbuild-linux-s390x@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.29.tgz#0f7310ff1daec463ead9b9e26b7aa083a9f9f1ee"
+  integrity sha512-o7EYajF1rC/4ho7kpSG3gENVx0o2SsHm7cJ5fvewWB/TEczWU7teDgusGSujxCYcMottE3zqa423VTglNTYhjg==
+
+esbuild-netbsd-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.29.tgz#ba9a0d9cb8aed73b684825126927f75d4fe44ff9"
+  integrity sha512-/esN6tb6OBSot6+JxgeOZeBk6P8V/WdR3GKBFeFpSqhgw4wx7xWUqPrdx4XNpBVO7X4Ipw9SAqgBrWHlXfddww==
+
+esbuild-openbsd-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.29.tgz#36dbe2c32d899106791b5f3af73f359213f71b8a"
+  integrity sha512-jUTdDzhEKrD0pLpjmk0UxwlfNJNg/D50vdwhrVcW/D26Vg0hVbthMfb19PJMatzclbK7cmgk1Nu0eNS+abzoHw==
+
+esbuild-sunos-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-sunos-64/-/esbuild-sunos-64-0.14.29.tgz#e5f857c121441ec63bf9b399a2131409a7d344e5"
+  integrity sha512-EfhQN/XO+TBHTbkxwsxwA7EfiTHFe+MNDfxcf0nj97moCppD9JHPq48MLtOaDcuvrTYOcrMdJVeqmmeQ7doTcg==
+
 esbuild-vue@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/esbuild-vue/-/esbuild-vue-0.2.0.tgz#8a3fde404bda57fe32b80e24917d14036e242bd3"
@@ -1391,10 +1476,46 @@ esbuild-vue@^0.2.0:
     piscina "^2.2.0"
     vue-template-compiler "^2.6.12"
 
-esbuild@^0.11.21:
-  version "0.11.21"
-  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.11.21.tgz#9220b0185ae40947811dcaff6bfcfb572bebac08"
-  integrity sha512-FqpYdJqiTeLDbj3vqxc/fG8UmHIEvQrDaUxSw1oJf4giLd/tnMDUUlXellCjOab7qGKQ5hUFD5eQgmO+tkZeow==
+esbuild-windows-32@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-windows-32/-/esbuild-windows-32-0.14.29.tgz#9c2f1ab071a828f3901d1d79d205982a74bdda6e"
+  integrity sha512-uoyb0YAJ6uWH4PYuYjfGNjvgLlb5t6b3zIaGmpWPOjgpr1Nb3SJtQiK4YCPGhONgfg2v6DcJgSbOteuKXhwqAw==
+
+esbuild-windows-64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-windows-64/-/esbuild-windows-64-0.14.29.tgz#85fbce7c2492521896451b98d649a7db93e52667"
+  integrity sha512-X9cW/Wl95QjsH8WUyr3NqbmfdU72jCp71cH3pwPvI4CgBM2IeOUDdbt6oIGljPu2bf5eGDIo8K3Y3vvXCCTd8A==
+
+esbuild-windows-arm64@0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.29.tgz#0aa7a9a1bc43a63350bcf574d94b639176f065b5"
+  integrity sha512-+O/PI+68fbUZPpl3eXhqGHTGK7DjLcexNnyJqtLZXOFwoAjaXlS5UBCvVcR3o2va+AqZTj8o6URaz8D2K+yfQQ==
+
+esbuild@^0.14.29:
+  version "0.14.29"
+  resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.14.29.tgz#24ad09c0674cbcb4aa2fe761485524eb1f6b1419"
+  integrity sha512-SQS8cO8xFEqevYlrHt6exIhK853Me4nZ4aMW6ieysInLa0FMAL+AKs87HYNRtR2YWRcEIqoXAHh+Ytt5/66qpg==
+  optionalDependencies:
+    esbuild-android-64 "0.14.29"
+    esbuild-android-arm64 "0.14.29"
+    esbuild-darwin-64 "0.14.29"
+    esbuild-darwin-arm64 "0.14.29"
+    esbuild-freebsd-64 "0.14.29"
+    esbuild-freebsd-arm64 "0.14.29"
+    esbuild-linux-32 "0.14.29"
+    esbuild-linux-64 "0.14.29"
+    esbuild-linux-arm "0.14.29"
+    esbuild-linux-arm64 "0.14.29"
+    esbuild-linux-mips64le "0.14.29"
+    esbuild-linux-ppc64le "0.14.29"
+    esbuild-linux-riscv64 "0.14.29"
+    esbuild-linux-s390x "0.14.29"
+    esbuild-netbsd-64 "0.14.29"
+    esbuild-openbsd-64 "0.14.29"
+    esbuild-sunos-64 "0.14.29"
+    esbuild-windows-32 "0.14.29"
+    esbuild-windows-64 "0.14.29"
+    esbuild-windows-arm64 "0.14.29"
 
 escalade@^3.1.1:
   version "3.1.1"
@@ -2846,9 +2967,9 @@ minimist-options@4.1.0:
     kind-of "^6.0.3"
 
 minimist@^1.2.0:
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
-  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
 minipass@^3.0.0:
   version "3.1.6"