perf: ~45% faster Desk first response (backport #17891) (#17897)

* fix(recorder): make whole order button clickable

(cherry picked from commit 04aeeabb2b)

* perf: ~33% faster Desk response

- hardcode `/app` resolution
- use cached website settings everywhere. It was mixing cache and DB
  everywhere and re-quering same thing (why ?)

(cherry picked from commit 786df3fbeb)

* perf: use is_virtual_doctype and remove limit

This reduces 1 query for each child table read

Removed limit cause with 1000+ doctypes in frappe+erpnext this cache
will just keep getting trashed for no reason. There's clear upper bound
on size so no need to limit it here.

(cherry picked from commit 61a9349789)

* perf: dont order by for uniq searches

(cherry picked from commit 48869d506f)

* perf: simpler/faster preload header computation

We parse entire response to find preload headers, instead just use
include_style and include_script to include assets directly into preload
headers. This shaves off ~13% overhead in response.

(cherry picked from commit 4241f8c8c0)

Co-authored-by: Ankush Menat <ankush@frappe.io>

frappe/__init__.py

@@ -240,6 +240,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None:
 	local.document_cache = {}
 	local.meta_cache = {}
 	local.form_dict = _dict()
+	local.preload_assets = {"style": [], "script": []}
 	local.session = _dict()
 	local.dev_server = _dev_server
 	local.qb = get_query_builder(local.conf.db_type or "mariadb")

frappe/desk/doctype/tag/tag.py

@@ -192,4 +192,4 @@ def get_documents_for_tag(tag):
 
 @frappe.whitelist()
 def get_tags_list_for_awesomebar():
-	return [t.name for t in frappe.get_list("Tag")]
+	return frappe.get_list("Tag", pluck="name", order_by=None)

frappe/desk/form/load.py

@@ -452,7 +452,9 @@ def get_title_values_for_link_and_dynamic_link_fields(doc, link_fields=None):
 		if not meta or not (meta.title_field and meta.show_title_field_in_link):
 			continue
 
-		link_title = frappe.db.get_value(doctype, doc.get(field.fieldname), meta.title_field, cache=True)
+		link_title = frappe.db.get_value(
+			doctype, doc.get(field.fieldname), meta.title_field, cache=True, order_by=None
+		)
 		link_titles.update({doctype + "::" + doc.get(field.fieldname): link_title})
 
 	return link_titles

frappe/model/document.py

@@ -15,6 +15,7 @@ from frappe.model import optional_fields, table_fields
 from frappe.model.base_document import BaseDocument, get_controller
 from frappe.model.docstatus import DocStatus
 from frappe.model.naming import set_new_name, validate_name
+from frappe.model.utils import is_virtual_doctype
 from frappe.model.workflow import set_workflow_state_on_action, validate_workflow
 from frappe.utils import cstr, date_diff, file_lock, flt, get_datetime_str, now
 from frappe.utils.data import get_absolute_url
@@ -154,11 +155,7 @@ class Document(BaseDocument):
 			# Make sure not to query the DB for a child table, if it is a virtual one.
 			# During frappe is installed, the property "is_virtual" is not available in tabDocType, so
 			# we need to filter those cases for the access to frappe.db.get_value() as it would crash otherwise.
-			if (
-				hasattr(self, "doctype")
-				and not hasattr(self, "module")
-				and frappe.db.get_value("DocType", df.options, "is_virtual", cache=True)
-			):
+			if hasattr(self, "doctype") and not hasattr(self, "module") and is_virtual_doctype(df.options):
 				self.set(df.fieldname, [])
 				continue
 

frappe/model/utils/__init__.py

@@ -128,6 +128,6 @@ def get_fetch_values(doctype, fieldname, value):
 	return result
 
 
-@site_cache(maxsize=128)
+@site_cache()
 def is_virtual_doctype(doctype):
 	return frappe.db.get_value("DocType", doctype, "is_virtual")

frappe/public/js/frappe/recorder/RequestDetail.vue

@@ -25,8 +25,8 @@
 							<li v-for="(column, index) in table_columns.filter(c => c.sortable)" :key="index" @click="query.sort = column.slug"><a class="option">{{ column.label }}</a></li>
 						</ul>
 					</div>
-					<button class="btn btn-default btn-xs btn-order">
-						<span class="octicon text-muted" :class="query.order == 'asc' ? 'octicon-arrow-down' : 'octicon-arrow-up'"  @click="query.order = (query.order == 'asc') ? 'desc' : 'asc'"></span>
+					<button class="btn btn-default btn-xs btn-order" @click="query.order = (query.order == 'asc') ? 'desc' : 'asc'">
+						<span class="octicon text-muted" :class="query.order == 'asc' ? 'octicon-arrow-down' : 'octicon-arrow-up'"></span>
 					</button>
 				</div>
 			</div>

frappe/utils/jinja_globals.py

@@ -95,13 +95,33 @@ def get_dom_id(seed=None):
 	return "id-" + generate_hash(seed, 12)
 
 
-def include_script(path):
+def include_script(path, preload=True):
+	"""Get path of bundled script files.
+
+	If preload is specified the path will be added to preload headers so browsers can prefetch
+	assets."""
 	path = bundled_asset(path)
+
+	if preload:
+		import frappe
+
+		frappe.local.preload_assets["script"].append(path)
+
 	return f'<script type="text/javascript" src="{path}"></script>'
 
 
-def include_style(path, rtl=None):
+def include_style(path, rtl=None, preload=True):
+	"""Get path of bundled style files.
+
+	If preload is specified the path will be added to preload headers so browsers can prefetch
+	assets."""
 	path = bundled_asset(path)
+
+	if preload:
+		import frappe
+
+		frappe.local.preload_assets["style"].append(path)
+
 	return f'<link type="text/css" rel="stylesheet" href="{path}">'
 
 

frappe/utils/user.py

@@ -273,7 +273,7 @@ def get_user_fullname(user: str) -> str:
 
 def get_fullname_and_avatar(user: str) -> _dict:
 	first_name, last_name, avatar, name = frappe.db.get_value(
-		"User", user, ["first_name", "last_name", "user_image", "name"]
+		"User", user, ["first_name", "last_name", "user_image", "name"], order_by=None
 	)
 	return _dict(
 		{

frappe/website/doctype/website_settings/website_settings.py

@@ -103,12 +103,12 @@ class WebsiteSettings(Document):
 def get_website_settings(context=None):
 	hooks = frappe.get_hooks()
 	context = frappe._dict(context or {})
-	settings: "WebsiteSettings" = frappe.get_single("Website Settings")
+	settings: "WebsiteSettings" = frappe.get_cached_doc("Website Settings")
 
 	context = context.update(
 		{
-			"top_bar_items": get_items("top_bar_items"),
-			"footer_items": get_items("footer_items"),
+			"top_bar_items": modify_header_footer_items(settings.top_bar_items),
+			"footer_items": modify_header_footer_items(settings.footer_items),
 			"post_login": [
 				{"label": _("My Account"), "url": "/me"},
 				{"label": _("Log out"), "url": "/?cmd=web_logout"},
@@ -203,22 +203,24 @@ def get_items(parentfield: str) -> list[dict]:
 		order_by="idx asc",
 		fields="*",
 	)
-	top_items = _items.copy()
+	return modify_header_footer_items(_items)
 
+
+def modify_header_footer_items(items: list):
+	top_items = items.copy()
 	# attach child items to top bar
-	for item in _items:
-		if not item["parent_label"]:
+	for item in items:
+		if not item.parent_label:
 			continue
 
 		for top_bar_item in top_items:
-			if top_bar_item["label"] != item["parent_label"]:
+			if top_bar_item.label != item.parent_label:
 				continue
 
-			if "child_items" not in top_bar_item:
+			if not top_bar_item.get("child_items"):
 				top_bar_item["child_items"] = []
 
-			top_bar_item["child_items"].append(item)
-
+			top_bar_item.child_items.append(item)
 			break
 
 	return top_items

frappe/website/doctype/website_theme/website_theme.py

@@ -133,9 +133,9 @@ class WebsiteTheme(Document):
 
 
 def get_active_theme() -> Optional["WebsiteTheme"]:
-	if website_theme := frappe.db.get_single_value("Website Settings", "website_theme"):
+	if website_theme := frappe.get_website_settings("website_theme"):
 		try:
-			return frappe.get_doc("Website Theme", website_theme)
+			return frappe.get_cached_doc("Website Theme", website_theme)
 		except frappe.DoesNotExistError:
 			pass
 

frappe/website/page_renderers/static_page.py

@@ -12,6 +12,8 @@ UNSUPPORTED_STATIC_PAGE_TYPES = ("html", "md", "js", "xml", "css", "txt", "py",
 
 
 class StaticPage(BaseRenderer):
+	__slots__ = ("path", "file_path")
+
 	def __init__(self, path, http_status_code=None):
 		super().__init__(path=path, http_status_code=http_status_code)
 		self.set_file_path()

frappe/website/path_resolver.py

@@ -17,6 +17,8 @@ from frappe.website.utils import can_cache, get_home_page
 
 
 class PathResolver:
+	__slots__ = ("path",)
+
 	def __init__(self, path):
 		self.path = path.strip("/ ")
 
@@ -36,6 +38,11 @@ class PathResolver:
 			return frappe.flags.redirect_location, RedirectPage(self.path)
 
 		endpoint = resolve_path(self.path)
+
+		# WARN: Hardcoded for better performance
+		if endpoint == "app":
+			return endpoint, TemplatePage(endpoint, 200)
+
 		custom_renderers = self.get_custom_page_renderers()
 		renderers = custom_renderers + [
 			StaticPage,
@@ -98,7 +105,7 @@ def resolve_redirect(path, query_string=None):
 	        ]
 	"""
 	redirects = frappe.get_hooks("website_redirects")
-	redirects += frappe.db.get_all("Website Route Redirect", ["source", "target"])
+	redirects += frappe.get_all("Website Route Redirect", ["source", "target"], order_by=None)
 
 	if not redirects:
 		return

frappe/website/utils.py

@@ -527,7 +527,8 @@ def build_response(path, data, http_status_code, headers: dict | None = None):
 	response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace")
 	response.headers["X-From-Cache"] = frappe.local.response.from_cache or False
 
-	add_preload_headers(response)
+	add_preload_for_bundled_assets(response)
+
 	if headers:
 		for key, val in headers.items():
 			response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace")
@@ -557,29 +558,18 @@ def set_content_type(response, data, path):
 	return data
 
 
-def add_preload_headers(response):
-	from bs4 import BeautifulSoup, SoupStrainer
-
-	try:
-		preload = []
-		strainer = SoupStrainer(re.compile("script|link"))
-		soup = BeautifulSoup(response.data, "html.parser", parse_only=strainer)
-		for elem in soup.find_all("script", src=re.compile(".*")):
-			preload.append(("script", elem.get("src")))
+def add_preload_for_bundled_assets(response):
 
-		for elem in soup.find_all("link", rel="stylesheet"):
-			preload.append(("style", elem.get("href")))
+	links = []
 
-		links = []
-		for _type, link in preload:
-			links.append(f"<{link}>; rel=preload; as={_type}")
+	for css in frappe.local.preload_assets["style"]:
+		links.append(f"<{css}>; rel=preload; as=style")
 
-		if links:
-			response.headers["Link"] = ",".join(links)
-	except Exception:
-		import traceback
+	for js in frappe.local.preload_assets["script"]:
+		links.append(f"<{js}>; rel=preload; as=script")
 
-		traceback.print_exc()
+	if links:
+		response.headers["Link"] = ",".join(links)
 
 
 @lru_cache

frappe/www/app.py

@@ -18,7 +18,9 @@ CLOSING_SCRIPT_TAG_PATTERN = re.compile(r"</script\>")
 def get_context(context):
 	if frappe.session.user == "Guest":
 		frappe.throw(_("Log in to access this page."), frappe.PermissionError)
-	elif frappe.db.get_value("User", frappe.session.user, "user_type") == "Website User":
+	elif (
+		frappe.db.get_value("User", frappe.session.user, "user_type", order_by=None) == "Website User"
+	):
 		frappe.throw(_("You are not permitted to access this page."), frappe.PermissionError)
 
 	hooks = frappe.get_hooks()