perf: drop unused indexes (backport #18118) (#18124)

* perf: drop many unused indexes

- reference owner - only used for activity page which is unusable
  *anyway*
- same for timeline and link doctypes

perf: drop unused index from Commment

Can't find a single  query that uses this index

perf: drop pointless "Is home folder" index

just why? It's checkbox index with 1 entry which has unique name already

Most pointless index I've seen no far.

perf: Drop "attached to doctype" separate index

There is already a composite index for this.

perf: drop indexes from web page view log

BOTH are totally unused in any of the queries related to this doctype.

Only "website analytics" report uses this which doesn't even show this
data lmao.

perf: drop unused index from EPS log

We never filter by doctype or type. pointless indexing.

perf: fix up notification log indexes

- drop unused index
- add "for user" index

(cherry picked from commit 3a5a45d8af)

* perf: drop all old `parent` indexes

- re-add parent index on custom_docperm

(cherry picked from commit 7453e8e3a1)

Co-authored-by: Ankush Menat <ankush@frappe.io>

frappe/core/doctype/activity_log/activity_log.json

@@ -102,8 +102,7 @@
    "fetch_from": "reference_name.owner",
    "fieldname": "reference_owner",
    "fieldtype": "Read Only",
-   "label": "Reference Owner",
-   "search_index": 1
+   "label": "Reference Owner"
   },
   {
    "fieldname": "column_break_14",
@@ -154,7 +153,7 @@
  "icon": "fa fa-comment",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-10-25 11:43:57.504565",
+ "modified": "2022-09-13 15:19:42.474114",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Activity Log",
@@ -181,6 +180,7 @@
  "search_fields": "subject",
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "subject",
  "track_seen": 1
-}
+}

frappe/core/doctype/activity_log/activity_log.py

@@ -37,7 +37,6 @@ def on_doctype_update():
 	"""Add indexes in `tabActivity Log`"""
 	frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"])
 	frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"])
-	frappe.db.add_index("Activity Log", ["link_doctype", "link_name"])
 
 
 def add_authentication_log(subject, user, operation="Login", status="Success"):

frappe/core/doctype/comment/comment.py

@@ -60,7 +60,6 @@ class Comment(Document):
 
 def on_doctype_update():
 	frappe.db.add_index("Comment", ["reference_doctype", "reference_name"])
-	frappe.db.add_index("Comment", ["link_doctype", "link_name"])
 
 
 def update_comment_in_doc(doc):

frappe/core/doctype/custom_docperm/custom_docperm.json

@@ -212,7 +212,8 @@
    "fieldname": "parent",
    "fieldtype": "Data",
    "label": "Reference Document Type",
-   "read_only": 1
+   "read_only": 1,
+   "search_index": 1
   },
   {
    "default": "0",

frappe/core/doctype/file/file.json

@@ -64,8 +64,7 @@
    "fieldname": "is_home_folder",
    "fieldtype": "Check",
    "hidden": 1,
-   "label": "Is Home Folder",
-   "search_index": 1
+   "label": "Is Home Folder"
   },
   {
    "default": "0",
@@ -125,8 +124,7 @@
    "in_standard_filter": 1,
    "label": "Attached To DocType",
    "options": "DocType",
-   "read_only": 1,
-   "search_index": 1
+   "read_only": 1
   },
   {
    "fieldname": "column_break_10",
@@ -136,8 +134,7 @@
    "fieldname": "attached_to_name",
    "fieldtype": "Data",
    "label": "Attached To Name",
-   "read_only": 1,
-   "search_index": 1
+   "read_only": 1
   },
   {
    "fieldname": "attached_to_field",
@@ -175,7 +172,7 @@
  "icon": "fa fa-file",
  "idx": 1,
  "links": [],
- "modified": "2020-06-28 12:21:30.772386",
+ "modified": "2022-09-13 15:50:15.508250",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "File",
@@ -210,6 +207,7 @@
  ],
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "title_field": "file_name",
  "track_changes": 1
 }

frappe/desk/doctype/notification_log/notification_log.json

@@ -22,18 +22,14 @@
    "fieldname": "subject",
    "fieldtype": "Text",
    "in_list_view": 1,
-   "label": "Subject",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Subject"
   },
   {
    "fieldname": "for_user",
    "fieldtype": "Link",
    "hidden": 1,
    "label": "For User",
-   "options": "User",
-   "show_days": 1,
-   "show_seconds": 1
+   "options": "User"
   },
   {
    "fieldname": "type",
@@ -42,36 +38,26 @@
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Type",
-   "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert",
-   "search_index": 1,
-   "show_days": 1,
-   "show_seconds": 1
+   "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert"
   },
   {
    "fieldname": "email_content",
    "fieldtype": "Text Editor",
-   "label": "Message",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Message"
   },
   {
    "fieldname": "document_type",
    "fieldtype": "Link",
    "hidden": 1,
    "label": "Document Type",
-   "options": "DocType",
-   "search_index": 1,
-   "show_days": 1,
-   "show_seconds": 1
+   "options": "DocType"
   },
   {
    "fieldname": "document_name",
    "fieldtype": "Data",
    "hidden": 1,
    "label": "Document Link",
-   "search_index": 1,
-   "show_days": 1,
-   "show_seconds": 1
+   "search_index": 1
   },
   {
    "fieldname": "from_user",
@@ -79,9 +65,7 @@
    "hidden": 1,
    "label": "From User",
    "options": "User",
-   "search_index": 1,
-   "show_days": 1,
-   "show_seconds": 1
+   "search_index": 1
   },
   {
    "default": "0",
@@ -89,38 +73,30 @@
    "fieldtype": "Check",
    "hidden": 1,
    "ignore_user_permissions": 1,
-   "label": "Read",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Read"
   },
   {
    "fieldname": "open_reference_document",
    "fieldtype": "Button",
-   "label": "Open Reference Document",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Open Reference Document"
   },
   {
    "fieldname": "attached_file",
    "fieldtype": "Code",
    "hidden": 1,
    "label": "Attached File",
-   "options": "JSON",
-   "show_days": 1,
-   "show_seconds": 1
+   "options": "JSON"
   },
   {
    "fieldname": "attachment_link",
    "fieldtype": "HTML",
-   "label": "Attachment Link",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Attachment Link"
   }
  ],
  "hide_toolbar": 1,
  "in_create": 1,
  "links": [],
- "modified": "2021-10-25 17:26:09.703215",
+ "modified": "2022-09-13 16:08:48.153934",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Notification Log",
@@ -138,6 +114,7 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "subject",
  "track_seen": 1
-}
+}

frappe/desk/doctype/notification_log/notification_log.py

@@ -136,7 +136,7 @@ def get_email_header(doc):
 @frappe.whitelist()
 def get_notification_logs(limit=20):
 	notification_logs = frappe.db.get_list(
-		"Notification Log", fields=["*"], limit=limit, order_by="creation desc"
+		"Notification Log", fields=["*"], limit=limit, order_by="modified desc"
 	)
 
 	users = [log.from_user for log in notification_logs]

frappe/patches.txt

@@ -212,3 +212,4 @@ frappe.patches.v14_0.delete_data_migration_tool
 frappe.patches.v14_0.set_suspend_email_queue_default
 frappe.patches.v14_0.different_encryption_key
 frappe.patches.v14_0.update_multistep_webforms
+frappe.patches.v14_0.drop_unused_indexes

frappe/patches/v14_0/drop_unused_indexes.py

@@ -0,0 +1,56 @@
+"""
+This patch just drops some known indexes which aren't being used anymore or never were used.
+"""
+
+import click
+
+import frappe
+
+UNUSED_INDEXES = [
+	("Comment", ["link_doctype", "link_name"]),
+	("Activity Log", ["link_doctype", "link_name"]),
+]
+
+
+def execute():
+	if frappe.db.db_type == "postgres":
+		return
+
+	db_tables = frappe.db.get_tables(cached=False)
+
+	# All parent indexes
+	parent_doctypes = frappe.get_all(
+		"DocType",
+		{"istable": 0, "is_virtual": 0, "issingle": 0},
+		pluck="name",
+	)
+	db_tables = frappe.db.get_tables(cached=False)
+
+	for doctype in parent_doctypes:
+		table = f"tab{doctype}"
+		if table not in db_tables:
+			continue
+		_drop_index_if_exists(table, "parent")
+
+	# Unused composite indexes
+	for doctype, index_fields in UNUSED_INDEXES:
+		table = f"tab{doctype}"
+		index_name = frappe.db.get_index_name(index_fields)
+		if table not in db_tables:
+			continue
+		_drop_index_if_exists(table, index_name)
+
+
+def _drop_index_if_exists(table: str, index: str):
+	if not frappe.db.has_index(table, index):
+		click.echo(f"- Skipped {index} index for {table} because it doesn't exist")
+		return
+
+	try:
+		frappe.db.sql_ddl(f"ALTER TABLE `{table}` DROP INDEX `{index}`")
+	except Exception as e:
+		frappe.log_error("Failed to drop index")
+		click.secho(f"x Failed to drop index {index} from {table}\n {str(e)}", fg="red")
+		return
+
+	click.echo(f"✓ dropped {index} index from {table}")

frappe/social/doctype/energy_point_log/energy_point_log.json

@@ -36,8 +36,7 @@
    "in_standard_filter": 1,
    "label": "Type",
    "options": "Auto\nAppreciation\nCriticism\nReview\nRevert",
-   "read_only": 1,
-   "search_index": 1
+   "read_only": 1
   },
   {
    "fieldname": "rule",
@@ -54,8 +53,7 @@
    "in_standard_filter": 1,
    "label": "Reference Document Type",
    "options": "DocType",
-   "read_only": 1,
-   "search_index": 1
+   "read_only": 1
   },
   {
    "fieldname": "reference_name",
@@ -112,7 +110,7 @@
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-10-25 17:25:40.477044",
+ "modified": "2022-09-13 16:10:37.747013",
  "modified_by": "Administrator",
  "module": "Social",
  "name": "Energy Point Log",
@@ -131,5 +129,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "user"
-}
+}

frappe/website/doctype/web_page_view/web_page_view.json

@@ -25,14 +25,12 @@
    "fieldname": "referrer",
    "fieldtype": "Data",
    "label": "Referrer",
-   "search_index": 1,
    "set_only_once": 1
   },
   {
    "fieldname": "browser",
    "fieldtype": "Data",
    "label": "Browser",
-   "search_index": 1,
    "set_only_once": 1
   },
   {
@@ -59,7 +57,7 @@
  ],
  "in_create": 1,
  "links": [],
- "modified": "2021-10-25 14:11:24.718770",
+ "modified": "2022-09-13 15:38:25.401797",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Web Page View",
@@ -82,5 +80,6 @@
  "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "path"
-}
+}