From dcb423a73b524476eca60848959e578124ce4c3c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 14:32:26 +0530 Subject: [PATCH] perf: drop unused indexes (backport #18118) (#18124) * perf: drop many unused indexes - reference owner - only used for activity page which is unusable *anyway* - same for timeline and link doctypes perf: drop unused index from Commment Can't find a single query that uses this index perf: drop pointless "Is home folder" index just why? It's checkbox index with 1 entry which has unique name already Most pointless index I've seen no far. perf: Drop "attached to doctype" separate index There is already a composite index for this. perf: drop indexes from web page view log BOTH are totally unused in any of the queries related to this doctype. Only "website analytics" report uses this which doesn't even show this data lmao. perf: drop unused index from EPS log We never filter by doctype or type. pointless indexing. perf: fix up notification log indexes - drop unused index - add "for user" index (cherry picked from commit 3a5a45d8af6e86742b466e06702d74cc8bade7e3) * perf: drop all old `parent` indexes - re-add parent index on custom_docperm (cherry picked from commit 7453e8e3a171c7d7ae05811d3a0548037e8f3c37) Co-authored-by: Ankush Menat --- .../doctype/activity_log/activity_log.json | 8 +-- .../core/doctype/activity_log/activity_log.py | 1 - frappe/core/doctype/comment/comment.py | 1 - .../custom_docperm/custom_docperm.json | 3 +- frappe/core/doctype/file/file.json | 12 ++-- .../notification_log/notification_log.json | 51 +++++------------ .../notification_log/notification_log.py | 2 +- frappe/patches.txt | 1 + frappe/patches/v14_0/drop_unused_indexes.py | 56 +++++++++++++++++++ .../energy_point_log/energy_point_log.json | 11 ++-- .../doctype/web_page_view/web_page_view.json | 7 +-- 11 files changed, 91 insertions(+), 62 deletions(-) create mode 100644 frappe/patches/v14_0/drop_unused_indexes.py diff --git a/frappe/core/doctype/activity_log/activity_log.json b/frappe/core/doctype/activity_log/activity_log.json index ad12246a95..910baceb5e 100644 --- a/frappe/core/doctype/activity_log/activity_log.json +++ b/frappe/core/doctype/activity_log/activity_log.json @@ -102,8 +102,7 @@ "fetch_from": "reference_name.owner", "fieldname": "reference_owner", "fieldtype": "Read Only", - "label": "Reference Owner", - "search_index": 1 + "label": "Reference Owner" }, { "fieldname": "column_break_14", @@ -154,7 +153,7 @@ "icon": "fa fa-comment", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-10-25 11:43:57.504565", + "modified": "2022-09-13 15:19:42.474114", "modified_by": "Administrator", "module": "Core", "name": "Activity Log", @@ -181,6 +180,7 @@ "search_fields": "subject", "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 468b7f4473..e819683d0a 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -37,7 +37,6 @@ def on_doctype_update(): """Add indexes in `tabActivity Log`""" frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"]) frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"]) - frappe.db.add_index("Activity Log", ["link_doctype", "link_name"]) def add_authentication_log(subject, user, operation="Login", status="Success"): diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index 985dde8ce2..3d886cf93b 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -60,7 +60,6 @@ class Comment(Document): def on_doctype_update(): frappe.db.add_index("Comment", ["reference_doctype", "reference_name"]) - frappe.db.add_index("Comment", ["link_doctype", "link_name"]) def update_comment_in_doc(doc): diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.json b/frappe/core/doctype/custom_docperm/custom_docperm.json index 93f5431903..2c594f5624 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.json +++ b/frappe/core/doctype/custom_docperm/custom_docperm.json @@ -212,7 +212,8 @@ "fieldname": "parent", "fieldtype": "Data", "label": "Reference Document Type", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "default": "0", diff --git a/frappe/core/doctype/file/file.json b/frappe/core/doctype/file/file.json index 3008e27aa0..cb727e48f0 100644 --- a/frappe/core/doctype/file/file.json +++ b/frappe/core/doctype/file/file.json @@ -64,8 +64,7 @@ "fieldname": "is_home_folder", "fieldtype": "Check", "hidden": 1, - "label": "Is Home Folder", - "search_index": 1 + "label": "Is Home Folder" }, { "default": "0", @@ -125,8 +124,7 @@ "in_standard_filter": 1, "label": "Attached To DocType", "options": "DocType", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "column_break_10", @@ -136,8 +134,7 @@ "fieldname": "attached_to_name", "fieldtype": "Data", "label": "Attached To Name", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "attached_to_field", @@ -175,7 +172,7 @@ "icon": "fa fa-file", "idx": 1, "links": [], - "modified": "2020-06-28 12:21:30.772386", + "modified": "2022-09-13 15:50:15.508250", "modified_by": "Administrator", "module": "Core", "name": "File", @@ -210,6 +207,7 @@ ], "sort_field": "modified", "sort_order": "ASC", + "states": [], "title_field": "file_name", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/notification_log/notification_log.json b/frappe/desk/doctype/notification_log/notification_log.json index e188708277..f24a6447b4 100644 --- a/frappe/desk/doctype/notification_log/notification_log.json +++ b/frappe/desk/doctype/notification_log/notification_log.json @@ -22,18 +22,14 @@ "fieldname": "subject", "fieldtype": "Text", "in_list_view": 1, - "label": "Subject", - "show_days": 1, - "show_seconds": 1 + "label": "Subject" }, { "fieldname": "for_user", "fieldtype": "Link", "hidden": 1, "label": "For User", - "options": "User", - "show_days": 1, - "show_seconds": 1 + "options": "User" }, { "fieldname": "type", @@ -42,36 +38,26 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Type", - "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert" }, { "fieldname": "email_content", "fieldtype": "Text Editor", - "label": "Message", - "show_days": 1, - "show_seconds": 1 + "label": "Message" }, { "fieldname": "document_type", "fieldtype": "Link", "hidden": 1, "label": "Document Type", - "options": "DocType", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "options": "DocType" }, { "fieldname": "document_name", "fieldtype": "Data", "hidden": 1, "label": "Document Link", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "search_index": 1 }, { "fieldname": "from_user", @@ -79,9 +65,7 @@ "hidden": 1, "label": "From User", "options": "User", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "search_index": 1 }, { "default": "0", @@ -89,38 +73,30 @@ "fieldtype": "Check", "hidden": 1, "ignore_user_permissions": 1, - "label": "Read", - "show_days": 1, - "show_seconds": 1 + "label": "Read" }, { "fieldname": "open_reference_document", "fieldtype": "Button", - "label": "Open Reference Document", - "show_days": 1, - "show_seconds": 1 + "label": "Open Reference Document" }, { "fieldname": "attached_file", "fieldtype": "Code", "hidden": 1, "label": "Attached File", - "options": "JSON", - "show_days": 1, - "show_seconds": 1 + "options": "JSON" }, { "fieldname": "attachment_link", "fieldtype": "HTML", - "label": "Attachment Link", - "show_days": 1, - "show_seconds": 1 + "label": "Attachment Link" } ], "hide_toolbar": 1, "in_create": 1, "links": [], - "modified": "2021-10-25 17:26:09.703215", + "modified": "2022-09-13 16:08:48.153934", "modified_by": "Administrator", "module": "Desk", "name": "Notification Log", @@ -138,6 +114,7 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index e3112b08a6..b46795dd8a 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -136,7 +136,7 @@ def get_email_header(doc): @frappe.whitelist() def get_notification_logs(limit=20): notification_logs = frappe.db.get_list( - "Notification Log", fields=["*"], limit=limit, order_by="creation desc" + "Notification Log", fields=["*"], limit=limit, order_by="modified desc" ) users = [log.from_user for log in notification_logs] diff --git a/frappe/patches.txt b/frappe/patches.txt index 3a902cb35e..907779098c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -212,3 +212,4 @@ frappe.patches.v14_0.delete_data_migration_tool frappe.patches.v14_0.set_suspend_email_queue_default frappe.patches.v14_0.different_encryption_key frappe.patches.v14_0.update_multistep_webforms +frappe.patches.v14_0.drop_unused_indexes diff --git a/frappe/patches/v14_0/drop_unused_indexes.py b/frappe/patches/v14_0/drop_unused_indexes.py new file mode 100644 index 0000000000..83fe1a4c37 --- /dev/null +++ b/frappe/patches/v14_0/drop_unused_indexes.py @@ -0,0 +1,56 @@ +""" +This patch just drops some known indexes which aren't being used anymore or never were used. +""" + +import click + +import frappe + +UNUSED_INDEXES = [ + ("Comment", ["link_doctype", "link_name"]), + ("Activity Log", ["link_doctype", "link_name"]), +] + + +def execute(): + if frappe.db.db_type == "postgres": + return + + db_tables = frappe.db.get_tables(cached=False) + + # All parent indexes + parent_doctypes = frappe.get_all( + "DocType", + {"istable": 0, "is_virtual": 0, "issingle": 0}, + pluck="name", + ) + db_tables = frappe.db.get_tables(cached=False) + + for doctype in parent_doctypes: + table = f"tab{doctype}" + if table not in db_tables: + continue + _drop_index_if_exists(table, "parent") + + # Unused composite indexes + for doctype, index_fields in UNUSED_INDEXES: + table = f"tab{doctype}" + index_name = frappe.db.get_index_name(index_fields) + if table not in db_tables: + continue + _drop_index_if_exists(table, index_name) + + +def _drop_index_if_exists(table: str, index: str): + if not frappe.db.has_index(table, index): + click.echo(f"- Skipped {index} index for {table} because it doesn't exist") + return + + try: + frappe.db.sql_ddl(f"ALTER TABLE `{table}` DROP INDEX `{index}`") + except Exception as e: + frappe.log_error("Failed to drop index") + click.secho(f"x Failed to drop index {index} from {table}\n {str(e)}", fg="red") + return + + click.echo(f"✓ dropped {index} index from {table}") diff --git a/frappe/social/doctype/energy_point_log/energy_point_log.json b/frappe/social/doctype/energy_point_log/energy_point_log.json index 8595783d1b..f24545407e 100644 --- a/frappe/social/doctype/energy_point_log/energy_point_log.json +++ b/frappe/social/doctype/energy_point_log/energy_point_log.json @@ -36,8 +36,7 @@ "in_standard_filter": 1, "label": "Type", "options": "Auto\nAppreciation\nCriticism\nReview\nRevert", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "rule", @@ -54,8 +53,7 @@ "in_standard_filter": 1, "label": "Reference Document Type", "options": "DocType", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "reference_name", @@ -112,7 +110,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-10-25 17:25:40.477044", + "modified": "2022-09-13 16:10:37.747013", "modified_by": "Administrator", "module": "Social", "name": "Energy Point Log", @@ -131,5 +129,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "user" -} +} \ No newline at end of file diff --git a/frappe/website/doctype/web_page_view/web_page_view.json b/frappe/website/doctype/web_page_view/web_page_view.json index e4b56183d8..7548ed5f83 100644 --- a/frappe/website/doctype/web_page_view/web_page_view.json +++ b/frappe/website/doctype/web_page_view/web_page_view.json @@ -25,14 +25,12 @@ "fieldname": "referrer", "fieldtype": "Data", "label": "Referrer", - "search_index": 1, "set_only_once": 1 }, { "fieldname": "browser", "fieldtype": "Data", "label": "Browser", - "search_index": 1, "set_only_once": 1 }, { @@ -59,7 +57,7 @@ ], "in_create": 1, "links": [], - "modified": "2021-10-25 14:11:24.718770", + "modified": "2022-09-13 15:38:25.401797", "modified_by": "Administrator", "module": "Website", "name": "Web Page View", @@ -82,5 +80,6 @@ "read_only": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "path" -} +} \ No newline at end of file