From 2ca5c390d062f42fca95dc61cce154090b749c8b Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Tue, 1 Feb 2022 23:00:41 +0530 Subject: [PATCH] fix: Use db insert for logs --- .../core/doctype/data_import/data_import.js | 9 +- .../core/doctype/data_import/data_import.json | 420 ++++++++---------- .../core/doctype/data_import/data_import.py | 25 +- frappe/core/doctype/data_import/importer.py | 21 +- .../data_import_log/data_import_log.json | 4 +- 5 files changed, 229 insertions(+), 250 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 6ea95fba02..1460f8698e 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -278,14 +278,15 @@ frappe.ui.form.on('Data Import', { } }) .then(r => { - let preview_data = r.message.preview_data; - let import_log = r.message.import_log; - frm.events.show_import_preview(frm, preview_data, import_log); + let preview_data = r.message; + frm.events.show_import_preview(frm, preview_data); frm.events.show_import_warnings(frm, preview_data); }); }, - show_import_preview(frm, preview_data, import_log) { + show_import_preview(frm, preview_data) { + let import_log = preview_data.import_log; + if ( frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index e8b8b7a546..9e948dac8c 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -1,227 +1,197 @@ { - "actions": [], - "autoname": "format:{reference_doctype} Import on {creation}", - "beta": 1, - "creation": "2019-08-04 14:16:08.318714", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "import_type", - "download_template", - "import_file", - "html_5", - "google_sheets_url", - "refresh_google_sheet", - "column_break_5", - "status", - "submit_after_import", - "mute_emails", - "template_options", - "import_warnings_section", - "template_warnings", - "import_warnings", - "section_import_preview", - "import_preview", - "import_log_section", - "show_failed_logs", - "import_log_preview" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1, - "set_only_once": 1, - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "import_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Import Type", - "options": "\nInsert New Records\nUpdate Existing Records", - "reqd": 1, - "set_only_once": 1, - "show_days": 1, - "show_seconds": 1 - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "import_file", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Import File", - "read_only_depends_on": "eval: ['Success', 'Partial Success', 'Partially Completed'].includes(doc.status)", - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "import_preview", - "fieldtype": "HTML", - "label": "Import Preview", - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "section_import_preview", - "fieldtype": "Section Break", - "label": "Preview", - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "column_break_5", - "fieldtype": "Column Break", - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "template_options", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Options", - "options": "JSON", - "read_only": 1, - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "import_log_section", - "fieldtype": "Section Break", - "label": "Import Log", - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "import_log_preview", - "fieldtype": "HTML", - "label": "Import Log Preview", - "show_days": 1, - "show_seconds": 1 - }, - { - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 1, - "label": "Status", - "options": "Pending\nSuccess\nPartial Success\nError", - "read_only": 1, - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "template_warnings", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Warnings", - "options": "JSON", - "show_days": 1, - "show_seconds": 1 - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit After Import", - "set_only_once": 1, - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "import_warnings_section", - "fieldtype": "Section Break", - "label": "Import File Errors and Warnings", - "show_days": 1, - "show_seconds": 1 - }, - { - "fieldname": "import_warnings", - "fieldtype": "HTML", - "label": "Import Warnings", - "show_days": 1, - "show_seconds": 1 - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "download_template", - "fieldtype": "Button", - "label": "Download Template", - "show_days": 1, - "show_seconds": 1 - }, - { - "default": "1", - "fieldname": "mute_emails", - "fieldtype": "Check", - "label": "Don't Send Emails", - "set_only_once": 1, - "show_days": 1, - "show_seconds": 1 - }, - { - "default": "0", - "fieldname": "show_failed_logs", - "fieldtype": "Check", - "label": "Show Failed Logs", - "show_days": 1, - "show_seconds": 1 - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file", - "fieldname": "html_5", - "fieldtype": "HTML", - "options": "
Or
", - "show_days": 1, - "show_seconds": 1 - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file\n", - "description": "Must be a publicly accessible Google Sheets URL", - "fieldname": "google_sheets_url", - "fieldtype": "Data", - "label": "Import from Google Sheets", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)", - "show_days": 1, - "show_seconds": 1 - }, - { - "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", - "fieldname": "refresh_google_sheet", - "fieldtype": "Button", - "label": "Refresh Google Sheet", - "show_days": 1, - "show_seconds": 1 - } - ], - "hide_toolbar": 1, - "links": [], - "modified": "2022-01-31 10:50:13.015426", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "actions": [], + "autoname": "format:{reference_doctype} Import on {creation}", + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "import_type", + "download_template", + "import_file", + "payload_count", + "html_5", + "google_sheets_url", + "refresh_google_sheet", + "column_break_5", + "status", + "submit_after_import", + "mute_emails", + "template_options", + "import_warnings_section", + "template_warnings", + "import_warnings", + "section_import_preview", + "import_preview", + "import_log_section", + "show_failed_logs", + "import_log_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "\nInsert New Records\nUpdate Existing Records", + "reqd": 1, + "set_only_once": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "template_options", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Options", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "label": "Status", + "options": "Pending\nSuccess\nPartial Success\nError", + "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Warnings", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import", + "set_only_once": 1 + }, + { + "fieldname": "import_warnings_section", + "fieldtype": "Section Break", + "label": "Import File Errors and Warnings" + }, + { + "fieldname": "import_warnings", + "fieldtype": "HTML", + "label": "Import Warnings" + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "default": "1", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails", + "set_only_once": 1 + }, + { + "default": "0", + "fieldname": "show_failed_logs", + "fieldtype": "Check", + "label": "Show Failed Logs" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file", + "fieldname": "html_5", + "fieldtype": "HTML", + "options": "
Or
" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file\n", + "description": "Must be a publicly accessible Google Sheets URL", + "fieldname": "google_sheets_url", + "fieldtype": "Data", + "label": "Import from Google Sheets", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", + "fieldname": "refresh_google_sheet", + "fieldtype": "Button", + "label": "Refresh Google Sheet" + }, + { + "fieldname": "payload_count", + "fieldtype": "Int", + "hidden": 1, + "label": "Payload Count", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "links": [], + "modified": "2022-02-01 20:08:37.624914", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 4aacb6925b..0c8ba072a1 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -28,6 +28,7 @@ class DataImport(Document): self.validate_import_file() self.validate_google_sheets_url() + self.set_payload_count() def validate_import_file(self): if self.import_file: @@ -39,6 +40,12 @@ class DataImport(Document): return validate_google_sheets_url(self.google_sheets_url) + def set_payload_count(self): + if self.import_file: + i = self.get_importer() + payloads = i.import_file.get_payloads_for_import() + self.payload_count = len(payloads) + @frappe.whitelist() def get_preview_from_template(self, import_file=None, google_sheets_url=None): if import_file: @@ -90,21 +97,10 @@ class DataImport(Document): @frappe.whitelist() def get_preview_from_template(data_import, import_file=None, google_sheets_url=None): - preview_data = frappe.get_doc("Data Import", data_import).get_preview_from_template( + return frappe.get_doc("Data Import", data_import).get_preview_from_template( import_file, google_sheets_url ) - # get first 10 import log if any - import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes"], - filters={"data_import": data_import}, - order_by="log_index", limit=10) - - return { - 'preview_data': preview_data, - 'import_log': import_log - } - - @frappe.whitelist() def form_start_import(data_import): return frappe.get_doc("Data Import", data_import).start_import() @@ -172,14 +168,15 @@ def get_import_status(data_import_name): filters={'data_import': data_import_name}, group_by='success') + total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count') + for log in logs: if log.get('success'): import_status['success'] = log.get('count') else: import_status['failed'] = log.get('count') - import_status['total_records'] = cint(import_status.get('success')) + \ - cint(import_status.get('failed')) + import_status['total_records'] = total_payload_count return import_status diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 8c98099c1a..d13f4c1389 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -47,7 +47,13 @@ class Importer: ) def get_data_for_import_preview(self): - return self.import_file.get_data_for_import_preview() + out = self.import_file.get_data_for_import_preview() + + out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index", limit=10) + + return out def before_import(self): # set user lang for translations @@ -85,7 +91,7 @@ class Importer: log_index = 0 # Do not remove rows in case of retry after an error or pending data import - if self.data_import.status == "Partial Success" and len(import_log) > len(payloads): + if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: # remove previous failures from import log only in case of retry after partial success import_log = [log for log in import_log if log.get("success")] @@ -93,7 +99,7 @@ class Importer: imported_rows = [] for log in import_log: log = frappe._dict(log) - if log.success or len(import_log) < len(payloads): + if log.success or len(import_log) < self.data_import.payload_count: imported_rows += json.loads(log.row_indexes) log_index = log.log_index @@ -182,6 +188,11 @@ class Importer: frappe.db.commit() log_index += 1 + # Logs are db inserted directly so will have to be fetched again + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + # set status failures = [log for log in import_log if not log.get("success")] if len(failures) == total_payload_count: @@ -1217,7 +1228,7 @@ def get_select_options(df): return [d for d in (df.options or "").split("\n") if d] def create_import_log(data_import, log_index, log_details): - return frappe.get_doc({ + frappe.get_doc({ 'doctype': 'Data Import Log', 'log_index': log_index, 'success': log_details.get('success'), @@ -1226,6 +1237,6 @@ def create_import_log(data_import, log_index, log_details): 'docname': log_details.get('docname'), 'messages': json.dumps(log_details.get('messages', '[]')), 'exception': log_details.get('exception') - }).insert(ignore_permissions=True) + }).db_insert() diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json index a39eb9be1d..8747728945 100644 --- a/frappe/core/doctype/data_import_log/data_import_log.json +++ b/frappe/core/doctype/data_import_log/data_import_log.json @@ -3,7 +3,7 @@ "creation": "2021-12-25 16:12:20.205889", "doctype": "DocType", "editable_grid": 1, - "engine": "InnoDB", + "engine": "MyISAM", "field_order": [ "data_import", "row_indexes", @@ -60,7 +60,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2021-12-27 11:19:19.646076", + "modified": "2021-12-28 11:19:19.646076", "modified_by": "Administrator", "module": "Core", "name": "Data Import Log",