diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 216db53c72..dfc560a98a 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -44,6 +44,7 @@ frappe.ui.form.on('Data Import', { } frm.dashboard.show_progress(__('Import Progress'), percent, message); frm.page.set_indicator(__('In Progress'), 'orange'); + frm.trigger('update_primary_action'); // hide progress when complete if (data.current === data.total) { @@ -80,7 +81,10 @@ frappe.ui.form.on('Data Import', { frm.trigger('show_import_log'); frm.trigger('show_import_warnings'); frm.trigger('toggle_submit_after_import'); - frm.trigger('show_import_status'); + + if (frm.doc.status != 'Pending') + frm.trigger('show_import_status'); + frm.trigger('show_report_error_button'); if (frm.doc.status === 'Partial Success') { @@ -128,40 +132,49 @@ frappe.ui.form.on('Data Import', { }, show_import_status(frm) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); - let successful_records = import_log.filter(log => log.success); - let failed_records = import_log.filter(log => !log.success); - if (successful_records.length === 0) return; - - let message; - if (failed_records.length === 0) { - let message_args = [successful_records.length]; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records.', message_args) - : __('Successfully imported {0} record.', message_args); - } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records.', message_args) - : __('Successfully updated {0} record.', message_args); - } - } else { - let message_args = [successful_records.length, import_log.length]; - if (frm.doc.import_type === 'Insert New Records') { - message = - successful_records.length > 1 - ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); - } else { - message = - successful_records.length > 1 - ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + frappe.call({ + 'method': 'frappe.core.doctype.data_import.data_import.get_import_status', + 'args': { + 'data_import_name': frm.doc.name + }, + 'callback': function(r) { + let successful_records = cint(r.message.success); + let failed_records = cint(r.message.failed); + let total_records = cint(r.message.total_records); + + if (!total_records) return; + + let message; + if (failed_records === 0) { + let message_args = [successful_records]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records > 1 + ? __('Successfully imported {0} records.', message_args) + : __('Successfully imported {0} record.', message_args); + } else { + message = + successful_records > 1 + ? __('Successfully updated {0} records.', message_args) + : __('Successfully updated {0} record.', message_args); + } + } else { + let message_args = [successful_records, total_records]; + if (frm.doc.import_type === 'Insert New Records') { + message = + successful_records > 1 + ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) + : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + } else { + message = + successful_records > 1 + ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) + : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + } + } + frm.dashboard.set_headline(message); } - } - frm.dashboard.set_headline(message); + }); }, show_report_error_button(frm) { @@ -275,7 +288,7 @@ frappe.ui.form.on('Data Import', { }, show_import_preview(frm, preview_data) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); + let import_log = preview_data.import_log; if ( frm.import_preview && @@ -316,6 +329,15 @@ frappe.ui.form.on('Data Import', { ); }, + export_import_log(frm) { + open_url_post( + '/api/method/frappe.core.doctype.data_import.data_import.download_import_log', + { + data_import_name: frm.doc.name + } + ); + }, + show_import_warnings(frm, preview_data) { let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || '[]'); @@ -391,92 +413,131 @@ frappe.ui.form.on('Data Import', { frm.trigger('show_import_log'); }, - show_import_log(frm) { - let import_log = JSON.parse(frm.doc.import_log || '[]'); - let logs = import_log; - frm.toggle_display('import_log', false); - frm.toggle_display('import_log_section', logs.length > 0); + render_import_log(frm) { + frappe.call({ + 'method': 'frappe.client.get_list', + 'args': { + 'doctype': 'Data Import Log', + 'filters': { + 'data_import': frm.doc.name + }, + 'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'], + 'limit_page_length': 5000, + 'order_by': 'log_index' + }, + callback: function(r) { + let logs = r.message; + + if (logs.length === 0) return; + + frm.toggle_display('import_log_section', true); + + let rows = logs + .map(log => { + let html = ''; + if (log.success) { + if (frm.doc.import_type === 'Insert New Records') { + html = __('Successfully imported {0}', [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}` + ]); + } else { + html = __('Successfully updated {0}', [ + `${frappe.utils.get_form_link( + frm.doc.reference_doctype, + log.docname, + true + )}` + ]); + } + } else { + let messages = (JSON.parse(log.messages || '[]')) + .map(JSON.parse) + .map(m => { + let title = m.title ? `${m.title}` : ''; + let message = m.message ? `
${m.message}
` : ''; + return title + message; + }) + .join(''); + let id = frappe.dom.get_unique_id(); + html = `${messages} + +
+
+ 
${log.exception}
+
+
`; + } + let indicator_color = log.success ? 'green' : 'red'; + let title = log.success ? __('Success') : __('Failure'); - if (logs.length === 0) { - frm.get_field('import_log_preview').$wrapper.empty(); - return; - } + if (frm.doc.show_failed_logs && log.success) { + return ''; + } - let rows = logs - .map(log => { - let html = ''; - if (log.success) { - if (frm.doc.import_type === 'Insert New Records') { - html = __('Successfully imported {0}', [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}` - ]); - } else { - html = __('Successfully updated {0}', [ - `${frappe.utils.get_form_link( - frm.doc.reference_doctype, - log.docname, - true - )}` - ]); - } - } else { - let messages = log.messages - .map(JSON.parse) - .map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `
${m.message}
` : ''; - return title + message; - }) - .join(''); - let id = frappe.dom.get_unique_id(); - html = `${messages} - -
-
- 
${log.exception}
-
-
`; - } - let indicator_color = log.success ? 'green' : 'red'; - let title = log.success ? __('Success') : __('Failure'); + return ` + ${JSON.parse(log.row_indexes).join(', ')} + +
${title}
+ + + ${html} + + `; + }) + .join(''); - if (frm.doc.show_failed_logs && log.success) { - return ''; + if (!rows && frm.doc.show_failed_logs) { + rows = ` + ${__('No failed logs')} + `; } - return ` - ${log.row_indexes.join(', ')} - -
${title}
- - - ${html} - - `; - }) - .join(''); + frm.get_field('import_log_preview').$wrapper.html(` + + + + + + + ${rows} +
${__('Row Number')}${__('Status')}${__('Message')}
+ `); + } + }); + }, + + show_import_log(frm) { + frm.toggle_display('import_log_section', false); - if (!rows && frm.doc.show_failed_logs) { - rows = ` - ${__('No failed logs')} - `; + if (frm.import_in_progress) { + return; } - frm.get_field('import_log_preview').$wrapper.html(` - - - - - - - ${rows} -
${__('Row Number')}${__('Status')}${__('Message')}
- `); + frappe.call({ + 'method': 'frappe.client.get_count', + 'args': { + 'doctype': 'Data Import Log', + 'filters': { + 'data_import': frm.doc.name + } + }, + 'callback': function(r) { + let count = r.message; + if (count < 5000) { + frm.trigger('render_import_log'); + } else { + frm.toggle_display('import_log_section', false); + frm.add_custom_button(__('Export Import Log'), () => + frm.trigger('export_import_log') + ); + } + } + }); }, }); diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index fe6fb90481..9e948dac8c 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -1,194 +1,197 @@ { - "actions": [], - "autoname": "format:{reference_doctype} Import on {creation}", - "beta": 1, - "creation": "2019-08-04 14:16:08.318714", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "import_type", - "download_template", - "import_file", - "html_5", - "google_sheets_url", - "refresh_google_sheet", - "column_break_5", - "status", - "submit_after_import", - "mute_emails", - "template_options", - "import_warnings_section", - "template_warnings", - "import_warnings", - "section_import_preview", - "import_preview", - "import_log_section", - "import_log", - "show_failed_logs", - "import_log_preview" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1, - "set_only_once": 1 - }, - { - "fieldname": "import_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Import Type", - "options": "\nInsert New Records\nUpdate Existing Records", - "reqd": 1, - "set_only_once": 1 - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "import_file", - "fieldtype": "Attach", - "in_list_view": 1, - "label": "Import File", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "fieldname": "import_preview", - "fieldtype": "HTML", - "label": "Import Preview" - }, - { - "fieldname": "section_import_preview", - "fieldtype": "Section Break", - "label": "Preview" - }, - { - "fieldname": "column_break_5", - "fieldtype": "Column Break" - }, - { - "fieldname": "template_options", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Options", - "options": "JSON", - "read_only": 1 - }, - { - "fieldname": "import_log", - "fieldtype": "Code", - "label": "Import Log", - "options": "JSON" - }, - { - "fieldname": "import_log_section", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "import_log_preview", - "fieldtype": "HTML", - "label": "Import Log Preview" - }, - { - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 1, - "label": "Status", - "options": "Pending\nSuccess\nPartial Success\nError", - "read_only": 1 - }, - { - "fieldname": "template_warnings", - "fieldtype": "Code", - "hidden": 1, - "label": "Template Warnings", - "options": "JSON" - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit After Import", - "set_only_once": 1 - }, - { - "fieldname": "import_warnings_section", - "fieldtype": "Section Break", - "label": "Import File Errors and Warnings" - }, - { - "fieldname": "import_warnings", - "fieldtype": "HTML", - "label": "Import Warnings" - }, - { - "depends_on": "eval:!doc.__islocal", - "fieldname": "download_template", - "fieldtype": "Button", - "label": "Download Template" - }, - { - "default": "1", - "fieldname": "mute_emails", - "fieldtype": "Check", - "label": "Don't Send Emails", - "set_only_once": 1 - }, - { - "default": "0", - "fieldname": "show_failed_logs", - "fieldtype": "Check", - "label": "Show Failed Logs" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file", - "fieldname": "html_5", - "fieldtype": "HTML", - "options": "
Or
" - }, - { - "depends_on": "eval:!doc.__islocal && !doc.import_file\n", - "description": "Must be a publicly accessible Google Sheets URL", - "fieldname": "google_sheets_url", - "fieldtype": "Data", - "label": "Import from Google Sheets", - "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" - }, - { - "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", - "fieldname": "refresh_google_sheet", - "fieldtype": "Button", - "label": "Refresh Google Sheet" - } - ], - "hide_toolbar": 1, - "links": [], - "modified": "2021-04-11 01:50:42.074623", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 + "actions": [], + "autoname": "format:{reference_doctype} Import on {creation}", + "beta": 1, + "creation": "2019-08-04 14:16:08.318714", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "import_type", + "download_template", + "import_file", + "payload_count", + "html_5", + "google_sheets_url", + "refresh_google_sheet", + "column_break_5", + "status", + "submit_after_import", + "mute_emails", + "template_options", + "import_warnings_section", + "template_warnings", + "import_warnings", + "section_import_preview", + "import_preview", + "import_log_section", + "show_failed_logs", + "import_log_preview" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1, + "set_only_once": 1 + }, + { + "fieldname": "import_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Import Type", + "options": "\nInsert New Records\nUpdate Existing Records", + "reqd": 1, + "set_only_once": 1 + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "import_file", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Import File", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "fieldname": "import_preview", + "fieldtype": "HTML", + "label": "Import Preview" + }, + { + "fieldname": "section_import_preview", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "template_options", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Options", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "import_log_section", + "fieldtype": "Section Break", + "label": "Import Log" + }, + { + "fieldname": "import_log_preview", + "fieldtype": "HTML", + "label": "Import Log Preview" + }, + { + "default": "Pending", + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "label": "Status", + "options": "Pending\nSuccess\nPartial Success\nError", + "read_only": 1 + }, + { + "fieldname": "template_warnings", + "fieldtype": "Code", + "hidden": 1, + "label": "Template Warnings", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "label": "Submit After Import", + "set_only_once": 1 + }, + { + "fieldname": "import_warnings_section", + "fieldtype": "Section Break", + "label": "Import File Errors and Warnings" + }, + { + "fieldname": "import_warnings", + "fieldtype": "HTML", + "label": "Import Warnings" + }, + { + "depends_on": "eval:!doc.__islocal", + "fieldname": "download_template", + "fieldtype": "Button", + "label": "Download Template" + }, + { + "default": "1", + "fieldname": "mute_emails", + "fieldtype": "Check", + "label": "Don't Send Emails", + "set_only_once": 1 + }, + { + "default": "0", + "fieldname": "show_failed_logs", + "fieldtype": "Check", + "label": "Show Failed Logs" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file", + "fieldname": "html_5", + "fieldtype": "HTML", + "options": "
Or
" + }, + { + "depends_on": "eval:!doc.__islocal && !doc.import_file\n", + "description": "Must be a publicly accessible Google Sheets URL", + "fieldname": "google_sheets_url", + "fieldtype": "Data", + "label": "Import from Google Sheets", + "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)" + }, + { + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)", + "fieldname": "refresh_google_sheet", + "fieldtype": "Button", + "label": "Refresh Google Sheet" + }, + { + "fieldname": "payload_count", + "fieldtype": "Int", + "hidden": 1, + "label": "Payload Count", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "links": [], + "modified": "2022-02-01 20:08:37.624914", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5935ddc4ba..5972e79b4d 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -27,6 +27,7 @@ class DataImport(Document): self.validate_import_file() self.validate_google_sheets_url() + self.set_payload_count() def validate_import_file(self): if self.import_file: @@ -38,6 +39,12 @@ class DataImport(Document): return validate_google_sheets_url(self.google_sheets_url) + def set_payload_count(self): + if self.import_file: + i = self.get_importer() + payloads = i.import_file.get_payloads_for_import() + self.payload_count = len(payloads) + @frappe.whitelist() def get_preview_from_template(self, import_file=None, google_sheets_url=None): if import_file: @@ -67,7 +74,7 @@ class DataImport(Document): enqueue( start_import, queue="default", - timeout=6000, + timeout=10000, event="data_import", job_name=self.name, data_import=self.name, @@ -80,6 +87,9 @@ class DataImport(Document): def export_errored_rows(self): return self.get_importer().export_errored_rows() + def download_import_log(self): + return self.get_importer().export_import_log() + def get_importer(self): return Importer(self.reference_doctype, data_import=self) @@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N import_file, google_sheets_url ) - @frappe.whitelist() def form_start_import(data_import): return frappe.get_doc("Data Import", data_import).start_import() @@ -145,6 +154,30 @@ def download_errored_template(data_import_name): data_import = frappe.get_doc("Data Import", data_import_name) data_import.export_errored_rows() +@frappe.whitelist() +def download_import_log(data_import_name): + data_import = frappe.get_doc("Data Import", data_import_name) + data_import.download_import_log() + +@frappe.whitelist() +def get_import_status(data_import_name): + import_status = {} + + logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'], + filters={'data_import': data_import_name}, + group_by='success') + + total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count') + + for log in logs: + if log.get('success'): + import_status['success'] = log.get('count') + else: + import_status['failed'] = log.get('count') + + import_status['total_records'] = total_payload_count + + return import_status def import_file( doctype, file_path, import_type, submit_after_import=False, console=False diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 0eb05aa354..6ab750ba25 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = { 'Error': 'red' }; let status = doc.status; + if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } if (status == 'Pending') { status = 'Not Started'; } + return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index b9b2050763..107c05a66a 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -47,7 +47,13 @@ class Importer: ) def get_data_for_import_preview(self): - return self.import_file.get_data_for_import_preview() + out = self.import_file.get_data_for_import_preview() + + out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index", limit=10) + + return out def before_import(self): # set user lang for translations @@ -58,7 +64,6 @@ class Importer: frappe.flags.in_import = True frappe.flags.mute_emails = self.data_import.mute_emails - self.data_import.db_set("status", "Pending") self.data_import.db_set("template_warnings", "") def import_data(self): @@ -79,20 +84,25 @@ class Importer: return # setup import log - if self.data_import.import_log: - import_log = frappe.parse_json(self.data_import.import_log) - else: - import_log = [] + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] - # remove previous failures from import log - import_log = [log for log in import_log if log.get("success")] + log_index = 0 + + # Do not remove rows in case of retry after an error or pending data import + if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: + # remove previous failures from import log only in case of retry after partial success + import_log = [log for log in import_log if log.get("success")] # get successfully imported rows imported_rows = [] for log in import_log: log = frappe._dict(log) - if log.success: - imported_rows += log.row_indexes + if log.success or len(import_log) < self.data_import.payload_count: + imported_rows += json.loads(log.row_indexes) + + log_index = log.log_index # start import total_payload_count = len(payloads) @@ -146,25 +156,41 @@ class Importer: }, ) - import_log.append( - frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) - ) + create_import_log(self.data_import.name, log_index, { + 'success': True, + 'docname': doc.name, + 'row_indexes': row_indexes + }) + + log_index += 1 + + if not self.data_import.status == "Partial Success": + self.data_import.db_set("status", "Partial Success") + # commit after every successful import frappe.db.commit() except Exception: - import_log.append( - frappe._dict( - success=False, - exception=frappe.get_traceback(), - messages=frappe.local.message_log, - row_indexes=row_indexes, - ) - ) + messages = frappe.local.message_log frappe.clear_messages() + # rollback if exception frappe.db.rollback() + create_import_log(self.data_import.name, log_index, { + 'success': False, + 'exception': frappe.get_traceback(), + 'messages': messages, + 'row_indexes': row_indexes + }) + + log_index += 1 + + # Logs are db inserted directly so will have to be fetched again + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + # set status failures = [log for log in import_log if not log.get("success")] if len(failures) == total_payload_count: @@ -178,7 +204,6 @@ class Importer: self.print_import_log(import_log) else: self.data_import.db_set("status", status) - self.data_import.db_set("import_log", json.dumps(import_log)) self.after_import() @@ -248,11 +273,14 @@ class Importer: if not self.data_import: return - import_log = frappe.parse_json(self.data_import.import_log or "[]") + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index") or [] + failures = [log for log in import_log if not log.get("success")] row_indexes = [] for f in failures: - row_indexes.extend(f.get("row_indexes", [])) + row_indexes.extend(json.loads(f.get("row_indexes", []))) # de duplicate row_indexes = list(set(row_indexes)) @@ -264,6 +292,30 @@ class Importer: build_csv_response(rows, _(self.doctype)) + def export_import_log(self): + from frappe.utils.csvutils import build_csv_response + + if not self.data_import: + return + + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": self.data_import.name}, + order_by="log_index") + + header_row = ["Row Numbers", "Status", "Message", "Exception"] + + rows = [header_row] + + for log in import_log: + row_number = json.loads(log.get("row_indexes"))[0] + status = "Success" if log.get('success') else "Failure" + message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \ + log.get("messages") + exception = frappe.utils.cstr(log.get("exception", '')) + rows += [[row_number, status, message, exception]] + + build_csv_response(rows, self.doctype) + def print_import_log(self, import_log): failed_records = [log for log in import_log if not log.success] successful_records = [log for log in import_log if log.success] @@ -1172,3 +1224,17 @@ def df_as_json(df): def get_select_options(df): return [d for d in (df.options or "").split("\n") if d] + +def create_import_log(data_import, log_index, log_details): + frappe.get_doc({ + 'doctype': 'Data Import Log', + 'log_index': log_index, + 'success': log_details.get('success'), + 'data_import': data_import, + 'row_indexes': json.dumps(log_details.get('row_indexes')), + 'docname': log_details.get('docname'), + 'messages': json.dumps(log_details.get('messages', '[]')), + 'exception': log_details.get('exception') + }).db_insert() + + diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index e1bc0e7ca5..3f78594dd2 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -60,15 +60,19 @@ class TestImporter(unittest.TestCase): frappe.local.message_log = [] data_import.start_import() data_import.reload() - import_log = frappe.parse_json(data_import.import_log) - self.assertEqual(import_log[0]['row_indexes'], [2,3]) + + import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + filters={"data_import": data_import.name}, + order_by="log_index") + + self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3]) expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error) + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error) expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error) + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error) - self.assertEqual(import_log[1]['row_indexes'], [4]) - self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required") + self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4]) + self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required") def test_data_import_update(self): existing_doc = frappe.get_doc( diff --git a/frappe/core/doctype/data_import_log/__init__.py b/frappe/core/doctype/data_import_log/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/data_import_log/data_import_log.js b/frappe/core/doctype/data_import_log/data_import_log.js new file mode 100644 index 0000000000..c376edeec9 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2021, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Import Log', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json new file mode 100644 index 0000000000..b1d991f099 --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.json @@ -0,0 +1,84 @@ +{ + "actions": [], + "creation": "2021-12-25 16:12:20.205889", + "doctype": "DocType", + "editable_grid": 1, + "engine": "MyISAM", + "field_order": [ + "data_import", + "row_indexes", + "success", + "docname", + "messages", + "exception", + "log_index" + ], + "fields": [ + { + "fieldname": "data_import", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Data Import", + "options": "Data Import" + }, + { + "fieldname": "docname", + "fieldtype": "Data", + "label": "Reference Name" + }, + { + "fieldname": "exception", + "fieldtype": "Text", + "label": "Exception" + }, + { + "fieldname": "row_indexes", + "fieldtype": "Code", + "label": "Row Indexes", + "options": "JSON" + }, + { + "default": "0", + "fieldname": "success", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Success" + }, + { + "fieldname": "log_index", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Log Index" + }, + { + "fieldname": "messages", + "fieldtype": "Code", + "label": "Messages", + "options": "JSON" + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2021-12-29 11:19:19.646076", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_log/data_import_log.py b/frappe/core/doctype/data_import_log/data_import_log.py new file mode 100644 index 0000000000..a71aefa8bc --- /dev/null +++ b/frappe/core/doctype/data_import_log/data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class DataImportLog(Document): + pass diff --git a/frappe/core/doctype/data_import_log/test_data_import_log.py b/frappe/core/doctype/data_import_log/test_data_import_log.py new file mode 100644 index 0000000000..244404936e --- /dev/null +++ b/frappe/core/doctype/data_import_log/test_data_import_log.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and Contributors +# See license.txt + +# import frappe +import unittest + +class TestDataImportLog(unittest.TestCase): + pass diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 2264042539..b153718c70 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview { is_row_imported(row) { let serial_no = row[0].content; return this.import_log.find(log => { - return log.success && log.row_indexes.includes(serial_no); + return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no); }); } };