Merge pull request #15829 from deepeshgarg007/data_import_enhancements_develop

fix: Ability to continue partially processed data imports

frappe/core/doctype/data_import/data_import.js

@@ -44,6 +44,7 @@ frappe.ui.form.on('Data Import', {
 			}
 			frm.dashboard.show_progress(__('Import Progress'), percent, message);
 			frm.page.set_indicator(__('In Progress'), 'orange');
+			frm.trigger('update_primary_action');
 
 			// hide progress when complete
 			if (data.current === data.total) {
@@ -80,7 +81,10 @@ frappe.ui.form.on('Data Import', {
 		frm.trigger('show_import_log');
 		frm.trigger('show_import_warnings');
 		frm.trigger('toggle_submit_after_import');
-		frm.trigger('show_import_status');
+
+		if (frm.doc.status != 'Pending')
+			frm.trigger('show_import_status');
+
 		frm.trigger('show_report_error_button');
 
 		if (frm.doc.status === 'Partial Success') {
@@ -128,40 +132,49 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	show_import_status(frm) {
-		let import_log = JSON.parse(frm.doc.import_log || '[]');
-		let successful_records = import_log.filter(log => log.success);
-		let failed_records = import_log.filter(log => !log.success);
-		if (successful_records.length === 0) return;
-
-		let message;
-		if (failed_records.length === 0) {
-			let message_args = [successful_records.length];
-			if (frm.doc.import_type === 'Insert New Records') {
-				message =
-					successful_records.length > 1
-						? __('Successfully imported {0} records.', message_args)
-						: __('Successfully imported {0} record.', message_args);
-			} else {
-				message =
-					successful_records.length > 1
-						? __('Successfully updated {0} records.', message_args)
-						: __('Successfully updated {0} record.', message_args);
-			}
-		} else {
-			let message_args = [successful_records.length, import_log.length];
-			if (frm.doc.import_type === 'Insert New Records') {
-				message =
-					successful_records.length > 1
-						? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
-						: __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
-			} else {
-				message =
-					successful_records.length > 1
-						? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
-						: __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
+		frappe.call({
+			'method': 'frappe.core.doctype.data_import.data_import.get_import_status',
+			'args': {
+				'data_import_name': frm.doc.name
+			},
+			'callback': function(r) {
+				let successful_records = cint(r.message.success);
+				let failed_records = cint(r.message.failed);
+				let total_records = cint(r.message.total_records);
+
+				if (!total_records) return;
+
+				let message;
+				if (failed_records === 0) {
+					let message_args = [successful_records];
+					if (frm.doc.import_type === 'Insert New Records') {
+						message =
+							successful_records > 1
+								? __('Successfully imported {0} records.', message_args)
+								: __('Successfully imported {0} record.', message_args);
+					} else {
+						message =
+							successful_records > 1
+								? __('Successfully updated {0} records.', message_args)
+								: __('Successfully updated {0} record.', message_args);
+					}
+				} else {
+					let message_args = [successful_records, total_records];
+					if (frm.doc.import_type === 'Insert New Records') {
+						message =
+							successful_records > 1
+								? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
+								: __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
+					} else {
+						message =
+							successful_records > 1
+								? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args)
+								: __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args);
+					}
+				}
+				frm.dashboard.set_headline(message);
 			}
-		}
-		frm.dashboard.set_headline(message);
+		});
 	},
 
 	show_report_error_button(frm) {
@@ -275,7 +288,7 @@ frappe.ui.form.on('Data Import', {
 	},
 
 	show_import_preview(frm, preview_data) {
-		let import_log = JSON.parse(frm.doc.import_log || '[]');
+		let import_log = preview_data.import_log;
 
 		if (
 			frm.import_preview &&
@@ -316,6 +329,15 @@ frappe.ui.form.on('Data Import', {
 		);
 	},
 
+	export_import_log(frm) {
+		open_url_post(
+			'/api/method/frappe.core.doctype.data_import.data_import.download_import_log',
+			{
+				data_import_name: frm.doc.name
+			}
+		);
+	},
+
 	show_import_warnings(frm, preview_data) {
 		let columns = preview_data.columns;
 		let warnings = JSON.parse(frm.doc.template_warnings || '[]');
@@ -391,92 +413,131 @@ frappe.ui.form.on('Data Import', {
 		frm.trigger('show_import_log');
 	},
 
-	show_import_log(frm) {
-		let import_log = JSON.parse(frm.doc.import_log || '[]');
-		let logs = import_log;
-		frm.toggle_display('import_log', false);
-		frm.toggle_display('import_log_section', logs.length > 0);
+	render_import_log(frm) {
+		frappe.call({
+			'method': 'frappe.client.get_list',
+			'args': {
+				'doctype': 'Data Import Log',
+				'filters': {
+					'data_import': frm.doc.name
+				},
+				'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'],
+				'limit_page_length': 5000,
+				'order_by': 'log_index'
+			},
+			callback: function(r) {
+				let logs = r.message;
+
+				if (logs.length === 0) return;
+
+				frm.toggle_display('import_log_section', true);
+
+				let rows = logs
+					.map(log => {
+						let html = '';
+						if (log.success) {
+							if (frm.doc.import_type === 'Insert New Records') {
+								html = __('Successfully imported {0}', [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`
+								]);
+							} else {
+								html = __('Successfully updated {0}', [
+									`<span class="underline">${frappe.utils.get_form_link(
+										frm.doc.reference_doctype,
+										log.docname,
+										true
+									)}<span>`
+								]);
+							}
+						} else {
+							let messages = (JSON.parse(log.messages || '[]'))
+								.map(JSON.parse)
+								.map(m => {
+									let title = m.title ? `<strong>${m.title}</strong>` : '';
+									let message = m.message ? `<div>${m.message}</div>` : '';
+									return title + message;
+								})
+								.join('');
+							let id = frappe.dom.get_unique_id();
+							html = `${messages}
+								<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
+									${__('Show Traceback')}
+								</button>
+								<div class="collapse" id="${id}" style="margin-top: 15px;">
+									<div class="well">
+										<pre>${log.exception}</pre>
+									</div>
+								</div>`;
+						}
+						let indicator_color = log.success ? 'green' : 'red';
+						let title = log.success ? __('Success') : __('Failure');
 
-		if (logs.length === 0) {
-			frm.get_field('import_log_preview').$wrapper.empty();
-			return;
-		}
+						if (frm.doc.show_failed_logs && log.success) {
+							return '';
+						}
 
-		let rows = logs
-			.map(log => {
-				let html = '';
-				if (log.success) {
-					if (frm.doc.import_type === 'Insert New Records') {
-						html = __('Successfully imported {0}', [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`
-						]);
-					} else {
-						html = __('Successfully updated {0}', [
-							`<span class="underline">${frappe.utils.get_form_link(
-								frm.doc.reference_doctype,
-								log.docname,
-								true
-							)}<span>`
-						]);
-					}
-				} else {
-					let messages = log.messages
-						.map(JSON.parse)
-						.map(m => {
-							let title = m.title ? `<strong>${m.title}</strong>` : '';
-							let message = m.message ? `<div>${m.message}</div>` : '';
-							return title + message;
-						})
-						.join('');
-					let id = frappe.dom.get_unique_id();
-					html = `${messages}
-						<button class="btn btn-default btn-xs" type="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false" aria-controls="${id}" style="margin-top: 15px;">
-							${__('Show Traceback')}
-						</button>
-						<div class="collapse" id="${id}" style="margin-top: 15px;">
-							<div class="well">
-								<pre>${log.exception}</pre>
-							</div>
-						</div>`;
-				}
-				let indicator_color = log.success ? 'green' : 'red';
-				let title = log.success ? __('Success') : __('Failure');
+						return `<tr>
+							<td>${JSON.parse(log.row_indexes).join(', ')}</td>
+							<td>
+								<div class="indicator ${indicator_color}">${title}</div>
+							</td>
+							<td>
+								${html}
+							</td>
+						</tr>`;
+					})
+					.join('');
 
-				if (frm.doc.show_failed_logs && log.success) {
-					return '';
+				if (!rows && frm.doc.show_failed_logs) {
+					rows = `<tr><td class="text-center text-muted" colspan=3>
+						${__('No failed logs')}
+					</td></tr>`;
 				}
 
-				return `<tr>
-					<td>${log.row_indexes.join(', ')}</td>
-					<td>
-						<div class="indicator ${indicator_color}">${title}</div>
-					</td>
-					<td>
-						${html}
-					</td>
-				</tr>`;
-			})
-			.join('');
+				frm.get_field('import_log_preview').$wrapper.html(`
+					<table class="table table-bordered">
+						<tr class="text-muted">
+							<th width="10%">${__('Row Number')}</th>
+							<th width="10%">${__('Status')}</th>
+							<th width="80%">${__('Message')}</th>
+						</tr>
+						${rows}
+					</table>
+				`);
+			}
+		});
+	},
+
+	show_import_log(frm) {
+		frm.toggle_display('import_log_section', false);
 
-		if (!rows && frm.doc.show_failed_logs) {
-			rows = `<tr><td class="text-center text-muted" colspan=3>
-				${__('No failed logs')}
-			</td></tr>`;
+		if (frm.import_in_progress) {
+			return;
 		}
 
-		frm.get_field('import_log_preview').$wrapper.html(`
-			<table class="table table-bordered">
-				<tr class="text-muted">
-					<th width="10%">${__('Row Number')}</th>
-					<th width="10%">${__('Status')}</th>
-					<th width="80%">${__('Message')}</th>
-				</tr>
-				${rows}
-			</table>
-		`);
+		frappe.call({
+			'method': 'frappe.client.get_count',
+			'args': {
+				'doctype': 'Data Import Log',
+				'filters': {
+					'data_import': frm.doc.name
+				}
+			},
+			'callback': function(r) {
+				let count = r.message;
+				if (count < 5000) {
+					frm.trigger('render_import_log');
+				} else {
+					frm.toggle_display('import_log_section', false);
+					frm.add_custom_button(__('Export Import Log'), () =>
+						frm.trigger('export_import_log')
+					);
+				}
+			}
+		});
 	},
 });

frappe/core/doctype/data_import/data_import.json

@@ -1,194 +1,197 @@
 {
- "actions": [],
- "autoname": "format:{reference_doctype} Import on {creation}",
- "beta": 1,
- "creation": "2019-08-04 14:16:08.318714",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "reference_doctype",
-  "import_type",
-  "download_template",
-  "import_file",
-  "html_5",
-  "google_sheets_url",
-  "refresh_google_sheet",
-  "column_break_5",
-  "status",
-  "submit_after_import",
-  "mute_emails",
-  "template_options",
-  "import_warnings_section",
-  "template_warnings",
-  "import_warnings",
-  "section_import_preview",
-  "import_preview",
-  "import_log_section",
-  "import_log",
-  "show_failed_logs",
-  "import_log_preview"
- ],
- "fields": [
-  {
-   "fieldname": "reference_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "reqd": 1,
-   "set_only_once": 1
-  },
-  {
-   "fieldname": "import_type",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Import Type",
-   "options": "\nInsert New Records\nUpdate Existing Records",
-   "reqd": 1,
-   "set_only_once": 1
-  },
-  {
-   "depends_on": "eval:!doc.__islocal",
-   "fieldname": "import_file",
-   "fieldtype": "Attach",
-   "in_list_view": 1,
-   "label": "Import File",
-   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-  },
-  {
-   "fieldname": "import_preview",
-   "fieldtype": "HTML",
-   "label": "Import Preview"
-  },
-  {
-   "fieldname": "section_import_preview",
-   "fieldtype": "Section Break",
-   "label": "Preview"
-  },
-  {
-   "fieldname": "column_break_5",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "template_options",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Template Options",
-   "options": "JSON",
-   "read_only": 1
-  },
-  {
-   "fieldname": "import_log",
-   "fieldtype": "Code",
-   "label": "Import Log",
-   "options": "JSON"
-  },
-  {
-   "fieldname": "import_log_section",
-   "fieldtype": "Section Break",
-   "label": "Import Log"
-  },
-  {
-   "fieldname": "import_log_preview",
-   "fieldtype": "HTML",
-   "label": "Import Log Preview"
-  },
-  {
-   "default": "Pending",
-   "fieldname": "status",
-   "fieldtype": "Select",
-   "hidden": 1,
-   "label": "Status",
-   "options": "Pending\nSuccess\nPartial Success\nError",
-   "read_only": 1
-  },
-  {
-   "fieldname": "template_warnings",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Template Warnings",
-   "options": "JSON"
-  },
-  {
-   "default": "0",
-   "fieldname": "submit_after_import",
-   "fieldtype": "Check",
-   "label": "Submit After Import",
-   "set_only_once": 1
-  },
-  {
-   "fieldname": "import_warnings_section",
-   "fieldtype": "Section Break",
-   "label": "Import File Errors and Warnings"
-  },
-  {
-   "fieldname": "import_warnings",
-   "fieldtype": "HTML",
-   "label": "Import Warnings"
-  },
-  {
-   "depends_on": "eval:!doc.__islocal",
-   "fieldname": "download_template",
-   "fieldtype": "Button",
-   "label": "Download Template"
-  },
-  {
-   "default": "1",
-   "fieldname": "mute_emails",
-   "fieldtype": "Check",
-   "label": "Don't Send Emails",
-   "set_only_once": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "show_failed_logs",
-   "fieldtype": "Check",
-   "label": "Show Failed Logs"
-  },
-  {
-   "depends_on": "eval:!doc.__islocal && !doc.import_file",
-   "fieldname": "html_5",
-   "fieldtype": "HTML",
-   "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
-  },
-  {
-   "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
-   "description": "Must be a publicly accessible Google Sheets URL",
-   "fieldname": "google_sheets_url",
-   "fieldtype": "Data",
-   "label": "Import from Google Sheets",
-   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
-  },
-  {
-   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
-   "fieldname": "refresh_google_sheet",
-   "fieldtype": "Button",
-   "label": "Refresh Google Sheet"
-  }
- ],
- "hide_toolbar": 1,
- "links": [],
- "modified": "2021-04-11 01:50:42.074623",
- "modified_by": "Administrator",
- "module": "Core",
- "name": "Data Import",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
+    "actions": [],
+    "autoname": "format:{reference_doctype} Import on {creation}",
+    "beta": 1,
+    "creation": "2019-08-04 14:16:08.318714",
+    "doctype": "DocType",
+    "editable_grid": 1,
+    "engine": "InnoDB",
+    "field_order": [
+        "reference_doctype",
+        "import_type",
+        "download_template",
+        "import_file",
+        "payload_count",
+        "html_5",
+        "google_sheets_url",
+        "refresh_google_sheet",
+        "column_break_5",
+        "status",
+        "submit_after_import",
+        "mute_emails",
+        "template_options",
+        "import_warnings_section",
+        "template_warnings",
+        "import_warnings",
+        "section_import_preview",
+        "import_preview",
+        "import_log_section",
+        "show_failed_logs",
+        "import_log_preview"
+    ],
+    "fields": [
+        {
+            "fieldname": "reference_doctype",
+            "fieldtype": "Link",
+            "in_list_view": 1,
+            "label": "Document Type",
+            "options": "DocType",
+            "reqd": 1,
+            "set_only_once": 1
+        },
+        {
+            "fieldname": "import_type",
+            "fieldtype": "Select",
+            "in_list_view": 1,
+            "label": "Import Type",
+            "options": "\nInsert New Records\nUpdate Existing Records",
+            "reqd": 1,
+            "set_only_once": 1
+        },
+        {
+            "depends_on": "eval:!doc.__islocal",
+            "fieldname": "import_file",
+            "fieldtype": "Attach",
+            "in_list_view": 1,
+            "label": "Import File",
+            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+        },
+        {
+            "fieldname": "import_preview",
+            "fieldtype": "HTML",
+            "label": "Import Preview"
+        },
+        {
+            "fieldname": "section_import_preview",
+            "fieldtype": "Section Break",
+            "label": "Preview"
+        },
+        {
+            "fieldname": "column_break_5",
+            "fieldtype": "Column Break"
+        },
+        {
+            "fieldname": "template_options",
+            "fieldtype": "Code",
+            "hidden": 1,
+            "label": "Template Options",
+            "options": "JSON",
+            "read_only": 1
+        },
+        {
+            "fieldname": "import_log_section",
+            "fieldtype": "Section Break",
+            "label": "Import Log"
+        },
+        {
+            "fieldname": "import_log_preview",
+            "fieldtype": "HTML",
+            "label": "Import Log Preview"
+        },
+        {
+            "default": "Pending",
+            "fieldname": "status",
+            "fieldtype": "Select",
+            "hidden": 1,
+            "label": "Status",
+            "options": "Pending\nSuccess\nPartial Success\nError",
+            "read_only": 1
+        },
+        {
+            "fieldname": "template_warnings",
+            "fieldtype": "Code",
+            "hidden": 1,
+            "label": "Template Warnings",
+            "options": "JSON"
+        },
+        {
+            "default": "0",
+            "fieldname": "submit_after_import",
+            "fieldtype": "Check",
+            "label": "Submit After Import",
+            "set_only_once": 1
+        },
+        {
+            "fieldname": "import_warnings_section",
+            "fieldtype": "Section Break",
+            "label": "Import File Errors and Warnings"
+        },
+        {
+            "fieldname": "import_warnings",
+            "fieldtype": "HTML",
+            "label": "Import Warnings"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal",
+            "fieldname": "download_template",
+            "fieldtype": "Button",
+            "label": "Download Template"
+        },
+        {
+            "default": "1",
+            "fieldname": "mute_emails",
+            "fieldtype": "Check",
+            "label": "Don't Send Emails",
+            "set_only_once": 1
+        },
+        {
+            "default": "0",
+            "fieldname": "show_failed_logs",
+            "fieldtype": "Check",
+            "label": "Show Failed Logs"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal && !doc.import_file",
+            "fieldname": "html_5",
+            "fieldtype": "HTML",
+            "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
+            "description": "Must be a publicly accessible Google Sheets URL",
+            "fieldname": "google_sheets_url",
+            "fieldtype": "Data",
+            "label": "Import from Google Sheets",
+            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+        },
+        {
+            "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
+            "fieldname": "refresh_google_sheet",
+            "fieldtype": "Button",
+            "label": "Refresh Google Sheet"
+        },
+        {
+            "fieldname": "payload_count",
+            "fieldtype": "Int",
+            "hidden": 1,
+            "label": "Payload Count",
+            "read_only": 1
+        }
+    ],
+    "hide_toolbar": 1,
+    "links": [],
+    "modified": "2022-02-01 20:08:37.624914",
+    "modified_by": "Administrator",
+    "module": "Core",
+    "name": "Data Import",
+    "naming_rule": "Expression",
+    "owner": "Administrator",
+    "permissions": [
+        {
+            "create": 1,
+            "delete": 1,
+            "email": 1,
+            "export": 1,
+            "print": 1,
+            "read": 1,
+            "report": 1,
+            "role": "System Manager",
+            "share": 1,
+            "write": 1
+        }
+    ],
+    "sort_field": "modified",
+    "sort_order": "DESC",
+    "states": [],
+    "track_changes": 1
 }

frappe/core/doctype/data_import/data_import.py

@@ -27,6 +27,7 @@ class DataImport(Document):
 
 		self.validate_import_file()
 		self.validate_google_sheets_url()
+		self.set_payload_count()
 
 	def validate_import_file(self):
 		if self.import_file:
@@ -38,6 +39,12 @@ class DataImport(Document):
 			return
 		validate_google_sheets_url(self.google_sheets_url)
 
+	def set_payload_count(self):
+		if self.import_file:
+			i = self.get_importer()
+			payloads = i.import_file.get_payloads_for_import()
+			self.payload_count = len(payloads)
+
 	@frappe.whitelist()
 	def get_preview_from_template(self, import_file=None, google_sheets_url=None):
 		if import_file:
@@ -67,7 +74,7 @@ class DataImport(Document):
 			enqueue(
 				start_import,
 				queue="default",
-				timeout=6000,
+				timeout=10000,
 				event="data_import",
 				job_name=self.name,
 				data_import=self.name,
@@ -80,6 +87,9 @@ class DataImport(Document):
 	def export_errored_rows(self):
 		return self.get_importer().export_errored_rows()
 
+	def download_import_log(self):
+		return self.get_importer().export_import_log()
+
 	def get_importer(self):
 		return Importer(self.reference_doctype, data_import=self)
 
@@ -90,7 +100,6 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N
 		import_file, google_sheets_url
 	)
 
-
 @frappe.whitelist()
 def form_start_import(data_import):
 	return frappe.get_doc("Data Import", data_import).start_import()
@@ -145,6 +154,30 @@ def download_errored_template(data_import_name):
 	data_import = frappe.get_doc("Data Import", data_import_name)
 	data_import.export_errored_rows()
 
+@frappe.whitelist()
+def download_import_log(data_import_name):
+	data_import = frappe.get_doc("Data Import", data_import_name)
+	data_import.download_import_log()
+
+@frappe.whitelist()
+def get_import_status(data_import_name):
+	import_status = {}
+
+	logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'],
+		filters={'data_import': data_import_name},
+		group_by='success')
+
+	total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count')
+
+	for log in logs:
+		if log.get('success'):
+			import_status['success'] = log.get('count')
+		else:
+			import_status['failed'] = log.get('count')
+
+	import_status['total_records'] = total_payload_count
+
+	return import_status
 
 def import_file(
 	doctype, file_path, import_type, submit_after_import=False, console=False

frappe/core/doctype/data_import/data_import_list.js

@@ -24,12 +24,14 @@ frappe.listview_settings['Data Import'] = {
 			'Error': 'red'
 		};
 		let status = doc.status;
+
 		if (imports_in_progress.includes(doc.name)) {
 			status = 'In Progress';
 		}
 		if (status == 'Pending') {
 			status = 'Not Started';
 		}
+
 		return [__(status), colors[status], 'status,=,' + doc.status];
 	},
 	formatters: {

frappe/core/doctype/data_import/importer.py

@@ -47,7 +47,13 @@ class Importer:
 		)
 
 	def get_data_for_import_preview(self):
-		return self.import_file.get_data_for_import_preview()
+		out = self.import_file.get_data_for_import_preview()
+
+		out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index", limit=10)
+
+		return out
 
 	def before_import(self):
 		# set user lang for translations
@@ -58,7 +64,6 @@ class Importer:
 		frappe.flags.in_import = True
 		frappe.flags.mute_emails = self.data_import.mute_emails
 
-		self.data_import.db_set("status", "Pending")
 		self.data_import.db_set("template_warnings", "")
 
 	def import_data(self):
@@ -79,20 +84,25 @@ class Importer:
 			return
 
 		# setup import log
-		if self.data_import.import_log:
-			import_log = frappe.parse_json(self.data_import.import_log)
-		else:
-			import_log = []
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
 
-		# remove previous failures from import log
-		import_log = [log for log in import_log if log.get("success")]
+		log_index = 0
+
+		# Do not remove rows in case of retry after an error or pending data import
+		if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
+			# remove previous failures from import log only in case of retry after partial success
+			import_log = [log for log in import_log if log.get("success")]
 
 		# get successfully imported rows
 		imported_rows = []
 		for log in import_log:
 			log = frappe._dict(log)
-			if log.success:
-				imported_rows += log.row_indexes
+			if log.success or len(import_log) < self.data_import.payload_count:
+				imported_rows += json.loads(log.row_indexes)
+
+			log_index = log.log_index
 
 		# start import
 		total_payload_count = len(payloads)
@@ -146,25 +156,41 @@ class Importer:
 							},
 						)
 
-					import_log.append(
-						frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes)
-					)
+					create_import_log(self.data_import.name, log_index, {
+						'success': True,
+						'docname': doc.name,
+						'row_indexes': row_indexes
+					})
+
+					log_index += 1
+
+					if not self.data_import.status == "Partial Success":
+						self.data_import.db_set("status", "Partial Success")
+					
 					# commit after every successful import
 					frappe.db.commit()
 
 				except Exception:
-					import_log.append(
-						frappe._dict(
-							success=False,
-							exception=frappe.get_traceback(),
-							messages=frappe.local.message_log,
-							row_indexes=row_indexes,
-						)
-					)
+					messages = frappe.local.message_log
 					frappe.clear_messages()
+
 					# rollback if exception
 					frappe.db.rollback()
 
+					create_import_log(self.data_import.name, log_index, {
+						'success': False,
+						'exception': frappe.get_traceback(),
+						'messages': messages,
+						'row_indexes': row_indexes
+					})
+
+					log_index += 1
+
+		# Logs are db inserted directly so will have to be fetched again
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
+
 		# set status
 		failures = [log for log in import_log if not log.get("success")]
 		if len(failures) == total_payload_count:
@@ -178,7 +204,6 @@ class Importer:
 			self.print_import_log(import_log)
 		else:
 			self.data_import.db_set("status", status)
-			self.data_import.db_set("import_log", json.dumps(import_log))
 
 		self.after_import()
 
@@ -248,11 +273,14 @@ class Importer:
 		if not self.data_import:
 			return
 
-		import_log = frappe.parse_json(self.data_import.import_log or "[]")
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
+
 		failures = [log for log in import_log if not log.get("success")]
 		row_indexes = []
 		for f in failures:
-			row_indexes.extend(f.get("row_indexes", []))
+			row_indexes.extend(json.loads(f.get("row_indexes", [])))
 
 		# de duplicate
 		row_indexes = list(set(row_indexes))
@@ -264,6 +292,30 @@ class Importer:
 
 		build_csv_response(rows, _(self.doctype))
 
+	def export_import_log(self):
+		from frappe.utils.csvutils import build_csv_response
+
+		if not self.data_import:
+			return
+
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index")
+
+		header_row = ["Row Numbers", "Status", "Message", "Exception"]
+
+		rows = [header_row]
+
+		for log in import_log:
+			row_number = json.loads(log.get("row_indexes"))[0]
+			status = "Success" if log.get('success') else "Failure"
+			message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \
+				log.get("messages")
+			exception = frappe.utils.cstr(log.get("exception", ''))
+			rows += [[row_number, status, message, exception]]
+
+		build_csv_response(rows, self.doctype)
+
 	def print_import_log(self, import_log):
 		failed_records = [log for log in import_log if not log.success]
 		successful_records = [log for log in import_log if log.success]
@@ -1172,3 +1224,17 @@ def df_as_json(df):
 
 def get_select_options(df):
 	return [d for d in (df.options or "").split("\n") if d]
+
+def create_import_log(data_import, log_index, log_details):
+	frappe.get_doc({
+		'doctype': 'Data Import Log',
+		'log_index': log_index,
+		'success': log_details.get('success'),
+		'data_import': data_import,
+		'row_indexes': json.dumps(log_details.get('row_indexes')),
+		'docname': log_details.get('docname'),
+		'messages': json.dumps(log_details.get('messages', '[]')),
+		'exception': log_details.get('exception')
+	}).db_insert()
+
+

frappe/core/doctype/data_import/test_importer.py

@@ -60,15 +60,19 @@ class TestImporter(unittest.TestCase):
 		frappe.local.message_log = []
 		data_import.start_import()
 		data_import.reload()
-		import_log = frappe.parse_json(data_import.import_log)
-		self.assertEqual(import_log[0]['row_indexes'], [2,3])
+
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"],
+			filters={"data_import": data_import.name},
+			order_by="log_index")
+
+		self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3])
 		expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #1: Value missing for: Child Title"
-		self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error)
+		self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error)
 		expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #2: Value missing for: Child Title"
-		self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error)
+		self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error)
 
-		self.assertEqual(import_log[1]['row_indexes'], [4])
-		self.assertEqual(frappe.parse_json(import_log[1]['messages'][0])['message'], "Title is required")
+		self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4])
+		self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required")
 
 	def test_data_import_update(self):
 		existing_doc = frappe.get_doc(

frappe/core/doctype/data_import_log/data_import_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2021, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Import Log', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/data_import_log/data_import_log.json

@@ -0,0 +1,84 @@
+{
+ "actions": [],
+ "creation": "2021-12-25 16:12:20.205889",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "MyISAM",
+ "field_order": [
+  "data_import",
+  "row_indexes",
+  "success",
+  "docname",
+  "messages",
+  "exception",
+  "log_index"
+ ],
+ "fields": [
+  {
+   "fieldname": "data_import",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Data Import",
+   "options": "Data Import"
+  },
+  {
+   "fieldname": "docname",
+   "fieldtype": "Data",
+   "label": "Reference Name"
+  },
+  {
+   "fieldname": "exception",
+   "fieldtype": "Text",
+   "label": "Exception"
+  },
+  {
+   "fieldname": "row_indexes",
+   "fieldtype": "Code",
+   "label": "Row Indexes",
+   "options": "JSON"
+  },
+  {
+   "default": "0",
+   "fieldname": "success",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Success"
+  },
+  {
+   "fieldname": "log_index",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Log Index"
+  },
+  {
+   "fieldname": "messages",
+   "fieldtype": "Code",
+   "label": "Messages",
+   "options": "JSON"
+  }
+ ],
+ "in_create": 1,
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2021-12-29 11:19:19.646076",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Data Import Log",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC"
+}

frappe/core/doctype/data_import_log/data_import_log.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+class DataImportLog(Document):
+	pass

frappe/core/doctype/data_import_log/test_data_import_log.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+import unittest
+
+class TestDataImportLog(unittest.TestCase):
+	pass

frappe/public/js/frappe/data_import/import_preview.js

@@ -331,7 +331,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
 	is_row_imported(row) {
 		let serial_no = row[0].content;
 		return this.import_log.find(log => {
-			return log.success && log.row_indexes.includes(serial_no);
+			return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no);
 		});
 	}
 };