fix: Use db insert for logs

frappe/core/doctype/data_import/data_import.js

@@ -278,14 +278,15 @@ frappe.ui.form.on('Data Import', {
 				}
 			})
 			.then(r => {
-				let preview_data = r.message.preview_data;
-				let import_log = r.message.import_log;
-				frm.events.show_import_preview(frm, preview_data, import_log);
+				let preview_data = r.message;
+				frm.events.show_import_preview(frm, preview_data);
 				frm.events.show_import_warnings(frm, preview_data);
 			});
 	},
 
-	show_import_preview(frm, preview_data, import_log) {
+	show_import_preview(frm, preview_data) {
+		let import_log = preview_data.import_log;
+
 		if (
 			frm.import_preview &&
 			frm.import_preview.doctype === frm.doc.reference_doctype

frappe/core/doctype/data_import/data_import.json

@@ -1,227 +1,197 @@
 {
- "actions": [],
- "autoname": "format:{reference_doctype} Import on {creation}",
- "beta": 1,
- "creation": "2019-08-04 14:16:08.318714",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "reference_doctype",
-  "import_type",
-  "download_template",
-  "import_file",
-  "html_5",
-  "google_sheets_url",
-  "refresh_google_sheet",
-  "column_break_5",
-  "status",
-  "submit_after_import",
-  "mute_emails",
-  "template_options",
-  "import_warnings_section",
-  "template_warnings",
-  "import_warnings",
-  "section_import_preview",
-  "import_preview",
-  "import_log_section",
-  "show_failed_logs",
-  "import_log_preview"
- ],
- "fields": [
-  {
-   "fieldname": "reference_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "reqd": 1,
-   "set_only_once": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "import_type",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Import Type",
-   "options": "\nInsert New Records\nUpdate Existing Records",
-   "reqd": 1,
-   "set_only_once": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval:!doc.__islocal",
-   "fieldname": "import_file",
-   "fieldtype": "Attach",
-   "in_list_view": 1,
-   "label": "Import File",
-   "read_only_depends_on": "eval: ['Success', 'Partial Success', 'Partially Completed'].includes(doc.status)",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "import_preview",
-   "fieldtype": "HTML",
-   "label": "Import Preview",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "section_import_preview",
-   "fieldtype": "Section Break",
-   "label": "Preview",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "column_break_5",
-   "fieldtype": "Column Break",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "template_options",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Template Options",
-   "options": "JSON",
-   "read_only": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "import_log_section",
-   "fieldtype": "Section Break",
-   "label": "Import Log",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "import_log_preview",
-   "fieldtype": "HTML",
-   "label": "Import Log Preview",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "default": "Pending",
-   "fieldname": "status",
-   "fieldtype": "Select",
-   "hidden": 1,
-   "label": "Status",
-   "options": "Pending\nSuccess\nPartial Success\nError",
-   "read_only": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "template_warnings",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Template Warnings",
-   "options": "JSON",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "submit_after_import",
-   "fieldtype": "Check",
-   "label": "Submit After Import",
-   "set_only_once": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "import_warnings_section",
-   "fieldtype": "Section Break",
-   "label": "Import File Errors and Warnings",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "import_warnings",
-   "fieldtype": "HTML",
-   "label": "Import Warnings",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval:!doc.__islocal",
-   "fieldname": "download_template",
-   "fieldtype": "Button",
-   "label": "Download Template",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "default": "1",
-   "fieldname": "mute_emails",
-   "fieldtype": "Check",
-   "label": "Don't Send Emails",
-   "set_only_once": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "show_failed_logs",
-   "fieldtype": "Check",
-   "label": "Show Failed Logs",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval:!doc.__islocal && !doc.import_file",
-   "fieldname": "html_5",
-   "fieldtype": "HTML",
-   "options": "<h5 class=\"text-muted uppercase\">Or</h5>",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
-   "description": "Must be a publicly accessible Google Sheets URL",
-   "fieldname": "google_sheets_url",
-   "fieldtype": "Data",
-   "label": "Import from Google Sheets",
-   "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
-   "fieldname": "refresh_google_sheet",
-   "fieldtype": "Button",
-   "label": "Refresh Google Sheet",
-   "show_days": 1,
-   "show_seconds": 1
-  }
- ],
- "hide_toolbar": 1,
- "links": [],
- "modified": "2022-01-31 10:50:13.015426",
- "modified_by": "Administrator",
- "module": "Core",
- "name": "Data Import",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
+    "actions": [],
+    "autoname": "format:{reference_doctype} Import on {creation}",
+    "beta": 1,
+    "creation": "2019-08-04 14:16:08.318714",
+    "doctype": "DocType",
+    "editable_grid": 1,
+    "engine": "InnoDB",
+    "field_order": [
+        "reference_doctype",
+        "import_type",
+        "download_template",
+        "import_file",
+        "payload_count",
+        "html_5",
+        "google_sheets_url",
+        "refresh_google_sheet",
+        "column_break_5",
+        "status",
+        "submit_after_import",
+        "mute_emails",
+        "template_options",
+        "import_warnings_section",
+        "template_warnings",
+        "import_warnings",
+        "section_import_preview",
+        "import_preview",
+        "import_log_section",
+        "show_failed_logs",
+        "import_log_preview"
+    ],
+    "fields": [
+        {
+            "fieldname": "reference_doctype",
+            "fieldtype": "Link",
+            "in_list_view": 1,
+            "label": "Document Type",
+            "options": "DocType",
+            "reqd": 1,
+            "set_only_once": 1
+        },
+        {
+            "fieldname": "import_type",
+            "fieldtype": "Select",
+            "in_list_view": 1,
+            "label": "Import Type",
+            "options": "\nInsert New Records\nUpdate Existing Records",
+            "reqd": 1,
+            "set_only_once": 1
+        },
+        {
+            "depends_on": "eval:!doc.__islocal",
+            "fieldname": "import_file",
+            "fieldtype": "Attach",
+            "in_list_view": 1,
+            "label": "Import File",
+            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+        },
+        {
+            "fieldname": "import_preview",
+            "fieldtype": "HTML",
+            "label": "Import Preview"
+        },
+        {
+            "fieldname": "section_import_preview",
+            "fieldtype": "Section Break",
+            "label": "Preview"
+        },
+        {
+            "fieldname": "column_break_5",
+            "fieldtype": "Column Break"
+        },
+        {
+            "fieldname": "template_options",
+            "fieldtype": "Code",
+            "hidden": 1,
+            "label": "Template Options",
+            "options": "JSON",
+            "read_only": 1
+        },
+        {
+            "fieldname": "import_log_section",
+            "fieldtype": "Section Break",
+            "label": "Import Log"
+        },
+        {
+            "fieldname": "import_log_preview",
+            "fieldtype": "HTML",
+            "label": "Import Log Preview"
+        },
+        {
+            "default": "Pending",
+            "fieldname": "status",
+            "fieldtype": "Select",
+            "hidden": 1,
+            "label": "Status",
+            "options": "Pending\nSuccess\nPartial Success\nError",
+            "read_only": 1
+        },
+        {
+            "fieldname": "template_warnings",
+            "fieldtype": "Code",
+            "hidden": 1,
+            "label": "Template Warnings",
+            "options": "JSON"
+        },
+        {
+            "default": "0",
+            "fieldname": "submit_after_import",
+            "fieldtype": "Check",
+            "label": "Submit After Import",
+            "set_only_once": 1
+        },
+        {
+            "fieldname": "import_warnings_section",
+            "fieldtype": "Section Break",
+            "label": "Import File Errors and Warnings"
+        },
+        {
+            "fieldname": "import_warnings",
+            "fieldtype": "HTML",
+            "label": "Import Warnings"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal",
+            "fieldname": "download_template",
+            "fieldtype": "Button",
+            "label": "Download Template"
+        },
+        {
+            "default": "1",
+            "fieldname": "mute_emails",
+            "fieldtype": "Check",
+            "label": "Don't Send Emails",
+            "set_only_once": 1
+        },
+        {
+            "default": "0",
+            "fieldname": "show_failed_logs",
+            "fieldtype": "Check",
+            "label": "Show Failed Logs"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal && !doc.import_file",
+            "fieldname": "html_5",
+            "fieldtype": "HTML",
+            "options": "<h5 class=\"text-muted uppercase\">Or</h5>"
+        },
+        {
+            "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
+            "description": "Must be a publicly accessible Google Sheets URL",
+            "fieldname": "google_sheets_url",
+            "fieldtype": "Data",
+            "label": "Import from Google Sheets",
+            "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+        },
+        {
+            "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved &&  ['Success', 'Partial Success'].includes(doc.status)",
+            "fieldname": "refresh_google_sheet",
+            "fieldtype": "Button",
+            "label": "Refresh Google Sheet"
+        },
+        {
+            "fieldname": "payload_count",
+            "fieldtype": "Int",
+            "hidden": 1,
+            "label": "Payload Count",
+            "read_only": 1
+        }
+    ],
+    "hide_toolbar": 1,
+    "links": [],
+    "modified": "2022-02-01 20:08:37.624914",
+    "modified_by": "Administrator",
+    "module": "Core",
+    "name": "Data Import",
+    "naming_rule": "Expression",
+    "owner": "Administrator",
+    "permissions": [
+        {
+            "create": 1,
+            "delete": 1,
+            "email": 1,
+            "export": 1,
+            "print": 1,
+            "read": 1,
+            "report": 1,
+            "role": "System Manager",
+            "share": 1,
+            "write": 1
+        }
+    ],
+    "sort_field": "modified",
+    "sort_order": "DESC",
+    "states": [],
+    "track_changes": 1
 }

frappe/core/doctype/data_import/data_import.py

@@ -28,6 +28,7 @@ class DataImport(Document):
 
 		self.validate_import_file()
 		self.validate_google_sheets_url()
+		self.set_payload_count()
 
 	def validate_import_file(self):
 		if self.import_file:
@@ -39,6 +40,12 @@ class DataImport(Document):
 			return
 		validate_google_sheets_url(self.google_sheets_url)
 
+	def set_payload_count(self):
+		if self.import_file:
+			i = self.get_importer()
+			payloads = i.import_file.get_payloads_for_import()
+			self.payload_count = len(payloads)
+
 	@frappe.whitelist()
 	def get_preview_from_template(self, import_file=None, google_sheets_url=None):
 		if import_file:
@@ -90,21 +97,10 @@ class DataImport(Document):
 
 @frappe.whitelist()
 def get_preview_from_template(data_import, import_file=None, google_sheets_url=None):
-	preview_data = frappe.get_doc("Data Import", data_import).get_preview_from_template(
+	return frappe.get_doc("Data Import", data_import).get_preview_from_template(
 		import_file, google_sheets_url
 	)
 
-	# get first 10 import log if any
-	import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes"],
-		filters={"data_import": data_import},
-		order_by="log_index", limit=10)
-
-	return {
-		'preview_data': preview_data,
-		'import_log': import_log
-	}
-
-
 @frappe.whitelist()
 def form_start_import(data_import):
 	return frappe.get_doc("Data Import", data_import).start_import()
@@ -172,14 +168,15 @@ def get_import_status(data_import_name):
 		filters={'data_import': data_import_name},
 		group_by='success')
 
+	total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count')
+
 	for log in logs:
 		if log.get('success'):
 			import_status['success'] = log.get('count')
 		else:
 			import_status['failed'] = log.get('count')
 
-	import_status['total_records'] = cint(import_status.get('success')) + \
-		cint(import_status.get('failed'))
+	import_status['total_records'] = total_payload_count
 
 	return import_status
 

frappe/core/doctype/data_import/importer.py

@@ -47,7 +47,13 @@ class Importer:
 		)
 
 	def get_data_for_import_preview(self):
-		return self.import_file.get_data_for_import_preview()
+		out = self.import_file.get_data_for_import_preview()
+
+		out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index", limit=10)
+
+		return out
 
 	def before_import(self):
 		# set user lang for translations
@@ -85,7 +91,7 @@ class Importer:
 		log_index = 0
 
 		# Do not remove rows in case of retry after an error or pending data import
-		if self.data_import.status == "Partial Success" and len(import_log) > len(payloads):
+		if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
 			# remove previous failures from import log only in case of retry after partial success
 			import_log = [log for log in import_log if log.get("success")]
 
@@ -93,7 +99,7 @@ class Importer:
 		imported_rows = []
 		for log in import_log:
 			log = frappe._dict(log)
-			if log.success or len(import_log) < len(payloads):
+			if log.success or len(import_log) < self.data_import.payload_count:
 				imported_rows += json.loads(log.row_indexes)
 
 			log_index = log.log_index
@@ -182,6 +188,11 @@ class Importer:
 					frappe.db.commit()
 					log_index += 1
 
+		# Logs are db inserted directly so will have to be fetched again
+		import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
+			filters={"data_import": self.data_import.name},
+			order_by="log_index") or []
+
 		# set status
 		failures = [log for log in import_log if not log.get("success")]
 		if len(failures) == total_payload_count:
@@ -1217,7 +1228,7 @@ def get_select_options(df):
 	return [d for d in (df.options or "").split("\n") if d]
 
 def create_import_log(data_import, log_index, log_details):
-	return frappe.get_doc({
+	frappe.get_doc({
 		'doctype': 'Data Import Log',
 		'log_index': log_index,
 		'success': log_details.get('success'),
@@ -1226,6 +1237,6 @@ def create_import_log(data_import, log_index, log_details):
 		'docname': log_details.get('docname'),
 		'messages': json.dumps(log_details.get('messages', '[]')),
 		'exception': log_details.get('exception')
-	}).insert(ignore_permissions=True)
+	}).db_insert()
 
 

frappe/core/doctype/data_import_log/data_import_log.json

@@ -3,7 +3,7 @@
  "creation": "2021-12-25 16:12:20.205889",
  "doctype": "DocType",
  "editable_grid": 1,
- "engine": "InnoDB",
+ "engine": "MyISAM",
  "field_order": [
   "data_import",
   "row_indexes",
@@ -60,7 +60,7 @@
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-12-27 11:19:19.646076",
+ "modified": "2021-12-28 11:19:19.646076",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Data Import Log",