diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js
index 6ea95fba02..1460f8698e 100644
--- a/frappe/core/doctype/data_import/data_import.js
+++ b/frappe/core/doctype/data_import/data_import.js
@@ -278,14 +278,15 @@ frappe.ui.form.on('Data Import', {
}
})
.then(r => {
- let preview_data = r.message.preview_data;
- let import_log = r.message.import_log;
- frm.events.show_import_preview(frm, preview_data, import_log);
+ let preview_data = r.message;
+ frm.events.show_import_preview(frm, preview_data);
frm.events.show_import_warnings(frm, preview_data);
});
},
- show_import_preview(frm, preview_data, import_log) {
+ show_import_preview(frm, preview_data) {
+ let import_log = preview_data.import_log;
+
if (
frm.import_preview &&
frm.import_preview.doctype === frm.doc.reference_doctype
diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json
index e8b8b7a546..9e948dac8c 100644
--- a/frappe/core/doctype/data_import/data_import.json
+++ b/frappe/core/doctype/data_import/data_import.json
@@ -1,227 +1,197 @@
{
- "actions": [],
- "autoname": "format:{reference_doctype} Import on {creation}",
- "beta": 1,
- "creation": "2019-08-04 14:16:08.318714",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
- "reference_doctype",
- "import_type",
- "download_template",
- "import_file",
- "html_5",
- "google_sheets_url",
- "refresh_google_sheet",
- "column_break_5",
- "status",
- "submit_after_import",
- "mute_emails",
- "template_options",
- "import_warnings_section",
- "template_warnings",
- "import_warnings",
- "section_import_preview",
- "import_preview",
- "import_log_section",
- "show_failed_logs",
- "import_log_preview"
- ],
- "fields": [
- {
- "fieldname": "reference_doctype",
- "fieldtype": "Link",
- "in_list_view": 1,
- "label": "Document Type",
- "options": "DocType",
- "reqd": 1,
- "set_only_once": 1,
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "import_type",
- "fieldtype": "Select",
- "in_list_view": 1,
- "label": "Import Type",
- "options": "\nInsert New Records\nUpdate Existing Records",
- "reqd": 1,
- "set_only_once": 1,
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "depends_on": "eval:!doc.__islocal",
- "fieldname": "import_file",
- "fieldtype": "Attach",
- "in_list_view": 1,
- "label": "Import File",
- "read_only_depends_on": "eval: ['Success', 'Partial Success', 'Partially Completed'].includes(doc.status)",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "import_preview",
- "fieldtype": "HTML",
- "label": "Import Preview",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "section_import_preview",
- "fieldtype": "Section Break",
- "label": "Preview",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "column_break_5",
- "fieldtype": "Column Break",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "template_options",
- "fieldtype": "Code",
- "hidden": 1,
- "label": "Template Options",
- "options": "JSON",
- "read_only": 1,
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "import_log_section",
- "fieldtype": "Section Break",
- "label": "Import Log",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "import_log_preview",
- "fieldtype": "HTML",
- "label": "Import Log Preview",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "default": "Pending",
- "fieldname": "status",
- "fieldtype": "Select",
- "hidden": 1,
- "label": "Status",
- "options": "Pending\nSuccess\nPartial Success\nError",
- "read_only": 1,
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "template_warnings",
- "fieldtype": "Code",
- "hidden": 1,
- "label": "Template Warnings",
- "options": "JSON",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "default": "0",
- "fieldname": "submit_after_import",
- "fieldtype": "Check",
- "label": "Submit After Import",
- "set_only_once": 1,
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "import_warnings_section",
- "fieldtype": "Section Break",
- "label": "Import File Errors and Warnings",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "fieldname": "import_warnings",
- "fieldtype": "HTML",
- "label": "Import Warnings",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "depends_on": "eval:!doc.__islocal",
- "fieldname": "download_template",
- "fieldtype": "Button",
- "label": "Download Template",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "default": "1",
- "fieldname": "mute_emails",
- "fieldtype": "Check",
- "label": "Don't Send Emails",
- "set_only_once": 1,
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "default": "0",
- "fieldname": "show_failed_logs",
- "fieldtype": "Check",
- "label": "Show Failed Logs",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "depends_on": "eval:!doc.__islocal && !doc.import_file",
- "fieldname": "html_5",
- "fieldtype": "HTML",
- "options": "Or
",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
- "description": "Must be a publicly accessible Google Sheets URL",
- "fieldname": "google_sheets_url",
- "fieldtype": "Data",
- "label": "Import from Google Sheets",
- "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)",
- "show_days": 1,
- "show_seconds": 1
- },
- {
- "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)",
- "fieldname": "refresh_google_sheet",
- "fieldtype": "Button",
- "label": "Refresh Google Sheet",
- "show_days": 1,
- "show_seconds": 1
- }
- ],
- "hide_toolbar": 1,
- "links": [],
- "modified": "2022-01-31 10:50:13.015426",
- "modified_by": "Administrator",
- "module": "Core",
- "name": "Data Import",
- "owner": "Administrator",
- "permissions": [
- {
- "create": 1,
- "delete": 1,
- "email": 1,
- "export": 1,
- "print": 1,
- "read": 1,
- "report": 1,
- "role": "System Manager",
- "share": 1,
- "write": 1
- }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
+ "actions": [],
+ "autoname": "format:{reference_doctype} Import on {creation}",
+ "beta": 1,
+ "creation": "2019-08-04 14:16:08.318714",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+ "reference_doctype",
+ "import_type",
+ "download_template",
+ "import_file",
+ "payload_count",
+ "html_5",
+ "google_sheets_url",
+ "refresh_google_sheet",
+ "column_break_5",
+ "status",
+ "submit_after_import",
+ "mute_emails",
+ "template_options",
+ "import_warnings_section",
+ "template_warnings",
+ "import_warnings",
+ "section_import_preview",
+ "import_preview",
+ "import_log_section",
+ "show_failed_logs",
+ "import_log_preview"
+ ],
+ "fields": [
+ {
+ "fieldname": "reference_doctype",
+ "fieldtype": "Link",
+ "in_list_view": 1,
+ "label": "Document Type",
+ "options": "DocType",
+ "reqd": 1,
+ "set_only_once": 1
+ },
+ {
+ "fieldname": "import_type",
+ "fieldtype": "Select",
+ "in_list_view": 1,
+ "label": "Import Type",
+ "options": "\nInsert New Records\nUpdate Existing Records",
+ "reqd": 1,
+ "set_only_once": 1
+ },
+ {
+ "depends_on": "eval:!doc.__islocal",
+ "fieldname": "import_file",
+ "fieldtype": "Attach",
+ "in_list_view": 1,
+ "label": "Import File",
+ "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+ },
+ {
+ "fieldname": "import_preview",
+ "fieldtype": "HTML",
+ "label": "Import Preview"
+ },
+ {
+ "fieldname": "section_import_preview",
+ "fieldtype": "Section Break",
+ "label": "Preview"
+ },
+ {
+ "fieldname": "column_break_5",
+ "fieldtype": "Column Break"
+ },
+ {
+ "fieldname": "template_options",
+ "fieldtype": "Code",
+ "hidden": 1,
+ "label": "Template Options",
+ "options": "JSON",
+ "read_only": 1
+ },
+ {
+ "fieldname": "import_log_section",
+ "fieldtype": "Section Break",
+ "label": "Import Log"
+ },
+ {
+ "fieldname": "import_log_preview",
+ "fieldtype": "HTML",
+ "label": "Import Log Preview"
+ },
+ {
+ "default": "Pending",
+ "fieldname": "status",
+ "fieldtype": "Select",
+ "hidden": 1,
+ "label": "Status",
+ "options": "Pending\nSuccess\nPartial Success\nError",
+ "read_only": 1
+ },
+ {
+ "fieldname": "template_warnings",
+ "fieldtype": "Code",
+ "hidden": 1,
+ "label": "Template Warnings",
+ "options": "JSON"
+ },
+ {
+ "default": "0",
+ "fieldname": "submit_after_import",
+ "fieldtype": "Check",
+ "label": "Submit After Import",
+ "set_only_once": 1
+ },
+ {
+ "fieldname": "import_warnings_section",
+ "fieldtype": "Section Break",
+ "label": "Import File Errors and Warnings"
+ },
+ {
+ "fieldname": "import_warnings",
+ "fieldtype": "HTML",
+ "label": "Import Warnings"
+ },
+ {
+ "depends_on": "eval:!doc.__islocal",
+ "fieldname": "download_template",
+ "fieldtype": "Button",
+ "label": "Download Template"
+ },
+ {
+ "default": "1",
+ "fieldname": "mute_emails",
+ "fieldtype": "Check",
+ "label": "Don't Send Emails",
+ "set_only_once": 1
+ },
+ {
+ "default": "0",
+ "fieldname": "show_failed_logs",
+ "fieldtype": "Check",
+ "label": "Show Failed Logs"
+ },
+ {
+ "depends_on": "eval:!doc.__islocal && !doc.import_file",
+ "fieldname": "html_5",
+ "fieldtype": "HTML",
+ "options": "Or
"
+ },
+ {
+ "depends_on": "eval:!doc.__islocal && !doc.import_file\n",
+ "description": "Must be a publicly accessible Google Sheets URL",
+ "fieldname": "google_sheets_url",
+ "fieldtype": "Data",
+ "label": "Import from Google Sheets",
+ "read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
+ },
+ {
+ "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)",
+ "fieldname": "refresh_google_sheet",
+ "fieldtype": "Button",
+ "label": "Refresh Google Sheet"
+ },
+ {
+ "fieldname": "payload_count",
+ "fieldtype": "Int",
+ "hidden": 1,
+ "label": "Payload Count",
+ "read_only": 1
+ }
+ ],
+ "hide_toolbar": 1,
+ "links": [],
+ "modified": "2022-02-01 20:08:37.624914",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Data Import",
+ "naming_rule": "Expression",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 1,
+ "print": 1,
+ "read": 1,
+ "report": 1,
+ "role": "System Manager",
+ "share": 1,
+ "write": 1
+ }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
}
\ No newline at end of file
diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py
index 4aacb6925b..0c8ba072a1 100644
--- a/frappe/core/doctype/data_import/data_import.py
+++ b/frappe/core/doctype/data_import/data_import.py
@@ -28,6 +28,7 @@ class DataImport(Document):
self.validate_import_file()
self.validate_google_sheets_url()
+ self.set_payload_count()
def validate_import_file(self):
if self.import_file:
@@ -39,6 +40,12 @@ class DataImport(Document):
return
validate_google_sheets_url(self.google_sheets_url)
+ def set_payload_count(self):
+ if self.import_file:
+ i = self.get_importer()
+ payloads = i.import_file.get_payloads_for_import()
+ self.payload_count = len(payloads)
+
@frappe.whitelist()
def get_preview_from_template(self, import_file=None, google_sheets_url=None):
if import_file:
@@ -90,21 +97,10 @@ class DataImport(Document):
@frappe.whitelist()
def get_preview_from_template(data_import, import_file=None, google_sheets_url=None):
- preview_data = frappe.get_doc("Data Import", data_import).get_preview_from_template(
+ return frappe.get_doc("Data Import", data_import).get_preview_from_template(
import_file, google_sheets_url
)
- # get first 10 import log if any
- import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes"],
- filters={"data_import": data_import},
- order_by="log_index", limit=10)
-
- return {
- 'preview_data': preview_data,
- 'import_log': import_log
- }
-
-
@frappe.whitelist()
def form_start_import(data_import):
return frappe.get_doc("Data Import", data_import).start_import()
@@ -172,14 +168,15 @@ def get_import_status(data_import_name):
filters={'data_import': data_import_name},
group_by='success')
+ total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count')
+
for log in logs:
if log.get('success'):
import_status['success'] = log.get('count')
else:
import_status['failed'] = log.get('count')
- import_status['total_records'] = cint(import_status.get('success')) + \
- cint(import_status.get('failed'))
+ import_status['total_records'] = total_payload_count
return import_status
diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py
index 8c98099c1a..d13f4c1389 100644
--- a/frappe/core/doctype/data_import/importer.py
+++ b/frappe/core/doctype/data_import/importer.py
@@ -47,7 +47,13 @@ class Importer:
)
def get_data_for_import_preview(self):
- return self.import_file.get_data_for_import_preview()
+ out = self.import_file.get_data_for_import_preview()
+
+ out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
+ filters={"data_import": self.data_import.name},
+ order_by="log_index", limit=10)
+
+ return out
def before_import(self):
# set user lang for translations
@@ -85,7 +91,7 @@ class Importer:
log_index = 0
# Do not remove rows in case of retry after an error or pending data import
- if self.data_import.status == "Partial Success" and len(import_log) > len(payloads):
+ if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
# remove previous failures from import log only in case of retry after partial success
import_log = [log for log in import_log if log.get("success")]
@@ -93,7 +99,7 @@ class Importer:
imported_rows = []
for log in import_log:
log = frappe._dict(log)
- if log.success or len(import_log) < len(payloads):
+ if log.success or len(import_log) < self.data_import.payload_count:
imported_rows += json.loads(log.row_indexes)
log_index = log.log_index
@@ -182,6 +188,11 @@ class Importer:
frappe.db.commit()
log_index += 1
+ # Logs are db inserted directly so will have to be fetched again
+ import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
+ filters={"data_import": self.data_import.name},
+ order_by="log_index") or []
+
# set status
failures = [log for log in import_log if not log.get("success")]
if len(failures) == total_payload_count:
@@ -1217,7 +1228,7 @@ def get_select_options(df):
return [d for d in (df.options or "").split("\n") if d]
def create_import_log(data_import, log_index, log_details):
- return frappe.get_doc({
+ frappe.get_doc({
'doctype': 'Data Import Log',
'log_index': log_index,
'success': log_details.get('success'),
@@ -1226,6 +1237,6 @@ def create_import_log(data_import, log_index, log_details):
'docname': log_details.get('docname'),
'messages': json.dumps(log_details.get('messages', '[]')),
'exception': log_details.get('exception')
- }).insert(ignore_permissions=True)
+ }).db_insert()
diff --git a/frappe/core/doctype/data_import_log/data_import_log.json b/frappe/core/doctype/data_import_log/data_import_log.json
index a39eb9be1d..8747728945 100644
--- a/frappe/core/doctype/data_import_log/data_import_log.json
+++ b/frappe/core/doctype/data_import_log/data_import_log.json
@@ -3,7 +3,7 @@
"creation": "2021-12-25 16:12:20.205889",
"doctype": "DocType",
"editable_grid": 1,
- "engine": "InnoDB",
+ "engine": "MyISAM",
"field_order": [
"data_import",
"row_indexes",
@@ -60,7 +60,7 @@
"in_create": 1,
"index_web_pages_for_search": 1,
"links": [],
- "modified": "2021-12-27 11:19:19.646076",
+ "modified": "2021-12-28 11:19:19.646076",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Import Log",