From 520bfc2ae4e840903db11e4a494d77063b137fbe Mon Sep 17 00:00:00 2001 From: Manas Solanki Date: Fri, 15 Dec 2017 12:17:24 +0530 Subject: [PATCH] New data import (#4601) * created/moved the files * added the model for the downloading the data * add the file with the error data * changes added other changes and fix codacy * changes in the config and utils files * fixed the test cases * minor changes in the data keys dict * changed the test file location * fixed the tests * set the route in the list view and show only erors * minor fixes in the childtable import and log tables rendering * Refactor Download dialog to use MultiCheck --- frappe/__init__.py | 8 +- frappe/commands/utils.py | 16 +- frappe/config/desktop.py | 2 +- frappe/config/setup.py | 8 +- .../data_import}/README.md | 0 frappe/core/doctype/data_import/__init__.py | 0 .../core/doctype/data_import/data_import.js | 260 +++++++ .../core/doctype/data_import/data_import.json | 661 ++++++++++++++++++ .../data_import/data_import.py} | 101 +-- .../doctype/data_import/data_import_list.js | 16 + .../doctype/data_import/export_template.html | 24 + .../data_import}/exporter.py | 2 +- .../data_import}/importer.py | 205 ++++-- .../core/doctype/data_import/log_details.html | 38 + .../doctype/data_import/test_data_import.py | 9 + frappe/core/page/data_import_tool/__init__.py | 4 - .../data_import_tool/data_import_main.html | 118 ---- .../data_import_tool/data_import_tool.css | 7 - .../page/data_import_tool/data_import_tool.js | 233 ------ .../data_import_tool/data_import_tool.json | 19 - .../data_import_tool_columns.html | 22 - frappe/patches.txt | 1 + frappe/public/css/controls.css | 6 + .../js/frappe/form/controls/multicheck.js | 7 +- frappe/public/js/frappe/list/list_view.js | 4 +- frappe/public/js/frappe/model/model.js | 3 + .../public/js/legacy/client_script_helpers.js | 9 +- frappe/public/less/controls.less | 9 +- frappe/tests/test_data_import.py | 4 +- .../test_exporter_fixtures.py | 2 +- frappe/translate.py | 8 + frappe/utils/fixtures.py | 2 +- 32 files changed, 1270 insertions(+), 538 deletions(-) rename frappe/core/{page/data_import_tool => doctype/data_import}/README.md (100%) create mode 100644 frappe/core/doctype/data_import/__init__.py create mode 100644 frappe/core/doctype/data_import/data_import.js create mode 100644 frappe/core/doctype/data_import/data_import.json rename frappe/core/{page/data_import_tool/data_import_tool.py => doctype/data_import/data_import.py} (63%) create mode 100644 frappe/core/doctype/data_import/data_import_list.js create mode 100644 frappe/core/doctype/data_import/export_template.html rename frappe/core/{page/data_import_tool => doctype/data_import}/exporter.py (99%) rename frappe/core/{page/data_import_tool => doctype/data_import}/importer.py (63%) create mode 100644 frappe/core/doctype/data_import/log_details.html create mode 100644 frappe/core/doctype/data_import/test_data_import.py delete mode 100644 frappe/core/page/data_import_tool/__init__.py delete mode 100644 frappe/core/page/data_import_tool/data_import_main.html delete mode 100644 frappe/core/page/data_import_tool/data_import_tool.css delete mode 100644 frappe/core/page/data_import_tool/data_import_tool.js delete mode 100644 frappe/core/page/data_import_tool/data_import_tool.json delete mode 100644 frappe/core/page/data_import_tool/data_import_tool_columns.html rename frappe/{core/page/data_import_tool => tests}/test_exporter_fixtures.py (99%) diff --git a/frappe/__init__.py b/frappe/__init__.py index e38f2d98ac..198b4b1fd4 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -976,9 +976,9 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp ps.insert() def import_doc(path, ignore_links=False, ignore_insert=False, insert=False): - """Import a file using Data Import Tool.""" - from frappe.core.page.data_import_tool import data_import_tool - data_import_tool.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert) + """Import a file using Data Import.""" + from frappe.core.doctype.data_import import data_import + data_import.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert) def copy_doc(doc, ignore_no_copy=True): """ No_copy fields also get copied.""" @@ -1366,7 +1366,7 @@ def logger(module=None, with_more_info=True): def log_error(message=None, title=None): '''Log error to Error Log''' - get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()), + return get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()), method=title)).insert(ignore_permissions=True) def get_desk_link(doctype, name): diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 1b042ed88c..0f68e7ddd6 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -162,12 +162,12 @@ def export_doc(context, doctype, docname): @pass_context def export_json(context, doctype, path, name=None): "Export doclist as json to the given path, use '-' as name for Singles." - from frappe.core.page.data_import_tool import data_import_tool + from frappe.core.doctype.data_import import data_import for site in context.sites: try: frappe.init(site=site) frappe.connect() - data_import_tool.export_json(doctype, path, name=name) + data_import.export_json(doctype, path, name=name) finally: frappe.destroy() @@ -177,12 +177,12 @@ def export_json(context, doctype, path, name=None): @pass_context def export_csv(context, doctype, path): "Export data import template with data for DocType" - from frappe.core.page.data_import_tool import data_import_tool + from frappe.core.doctype.data_import import data_import for site in context.sites: try: frappe.init(site=site) frappe.connect() - data_import_tool.export_csv(doctype, path) + data_import.export_csv(doctype, path) finally: frappe.destroy() @@ -204,7 +204,7 @@ def export_fixtures(context): @pass_context def import_doc(context, path, force=False): "Import (insert/update) doclist. If the argument is a directory, all files ending with .json are imported" - from frappe.core.page.data_import_tool import data_import_tool + from frappe.core.doctype.data_import import data_import if not os.path.exists(path): path = os.path.join('..', path) @@ -216,7 +216,7 @@ def import_doc(context, path, force=False): try: frappe.init(site=site) frappe.connect() - data_import_tool.import_doc(path, overwrite=context.force) + data_import.import_doc(path, overwrite=context.force) finally: frappe.destroy() @@ -229,8 +229,8 @@ def import_doc(context, path, force=False): @pass_context def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): - "Import CSV using data import tool" - from frappe.core.page.data_import_tool import importer + "Import CSV using data import" + from frappe.core.doctype.data_import import importer from frappe.utils.csvutils import read_csv_content site = get_site(context) diff --git a/frappe/config/desktop.py b/frappe/config/desktop.py index 5ac41b59dd..76f02708b5 100644 --- a/frappe/config/desktop.py +++ b/frappe/config/desktop.py @@ -70,5 +70,5 @@ def get_data(): "icon": "octicon octicon-book", "color": '#FFAEDB', "hidden": 1, - }, + } ] diff --git a/frappe/config/setup.py b/frappe/config/setup.py index 142c8ab722..a3e183cb67 100644 --- a/frappe/config/setup.py +++ b/frappe/config/setup.py @@ -93,11 +93,11 @@ def get_data(): "icon": "fa fa-th", "items": [ { - "type": "page", - "name": "data-import-tool", + "type": "doctype", + "name": "Data Import", "label": _("Import / Export Data"), - "icon": "fa fa-upload", - "description": _("Import / Export Data from .csv files.") + "icon": "octicon octicon-cloud-upload", + "description": _("Import / Export Data from CSV and Excel files.") }, { "type": "doctype", diff --git a/frappe/core/page/data_import_tool/README.md b/frappe/core/doctype/data_import/README.md similarity index 100% rename from frappe/core/page/data_import_tool/README.md rename to frappe/core/doctype/data_import/README.md diff --git a/frappe/core/doctype/data_import/__init__.py b/frappe/core/doctype/data_import/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js new file mode 100644 index 0000000000..731bcc5bec --- /dev/null +++ b/frappe/core/doctype/data_import/data_import.js @@ -0,0 +1,260 @@ +// Copyright (c) 2017, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Data Import', { + onload: function(frm) { + frm.set_query("reference_doctype", function() { + return { + "filters": { + "issingle": 0, + "istable": 0, + "name": ['in', frappe.boot.user.can_import] + } + }; + }); + + frappe.realtime.on("data_import_progress", function(data) { + if (data.data_import === frm.doc.name) { + if (data.reload && data.reload === true) { + frm.reload_doc(); + } + if (data.progress) { + let progress_bar = $(frm.dashboard.progress_area).find(".progress-bar"); + if (progress_bar) { + $(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped"); + $(progress_bar).css("width", data.progress+"%"); + } + } + } + }); + }, + + refresh: function(frm) { + frm.disable_save(); + frm.dashboard.clear_headline(); + if (frm.doc.reference_doctype && !frm.doc.import_file) { + frm.dashboard.add_comment(__('Please attach a file to import')); + } else { + if (frm.doc.import_status) { + frm.dashboard.add_comment(frm.doc.import_status); + + if (frm.doc.import_status==="In Progress") { + frm.dashboard.add_progress("Data Import Progress", "0"); + frm.set_read_only(true); + } + } + } + + if (frm.doc.reference_doctype) { + frappe.model.with_doctype(frm.doc.reference_doctype); + } + + frm.add_custom_button(__("Help"), function() { + frappe.help.show_video("6wiriRKPhmg"); + }); + + if(frm.doc.reference_doctype && frm.doc.docstatus === 0) { + frm.add_custom_button(__("Download template"), function() { + frappe.data_import.download_dialog(frm).show(); + }); + } + + if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows && frm.doc.docstatus === 0) { + frm.page.set_primary_action(__("Start Import"), function() { + frappe.call({ + method: "frappe.core.doctype.data_import.data_import.import_data", + args: { + data_import: frm.doc.name + } + }); + }).addClass('btn btn-primary'); + } + + if (frm.doc.log_details) { + frm.events.create_log_table(frm); + } else { + $(frm.fields_dict.import_log.wrapper).empty(); + } + }, + + reference_doctype: function(frm) { + if (frm.doc.reference_doctype) { + frm.save(); + } + }, + + // import_file: function(frm) { + // frm.save(); + // }, + + overwrite: function(frm) { + if (frm.doc.overwrite === 0) { + frm.doc.only_update = 0; + } + frm.save(); + }, + + only_update: function(frm) { + frm.save(); + }, + + submit_after_import: function(frm) { + frm.save(); + }, + + skip_errors: function(frm) { + frm.save(); + }, + + ignore_encoding_errors: function(frm) { + frm.save(); + }, + + no_email: function(frm) { + frm.save(); + }, + + show_only_errors: function(frm) { + frm.events.create_log_table(frm); + }, + + create_log_table: function(frm) { + let msg = JSON.parse(frm.doc.log_details); + var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty(); + $(frappe.render_template("log_details", {data: msg.messages, show_only_errors: frm.doc.show_only_errors, + import_status: frm.doc.import_status})).appendTo($log_wrapper); + } +}); + +frappe.provide('frappe.data_import'); +frappe.data_import.download_dialog = function(frm) { + var dialog; + const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden; + const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields); + + const get_doctypes = parentdt => { + return [parentdt].concat( + frappe.meta.get_table_fields(parentdt).map(df => df.options) + ); + }; + + const get_doctype_checkbox_fields = () => { + return dialog.fields.filter(df => df.fieldname.endsWith('_fields')) + .map(df => dialog.fields_dict[df.fieldname]); + } + + const doctype_fields = get_fields(frm.doc.reference_doctype) + .map(df => ({ + label: df.label, + value: df.fieldname, + checked: 1 + })); + + let fields = [ + { + "label": __("Select Columns"), + "fieldname": "select_columns", + "fieldtype": "Select", + "options": "All\nMandatory\nManually", + "reqd": 1, + "onchange": function() { + const fields = get_doctype_checkbox_fields(); + fields.map(f => f.toggle(this.value === 'Manually')); + } + }, + { + "label": __("File Type"), + "fieldname": "file_type", + "fieldtype": "Select", + "options": "Excel\nCSV", + "default": "Excel" + }, + { + "label": __("Download with Data"), + "fieldname": "with_data", + "fieldtype": "Check" + }, + { + "label": frm.doc.reference_doctype, + "fieldname": "doctype_fields", + "fieldtype": "MultiCheck", + "options": doctype_fields, + "columns": 2, + "hidden": 1 + } + ]; + + const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype) + .map(df => { + return { + "label": df.options, + "fieldname": df.fieldname + '_fields', + "fieldtype": "MultiCheck", + "options": frappe.meta.get_docfields(df.options) + .filter(filter_fields) + .map(df => ({ + label: df.label, + value: df.fieldname, + checked: 1 + })), + "columns": 2, + "hidden": 1 + }; + }); + + fields = fields.concat(child_table_fields); + + dialog = new frappe.ui.Dialog({ + title: __('Download Template'), + fields: fields, + primary_action: function(values) { + var data = values; + if (frm.doc.reference_doctype) { + var export_params = () => { + let columns = {}; + if (values.select_columns === 'All') { + columns = get_doctypes(frm.doc.reference_doctype).reduce((columns, doctype) => { + columns[doctype] = get_fields(doctype).map(df => df.fieldname); + return columns; + }, {}); + } else if (values.select_columns === 'Mandatory') { + // only reqd child tables + const doctypes = [frm.doc.reference_doctype].concat( + frappe.meta.get_table_fields(frm.doc.reference_doctype) + .filter(df => df.reqd).map(df => df.options) + ); + + columns = doctypes.reduce((columns, doctype) => { + columns[doctype] = get_fields(doctype).filter(df => df.reqd).map(df => df.fieldname); + return columns; + }, {}); + } else if (values.select_columns === 'Manually') { + columns = get_doctype_checkbox_fields().reduce((columns, field) => { + const options = field.get_checked_options(); + columns[field.df.label] = options; + return columns; + }, {}); + } + + return { + doctype: frm.doc.reference_doctype, + parent_doctype: frm.doc.reference_doctype, + select_columns: JSON.stringify(columns), + with_data: data.with_data ? 'Yes' : 'No', + all_doctypes: 'Yes', + from_data_import: 'Yes', + excel_format: data.file_type === 'Excel' ? 'Yes' : 'No' + }; + }; + let get_template_url = '/api/method/frappe.core.doctype.data_import.exporter.get_template'; + open_url_post(get_template_url, export_params()); + } else { + frappe.msgprint(__("Please select the Document Type.")); + } + dialog.hide(); + }, + primary_action_label: __('Download') + }); + + return dialog; +}; diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json new file mode 100644 index 0000000000..69e7e02d4d --- /dev/null +++ b/frappe/core/doctype/data_import/data_import.json @@ -0,0 +1,661 @@ +{ + "allow_copy": 1, + "allow_guest_to_view": 0, + "allow_import": 0, + "allow_rename": 0, + "autoname": "", + "beta": 0, + "creation": "2016-12-09 14:27:32.720061", + "custom": 0, + "docstatus": 0, + "doctype": "DocType", + "document_type": "Document", + "editable_grid": 1, + "engine": "InnoDB", + "fields": [ + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "depends_on": "", + "fieldname": "reference_doctype", + "fieldtype": "Link", + "hidden": 0, + "ignore_user_permissions": 1, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 1, + "in_standard_filter": 0, + "label": "Document Type", + "length": 0, + "no_copy": 0, + "options": "DocType", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "collapsible_depends_on": "", + "columns": 0, + "depends_on": "eval:(!doc.__islocal)", + "fieldname": "section_break_4", + "fieldtype": "Section Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "depends_on": "", + "fieldname": "import_file", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Attach file for Import", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "column_break_4", + "fieldtype": "Column Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "depends_on": "eval: doc.import_status == \"Partially Successful\"", + "description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.", + "fieldname": "error_file", + "fieldtype": "Attach", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Generated File", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "collapsible_depends_on": "", + "columns": 0, + "depends_on": "eval:(!doc.__islocal)", + "fieldname": "section_break_6", + "fieldtype": "Section Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "0", + "depends_on": "", + "description": "If you are updating/overwriting already created records.", + "fieldname": "overwrite", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Update records", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "0", + "depends_on": "overwrite", + "description": "If you don't want to create any new records while updating the older records.", + "fieldname": "only_update", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Don't create new records", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.\n", + "fieldname": "skip_errors", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Skip rows with errors", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "0", + "depends_on": "", + "fieldname": "submit_after_import", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Submit after importing", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "0", + "depends_on": "", + "fieldname": "ignore_encoding_errors", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Ignore encoding errors", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "1", + "depends_on": "", + "fieldname": "no_email", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Do not send Emails", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 1, + "collapsible_depends_on": "eval: doc.import_status == \"Failed\"", + "columns": 0, + "depends_on": "import_status", + "fieldname": "import_detail", + "fieldtype": "Section Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Import Log", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "depends_on": "", + "fieldname": "import_status", + "fieldtype": "Select", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 1, + "in_standard_filter": 0, + "label": "Import Status", + "length": 0, + "no_copy": 0, + "options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 1, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "1", + "fieldname": "show_only_errors", + "fieldtype": "Check", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Show only errors", + "length": 0, + "no_copy": 1, + "permlevel": 0, + "precision": "", + "print_hide": 1, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 1, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "", + "depends_on": "import_status", + "fieldname": "import_log", + "fieldtype": "HTML", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Import Log", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 1, + "bold": 0, + "collapsible": 0, + "columns": 0, + "depends_on": "", + "fieldname": "log_details", + "fieldtype": "Code", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Log Details", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "amended_from", + "fieldtype": "Link", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Amended From", + "length": 0, + "no_copy": 1, + "options": "Data Import", + "permlevel": 0, + "print_hide": 1, + "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "total_rows", + "fieldtype": "Int", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Total Rows", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + } + ], + "has_web_view": 0, + "hide_heading": 0, + "hide_toolbar": 0, + "idx": 0, + "image_view": 0, + "in_create": 0, + "is_submittable": 1, + "issingle": 0, + "istable": 0, + "max_attachments": 1, + "modified": "2017-12-14 16:27:37.683505", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Import", + "name_case": "", + "owner": "Administrator", + "permissions": [ + { + "amend": 0, + "apply_user_permissions": 0, + "cancel": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 0, + "if_owner": 0, + "import": 0, + "permlevel": 0, + "print": 0, + "read": 1, + "report": 0, + "role": "System Manager", + "set_user_permissions": 0, + "share": 1, + "submit": 1, + "write": 1 + } + ], + "quick_entry": 0, + "read_only": 0, + "read_only_onload": 0, + "show_name_in_global_search": 0, + "sort_field": "modified", + "sort_order": "DESC", + "title_field": "", + "track_changes": 1, + "track_seen": 1 +} \ No newline at end of file diff --git a/frappe/core/page/data_import_tool/data_import_tool.py b/frappe/core/doctype/data_import/data_import.py similarity index 63% rename from frappe/core/page/data_import_tool/data_import_tool.py rename to frappe/core/doctype/data_import/data_import.py index 8338848561..7e0b8cc605 100644 --- a/frappe/core/page/data_import_tool/data_import_tool.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -1,44 +1,67 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals, print_function +# -*- coding: utf-8 -*- +# Copyright (c) 2017, Frappe Technologies and contributors +# For license information, please see license.txt +from __future__ import unicode_literals import frappe, os from frappe import _ import frappe.modules.import_file +from frappe.model.document import Document +from frappe.utils.data import format_datetime +from frappe.core.doctype.data_import.importer import upload +from frappe.utils.background_jobs import enqueue -@frappe.whitelist() -def get_data_keys(): - return frappe._dict({ - "data_separator": _('Start entering data below this line'), - "main_table": _("Table") + ":", - "parent_table": _("Parent Table") + ":", - "columns": _("Column Name") + ":", - "doctype": _("DocType") + ":" - }) -@frappe.whitelist() -def get_doctypes(): - return frappe.get_user()._get("can_import") +class DataImport(Document): + def autoname(self): + self.name = "Import on "+ format_datetime(self.creation) + + def validate(self): + if not self.import_file: + self.db_set("total_rows", 0) + if self.import_status == "In Progress": + frappe.throw(_("Can't save the form as data import is in progress.")) + + # validate the template just after the upload + # if there is total_rows in the doc, it means that the template is already validated and error free + if self.import_file and not self.total_rows: + upload(data_import_doc=self, from_data_import="Yes", validate_template=True) + @frappe.whitelist() -def get_doctype_options(): - doctype = frappe.form_dict['doctype'] - return [doctype] + [d.options for d in frappe.get_meta(doctype).get_table_fields()] +def import_data(data_import): + frappe.db.set_value("Data Import", data_import, "import_status", "In Progress") + frappe.publish_realtime("data_import_progress", {"progress": "0", + "data_import": data_import, "reload": True}, user=frappe.session.user) + enqueue(upload, queue='default', timeout=6000, event='data_import', + data_import_doc=data_import, from_data_import="Yes", validate_template=False) + + +def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False, + insert=False, submit=False, pre_process=None): + if os.path.isdir(path): + files = [os.path.join(path, f) for f in os.listdir(path)] + else: + files = [path] + + for f in files: + if f.endswith(".json"): + frappe.flags.mute_emails = True + frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True) + frappe.flags.mute_emails = False + frappe.db.commit() + elif f.endswith(".csv"): + import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process) + frappe.db.commit() + def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True): from frappe.utils.csvutils import read_csv_content - from frappe.core.page.data_import_tool.importer import upload print("Importing " + path) with open(path, "r") as infile: upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite, - submit_after_import=submit, pre_process=pre_process) + submit_after_import=submit, pre_process=pre_process) -def export_csv(doctype, path): - from frappe.core.page.data_import_tool.exporter import get_template - with open(path, "wb") as csvfile: - get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes") - csvfile.write(frappe.response.result.encode("utf-8")) def export_json(doctype, path, filters=None, or_filters=None, name=None): def post_process(out): @@ -71,6 +94,14 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None): with open(path, "w") as outfile: outfile.write(frappe.as_json(out)) + +def export_csv(doctype, path): + from frappe.core.doctype.data_import.exporter import get_template + with open(path, "wb") as csvfile: + get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes") + csvfile.write(frappe.response.result.encode("utf-8")) + + @frappe.whitelist() def export_fixture(doctype, app): if frappe.session.user != "Administrator": @@ -80,21 +111,3 @@ def export_fixture(doctype, app): os.mkdir(frappe.get_app_path(app, "fixtures")) export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json")) - - -def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False, - insert=False, submit=False, pre_process=None): - if os.path.isdir(path): - files = [os.path.join(path, f) for f in os.listdir(path)] - else: - files = [path] - - for f in files: - if f.endswith(".json"): - frappe.flags.mute_emails = True - frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True) - frappe.flags.mute_emails = False - frappe.db.commit() - elif f.endswith(".csv"): - import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process) - frappe.db.commit() diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js new file mode 100644 index 0000000000..cb5c357c80 --- /dev/null +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -0,0 +1,16 @@ +frappe.listview_settings['Data Import'] = { + add_fields: ["import_status"], + get_indicator: function(doc) { + if (doc.import_status=="Successful") { + return [__("Data imported"), "blue", "import_status,=,Successful"]; + } else if(doc.import_status == "Partially Successful") { + return [__("Data partially imported"), "blue", "import_status,=,Partially Successful"]; + } else if(doc.import_status == "In Process") { + return [__("Data import in progress"), "orange", "import_status,=,In Process"]; + } else if(doc.import_status == "Failed") { + return [__("Data import failed"), "red", "import_status,=,Failed"]; + } else { + return [__("Data import pending"), "green", "import_status,=,"]; + } + } +}; \ No newline at end of file diff --git a/frappe/core/doctype/data_import/export_template.html b/frappe/core/doctype/data_import/export_template.html new file mode 100644 index 0000000000..6d043821a6 --- /dev/null +++ b/frappe/core/doctype/data_import/export_template.html @@ -0,0 +1,24 @@ +
+

{{ __("Select Columns Manually") }}

+ {% for doctype in doctype_list %} +
{{ doctype.name }}
+
+ {% for f in doctype.fields %} + {% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %} + {% doctype.reqd||(f.reqd=0);%} +
+
+ +
+
+ {% endif %} + {% endfor %} +
+ {% endfor %} +
+ \ No newline at end of file diff --git a/frappe/core/page/data_import_tool/exporter.py b/frappe/core/doctype/data_import/exporter.py similarity index 99% rename from frappe/core/page/data_import_tool/exporter.py rename to frappe/core/doctype/data_import/exporter.py index 7920059957..4a0a133651 100644 --- a/frappe/core/page/data_import_tool/exporter.py +++ b/frappe/core/doctype/data_import/exporter.py @@ -9,7 +9,7 @@ import frappe.permissions import re, csv, os from frappe.utils.csvutils import UnicodeWriter from frappe.utils import cstr, formatdate, format_datetime -from frappe.core.page.data_import_tool.data_import_tool import get_data_keys +from frappe.core.doctype.data_import.importer import get_data_keys from six import string_types reflags = { diff --git a/frappe/core/page/data_import_tool/importer.py b/frappe/core/doctype/data_import/importer.py similarity index 63% rename from frappe/core/page/data_import_tool/importer.py rename to frappe/core/doctype/data_import/importer.py index 158d460bee..28cc5dbe58 100644 --- a/frappe/core/page/data_import_tool/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -15,35 +15,54 @@ from frappe.utils.csvutils import getlink from frappe.utils.dateutils import parse_date from frappe.utils.file_manager import save_url -from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url -from frappe.core.page.data_import_tool.data_import_tool import get_data_keys +from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_url_to_form from six import text_type, string_types + @frappe.whitelist() -def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, - update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", - skip_errors = True): - """upload data""" +def get_data_keys(): + return frappe._dict({ + "data_separator": _('Start entering data below this line'), + "main_table": _("Table") + ":", + "parent_table": _("Parent Table") + ":", + "columns": _("Column Name") + ":", + "doctype": _("DocType") + ":" + }) - frappe.flags.in_import = True - # extra input params - params = json.loads(frappe.form_dict.get("params") or '{}') +@frappe.whitelist() +def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, + update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", + skip_errors = True, data_import_doc=None, validate_template=False): + """upload data""" - if params.get("submit_after_import"): - submit_after_import = True - if params.get("ignore_encoding_errors"): - ignore_encoding_errors = True - if not params.get("no_email"): - no_email = False - if params.get('update_only'): - update_only = True - if params.get('from_data_import'): - from_data_import = params.get('from_data_import') - if not params.get('skip_errors'): - skip_errors = params.get('skip_errors') + if data_import_doc and isinstance(data_import_doc, string_types): + data_import_doc = frappe.get_doc("Data Import", data_import_doc) + if data_import_doc and from_data_import == "Yes": + no_email = data_import_doc.no_email + ignore_encoding_errors = data_import_doc.ignore_encoding_errors + update_only = data_import_doc.only_update + submit_after_import = data_import_doc.submit_after_import + overwrite = data_import_doc.overwrite + skip_errors = data_import_doc.skip_errors + else: + # extra input params + params = json.loads(frappe.form_dict.get("params") or '{}') + if params.get("submit_after_import"): + submit_after_import = True + if params.get("ignore_encoding_errors"): + ignore_encoding_errors = True + if not params.get("no_email"): + no_email = False + if params.get('update_only'): + update_only = True + if params.get('from_data_import'): + from_data_import = params.get('from_data_import') + if not params.get('skip_errors'): + skip_errors = params.get('skip_errors') + frappe.flags.in_import = True frappe.flags.mute_emails = no_email def get_data_keys_definition(): @@ -55,9 +74,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, def check_data_length(): max_rows = 5000 if not data: - frappe.throw(_("No data found")) - elif not via_console and len(data) > max_rows: - frappe.throw(_("Only allowed {0} rows in one import").format(max_rows)) + frappe.throw(_("No data found in the file. Please reattach the new file with data.")) def get_start_row(): for i, row in enumerate(rows): @@ -111,6 +128,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, if doctypes: doc = {} for idx in range(start_idx, len(rows)): + last_error_row_idx = idx # pylint: disable=W0612 if (not doc) or main_doc_empty(rows[idx]): for dt, parentfield in doctypes: d = {} @@ -119,7 +137,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx] fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx] - if not fieldname: + if not fieldname or not rows[idx][column_idx]: continue d[fieldname] = rows[idx][column_idx] @@ -176,10 +194,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, return doc def main_doc_empty(row): - return not (row and filter(None, row)) + return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2]))) def validate_naming(doc): - autoname = frappe.get_meta(doc['doctype']).autoname + autoname = frappe.get_meta(doctype).autoname if ".#" in autoname or "hash" in autoname: autoname = "" @@ -238,19 +256,18 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, save_url(file_url, None, doctype, docname, "Home/Attachments", 0) # header + filename, file_extension = ['',''] if not rows: - from frappe.utils.file_manager import get_file_doc - file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1) - filename, file_extension = os.path.splitext(file_doc.file_name) + from frappe.utils.file_manager import get_file # get_file_doc + fname, fcontent = get_file(data_import_doc.import_file) + filename, file_extension = os.path.splitext(fname) if file_extension == '.xlsx' and from_data_import == 'Yes': from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file - rows = read_xlsx_file_from_attached_file(file_id=file_doc.name) + rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file) elif file_extension == '.csv': - from frappe.utils.file_manager import get_file from frappe.utils.csvutils import read_csv_content - fname, fcontent = get_file(file_doc.name) rows = read_csv_content(fcontent, ignore_encoding_errors) else: @@ -266,6 +283,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, column_idx_to_fieldtype = {} attachments = [] + if skip_errors: + last_error_row_idx = None + data_rows_with_error = header + if submit_after_import and not cint(frappe.db.get_value("DocType", doctype, "is_submittable")): submit_after_import = False @@ -280,14 +301,19 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, frappe.flags.mute_emails = False return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True} - # allow limit rows to be uploaded + # Throw expception in case of the empty data file check_data_length() make_column_map() + total = len(data) + + if validate_template: + if total: + data_import_doc.total_rows = total + return True if overwrite==None: overwrite = params.get('overwrite') - # delete child rows (if parenttype) parentfield = None if parenttype: @@ -296,13 +322,12 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, if overwrite: delete_child_rows(data, doctype) - ret = [] - - def log(msg): + import_log = [] + def log(**kwargs): if via_console: - print(msg.encode('utf-8')) + print((kwargs.get("title") + kwargs.get("message")).encode('utf-8')) else: - ret.append(msg) + import_log.append(kwargs) def as_link(doctype, name): if via_console: @@ -310,8 +335,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, else: return getlink(doctype, name) - error = False - total = len(data) + # publish realtime task update + def publish_progress(achieved, reload=False): + if data_import_doc: + frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)), + "data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user) + + + error_flag = rollback_flag = False for i, row in enumerate(data): # bypass empty rows if main_doc_empty(row): @@ -320,9 +351,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, row_idx = i + start_row doc = None - # publish task_update - frappe.publish_realtime("data_import_progress", {"progress": [i, total]}, - user=frappe.session.user) + publish_progress(i) try: doc = get_doc(row_idx) @@ -330,12 +359,11 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, if pre_process: pre_process(doc) + original = None if parentfield: parent = frappe.get_doc(parenttype, doc["parent"]) doc = parent.append(parentfield, doc) parent.save() - log('Inserted row for %s at #%s' % (as_link(parenttype, - doc.parent),text_type(doc.idx))) else: if overwrite and doc["name"] and frappe.db.exists(doctype, doc["name"]): original = frappe.get_doc(doctype, doc["name"]) @@ -345,7 +373,6 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, original.name = original_name original.flags.ignore_links = ignore_links original.save() - log('Updated row (#%d) %s' % (row_idx + 1, as_link(original.doctype, original.name))) doc = original else: if not update_only: @@ -353,29 +380,50 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, prepare_for_insert(doc) doc.flags.ignore_links = ignore_links doc.insert() - log('Inserted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name))) - else: - log('Ignored row (#%d) %s' % (row_idx + 1, row[1])) if attachments: # check file url and create a File document for file_url in attachments: attach_file_to_doc(doc.doctype, doc.name, file_url) if submit_after_import: doc.submit() - log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name))) + + # log errors + if parentfield: + log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)), + "link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"}) + elif submit_after_import: + log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)), + "message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"}) + elif original: + log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)), + "message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"}) + elif not update_only: + log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)), + "message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"}) + else: + log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None, + "message": "Document updation ignored", "indicator": "orange"}) + except Exception as e: - if not skip_errors: - error = True - if doc: - frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict()) - err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e) - log('Error for row (#%d) %s : %s' % (row_idx + 1, - len(row)>1 and row[1] or "", err_msg)) - frappe.errprint(frappe.get_traceback()) + error_flag = True + err_msg = frappe.local.message_log and "\n".join([json.loads(msg).get('message') for msg in frappe.local.message_log]) or cstr(e) + error_trace = frappe.get_traceback() + if error_trace: + error_log_doc = frappe.log_error(error_trace) + error_link = get_url_to_form("Error Log", error_log_doc.name) + else: + error_link = None + log(**{"row": row_idx + 1, "title":'Error for row %s' % (len(row)>1 and row[1] or ""), "message": err_msg, + "indicator": "red", "link":error_link}) + # data with error to create a new file + if skip_errors: + data_rows_with_error += data[row_idx:last_error_row_idx] + else: + rollback_flag = True finally: frappe.local.message_log = [] - if error: + if rollback_flag: frappe.db.rollback() else: frappe.db.commit() @@ -383,7 +431,40 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, frappe.flags.mute_emails = False frappe.flags.in_import = False - return {"messages": ret, "error": error} + log_message = {"messages": import_log, "error": error_flag} + if data_import_doc: + data_import_doc.log_details = json.dumps(log_message) + + import_status = None + if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error): + import_status = "Partially Successful" + # write the file with the faulty row + from frappe.utils.file_manager import save_file + file_name = 'error_' + filename + file_extension + if file_extension == '.xlsx': + from frappe.utils.xlsxutils import make_xlsx + xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template") + file_data = xlsx_file.getvalue() + else: + from frappe.utils.csvutils import to_csv + file_data = to_csv(data_rows_with_error) + error_data_file = save_file(file_name, file_data, "Data Import", + data_import_doc.name, "Home/Attachments") + data_import_doc.error_file = error_data_file.file_url + + elif error_flag: + import_status = "Failed" + else: + import_status = "Successful" + + data_import_doc.import_status = import_status + data_import_doc.save() + if data_import_doc.import_status in ["Successful", "Partially Successful"]: + data_import_doc.submit() + frappe.db.commit() + publish_progress(100, True) + else: + return log_message def get_parent_field(doctype, parenttype): parentfield = None diff --git a/frappe/core/doctype/data_import/log_details.html b/frappe/core/doctype/data_import/log_details.html new file mode 100644 index 0000000000..ae6c02ac04 --- /dev/null +++ b/frappe/core/doctype/data_import/log_details.html @@ -0,0 +1,38 @@ +
+
+ + + + + + + + {% for row in data %} + {% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %} + + + + + + {% endif %} + {% endfor %} +
{{ __("Row No") }} {{ __("Row Status") }} {{ __("Message") }}
+ {{ row.row }} + + {{ row.title }} + + {% if (import_status != "Failed" || (row.indicator == "red")) { %} + {{ row.message }} + {% if row.link %} + + + + + + {% endif %} + {% } else { %} + {{ __("Document can't saved.") }} + {% } %} +
+
+
\ No newline at end of file diff --git a/frappe/core/doctype/data_import/test_data_import.py b/frappe/core/doctype/data_import/test_data_import.py new file mode 100644 index 0000000000..30dc9d173c --- /dev/null +++ b/frappe/core/doctype/data_import/test_data_import.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +import unittest + +class TestDataImport(unittest.TestCase): + pass diff --git a/frappe/core/page/data_import_tool/__init__.py b/frappe/core/page/data_import_tool/__init__.py deleted file mode 100644 index 4dbcd0d163..0000000000 --- a/frappe/core/page/data_import_tool/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -from __future__ import unicode_literals diff --git a/frappe/core/page/data_import_tool/data_import_main.html b/frappe/core/page/data_import_tool/data_import_main.html deleted file mode 100644 index 549bebcdfd..0000000000 --- a/frappe/core/page/data_import_tool/data_import_main.html +++ /dev/null @@ -1,118 +0,0 @@ -
-
-

{%= __("Export Template") %}

-

{%= __("To import or update records, you must first download the template for importing.") %}

-
{%= __("Select Type of Document to Download") %}
-
- -
-
-
-
-

{{ __("1. Select Columns") }}

-

- - {%= __("Select All") %} - - {%= __("Select Mandatory") %} - - {%= __("Unselect All") %} -

-
-
-
-

{{ __("2. Download") }}

-
-
-

- {%= __("Download Blank Template") %}

-
-
-
{%= __("Recommended for inserting new records.") %}
-
-
-
-
-

- {%= __("Download with Data") %}

-
-
-
{%= __("Recommended bulk editing records via import, or understanding the import format.") %}
-
-
-
-
-
- -
-
-
-
-
-
-

{%= __("Import") %}

-

{%= __("Update the template and save in downloaded format before attaching.") %}

-
-
-
-

{{ __("1. Select File") }}

-
-
- -

{{ __("2. Upload") }}

-
- -
-
- -
-
- -
-
- -
-
- -
-
- -
-

- -

-
-
-

Import Log

-
-
-
-
-
diff --git a/frappe/core/page/data_import_tool/data_import_tool.css b/frappe/core/page/data_import_tool/data_import_tool.css deleted file mode 100644 index 67841d5014..0000000000 --- a/frappe/core/page/data_import_tool/data_import_tool.css +++ /dev/null @@ -1,7 +0,0 @@ -.data-import-tool { - padding: 15px; -} - -.data-import-tool hr { - margin: 10px -15px; -} diff --git a/frappe/core/page/data_import_tool/data_import_tool.js b/frappe/core/page/data_import_tool/data_import_tool.js deleted file mode 100644 index b7e9dab79a..0000000000 --- a/frappe/core/page/data_import_tool/data_import_tool.js +++ /dev/null @@ -1,233 +0,0 @@ - - -frappe.DataImportTool = Class.extend({ - init: function(parent) { - this.page = frappe.ui.make_app_page({ - parent: parent, - title: __("Data Import Tool"), - single_column: true - }); - this.page.add_inner_button(__("Help"), function() { - frappe.help.show_video("6wiriRKPhmg"); - }); - this.make(); - this.make_upload(); - }, - set_route_options: function() { - var doctype = null; - if(frappe.get_route()[1]) { - doctype = frappe.get_route()[1]; - } else if(frappe.route_options && frappe.route_options.doctype) { - doctype = frappe.route_options.doctype; - } - - if(in_list(frappe.boot.user.can_import, doctype)) { - this.select.val(doctype).change(); - } - - frappe.route_options = null; - }, - make: function() { - var me = this; - frappe.boot.user.can_import = frappe.boot.user.can_import.sort(); - - $(frappe.render_template("data_import_main", this)).appendTo(this.page.main); - - this.select = this.page.main.find("select.doctype"); - this.select_columns = this.page.main.find('.select-columns'); - this.select.on("change", function() { - me.doctype = $(this).val(); - frappe.model.with_doctype(me.doctype, function() { - me.page.main.find(".export-import-section").toggleClass(!!me.doctype); - if(me.doctype) { - - // render select columns - var parent_doctype = frappe.get_doc('DocType', me.doctype); - parent_doctype["reqd"] = true; - var doctype_list = [parent_doctype]; - - frappe.meta.get_table_fields(me.doctype).forEach(function(df) { - var d = frappe.get_doc('DocType', df.options); - d["reqd"]=df.reqd; - doctype_list.push(d); - }); - $(frappe.render_template("data_import_tool_columns", {doctype_list: doctype_list})) - .appendTo(me.select_columns.empty()); - } - }); - }); - - this.page.main.find('.btn-select-all').on('click', function() { - me.select_columns.find('.select-column-check').prop('checked', true); - }); - - this.page.main.find('.btn-unselect-all').on('click', function() { - me.select_columns.find('.select-column-check').prop('checked', false); - }); - - this.page.main.find('.btn-select-mandatory').on('click', function() { - me.select_columns.find('.select-column-check').prop('checked', false); - me.select_columns.find('.select-column-check[data-reqd="1"]').prop('checked', true); - }); - - var get_template_url = '/api/method/frappe.core.page.data_import_tool.exporter.get_template'; - - this.page.main.find(".btn-download-template").on('click', function() { - open_url_post(get_template_url, me.get_export_params(false)); - }); - - this.page.main.find(".btn-download-data").on('click', function() { - open_url_post(get_template_url, me.get_export_params(true)); - }); - - }, - get_export_params: function(with_data) { - var doctype = this.select.val(); - var columns = {}; - - this.select_columns.find('.select-column-check:checked').each(function() { - var _doctype = $(this).attr('data-doctype'); - var _fieldname = $(this).attr('data-fieldname'); - if(!columns[_doctype]) { - columns[_doctype] = []; - } - columns[_doctype].push(_fieldname); - }); - - return { - doctype: doctype, - parent_doctype: doctype, - select_columns: JSON.stringify(columns), - with_data: with_data ? 'Yes' : 'No', - all_doctypes: 'Yes', - from_data_import: 'Yes', - excel_format: this.page.main.find(".excel-check").is(":checked") ? 'Yes' : 'No' - } - }, - make_upload: function() { - var me = this; - frappe.upload.make({ - no_socketio: true, - parent: this.page.main.find(".upload-area"), - btn: this.page.main.find(".btn-import"), - get_params: function() { - return { - submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"), - ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"), - skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"), - overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"), - update_only: me.page.main.find('[name="update_only"]').prop("checked"), - no_email: me.page.main.find('[name="no_email"]').prop("checked"), - from_data_import: 'Yes' - } - }, - args: { - method: 'frappe.core.page.data_import_tool.importer.upload', - }, - allow_multiple: 0, - onerror: function(r) { - me.onerror(r); - }, - queued: function() { - // async, show queued - msg_dialog.clear(); - frappe.msgprint(__("Import Request Queued. This may take a few moments, please be patient.")); - }, - running: function() { - // update async status as running - msg_dialog.clear(); - frappe.msgprint(__("Importing...")); - me.write_messages([__("Importing")]); - me.has_progress = false; - }, - progress: function(data) { - // show callback if async - if(data.progress) { - frappe.hide_msgprint(true); - me.has_progress = true; - frappe.show_progress(__("Importing"), data.progress[0], - data.progress[1]); - } - }, - callback: function(attachment, r) { - if(r.message.error || r.message.messages.length==0) { - me.onerror(r); - } else { - if(me.has_progress) { - frappe.show_progress(__("Importing"), 1, 1); - setTimeout(frappe.hide_progress, 1000); - } - - r.messages = ["
" + __("Import Successful!") + "
"]. - concat(r.message.messages) - - me.write_messages(r.messages); - } - }, - is_private: true - }); - - frappe.realtime.on("data_import_progress", function(data) { - if(data.progress) { - frappe.hide_msgprint(true); - me.has_progress = true; - frappe.show_progress(__("Importing"), data.progress[0], - data.progress[1]); - } - }) - - }, - write_messages: function(data) { - this.page.main.find(".import-log").removeClass("hide"); - var parent = this.page.main.find(".import-log-messages").empty(); - - // TODO render using template! - for (var i=0, l=data.length; i

').html(frappe.markdown(v)).appendTo(parent); - if(v.substr(0,5)=='Error') { - $p.css('color', 'red'); - } else if(v.substr(0,8)=='Inserted') { - $p.css('color', 'green'); - } else if(v.substr(0,7)=='Updated') { - $p.css('color', 'green'); - } else if(v.substr(0,5)=='Valid') { - $p.css('color', '#777'); - } else if(v.substr(0,7)=='Ignored') { - $p.css('color', '#777'); - } - } - }, - onerror: function(r) { - if(r.message) { - // bad design: moves r.messages to r.message.messages - r.messages = $.map(r.message.messages, function(v) { - var msg = v.replace("Inserted", "Valid") - .replace("Updated", "Valid").split("<"); - if (msg.length > 1) { - v = msg[0] + (msg[1].split(">").slice(-1)[0]); - } else { - v = msg[0]; - } - return v; - }); - - r.messages = ["

" + __("Import Failed") + "

"] - .concat(r.messages); - - r.messages.push("Please correct the format of the file and import again."); - - frappe.show_progress(__("Importing"), 1, 1); - - this.write_messages(r.messages); - } - } -}); - -frappe.pages['data-import-tool'].on_page_load = function(wrapper) { - frappe.data_import_tool = new frappe.DataImportTool(wrapper); -} - -frappe.pages['data-import-tool'].on_page_show = function(wrapper) { - frappe.data_import_tool && frappe.data_import_tool.set_route_options(); -} diff --git a/frappe/core/page/data_import_tool/data_import_tool.json b/frappe/core/page/data_import_tool/data_import_tool.json deleted file mode 100644 index f8e05ef598..0000000000 --- a/frappe/core/page/data_import_tool/data_import_tool.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "content": null, - "creation": "2012-06-14 15:07:25", - "docstatus": 0, - "doctype": "Page", - "icon": "fa fa-upload", - "idx": 1, - "modified": "2016-05-11 03:37:53.385693", - "modified_by": "Administrator", - "module": "Core", - "name": "data-import-tool", - "owner": "Administrator", - "page_name": "data-import-tool", - "roles": [], - "script": null, - "standard": "Yes", - "style": null, - "title": "Data Import Tool" -} \ No newline at end of file diff --git a/frappe/core/page/data_import_tool/data_import_tool_columns.html b/frappe/core/page/data_import_tool/data_import_tool_columns.html deleted file mode 100644 index 8b7575d47f..0000000000 --- a/frappe/core/page/data_import_tool/data_import_tool_columns.html +++ /dev/null @@ -1,22 +0,0 @@ -
-{% for doctype in doctype_list %} -
{{ doctype.name }}
-
- {% for f in doctype.fields %} - {% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %} - {% doctype.reqd||(f.reqd=0);%} -
-
- -
-
- {% endif %} - {% endfor %} -
-{% endfor %} -
diff --git a/frappe/patches.txt b/frappe/patches.txt index 2bd2eb7705..2ba0551b97 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -200,3 +200,4 @@ frappe.patches.v9_1.add_sms_sender_name_as_parameters frappe.patches.v9_1.resave_domain_settings frappe.patches.v9_1.revert_domain_settings frappe.patches.v9_1.move_feed_to_activity_log +execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True) \ No newline at end of file diff --git a/frappe/public/css/controls.css b/frappe/public/css/controls.css index 9f10884fcc..ab97425429 100644 --- a/frappe/public/css/controls.css +++ b/frappe/public/css/controls.css @@ -1,5 +1,11 @@ .unit-checkbox { color: #36414c; + margin-top: 5px; + margin-bottom: 5px; +} +.unit-checkbox + .checkbox { + margin-top: 5px; + margin-bottom: 5px; } .frappe-control .select-all { margin-right: 5px; diff --git a/frappe/public/js/frappe/form/controls/multicheck.js b/frappe/public/js/frappe/form/controls/multicheck.js index 1d871e49da..2feb180ff5 100644 --- a/frappe/public/js/frappe/form/controls/multicheck.js +++ b/frappe/public/js/frappe/form/controls/multicheck.js @@ -5,12 +5,12 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({ make() { this._super(); - // this.$label = $(``).appendTo(this.wrapper); + this.$label = $(``).appendTo(this.wrapper); this.$load_state = $('
' + __("Loading") + '...
'); this.$select_buttons = this.get_select_buttons().appendTo(this.wrapper); this.$load_state.appendTo(this.wrapper); - this.$checkbox_area = $('
').appendTo(this.wrapper); + this.$checkbox_area = $('
').appendTo(this.wrapper); this.set_options(); this.bind_checkboxes(); }, @@ -120,8 +120,9 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({ }, get_checkbox_element(option) { + const column_size = 12 / (this.df.columns || 1); return $(` -
+