New data import (#4601)

* created/moved the files

* added the model for the downloading the data

* add the file with the error data

* changes added other changes and fix codacy

* changes in the config and utils files

* fixed the test cases

* minor changes in the data keys dict

* changed the test file location

* fixed the tests

* set the route in the list view and show only erors

* minor fixes in the childtable import and log tables rendering

* Refactor Download dialog to use MultiCheck

frappe/__init__.py

@@ -976,9 +976,9 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp
 		ps.insert()
 
 def import_doc(path, ignore_links=False, ignore_insert=False, insert=False):
-	"""Import a file using Data Import Tool."""
-	from frappe.core.page.data_import_tool import data_import_tool
-	data_import_tool.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
+	"""Import a file using Data Import."""
+	from frappe.core.doctype.data_import import data_import
+	data_import.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
 
 def copy_doc(doc, ignore_no_copy=True):
 	""" No_copy fields also get copied."""
@@ -1366,7 +1366,7 @@ def logger(module=None, with_more_info=True):
 
 def log_error(message=None, title=None):
 	'''Log error to Error Log'''
-	get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
+	return get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
 		method=title)).insert(ignore_permissions=True)
 
 def get_desk_link(doctype, name):

frappe/commands/utils.py

@@ -162,12 +162,12 @@ def export_doc(context, doctype, docname):
 @pass_context
 def export_json(context, doctype, path, name=None):
 	"Export doclist as json to the given path, use '-' as name for Singles."
-	from frappe.core.page.data_import_tool import data_import_tool
+	from frappe.core.doctype.data_import import data_import
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			data_import_tool.export_json(doctype, path, name=name)
+			data_import.export_json(doctype, path, name=name)
 		finally:
 			frappe.destroy()
 
@@ -177,12 +177,12 @@ def export_json(context, doctype, path, name=None):
 @pass_context
 def export_csv(context, doctype, path):
 	"Export data import template with data for DocType"
-	from frappe.core.page.data_import_tool import data_import_tool
+	from frappe.core.doctype.data_import import data_import
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			data_import_tool.export_csv(doctype, path)
+			data_import.export_csv(doctype, path)
 		finally:
 			frappe.destroy()
 
@@ -204,7 +204,7 @@ def export_fixtures(context):
 @pass_context
 def import_doc(context, path, force=False):
 	"Import (insert/update) doclist. If the argument is a directory, all files ending with .json are imported"
-	from frappe.core.page.data_import_tool import data_import_tool
+	from frappe.core.doctype.data_import import data_import
 
 	if not os.path.exists(path):
 		path = os.path.join('..', path)
@@ -216,7 +216,7 @@ def import_doc(context, path, force=False):
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			data_import_tool.import_doc(path, overwrite=context.force)
+			data_import.import_doc(path, overwrite=context.force)
 		finally:
 			frappe.destroy()
 
@@ -229,8 +229,8 @@ def import_doc(context, path, force=False):
 
 @pass_context
 def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
-	"Import CSV using data import tool"
-	from frappe.core.page.data_import_tool import importer
+	"Import CSV using data import"
+	from frappe.core.doctype.data_import import importer
 	from frappe.utils.csvutils import read_csv_content
 	site = get_site(context)
 

frappe/config/desktop.py

@@ -70,5 +70,5 @@ def get_data():
 			"icon": "octicon octicon-book",
 			"color": '#FFAEDB',
 			"hidden": 1,
-		},
+		}
 	]

frappe/config/setup.py

@@ -93,11 +93,11 @@ def get_data():
 			"icon": "fa fa-th",
 			"items": [
 				{
-					"type": "page",
-					"name": "data-import-tool",
+					"type": "doctype",
+					"name": "Data Import",
 					"label": _("Import / Export Data"),
-					"icon": "fa fa-upload",
-					"description": _("Import / Export Data from .csv files.")
+					"icon": "octicon octicon-cloud-upload",
+					"description": _("Import / Export Data from CSV and Excel files.")
 				},
 				{
 					"type": "doctype",

frappe/core/doctype/data_import/data_import.js

@@ -0,0 +1,260 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Import', {
+	onload: function(frm) {
+		frm.set_query("reference_doctype", function() {
+			return {
+				"filters": {
+					"issingle": 0,
+					"istable": 0,
+					"name": ['in', frappe.boot.user.can_import]
+				}
+			};
+		});
+
+		frappe.realtime.on("data_import_progress", function(data) {
+			if (data.data_import === frm.doc.name) {
+				if (data.reload && data.reload === true) {
+					frm.reload_doc();
+				}
+				if (data.progress) {
+					let progress_bar = $(frm.dashboard.progress_area).find(".progress-bar");
+					if (progress_bar) {
+						$(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped");
+						$(progress_bar).css("width", data.progress+"%");
+					}
+				}
+			}
+		});
+	},
+
+	refresh: function(frm) {
+		frm.disable_save();
+		frm.dashboard.clear_headline();
+		if (frm.doc.reference_doctype && !frm.doc.import_file) {
+			frm.dashboard.add_comment(__('Please attach a file to import'));
+		} else {
+			if (frm.doc.import_status) {
+				frm.dashboard.add_comment(frm.doc.import_status);
+
+				if (frm.doc.import_status==="In Progress") {
+					frm.dashboard.add_progress("Data Import Progress", "0");
+					frm.set_read_only(true);
+				}
+			}
+		}
+
+		if (frm.doc.reference_doctype) {
+			frappe.model.with_doctype(frm.doc.reference_doctype);
+		}
+
+		frm.add_custom_button(__("Help"), function() {
+			frappe.help.show_video("6wiriRKPhmg");
+		});
+
+		if(frm.doc.reference_doctype && frm.doc.docstatus === 0) {
+			frm.add_custom_button(__("Download template"), function() {
+				frappe.data_import.download_dialog(frm).show();
+			});
+		}
+
+		if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows  && frm.doc.docstatus === 0) {
+			frm.page.set_primary_action(__("Start Import"), function() {
+				frappe.call({
+					method: "frappe.core.doctype.data_import.data_import.import_data",
+					args: {
+						data_import: frm.doc.name
+					}
+				});
+			}).addClass('btn btn-primary');
+		}
+
+		if (frm.doc.log_details) {
+			frm.events.create_log_table(frm);
+		} else {
+			$(frm.fields_dict.import_log.wrapper).empty();
+		}
+	},
+
+	reference_doctype: function(frm) {
+		if (frm.doc.reference_doctype) {
+			frm.save();
+		}
+	},
+
+	// import_file: function(frm) {
+	// 	frm.save();
+	// },
+
+	overwrite: function(frm) {
+		if (frm.doc.overwrite === 0) {
+			frm.doc.only_update = 0;
+		}
+		frm.save();
+	},
+
+	only_update: function(frm) {
+		frm.save();
+	},
+
+	submit_after_import: function(frm) {
+		frm.save();
+	},
+
+	skip_errors: function(frm) {
+		frm.save();
+	},
+
+	ignore_encoding_errors: function(frm) {
+		frm.save();
+	},
+
+	no_email: function(frm) {
+		frm.save();
+	},
+
+	show_only_errors: function(frm) {
+		frm.events.create_log_table(frm);
+	},
+
+	create_log_table: function(frm) {
+		let msg = JSON.parse(frm.doc.log_details);
+		var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty();
+		$(frappe.render_template("log_details", {data: msg.messages, show_only_errors: frm.doc.show_only_errors,
+			import_status: frm.doc.import_status})).appendTo($log_wrapper);
+	}
+});
+
+frappe.provide('frappe.data_import');
+frappe.data_import.download_dialog = function(frm) {
+	var dialog;
+	const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden;
+	const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields);
+
+	const get_doctypes = parentdt => {
+		return [parentdt].concat(
+			frappe.meta.get_table_fields(parentdt).map(df => df.options)
+		);
+	};
+
+	const get_doctype_checkbox_fields = () => {
+		return dialog.fields.filter(df => df.fieldname.endsWith('_fields'))
+			.map(df => dialog.fields_dict[df.fieldname]);
+	}
+
+	const doctype_fields = get_fields(frm.doc.reference_doctype)
+		.map(df => ({
+			label: df.label,
+			value: df.fieldname,
+			checked: 1
+		}));
+
+	let fields = [
+		{
+			"label": __("Select Columns"),
+			"fieldname": "select_columns",
+			"fieldtype": "Select",
+			"options": "All\nMandatory\nManually",
+			"reqd": 1,
+			"onchange": function() {
+				const fields = get_doctype_checkbox_fields();
+				fields.map(f => f.toggle(this.value === 'Manually'));
+			}
+		},
+		{
+			"label": __("File Type"),
+			"fieldname": "file_type",
+			"fieldtype": "Select",
+			"options": "Excel\nCSV",
+			"default": "Excel"
+		},
+		{
+			"label": __("Download with Data"),
+			"fieldname": "with_data",
+			"fieldtype": "Check"
+		},
+		{
+			"label": frm.doc.reference_doctype,
+			"fieldname": "doctype_fields",
+			"fieldtype": "MultiCheck",
+			"options": doctype_fields,
+			"columns": 2,
+			"hidden": 1
+		}
+	];
+
+	const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype)
+		.map(df => {
+			return {
+				"label": df.options,
+				"fieldname": df.fieldname + '_fields',
+				"fieldtype": "MultiCheck",
+				"options": frappe.meta.get_docfields(df.options)
+					.filter(filter_fields)
+					.map(df => ({
+						label: df.label,
+						value: df.fieldname,
+						checked: 1
+					})),
+				"columns": 2,
+				"hidden": 1
+			};
+		});
+
+	fields = fields.concat(child_table_fields);
+
+	dialog = new frappe.ui.Dialog({
+		title: __('Download Template'),
+		fields: fields,
+		primary_action: function(values) {
+			var data = values;
+			if (frm.doc.reference_doctype) {
+				var export_params = () => {
+					let columns = {};
+					if (values.select_columns === 'All') {
+						columns = get_doctypes(frm.doc.reference_doctype).reduce((columns, doctype) => {
+							columns[doctype] = get_fields(doctype).map(df => df.fieldname);
+							return columns;
+						}, {});
+					} else if (values.select_columns === 'Mandatory') {
+						// only reqd child tables
+						const doctypes = [frm.doc.reference_doctype].concat(
+							frappe.meta.get_table_fields(frm.doc.reference_doctype)
+								.filter(df => df.reqd).map(df => df.options)
+						);
+
+						columns = doctypes.reduce((columns, doctype) => {
+							columns[doctype] = get_fields(doctype).filter(df => df.reqd).map(df => df.fieldname);
+							return columns;
+						}, {});
+					} else if (values.select_columns === 'Manually') {
+						columns = get_doctype_checkbox_fields().reduce((columns, field) => {
+							const options = field.get_checked_options();
+							columns[field.df.label] = options;
+							return columns;
+						}, {});
+					}
+
+					return {
+						doctype: frm.doc.reference_doctype,
+						parent_doctype: frm.doc.reference_doctype,
+						select_columns: JSON.stringify(columns),
+						with_data: data.with_data ? 'Yes' : 'No',
+						all_doctypes: 'Yes',
+						from_data_import: 'Yes',
+						excel_format: data.file_type === 'Excel' ? 'Yes' : 'No'
+					};
+				};
+				let get_template_url = '/api/method/frappe.core.doctype.data_import.exporter.get_template';
+				open_url_post(get_template_url, export_params());
+			} else {
+				frappe.msgprint(__("Please select the Document Type."));
+			}
+			dialog.hide();
+		},
+		primary_action_label: __('Download')
+	});
+
+	return dialog;
+};

frappe/core/doctype/data_import/data_import.json

@@ -0,0 +1,661 @@
+{
+ "allow_copy": 1, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "", 
+ "beta": 0, 
+ "creation": "2016-12-09 14:27:32.720061", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "Document", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "reference_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 1, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Document Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "collapsible_depends_on": "", 
+   "columns": 0, 
+   "depends_on": "eval:(!doc.__islocal)", 
+   "fieldname": "section_break_4", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "import_file", 
+   "fieldtype": "Attach", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Attach file for Import", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_4", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval: doc.import_status == \"Partially Successful\"", 
+   "description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.", 
+   "fieldname": "error_file", 
+   "fieldtype": "Attach", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Generated File", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "collapsible_depends_on": "", 
+   "columns": 0, 
+   "depends_on": "eval:(!doc.__islocal)", 
+   "fieldname": "section_break_6", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "", 
+   "description": "If you are updating/overwriting already created records.", 
+   "fieldname": "overwrite", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Update records", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "overwrite", 
+   "description": "If you don't want to create any new records while updating the older records.", 
+   "fieldname": "only_update", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Don't create new records", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.\n", 
+   "fieldname": "skip_errors", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Skip rows with errors", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "", 
+   "fieldname": "submit_after_import", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Submit after importing", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "", 
+   "fieldname": "ignore_encoding_errors", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Ignore encoding errors", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "1", 
+   "depends_on": "", 
+   "fieldname": "no_email", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Do not send Emails", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 1, 
+   "collapsible_depends_on": "eval: doc.import_status == \"Failed\"", 
+   "columns": 0, 
+   "depends_on": "import_status", 
+   "fieldname": "import_detail", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Import Log", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "import_status", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Import Status", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 1, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "1", 
+   "fieldname": "show_only_errors", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Show only errors", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 1, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 1, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "", 
+   "depends_on": "import_status", 
+   "fieldname": "import_log", 
+   "fieldtype": "HTML", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Import Log", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 1, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "log_details", 
+   "fieldtype": "Code", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Log Details", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "amended_from", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Amended From", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "Data Import", 
+   "permlevel": 0, 
+   "print_hide": 1, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "total_rows", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Total Rows", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 1, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 1, 
+ "modified": "2017-12-14 16:27:37.683505", 
+ "modified_by": "Administrator", 
+ "module": "Core", 
+ "name": "Data Import", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 0, 
+   "read": 1, 
+   "report": 0, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 1, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "title_field": "", 
+ "track_changes": 1, 
+ "track_seen": 1
+}

frappe/core/page/data_import_tool/data_import_tool.py → frappe/core/doctype/data_import/data_import.py

@@ -1,44 +1,67 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals, print_function
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
 
+from __future__ import unicode_literals
 import frappe, os
 from frappe import _
 import frappe.modules.import_file
+from frappe.model.document import Document
+from frappe.utils.data import format_datetime
+from frappe.core.doctype.data_import.importer import upload
+from frappe.utils.background_jobs import enqueue
 
-@frappe.whitelist()
-def get_data_keys():
-    return frappe._dict({
-        "data_separator": _('Start entering data below this line'),
-        "main_table": _("Table") + ":",
-        "parent_table": _("Parent Table") + ":",
-        "columns": _("Column Name") + ":",
-        "doctype": _("DocType") + ":"
-    })
 
-@frappe.whitelist()
-def get_doctypes():
-	return frappe.get_user()._get("can_import")
+class DataImport(Document):
+	def autoname(self):
+		self.name = "Import on "+ format_datetime(self.creation)
+
+	def validate(self):
+		if not self.import_file:
+			self.db_set("total_rows", 0)
+		if self.import_status == "In Progress":
+			frappe.throw(_("Can't save the form as data import is in progress."))
+
+		# validate the template just after the upload
+		# if there is total_rows in the doc, it means that the template is already validated and error free
+		if self.import_file and not self.total_rows:
+			upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
+
 
 @frappe.whitelist()
-def get_doctype_options():
-	doctype = frappe.form_dict['doctype']
-	return [doctype] + [d.options for d in frappe.get_meta(doctype).get_table_fields()]
+def import_data(data_import):
+	frappe.db.set_value("Data Import", data_import, "import_status", "In Progress")
+	frappe.publish_realtime("data_import_progress", {"progress": "0",
+		"data_import": data_import, "reload": True}, user=frappe.session.user)
+	enqueue(upload, queue='default', timeout=6000, event='data_import',
+		data_import_doc=data_import, from_data_import="Yes", validate_template=False)
+
+
+def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
+	insert=False, submit=False, pre_process=None):
+	if os.path.isdir(path):
+		files = [os.path.join(path, f) for f in os.listdir(path)]
+	else:
+		files = [path]
+
+	for f in files:
+		if f.endswith(".json"):
+			frappe.flags.mute_emails = True
+			frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
+			frappe.flags.mute_emails = False
+			frappe.db.commit()
+		elif f.endswith(".csv"):
+			import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
+			frappe.db.commit()
+
 
 def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True):
 	from frappe.utils.csvutils import read_csv_content
-	from frappe.core.page.data_import_tool.importer import upload
 	print("Importing " + path)
 	with open(path, "r") as infile:
 		upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
-            submit_after_import=submit, pre_process=pre_process)
+			submit_after_import=submit, pre_process=pre_process)
 
-def export_csv(doctype, path):
-	from frappe.core.page.data_import_tool.exporter import get_template
-	with open(path, "wb") as csvfile:
-		get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
-		csvfile.write(frappe.response.result.encode("utf-8"))
 
 def export_json(doctype, path, filters=None, or_filters=None, name=None):
 	def post_process(out):
@@ -71,6 +94,14 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None):
 	with open(path, "w") as outfile:
 		outfile.write(frappe.as_json(out))
 
+
+def export_csv(doctype, path):
+	from frappe.core.doctype.data_import.exporter import get_template
+	with open(path, "wb") as csvfile:
+		get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
+		csvfile.write(frappe.response.result.encode("utf-8"))
+
+
 @frappe.whitelist()
 def export_fixture(doctype, app):
 	if frappe.session.user != "Administrator":
@@ -80,21 +111,3 @@ def export_fixture(doctype, app):
 		os.mkdir(frappe.get_app_path(app, "fixtures"))
 
 	export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"))
-
-
-def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
-    insert=False, submit=False, pre_process=None):
-	if os.path.isdir(path):
-		files = [os.path.join(path, f) for f in os.listdir(path)]
-	else:
-		files = [path]
-
-	for f in files:
-		if f.endswith(".json"):
-			frappe.flags.mute_emails = True
-			frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
-			frappe.flags.mute_emails = False
-			frappe.db.commit()
-		elif f.endswith(".csv"):
-			import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
-			frappe.db.commit()

frappe/core/doctype/data_import/data_import_list.js

@@ -0,0 +1,16 @@
+frappe.listview_settings['Data Import'] = {
+	add_fields: ["import_status"],
+	get_indicator: function(doc) {
+		if (doc.import_status=="Successful") {
+			return [__("Data imported"), "blue", "import_status,=,Successful"];
+		} else if(doc.import_status == "Partially Successful") {
+			return [__("Data partially imported"), "blue", "import_status,=,Partially Successful"];
+		} else if(doc.import_status == "In Process") {
+			return [__("Data import in progress"), "orange", "import_status,=,In Process"];
+		} else if(doc.import_status == "Failed") {
+			return [__("Data import failed"), "red", "import_status,=,Failed"];
+		} else {
+			return [__("Data import pending"), "green", "import_status,=,"];
+		}
+	}
+};

frappe/core/doctype/data_import/export_template.html

@@ -0,0 +1,24 @@
+<div style="margin: 15px 0px;">
+	<h4>{{ __("Select Columns Manually") }}</h4>
+	{% for doctype in doctype_list %}
+		<h5 style="margin-top: 25px; margin-bottom: 5px;">{{ doctype.name }}</h5>
+		<div class="row">
+			{% for f in doctype.fields %}
+			{% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %}
+			{% doctype.reqd||(f.reqd=0);%}		
+			<div class="col-sm-6">
+				<div class="checkbox" style="margin: 5px 0px;">
+					<label>
+						<input type="checkbox" class="select-column-check"
+							data-fieldname="{{ f.fieldname }}" data-reqd="{{ f.reqd }}"
+							data-doctype="{{ doctype.name }}" checked>
+							<small>{{ __(f.label) }}</small>
+					</label>
+				</div>
+			</div>
+			{% endif %}
+			{% endfor %}
+		</div>
+	{% endfor %}
+	</div>
+	

frappe/core/page/data_import_tool/exporter.py → frappe/core/doctype/data_import/exporter.py

@@ -9,7 +9,7 @@ import frappe.permissions
 import re, csv, os
 from frappe.utils.csvutils import UnicodeWriter
 from frappe.utils import cstr, formatdate, format_datetime
-from  frappe.core.page.data_import_tool.data_import_tool import get_data_keys
+from frappe.core.doctype.data_import.importer import get_data_keys
 from six import string_types
 
 reflags = {

frappe/core/page/data_import_tool/importer.py → frappe/core/doctype/data_import/importer.py

@@ -15,35 +15,54 @@ from frappe.utils.csvutils import getlink
 from frappe.utils.dateutils import parse_date
 from frappe.utils.file_manager import save_url
 
-from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url
-from frappe.core.page.data_import_tool.data_import_tool import get_data_keys
+from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_url_to_form
 from six import text_type, string_types
 
+
 @frappe.whitelist()
-def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
-	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
-	skip_errors = True):
-	"""upload data"""
+def get_data_keys():
+	return frappe._dict({
+		"data_separator": _('Start entering data below this line'),
+		"main_table": _("Table") + ":",
+		"parent_table": _("Parent Table") + ":",
+		"columns": _("Column Name") + ":",
+		"doctype": _("DocType") + ":"
+	})
 
-	frappe.flags.in_import = True
 
-	# extra input params
-	params = json.loads(frappe.form_dict.get("params") or '{}')
 
+@frappe.whitelist()
+def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
+	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
+	skip_errors = True, data_import_doc=None, validate_template=False):
+	"""upload data"""
 
-	if params.get("submit_after_import"):
-		submit_after_import = True
-	if params.get("ignore_encoding_errors"):
-		ignore_encoding_errors = True
-	if not params.get("no_email"):
-		no_email = False
-	if params.get('update_only'):
-		update_only = True
-	if params.get('from_data_import'):
-		from_data_import = params.get('from_data_import')
-	if not params.get('skip_errors'):
-		skip_errors = params.get('skip_errors')
+	if data_import_doc and isinstance(data_import_doc, string_types):
+		data_import_doc = frappe.get_doc("Data Import", data_import_doc)
+	if data_import_doc and from_data_import == "Yes":
+		no_email = data_import_doc.no_email
+		ignore_encoding_errors = data_import_doc.ignore_encoding_errors
+		update_only = data_import_doc.only_update
+		submit_after_import = data_import_doc.submit_after_import
+		overwrite = data_import_doc.overwrite
+		skip_errors = data_import_doc.skip_errors
+	else:
+		# extra input params
+		params = json.loads(frappe.form_dict.get("params") or '{}')
+		if params.get("submit_after_import"):
+			submit_after_import = True
+		if params.get("ignore_encoding_errors"):
+			ignore_encoding_errors = True
+		if not params.get("no_email"):
+			no_email = False
+		if params.get('update_only'):
+			update_only = True
+		if params.get('from_data_import'):
+			from_data_import = params.get('from_data_import')
+		if not params.get('skip_errors'):
+			skip_errors = params.get('skip_errors')
 
+	frappe.flags.in_import = True
 	frappe.flags.mute_emails = no_email
 
 	def get_data_keys_definition():
@@ -55,9 +74,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	def check_data_length():
 		max_rows = 5000
 		if not data:
-			frappe.throw(_("No data found"))
-		elif not via_console and len(data) > max_rows:
-			frappe.throw(_("Only allowed {0} rows in one import").format(max_rows))
+			frappe.throw(_("No data found in the file. Please reattach the new file with data."))
 
 	def get_start_row():
 		for i, row in enumerate(rows):
@@ -111,6 +128,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		if doctypes:
 			doc = {}
 			for idx in range(start_idx, len(rows)):
+				last_error_row_idx = idx	# pylint: disable=W0612
 				if (not doc) or main_doc_empty(rows[idx]):
 					for dt, parentfield in doctypes:
 						d = {}
@@ -119,7 +137,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 								fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
 								fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
 
-								if not fieldname:
+								if not fieldname or not rows[idx][column_idx]:
 									continue
 
 								d[fieldname] = rows[idx][column_idx]
@@ -176,10 +194,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 			return doc
 
 	def main_doc_empty(row):
-		return not (row and filter(None, row))
+		return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2])))
 
 	def validate_naming(doc):
-		autoname = frappe.get_meta(doc['doctype']).autoname
+		autoname = frappe.get_meta(doctype).autoname
 
 		if ".#" in autoname or "hash" in autoname:
 			autoname = ""
@@ -238,19 +256,18 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
 
 	# header
+	filename, file_extension = ['','']
 	if not rows:
-		from frappe.utils.file_manager import get_file_doc
-		file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1)
-		filename, file_extension = os.path.splitext(file_doc.file_name)
+		from frappe.utils.file_manager import get_file # get_file_doc
+		fname, fcontent = get_file(data_import_doc.import_file)
+		filename, file_extension = os.path.splitext(fname)
 
 		if file_extension == '.xlsx' and from_data_import == 'Yes':
 			from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
-			rows = read_xlsx_file_from_attached_file(file_id=file_doc.name)
+			rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file)
 
 		elif file_extension == '.csv':
-			from frappe.utils.file_manager import get_file
 			from frappe.utils.csvutils import read_csv_content
-			fname, fcontent = get_file(file_doc.name)
 			rows = read_csv_content(fcontent, ignore_encoding_errors)
 
 		else:
@@ -266,6 +283,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	column_idx_to_fieldtype = {}
 	attachments = []
 
+	if skip_errors:
+		last_error_row_idx = None
+		data_rows_with_error = header
+
 	if submit_after_import and not cint(frappe.db.get_value("DocType",
 			doctype, "is_submittable")):
 		submit_after_import = False
@@ -280,14 +301,19 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		frappe.flags.mute_emails = False
 		return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
 
-	# allow limit rows to be uploaded
+	# Throw expception in case of the empty data file
 	check_data_length()
 	make_column_map()
+	total = len(data)
+
+	if validate_template:
+		if total:
+			data_import_doc.total_rows = total
+		return True
 
 	if overwrite==None:
 		overwrite = params.get('overwrite')
 
-
 	# delete child rows (if parenttype)
 	parentfield = None
 	if parenttype:
@@ -296,13 +322,12 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		if overwrite:
 			delete_child_rows(data, doctype)
 
-	ret = []
-
-	def log(msg):
+	import_log = []
+	def log(**kwargs):
 		if via_console:
-			print(msg.encode('utf-8'))
+			print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
 		else:
-			ret.append(msg)
+			import_log.append(kwargs)
 
 	def as_link(doctype, name):
 		if via_console:
@@ -310,8 +335,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		else:
 			return getlink(doctype, name)
 
-	error = False
-	total = len(data)
+	# publish realtime task update
+	def publish_progress(achieved, reload=False):
+		if data_import_doc:
+			frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
+				"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
+
+
+	error_flag = rollback_flag = False
 	for i, row in enumerate(data):
 		# bypass empty rows
 		if main_doc_empty(row):
@@ -320,9 +351,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		row_idx = i + start_row
 		doc = None
 
-		# publish task_update
-		frappe.publish_realtime("data_import_progress", {"progress": [i, total]},
-			user=frappe.session.user)
+		publish_progress(i)
 
 		try:
 			doc = get_doc(row_idx)
@@ -330,12 +359,11 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 			if pre_process:
 				pre_process(doc)
 
+			original = None
 			if parentfield:
 				parent = frappe.get_doc(parenttype, doc["parent"])
 				doc = parent.append(parentfield, doc)
 				parent.save()
-				log('Inserted row for %s at #%s' % (as_link(parenttype,
-					doc.parent),text_type(doc.idx)))
 			else:
 				if overwrite and doc["name"] and frappe.db.exists(doctype, doc["name"]):
 					original = frappe.get_doc(doctype, doc["name"])
@@ -345,7 +373,6 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 					original.name = original_name
 					original.flags.ignore_links = ignore_links
 					original.save()
-					log('Updated row (#%d) %s' % (row_idx + 1, as_link(original.doctype, original.name)))
 					doc = original
 				else:
 					if not update_only:
@@ -353,29 +380,50 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 						prepare_for_insert(doc)
 						doc.flags.ignore_links = ignore_links
 						doc.insert()
-						log('Inserted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
-					else:
-						log('Ignored row (#%d) %s' % (row_idx + 1, row[1]))
 				if attachments:
 					# check file url and create a File document
 					for file_url in attachments:
 						attach_file_to_doc(doc.doctype, doc.name, file_url)
 				if submit_after_import:
 					doc.submit()
-					log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
+
+			# log errors
+			if parentfield:
+				log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
+					"link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
+			elif submit_after_import:
+				log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
+					"message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"})
+			elif original:
+				log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
+					"message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
+			elif not update_only:
+				log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
+					"message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
+			else:
+				log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
+					"message": "Document updation ignored", "indicator": "orange"})
+
 		except Exception as e:
-			if not skip_errors:
-				error = True
-				if doc:
-					frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
-				err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
-				log('Error for row (#%d) %s : %s' % (row_idx + 1,
-					len(row)>1 and row[1] or "", err_msg))
-				frappe.errprint(frappe.get_traceback())
+			error_flag = True
+			err_msg = frappe.local.message_log and "\n".join([json.loads(msg).get('message') for msg in frappe.local.message_log]) or cstr(e)
+			error_trace = frappe.get_traceback()
+			if error_trace:
+				error_log_doc = frappe.log_error(error_trace)
+				error_link = get_url_to_form("Error Log", error_log_doc.name)
+			else:
+				error_link = None
+			log(**{"row": row_idx + 1, "title":'Error for row %s' % (len(row)>1 and row[1] or ""), "message": err_msg,
+				"indicator": "red", "link":error_link})
+			# data with error to create a new file
+			if skip_errors:
+				data_rows_with_error += data[row_idx:last_error_row_idx]
+			else:
+				rollback_flag = True
 		finally:
 			frappe.local.message_log = []
 
-	if error:
+	if rollback_flag:
 		frappe.db.rollback()
 	else:
 		frappe.db.commit()
@@ -383,7 +431,40 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	frappe.flags.mute_emails = False
 	frappe.flags.in_import = False
 
-	return {"messages": ret, "error": error}
+	log_message = {"messages": import_log, "error": error_flag}
+	if data_import_doc:
+		data_import_doc.log_details = json.dumps(log_message)
+
+		import_status = None
+		if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
+			import_status = "Partially Successful"
+			# write the file with the faulty row
+			from frappe.utils.file_manager import save_file
+			file_name = 'error_' + filename + file_extension
+			if file_extension == '.xlsx':
+				from frappe.utils.xlsxutils import make_xlsx
+				xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
+				file_data = xlsx_file.getvalue()
+			else:
+				from frappe.utils.csvutils import to_csv
+				file_data = to_csv(data_rows_with_error)
+			error_data_file = save_file(file_name, file_data, "Data Import",
+				data_import_doc.name,  "Home/Attachments")
+			data_import_doc.error_file = error_data_file.file_url
+
+		elif error_flag:
+			import_status = "Failed"
+		else:
+			import_status = "Successful"
+
+		data_import_doc.import_status = import_status
+		data_import_doc.save()
+		if data_import_doc.import_status in ["Successful", "Partially Successful"]:
+			data_import_doc.submit()
+		frappe.db.commit()
+		publish_progress(100, True)
+	else:
+		return log_message
 
 def get_parent_field(doctype, parenttype):
 	parentfield = None

frappe/core/doctype/data_import/log_details.html

@@ -0,0 +1,38 @@
+<div>
+	<div class="table-responsive">
+		<table class="table table-bordered table-hover log-details-table">
+				<tr>
+					<th style="width:10%"> {{ __("Row No") }} </th>
+					<th style="width:40%"> {{ __("Row Status") }} </th>
+					<th style="width:50%"> {{ __("Message") }} </th>
+				</tr>
+			
+			{% for row in data %}
+				{% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %}
+				<tr>
+					<td>
+						<span>{{ row.row }} </span> 
+					</td>
+					<td>
+						<span class="indicator {{ row.indicator }}"> {{ row.title }} </span>
+					</td>
+					<td>
+						{% if (import_status != "Failed" || (row.indicator == "red")) { %}
+							<span> {{ row.message }} </span>
+							{% if row.link %}
+								<span style="width: 10%; float:right;">
+									<a class="btn-open no-decoration" title="Open Link" href="{{ row.link }}">
+										<i class="octicon octicon-arrow-right"></i>
+									</a>
+								</span>
+							{% endif %}
+						{% } else { %}
+							<span> {{ __("Document can't saved.") }} </span>
+						{% } %}
+					</td>
+				</tr>
+				{% endif %}
+			{% endfor %}
+		</table>
+	</div>
+</div>

frappe/core/doctype/data_import/test_data_import.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import unittest
+
+class TestDataImport(unittest.TestCase):
+	pass

frappe/core/page/data_import_tool/__init__.py

@@ -1,4 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals

frappe/core/page/data_import_tool/data_import_main.html

@@ -1,118 +0,0 @@
-<div class="data-import-tool">
-    <div class="data-import-selector">
-        <h3>{%= __("Export Template") %}</h3>
-        <p class="text-muted">{%= __("To import or update records, you must first download the template for importing.") %}</p>
-        <h6>{%= __("Select Type of Document to Download") %}</h6>
-    	<div>
-            <select class="form-control doctype" style="width: 200px" placeholder="{%= __("Select Type") %}">
-                <option value=""></option>
-                {% for (var i=0, l= frappe.boot.user.can_import.length; i < l; i++) {
-                    var doctype = frappe.boot.user.can_import[i]; %}
-                <option value="{%= doctype %}">{%= __(doctype) %}</option>
-                {% } %}
-            </select>
-            <br>
-        </div>
-    </div>
-    <div class="export-import-section hide"  style="max-width: 700px;">
-		<h4>{{ __("1. Select Columns") }}</h4>
-        <p>
-			<a class="btn btn-default btn-xs btn-select-all" style="margin-right: 7px;">
-            {%= __("Select All") %}</a>
-			<a class="btn btn-default btn-xs btn-select-mandatory" style="margin-right: 7px;">
-            {%= __("Select Mandatory") %}</a>
-			<a class="btn btn-default btn-xs btn-unselect-all">
-            {%= __("Unselect All") %}</a>
-		</p>
-		<div class="select-columns">
-		</div>
-		<br>
-		<h4>{{ __("2. Download") }}</h4>
-		<div class="row">
-            <div class="col-sm-4">
-                <p><a class="btn btn-primary btn-xs btn-download-template">
-                    {%= __("Download Blank Template") %}</a></p>
-            </div>
-            <div class="col-sm-8">
-                <h6 class="text-muted">{%= __("Recommended for inserting new records.") %}</h6>
-            </div>
-        </div>
-        <div class="row">
-            <div class="col-sm-4">
-                <p><a class="btn btn-primary btn-xs btn-download-data">
-                    {%= __("Download with Data") %}</a></p>
-            </div>
-            <div class="col-sm-8">
-                <h6 class="text-muted">{%= __("Recommended bulk editing records via import, or understanding the import format.") %}</h6>
-            </div>
-        </div>
-        <div class="row">
-            <div class="col-sm-4">
-                <div class="checkbox" style="margin: 5px 0px;">
-                    <label>
-                        <input type="checkbox" class="excel-check" data-fieldname="excel_check" checked>
-                            <small>{%= __("Download in Excel File Format") %}</small>
-                    </label>
-                </div>
-            </div>
-        </div>
-    </div>
-    <div>
-        <hr style="margin-top: 50px;">
-        <h3>{%= __("Import") %}</h3>
-        <p class="text-muted">{%= __("Update the template and save in downloaded format before attaching.") %}</p>
-        <div class="row">
-            <div class="col-md-6">
-				<br>
-				<h4>{{ __("1. Select File") }}</h4>
-                <div class="upload-area"></div>
-				<br>
-
-				<h4>{{ __("2. Upload") }}</h4>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="always_insert">
-                            {%= __("Do not update, but insert new records.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="update_only">
-                            {%= __("Update only, do not insert new records.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="submit_after_import">
-                            {%= __("Submit after importing.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="ignore_encoding_errors">
-                            {%= __("Ignore encoding errors.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="skip_errors">
-                            {%= __("Skip rows with errors.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="no_email" checked>
-                            {%= __("Do not send emails.") %}
-                    </label>
-                </div>
-                <p>
-                    <button class="btn btn-sm btn-primary btn-import">Import</button>
-                </p>
-            </div>
-            <div class="import-log hide col-md-6">
-                <h3>Import Log</h3>
-                <div class="import-log-messages"></div>
-            </div>
-        </div>
-    </div>
-</div>

frappe/core/page/data_import_tool/data_import_tool.css

@@ -1,7 +0,0 @@
-.data-import-tool {
-	padding: 15px;
-}
-
-.data-import-tool hr {
-	margin: 10px -15px;
-}

frappe/core/page/data_import_tool/data_import_tool.js

@@ -1,233 +0,0 @@
-
-
-frappe.DataImportTool = Class.extend({
-	init: function(parent) {
-		this.page = frappe.ui.make_app_page({
-			parent: parent,
-			title: __("Data Import Tool"),
-			single_column: true
-		});
-		this.page.add_inner_button(__("Help"), function() {
-			frappe.help.show_video("6wiriRKPhmg");
-		});
-		this.make();
-		this.make_upload();
-	},
-	set_route_options: function() {
-		var doctype = null;
-		if(frappe.get_route()[1]) {
-			doctype = frappe.get_route()[1];
-		} else if(frappe.route_options && frappe.route_options.doctype) {
-			doctype = frappe.route_options.doctype;
-		}
-
-		if(in_list(frappe.boot.user.can_import, doctype)) {
-			this.select.val(doctype).change();
-		}
-
-		frappe.route_options = null;
-	},
-	make: function() {
-		var me = this;
-		frappe.boot.user.can_import = frappe.boot.user.can_import.sort();
-
-		$(frappe.render_template("data_import_main", this)).appendTo(this.page.main);
-
-		this.select = this.page.main.find("select.doctype");
-		this.select_columns = this.page.main.find('.select-columns');
-		this.select.on("change", function() {
-			me.doctype = $(this).val();
-			frappe.model.with_doctype(me.doctype, function() {
-				me.page.main.find(".export-import-section").toggleClass(!!me.doctype);
-				if(me.doctype) {
-
-					// render select columns
-					var parent_doctype = frappe.get_doc('DocType', me.doctype);
-					parent_doctype["reqd"] = true;
-					var doctype_list = [parent_doctype];
-
-					frappe.meta.get_table_fields(me.doctype).forEach(function(df) {
-						var d = frappe.get_doc('DocType', df.options);
-						d["reqd"]=df.reqd;
-						doctype_list.push(d);
-					});
-					$(frappe.render_template("data_import_tool_columns", {doctype_list: doctype_list}))
-						.appendTo(me.select_columns.empty());
-				}
-			});
-		});
-
-		this.page.main.find('.btn-select-all').on('click', function() {
-			me.select_columns.find('.select-column-check').prop('checked', true);
-		});
-
-		this.page.main.find('.btn-unselect-all').on('click', function() {
-			me.select_columns.find('.select-column-check').prop('checked', false);
-		});
-
-		this.page.main.find('.btn-select-mandatory').on('click', function() {
-			me.select_columns.find('.select-column-check').prop('checked', false);
-			me.select_columns.find('.select-column-check[data-reqd="1"]').prop('checked', true);
-		});
-
-		var get_template_url = '/api/method/frappe.core.page.data_import_tool.exporter.get_template';
-
-		this.page.main.find(".btn-download-template").on('click', function() {
-			open_url_post(get_template_url, me.get_export_params(false));
-		});
-
-		this.page.main.find(".btn-download-data").on('click', function() {
-			open_url_post(get_template_url, me.get_export_params(true));
-		});
-
-	},
-	get_export_params: function(with_data) {
-		var doctype = this.select.val();
-		var columns = {};
-
-		this.select_columns.find('.select-column-check:checked').each(function() {
-			var _doctype = $(this).attr('data-doctype');
-			var _fieldname = $(this).attr('data-fieldname');
-			if(!columns[_doctype]) {
-				columns[_doctype] = [];
-			}
-			columns[_doctype].push(_fieldname);
-		});
-
-		return {
-			doctype: doctype,
-			parent_doctype: doctype,
-			select_columns: JSON.stringify(columns),
-			with_data: with_data ? 'Yes' : 'No',
-			all_doctypes: 'Yes',
-			from_data_import: 'Yes',
-			excel_format: this.page.main.find(".excel-check").is(":checked") ? 'Yes' : 'No'
-		}
-	},
-	make_upload: function() {
-		var me = this;
-		frappe.upload.make({
-			no_socketio: true,
-			parent: this.page.main.find(".upload-area"),
-			btn: this.page.main.find(".btn-import"),
-			get_params: function() {
-				return {
-					submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"),
-					ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
-					skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"),
-					overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
-					update_only: me.page.main.find('[name="update_only"]').prop("checked"),
-					no_email: me.page.main.find('[name="no_email"]').prop("checked"),
-					from_data_import: 'Yes'
-				}
-			},
-			args: {
-				method: 'frappe.core.page.data_import_tool.importer.upload',
-			},
-			allow_multiple: 0,
-			onerror: function(r) {
-				me.onerror(r);
-			},
-			queued: function() {
-				// async, show queued
-				msg_dialog.clear();
-				frappe.msgprint(__("Import Request Queued. This may take a few moments, please be patient."));
-			},
-			running: function() {
-				// update async status as running
-				msg_dialog.clear();
-				frappe.msgprint(__("Importing..."));
-				me.write_messages([__("Importing")]);
-				me.has_progress = false;
-			},
-			progress: function(data) {
-				// show callback if async
-				if(data.progress) {
-					frappe.hide_msgprint(true);
-					me.has_progress = true;
-					frappe.show_progress(__("Importing"), data.progress[0],
-						data.progress[1]);
-				}
-			},
-			callback: function(attachment, r) {
-				if(r.message.error || r.message.messages.length==0) {
-					me.onerror(r);
-				} else {
-					if(me.has_progress) {
-						frappe.show_progress(__("Importing"), 1, 1);
-						setTimeout(frappe.hide_progress, 1000);
-					}
-
-					r.messages = ["<h5 style='color:green'>" + __("Import Successful!") + "</h5>"].
-						concat(r.message.messages)
-
-					me.write_messages(r.messages);
-				}
-			},
-			is_private: true
-		});
-
-		frappe.realtime.on("data_import_progress", function(data) {
-			if(data.progress) {
-				frappe.hide_msgprint(true);
-				me.has_progress = true;
-				frappe.show_progress(__("Importing"), data.progress[0],
-					data.progress[1]);
-			}
-		})
-
-	},
-	write_messages: function(data) {
-		this.page.main.find(".import-log").removeClass("hide");
-		var parent = this.page.main.find(".import-log-messages").empty();
-
-		// TODO render using template!
-		for (var i=0, l=data.length; i<l; i++) {
-			var v = data[i];
-			var $p = $('<p></p>').html(frappe.markdown(v)).appendTo(parent);
-			if(v.substr(0,5)=='Error') {
-				$p.css('color', 'red');
-			} else if(v.substr(0,8)=='Inserted') {
-				$p.css('color', 'green');
-			} else if(v.substr(0,7)=='Updated') {
-				$p.css('color', 'green');
-			} else if(v.substr(0,5)=='Valid') {
-				$p.css('color', '#777');
-			} else if(v.substr(0,7)=='Ignored') {
-				$p.css('color', '#777');
-			}
-		}
-	},
-	onerror: function(r) {
-		if(r.message) {
-			// bad design: moves r.messages to r.message.messages
-			r.messages = $.map(r.message.messages, function(v) {
-				var msg = v.replace("Inserted", "Valid")
-					.replace("Updated", "Valid").split("<");
-				if (msg.length > 1) {
-					v = msg[0] + (msg[1].split(">").slice(-1)[0]);
-				} else {
-					v = msg[0];
-				}
-				return v;
-			});
-
-			r.messages = ["<h4 style='color:red'>" + __("Import Failed") + "</h4>"]
-				.concat(r.messages);
-
-			r.messages.push("Please correct the format of the file and import again.");
-
-			frappe.show_progress(__("Importing"), 1, 1);
-
-			this.write_messages(r.messages);
-		}
-	}
-});
-
-frappe.pages['data-import-tool'].on_page_load = function(wrapper) {
-	frappe.data_import_tool = new frappe.DataImportTool(wrapper);
-}
-
-frappe.pages['data-import-tool'].on_page_show = function(wrapper) {
-	frappe.data_import_tool && frappe.data_import_tool.set_route_options();
-}

frappe/core/page/data_import_tool/data_import_tool.json

@@ -1,19 +0,0 @@
-{
- "content": null, 
- "creation": "2012-06-14 15:07:25", 
- "docstatus": 0, 
- "doctype": "Page", 
- "icon": "fa fa-upload", 
- "idx": 1, 
- "modified": "2016-05-11 03:37:53.385693", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "data-import-tool", 
- "owner": "Administrator", 
- "page_name": "data-import-tool", 
- "roles": [], 
- "script": null, 
- "standard": "Yes", 
- "style": null, 
- "title": "Data Import Tool"
-}

frappe/core/page/data_import_tool/data_import_tool_columns.html

@@ -1,22 +0,0 @@
-<div style="margin: 15px 0px;">
-{% for doctype in doctype_list %}
-	<h5 style="margin-top: 25px; margin-bottom: 5px;">{{ doctype.name }}</h5>
-	<div class="row">
-		{% for f in doctype.fields %}
-		{% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %}
-		{% doctype.reqd||(f.reqd=0);%}		
-		<div class="col-sm-4">
-            <div class="checkbox" style="margin: 5px 0px;">
-                <label>
-                    <input type="checkbox" class="select-column-check"
-						data-fieldname="{{ f.fieldname }}" data-reqd="{{ f.reqd }}"
-						data-doctype="{{ doctype.name }}" checked>
-						<small>{{ __(f.label) }}</small>
-                </label>
-            </div>
-		</div>
-		{% endif %}
-		{% endfor %}
-	</div>
-{% endfor %}
-</div>

frappe/patches.txt

@@ -200,3 +200,4 @@ frappe.patches.v9_1.add_sms_sender_name_as_parameters
 frappe.patches.v9_1.resave_domain_settings
 frappe.patches.v9_1.revert_domain_settings
 frappe.patches.v9_1.move_feed_to_activity_log
+execute:frappe.delete_doc('Page', 'data-import-tool', ignore_missing=True)

frappe/public/css/controls.css

@@ -1,5 +1,11 @@
 .unit-checkbox {
   color: #36414c;
+  margin-top: 5px;
+  margin-bottom: 5px;
+}
+.unit-checkbox + .checkbox {
+  margin-top: 5px;
+  margin-bottom: 5px;
 }
 .frappe-control .select-all {
   margin-right: 5px;

frappe/public/js/frappe/form/controls/multicheck.js

@@ -5,12 +5,12 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({
 
 	make() {
 		this._super();
-		// this.$label = $(`<label class="control-label">${this.df.label}</label>`).appendTo(this.wrapper);
+		this.$label = $(`<label class="control-label">${this.df.label || ''}</label>`).appendTo(this.wrapper);
 		this.$load_state = $('<div class="load-state text-muted small">' + __("Loading") + '...</div>');
 		this.$select_buttons = this.get_select_buttons().appendTo(this.wrapper);
 		this.$load_state.appendTo(this.wrapper);
 
-		this.$checkbox_area = $('<div class="checkbox-options"></div>').appendTo(this.wrapper);
+		this.$checkbox_area = $('<div class="checkbox-options row"></div>').appendTo(this.wrapper);
 		this.set_options();
 		this.bind_checkboxes();
 	},
@@ -120,8 +120,9 @@ frappe.ui.form.ControlMultiCheck = frappe.ui.form.Control.extend({
 	},
 
 	get_checkbox_element(option) {
+		const column_size = 12 / (this.df.columns || 1);
 		return $(`
-			<div class="checkbox unit-checkbox">
+			<div class="checkbox unit-checkbox col-sm-${column_size}">
 				<label>
 				<input type="checkbox" data-unit="${option.value}">
 				</input>

frappe/public/js/frappe/list/list_view.js

@@ -601,8 +601,8 @@ frappe.views.ListView = frappe.ui.BaseList.extend({
 
 		if (frappe.model.can_import(this.doctype)) {
 			this.page.add_menu_item(__('Import'), function () {
-				frappe.set_route('data-import-tool', {
-					doctype: me.doctype
+				frappe.set_route('List', 'Data Import', {
+					reference_doctype: me.doctype
 				});
 			}, true);
 		}

frappe/public/js/frappe/model/model.js

@@ -67,6 +67,9 @@ $.extend(frappe.model, {
 	},
 
 	is_value_type: function(fieldtype) {
+		if (typeof fieldtype == 'object') {
+			fieldtype = fieldtype.fieldtype;
+		}
 		// not in no-value type
 		return frappe.model.no_value_type.indexOf(fieldtype)===-1;
 	},

frappe/public/js/legacy/client_script_helpers.js

@@ -359,7 +359,7 @@ _f.Frm.prototype.get_field = function(field) {
 };
 
 
-_f.Frm.prototype.set_read_only = function() {
+_f.Frm.prototype.set_read_only = function(refresh_fields = false) {
 	var perm = [];
 	var docperms = frappe.perm.get_perm(this.doc.doctype);
 	for (var i=0, l=docperms.length; i<l; i++) {
@@ -367,6 +367,13 @@ _f.Frm.prototype.set_read_only = function() {
 		perm[p.permlevel || 0] = {read:1, print:1, cancel:1};
 	}
 	this.perm = perm;
+
+	if (refresh_fields) {
+		this.fields.map(f => {
+			f.perm = this.perm;
+			f.refresh();
+		});
+	}
 }
 
 _f.Frm.prototype.trigger = function(event) {

frappe/public/less/controls.less

@@ -1,5 +1,12 @@
-.unit-checkbox{
+.unit-checkbox {
 	color: #36414c;
+	margin-top: 5px;
+	margin-bottom: 5px;
+
+	& + .checkbox {
+		margin-top: 5px;
+		margin-bottom: 5px;
+	}
 }
 
 .frappe-control .select-all {

frappe/tests/test_data_import.py

@@ -3,8 +3,8 @@
 from __future__ import unicode_literals
 
 import frappe, unittest
-from frappe.core.page.data_import_tool import exporter
-from frappe.core.page.data_import_tool import importer
+from frappe.core.doctype.data_import import exporter
+from frappe.core.doctype.data_import import importer
 from frappe.utils.csvutils import read_csv_content
 
 class TestDataImport(unittest.TestCase):

frappe/core/page/data_import_tool/test_exporter_fixtures.py → frappe/tests/test_exporter_fixtures.py

@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 
 import frappe
 import frappe.defaults
-from frappe.core.page.data_import_tool.data_import_tool import export_csv
+from frappe.core.doctype.data_import.data_import import export_csv
 import unittest
 import os
 

frappe/translate.py

@@ -98,6 +98,8 @@ def get_dict(fortype, name=None):
 	asset_key = fortype + ":" + (name or "-")
 	translation_assets = cache.hget("translation_assets", frappe.local.lang, shared=True) or {}
 
+	print(translation_assets)
+
 	if not asset_key in translation_assets:
 		if fortype=="doctype":
 			messages = get_messages_from_doctype(name)
@@ -121,9 +123,13 @@ def get_dict(fortype, name=None):
 		message_dict = make_dict_from_messages(messages)
 		message_dict.update(get_dict_from_hooks(fortype, name))
 
+		print(message_dict)
+
 		# remove untranslated
 		message_dict = {k:v for k, v in iteritems(message_dict) if k!=v}
 
+		print(message_dict)
+
 		translation_assets[asset_key] = message_dict
 
 		cache.hset("translation_assets", frappe.local.lang, translation_assets, shared=True)
@@ -163,6 +169,8 @@ def make_dict_from_messages(messages, full_dict=None):
 	for m in messages:
 		if m[1] in full_dict:
 			out[m[1]] = full_dict[m[1]]
+
+	print(out)
 	return out
 
 def get_lang_js(fortype, name):

frappe/utils/fixtures.py

@@ -4,7 +4,7 @@
 from __future__ import unicode_literals, print_function
 
 import frappe, os
-from frappe.core.page.data_import_tool.data_import_tool import import_doc, export_json
+from frappe.core.doctype.data_import.data_import import import_doc, export_json
 
 def sync_fixtures(app=None):
 	"""Import, overwrite fixtures from `[app]/fixtures`"""