diff --git a/frappe/core/page/data_import_tool/README.md b/frappe/core/page/data_import_tool/README.md index 6bc9920313..7bd4ac809b 100644 --- a/frappe/core/page/data_import_tool/README.md +++ b/frappe/core/page/data_import_tool/README.md @@ -1 +1 @@ -Bulk import / update of data via file upload in CSV. \ No newline at end of file +Bulk import / update of data via file upload in Excel or CSV. \ No newline at end of file diff --git a/frappe/core/page/data_import_tool/data_import_main.html b/frappe/core/page/data_import_tool/data_import_main.html index b3fcf96062..db44a06e70 100644 --- a/frappe/core/page/data_import_tool/data_import_main.html +++ b/frappe/core/page/data_import_tool/data_import_main.html @@ -46,11 +46,21 @@
{%= __("Recommended bulk editing records via import, or understanding the import format.") %}
+
+
+
+ +
+
+

{%= __("Import") %}

-

{%= __("Update the template and save in CSV (Comma Separate Values) format before attaching.") %}

+

{%= __("Update the template and save in downloaded format before attaching.") %}


diff --git a/frappe/core/page/data_import_tool/data_import_tool.js b/frappe/core/page/data_import_tool/data_import_tool.js index e461203fdd..c0ec56c0e1 100644 --- a/frappe/core/page/data_import_tool/data_import_tool.js +++ b/frappe/core/page/data_import_tool/data_import_tool.js @@ -99,7 +99,9 @@ frappe.DataImportTool = Class.extend({ parent_doctype: doctype, select_columns: JSON.stringify(columns), with_data: with_data ? 'Yes' : 'No', - all_doctypes: 'Yes' + all_doctypes: 'Yes', + from_data_import: 'Yes', + excel_format: this.page.main.find(".excel-check").is(":checked") ? 'Yes' : 'No' } }, make_upload: function() { @@ -113,7 +115,8 @@ frappe.DataImportTool = Class.extend({ ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"), overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"), update_only: me.page.main.find('[name="update_only"]').prop("checked"), - no_email: me.page.main.find('[name="no_email"]').prop("checked") + no_email: me.page.main.find('[name="no_email"]').prop("checked"), + from_data_import: 'Yes' } }, args: { diff --git a/frappe/core/page/data_import_tool/exporter.py b/frappe/core/page/data_import_tool/exporter.py index 2b769c00a6..d76ba01bff 100644 --- a/frappe/core/page/data_import_tool/exporter.py +++ b/frappe/core/page/data_import_tool/exporter.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals import frappe, json from frappe import _ import frappe.permissions -import re +import re, csv, os from frappe.utils.csvutils import UnicodeWriter from frappe.utils import cstr, formatdate, format_datetime from frappe.core.page.data_import_tool.data_import_tool import get_data_keys @@ -22,7 +22,8 @@ reflags = { } @frappe.whitelist() -def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None): +def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None, + from_data_import="No", excel_format="No"): all_doctypes = all_doctypes=="Yes" if select_columns: select_columns = json.loads(select_columns); @@ -280,7 +281,26 @@ def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data add_field_headings() add_data() - # write out response as a type csv - frappe.response['result'] = cstr(w.getvalue()) - frappe.response['type'] = 'csv' - frappe.response['doctype'] = doctype + if from_data_import == "Yes" and excel_format == "Yes": + filename = frappe.generate_hash("", 10) + with open(filename, 'wb') as f: + f.write(cstr(w.getvalue()).encode("utf-8")) + f = open(filename) + reader = csv.reader(f) + + from frappe.utils.xlsxutils import make_xlsx + xlsx_file = make_xlsx(reader, "Data Import Template") + + f.close() + os.remove(filename) + + # write out response as a xlsx type + frappe.response['filename'] = doctype + '.xlsx' + frappe.response['filecontent'] = xlsx_file.getvalue() + frappe.response['type'] = 'binary' + + else: + # write out response as a type csv + frappe.response['result'] = cstr(w.getvalue()) + frappe.response['type'] = 'csv' + frappe.response['doctype'] = doctype diff --git a/frappe/core/page/data_import_tool/importer.py b/frappe/core/page/data_import_tool/importer.py index d3fa4031d9..0eba40f2c9 100644 --- a/frappe/core/page/data_import_tool/importer.py +++ b/frappe/core/page/data_import_tool/importer.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals, print_function from six.moves import range import requests -import frappe, json +import frappe, json, os import frappe.permissions import frappe.async @@ -20,7 +20,7 @@ from frappe.core.page.data_import_tool.data_import_tool import get_data_keys @frappe.whitelist() def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, - update_only = None, ignore_links=False, pre_process=None, via_console=False): + update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No"): """upload data""" frappe.flags.in_import = True @@ -37,11 +37,11 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email = False if params.get('update_only'): update_only = True + if params.get('from_data_import'): + from_data_import = params.get('from_data_import') frappe.flags.mute_emails = no_email - from frappe.utils.csvutils import read_csv_content_from_uploaded_file - def get_data_keys_definition(): return get_data_keys() @@ -207,7 +207,23 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, # header if not rows: - rows = read_csv_content_from_uploaded_file(ignore_encoding_errors) + from frappe.utils.file_manager import save_uploaded + file_doc = save_uploaded(dt=None, dn="Data Import", folder='Home', is_private=1) + filename, file_extension = os.path.splitext(file_doc.file_name) + + if file_extension == '.xlsx' and from_data_import == 'Yes': + from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file + rows = read_xlsx_file_from_attached_file(file_id=file_doc.name) + + elif file_extension == '.csv': + from frappe.utils.file_manager import get_file + from frappe.utils.csvutils import read_csv_content + fname, fcontent = get_file(file_doc.names) + rows = read_csv_content(fcontent, ignore_encoding_errors) + + else: + frappe.throw(_("Unsupported File Format")) + start_row = get_start_row() header = rows[:start_row] data = rows[start_row:] diff --git a/frappe/tests/test_data_import.py b/frappe/tests/test_data_import.py index c1ec026aa8..3c332e1c21 100644 --- a/frappe/tests/test_data_import.py +++ b/frappe/tests/test_data_import.py @@ -85,3 +85,14 @@ class TestDataImport(unittest.TestCase): importer.upload(content) ev = frappe.get_doc("Event", {"subject":"__Test Event with children"}) + + def test_excel_import(self): + if frappe.db.exists("Event", "EV00001"): + frappe.delete_doc("Event", "EV00001") + + exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes") + from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file + content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent) + content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"]) + importer.upload(content) + self.assertTrue(frappe.db.get_value("Event", "EV00001", "subject"), "_test") diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 6728222535..bf43219608 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -8,6 +8,7 @@ from frappe.utils import encode, cstr, cint, flt, comma_or import openpyxl from cStringIO import StringIO from openpyxl.styles import Font +from openpyxl import load_workbook # return xlsx file object @@ -22,7 +23,7 @@ def make_xlsx(data, sheet_name): for row in data: clean_row = [] for item in row: - if isinstance(item, basestring): + if isinstance(item, basestring) and sheet_name != "Data Import Template": value = handle_html(item) else: value = item @@ -36,7 +37,6 @@ def make_xlsx(data, sheet_name): def handle_html(data): - # import html2text from html2text import unescape, HTML2Text h = HTML2Text() @@ -53,3 +53,24 @@ def handle_html(data): return value[0] else: return value[1] + + +def read_xlsx_file_from_attached_file(file_id=None, fcontent=None): + if file_id: + from frappe.utils.file_manager import get_file_path + filename = get_file_path(file_id) + elif fcontent: + from io import BytesIO + filename = BytesIO(fcontent) + else: + return + + rows = [] + wb1 = load_workbook(filename=filename, read_only=True) + ws1 = wb1.active + for row in ws1.iter_rows(): + tmp_list = [] + for cell in row: + tmp_list.append(cell.value) + rows.append(tmp_list) + return rows