From e83959a3450ed8fccf05f5d73c112627e089d992 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 9 Oct 2017 13:08:50 +0530 Subject: [PATCH] Fixes for Data Migration Tool (#4275) --- .../data_migration_connector.json | 40 ++++++++++++++++--- .../data_migration_connector.py | 24 +++++++++-- .../data_migration_plan.py | 5 ++- .../data_migration_run/data_migration_run.py | 10 ++++- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json index 3632531873..0a46c464e5 100644 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json +++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json @@ -69,7 +69,37 @@ "read_only": 0, "remember_last_selected_value": 0, "report_hide": 0, - "reqd": 1, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "python_module", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Python Module", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, "search_index": 0, "set_only_once": 0, "unique": 0 @@ -172,7 +202,7 @@ "collapsible": 0, "columns": 0, "fieldname": "password", - "fieldtype": "Data", + "fieldtype": "Password", "hidden": 0, "ignore_user_permissions": 0, "ignore_xss_filter": 0, @@ -206,8 +236,8 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2017-09-07 12:40:15.008483", - "modified_by": "faris@erpnext.com", + "modified": "2017-10-08 14:34:30.603690", + "modified_by": "Administrator", "module": "Data Migration", "name": "Data Migration Connector", "name_case": "", @@ -234,7 +264,7 @@ "write": 1 } ], - "quick_entry": 1, + "quick_entry": 0, "read_only": 0, "read_only_onload": 0, "show_name_in_global_search": 0, diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py index a48842debf..13094bb54d 100644 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py @@ -3,16 +3,32 @@ # For license information, please see license.txt from __future__ import unicode_literals +import frappe from frappe.model.document import Document +from frappe import _ from .connectors.postgres import PostGresConnection from .connectors.frappe_connection import FrappeConnection class DataMigrationConnector(Document): + def validate(self): + if not (self.python_module or self.connector_type): + frappe.throw(_('Enter python module or select connector type')) + + if self.python_module: + try: + frappe.get_module(self.python_module) + except: + frappe.throw(frappe._('Invalid module path')) + def get_connection(self): - if self.connector_type == 'Frappe': - self.connection = FrappeConnection(self) - elif self.connector_type == 'PostGres': - self.connection = PostGresConnection(self.as_dict()) + if self.python_module: + module = frappe.get_module(self.python_module) + return module.get_connection(self) + else: + if self.connector_type == 'Frappe': + self.connection = FrappeConnection(self) + elif self.connector_type == 'PostGres': + self.connection = PostGresConnection(self.as_dict()) return self.connection diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py index 45883d0669..313ca5943e 100644 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py @@ -10,10 +10,11 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.model.document import Document class DataMigrationPlan(Document): - def after_insert(self): - self.make_custom_fields_for_mappings() def on_update(self): + # update custom fields in mappings + self.make_custom_fields_for_mappings() + if frappe.flags.in_import or frappe.flags.in_test: return diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py index c39be68d8d..591365b1cd 100644 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py @@ -380,8 +380,16 @@ class DataMigrationRun(Document): pull_update = self.get_log('pull_update', 0) pull_failed = self.get_log('pull_failed', []) + def get_migration_id_value(source, key): + value = None + try: + value = source[key] + except: + value = getattr(source, key) + return value + for d in data: - migration_id_value = d[connection.name_field] + migration_id_value = get_migration_id_value(d, connection.name_field) doc = self.pre_process_doc(d) doc = mapping.get_mapped_record(doc)