* refactor: Remove Data Migration Tool * chore: unnecessary importsversion-14
@@ -1,24 +0,0 @@ | |||||
from abc import ABCMeta, abstractmethod | |||||
from frappe.utils.password import get_decrypted_password | |||||
class BaseConnection(metaclass=ABCMeta): | |||||
@abstractmethod | |||||
def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): | |||||
pass | |||||
@abstractmethod | |||||
def insert(self, doctype, doc): | |||||
pass | |||||
@abstractmethod | |||||
def update(self, doctype, doc, migration_id): | |||||
pass | |||||
@abstractmethod | |||||
def delete(self, doctype, migration_id): | |||||
pass | |||||
def get_password(self): | |||||
return get_decrypted_password("Data Migration Connector", self.connector.name) |
@@ -1,32 +0,0 @@ | |||||
import frappe | |||||
from frappe.frappeclient import FrappeClient | |||||
from .base import BaseConnection | |||||
class FrappeConnection(BaseConnection): | |||||
def __init__(self, connector): | |||||
self.connector = connector | |||||
self.connection = FrappeClient( | |||||
self.connector.hostname, self.connector.username, self.get_password() | |||||
) | |||||
self.name_field = "name" | |||||
def insert(self, doctype, doc): | |||||
doc = frappe._dict(doc) | |||||
doc.doctype = doctype | |||||
return self.connection.insert(doc) | |||||
def update(self, doctype, doc, migration_id): | |||||
doc = frappe._dict(doc) | |||||
doc.doctype = doctype | |||||
doc.name = migration_id | |||||
return self.connection.update(doc) | |||||
def delete(self, doctype, migration_id): | |||||
return self.connection.delete(doctype, migration_id) | |||||
def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): | |||||
return self.connection.get_list( | |||||
doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length | |||||
) |
@@ -1,47 +0,0 @@ | |||||
// Copyright (c) 2017, Frappe Technologies and contributors | |||||
// For license information, please see license.txt | |||||
frappe.ui.form.on('Data Migration Connector', { | |||||
onload(frm) { | |||||
if(frappe.boot.developer_mode) { | |||||
frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm)); | |||||
} | |||||
}, | |||||
new_connection(frm) { | |||||
const d = new frappe.ui.Dialog({ | |||||
title: __('New Connection'), | |||||
fields: [ | |||||
{ label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 }, | |||||
{ label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 }, | |||||
], | |||||
primary_action_label: __('Create'), | |||||
primary_action: (values) => { | |||||
let { module, connection_name } = values; | |||||
frm.events.create_new_connection(module, connection_name) | |||||
.then(r => { | |||||
if (r.message) { | |||||
const connector_name = connection_name | |||||
.replace('connection', 'Connector') | |||||
.replace('Connection', 'Connector') | |||||
.trim(); | |||||
frm.set_value('connector_name', connector_name); | |||||
frm.set_value('connector_type', 'Custom'); | |||||
frm.set_value('python_module', r.message); | |||||
frm.save(); | |||||
frappe.show_alert(__("New module created {0}", [r.message])); | |||||
d.hide(); | |||||
} | |||||
}); | |||||
} | |||||
}); | |||||
d.show(); | |||||
}, | |||||
create_new_connection(module, connection_name) { | |||||
return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', { | |||||
module, connection_name | |||||
}); | |||||
} | |||||
}); |
@@ -1,307 +0,0 @@ | |||||
{ | |||||
"allow_copy": 0, | |||||
"allow_guest_to_view": 0, | |||||
"allow_import": 0, | |||||
"allow_rename": 1, | |||||
"autoname": "field:connector_name", | |||||
"beta": 1, | |||||
"creation": "2017-08-11 05:03:27.091416", | |||||
"custom": 0, | |||||
"docstatus": 0, | |||||
"doctype": "DocType", | |||||
"document_type": "", | |||||
"editable_grid": 1, | |||||
"engine": "InnoDB", | |||||
"fields": [ | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "connector_name", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Connector Name", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"depends_on": "eval:!doc.is_custom", | |||||
"fieldname": "connector_type", | |||||
"fieldtype": "Select", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Connector Type", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "\nFrappe\nCustom", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"depends_on": "eval:doc.connector_type == 'Custom'", | |||||
"fieldname": "python_module", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Python Module", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "authentication_credentials", | |||||
"fieldtype": "Section Break", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Authentication Credentials", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"default": "", | |||||
"fieldname": "hostname", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Hostname", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "database_name", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Database Name", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "username", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Username", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "password", | |||||
"fieldtype": "Password", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Password", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
} | |||||
], | |||||
"has_web_view": 0, | |||||
"hide_heading": 0, | |||||
"hide_toolbar": 0, | |||||
"idx": 0, | |||||
"image_view": 0, | |||||
"in_create": 0, | |||||
"is_submittable": 0, | |||||
"issingle": 0, | |||||
"istable": 0, | |||||
"max_attachments": 0, | |||||
"modified": "2017-12-01 13:38:55.992499", | |||||
"modified_by": "Administrator", | |||||
"module": "Data Migration", | |||||
"name": "Data Migration Connector", | |||||
"name_case": "", | |||||
"owner": "Administrator", | |||||
"permissions": [ | |||||
{ | |||||
"amend": 0, | |||||
"apply_user_permissions": 0, | |||||
"cancel": 0, | |||||
"create": 1, | |||||
"delete": 1, | |||||
"email": 1, | |||||
"export": 1, | |||||
"if_owner": 0, | |||||
"import": 0, | |||||
"permlevel": 0, | |||||
"print": 1, | |||||
"read": 1, | |||||
"report": 1, | |||||
"role": "System Manager", | |||||
"set_user_permissions": 0, | |||||
"share": 1, | |||||
"submit": 0, | |||||
"write": 1 | |||||
} | |||||
], | |||||
"quick_entry": 0, | |||||
"read_only": 0, | |||||
"read_only_onload": 0, | |||||
"show_name_in_global_search": 0, | |||||
"sort_field": "modified", | |||||
"sort_order": "DESC", | |||||
"track_changes": 1, | |||||
"track_seen": 0 | |||||
} |
@@ -1,107 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and contributors | |||||
# License: MIT. See LICENSE | |||||
import os | |||||
import frappe | |||||
from frappe import _ | |||||
from frappe.model.document import Document | |||||
from frappe.modules.export_file import create_init_py | |||||
from .connectors.base import BaseConnection | |||||
from .connectors.frappe_connection import FrappeConnection | |||||
class DataMigrationConnector(Document): | |||||
def validate(self): | |||||
if not (self.python_module or self.connector_type): | |||||
frappe.throw(_("Enter python module or select connector type")) | |||||
if self.python_module: | |||||
try: | |||||
get_connection_class(self.python_module) | |||||
except: | |||||
frappe.throw(frappe._("Invalid module path")) | |||||
def get_connection(self): | |||||
if self.python_module: | |||||
_class = get_connection_class(self.python_module) | |||||
return _class(self) | |||||
else: | |||||
self.connection = FrappeConnection(self) | |||||
return self.connection | |||||
@frappe.whitelist() | |||||
def create_new_connection(module, connection_name): | |||||
if not frappe.conf.get("developer_mode"): | |||||
frappe.msgprint(_("Please enable developer mode to create new connection")) | |||||
return | |||||
# create folder | |||||
module_path = frappe.get_module_path(module) | |||||
connectors_folder = os.path.join(module_path, "connectors") | |||||
frappe.create_folder(connectors_folder) | |||||
# create init py | |||||
create_init_py(module_path, "connectors", "") | |||||
connection_class = connection_name.replace(" ", "") | |||||
file_name = frappe.scrub(connection_name) + ".py" | |||||
file_path = os.path.join(module_path, "connectors", file_name) | |||||
# create boilerplate file | |||||
with open(file_path, "w") as f: | |||||
f.write(connection_boilerplate.format(connection_class=connection_class)) | |||||
# get python module string from file_path | |||||
app_name = frappe.db.get_value("Module Def", module, "app_name") | |||||
python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( | |||||
os.path.sep, "." | |||||
)[:-3] | |||||
return python_module | |||||
def get_connection_class(python_module): | |||||
filename = python_module.rsplit(".", 1)[-1] | |||||
classname = frappe.unscrub(filename).replace(" ", "") | |||||
module = frappe.get_module(python_module) | |||||
raise_error = False | |||||
if hasattr(module, classname): | |||||
_class = getattr(module, classname) | |||||
if not issubclass(_class, BaseConnection): | |||||
raise_error = True | |||||
else: | |||||
raise_error = True | |||||
if raise_error: | |||||
raise ImportError(filename) | |||||
return _class | |||||
connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection | |||||
class {connection_class}(BaseConnection): | |||||
def __init__(self, connector): | |||||
# self.connector = connector | |||||
# self.connection = YourModule(self.connector.username, self.get_password()) | |||||
# self.name_field = 'id' | |||||
pass | |||||
def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): | |||||
pass | |||||
def insert(self, doctype, doc): | |||||
pass | |||||
def update(self, doctype, doc, migration_id): | |||||
pass | |||||
def delete(self, doctype, migration_id): | |||||
pass | |||||
""" |
@@ -1,8 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and Contributors | |||||
# License: MIT. See LICENSE | |||||
import unittest | |||||
class TestDataMigrationConnector(unittest.TestCase): | |||||
pass |
@@ -1,8 +0,0 @@ | |||||
// Copyright (c) 2017, Frappe Technologies and contributors | |||||
// For license information, please see license.txt | |||||
frappe.ui.form.on('Data Migration Mapping', { | |||||
refresh: function() { | |||||
} | |||||
}); |
@@ -1,456 +0,0 @@ | |||||
{ | |||||
"allow_copy": 0, | |||||
"allow_guest_to_view": 0, | |||||
"allow_import": 0, | |||||
"allow_rename": 1, | |||||
"autoname": "field:mapping_name", | |||||
"beta": 1, | |||||
"creation": "2017-08-11 05:11:49.975801", | |||||
"custom": 0, | |||||
"docstatus": 0, | |||||
"doctype": "DocType", | |||||
"document_type": "", | |||||
"editable_grid": 1, | |||||
"engine": "InnoDB", | |||||
"fields": [ | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "mapping_name", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Mapping Name", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "remote_objectname", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Remote Objectname", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "remote_primary_key", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Remote Primary Key", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "local_doctype", | |||||
"fieldtype": "Link", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Local DocType", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "DocType", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "local_primary_key", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Local Primary Key", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "column_break_5", | |||||
"fieldtype": "Column Break", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "mapping_type", | |||||
"fieldtype": "Select", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Mapping Type", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Push\nPull\nSync", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"default": "10", | |||||
"fieldname": "page_length", | |||||
"fieldtype": "Int", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Page Length", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "migration_id_field", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Migration ID Field", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "mapping", | |||||
"fieldtype": "Section Break", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Mapping", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "fields", | |||||
"fieldtype": "Table", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Field Maps", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Data Migration Mapping Detail", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 1, | |||||
"columns": 0, | |||||
"fieldname": "condition_detail", | |||||
"fieldtype": "Section Break", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Condition Detail", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "condition", | |||||
"fieldtype": "Code", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Condition", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
} | |||||
], | |||||
"has_web_view": 0, | |||||
"hide_heading": 0, | |||||
"hide_toolbar": 0, | |||||
"idx": 0, | |||||
"image_view": 0, | |||||
"in_create": 0, | |||||
"is_submittable": 0, | |||||
"issingle": 0, | |||||
"istable": 0, | |||||
"max_attachments": 0, | |||||
"modified": "2017-09-27 18:06:43.275207", | |||||
"modified_by": "Administrator", | |||||
"module": "Data Migration", | |||||
"name": "Data Migration Mapping", | |||||
"name_case": "", | |||||
"owner": "Administrator", | |||||
"permissions": [ | |||||
{ | |||||
"amend": 0, | |||||
"apply_user_permissions": 0, | |||||
"cancel": 0, | |||||
"create": 1, | |||||
"delete": 1, | |||||
"email": 1, | |||||
"export": 1, | |||||
"if_owner": 0, | |||||
"import": 0, | |||||
"permlevel": 0, | |||||
"print": 1, | |||||
"read": 1, | |||||
"report": 1, | |||||
"role": "System Manager", | |||||
"set_user_permissions": 0, | |||||
"share": 1, | |||||
"submit": 0, | |||||
"write": 1 | |||||
} | |||||
], | |||||
"quick_entry": 1, | |||||
"read_only": 0, | |||||
"read_only_onload": 0, | |||||
"show_name_in_global_search": 0, | |||||
"sort_field": "modified", | |||||
"sort_order": "DESC", | |||||
"track_changes": 1, | |||||
"track_seen": 0 | |||||
} |
@@ -1,83 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and contributors | |||||
# License: MIT. See LICENSE | |||||
import frappe | |||||
from frappe.model.document import Document | |||||
from frappe.utils.safe_exec import get_safe_globals | |||||
class DataMigrationMapping(Document): | |||||
def get_filters(self): | |||||
if self.condition: | |||||
return frappe.safe_eval(self.condition, get_safe_globals()) | |||||
def get_fields(self): | |||||
fields = [] | |||||
for f in self.fields: | |||||
if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): | |||||
fields.append(f.local_fieldname) | |||||
if frappe.db.has_column(self.local_doctype, self.migration_id_field): | |||||
fields.append(self.migration_id_field) | |||||
if "name" not in fields: | |||||
fields.append("name") | |||||
return fields | |||||
def get_mapped_record(self, doc): | |||||
"""Build a mapped record using information from the fields table""" | |||||
mapped = frappe._dict() | |||||
key_fieldname = "remote_fieldname" | |||||
value_fieldname = "local_fieldname" | |||||
if self.mapping_type == "Pull": | |||||
key_fieldname, value_fieldname = value_fieldname, key_fieldname | |||||
for field_map in self.fields: | |||||
key = get_source_value(field_map, key_fieldname) | |||||
if not field_map.is_child_table: | |||||
# field to field mapping | |||||
value = get_value_from_fieldname(field_map, value_fieldname, doc) | |||||
else: | |||||
# child table mapping | |||||
mapping_name = field_map.child_table_mapping | |||||
value = get_mapped_child_records( | |||||
mapping_name, doc.get(get_source_value(field_map, value_fieldname)) | |||||
) | |||||
mapped[key] = value | |||||
return mapped | |||||
def get_mapped_child_records(mapping_name, child_docs): | |||||
mapped_child_docs = [] | |||||
mapping = frappe.get_doc("Data Migration Mapping", mapping_name) | |||||
for child_doc in child_docs: | |||||
mapped_child_docs.append(mapping.get_mapped_record(child_doc)) | |||||
return mapped_child_docs | |||||
def get_value_from_fieldname(field_map, fieldname_field, doc): | |||||
field_name = get_source_value(field_map, fieldname_field) | |||||
if field_name.startswith("eval:"): | |||||
value = frappe.safe_eval(field_name[5:], get_safe_globals()) | |||||
elif field_name[0] in ('"', "'"): | |||||
value = field_name[1:-1] | |||||
else: | |||||
value = get_source_value(doc, field_name) | |||||
return value | |||||
def get_source_value(source, key): | |||||
"""Get value from source (object or dict) based on key""" | |||||
if isinstance(source, dict): | |||||
return source.get(key) | |||||
else: | |||||
return getattr(source, key) |
@@ -1,8 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and Contributors | |||||
# License: MIT. See LICENSE | |||||
import unittest | |||||
class TestDataMigrationMapping(unittest.TestCase): | |||||
pass |
@@ -1,163 +0,0 @@ | |||||
{ | |||||
"allow_copy": 0, | |||||
"allow_guest_to_view": 0, | |||||
"allow_import": 0, | |||||
"allow_rename": 0, | |||||
"beta": 0, | |||||
"creation": "2017-08-11 05:09:10.900237", | |||||
"custom": 0, | |||||
"docstatus": 0, | |||||
"doctype": "DocType", | |||||
"document_type": "", | |||||
"editable_grid": 1, | |||||
"engine": "InnoDB", | |||||
"fields": [ | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "remote_fieldname", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Remote Fieldname", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "local_fieldname", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Local Fieldname", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "is_child_table", | |||||
"fieldtype": "Check", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Is Child Table", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"depends_on": "is_child_table", | |||||
"fieldname": "child_table_mapping", | |||||
"fieldtype": "Link", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Child Table Mapping", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Data Migration Mapping", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
} | |||||
], | |||||
"has_web_view": 0, | |||||
"hide_heading": 0, | |||||
"hide_toolbar": 0, | |||||
"idx": 0, | |||||
"image_view": 0, | |||||
"in_create": 0, | |||||
"is_submittable": 0, | |||||
"issingle": 0, | |||||
"istable": 1, | |||||
"max_attachments": 0, | |||||
"modified": "2017-09-28 17:13:31.337005", | |||||
"modified_by": "Administrator", | |||||
"module": "Data Migration", | |||||
"name": "Data Migration Mapping Detail", | |||||
"name_case": "", | |||||
"owner": "Administrator", | |||||
"permissions": [], | |||||
"quick_entry": 1, | |||||
"read_only": 0, | |||||
"read_only_onload": 0, | |||||
"show_name_in_global_search": 0, | |||||
"sort_field": "modified", | |||||
"sort_order": "DESC", | |||||
"track_changes": 1, | |||||
"track_seen": 0 | |||||
} |
@@ -1,9 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and contributors | |||||
# License: MIT. See LICENSE | |||||
from frappe.model.document import Document | |||||
class DataMigrationMappingDetail(Document): | |||||
pass |
@@ -1,10 +0,0 @@ | |||||
// Copyright (c) 2017, Frappe Technologies and contributors | |||||
// For license information, please see license.txt | |||||
frappe.ui.form.on('Data Migration Plan', { | |||||
onload(frm) { | |||||
frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', { | |||||
data_migration_plan: frm.doc.name | |||||
})); | |||||
} | |||||
}); |
@@ -1,224 +0,0 @@ | |||||
{ | |||||
"allow_copy": 0, | |||||
"allow_guest_to_view": 0, | |||||
"allow_import": 0, | |||||
"allow_rename": 0, | |||||
"autoname": "field:plan_name", | |||||
"beta": 0, | |||||
"creation": "2017-08-11 05:15:51.482165", | |||||
"custom": 0, | |||||
"docstatus": 0, | |||||
"doctype": "DocType", | |||||
"document_type": "", | |||||
"editable_grid": 1, | |||||
"engine": "InnoDB", | |||||
"fields": [ | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "plan_name", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Plan Name", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 1 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "module", | |||||
"fieldtype": "Link", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Module", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Module Def", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "mappings", | |||||
"fieldtype": "Table", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Mappings", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Data Migration Plan Mapping", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "preprocess_method", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Preprocess Method", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "postprocess_method", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Postprocess Method", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
} | |||||
], | |||||
"has_web_view": 0, | |||||
"hide_heading": 0, | |||||
"hide_toolbar": 0, | |||||
"idx": 0, | |||||
"image_view": 0, | |||||
"in_create": 0, | |||||
"is_submittable": 0, | |||||
"issingle": 0, | |||||
"istable": 0, | |||||
"max_attachments": 0, | |||||
"modified": "2020-09-18 17:26:09.703215", | |||||
"modified_by": "Administrator", | |||||
"module": "Data Migration", | |||||
"name": "Data Migration Plan", | |||||
"name_case": "", | |||||
"owner": "Administrator", | |||||
"permissions": [ | |||||
{ | |||||
"amend": 0, | |||||
"cancel": 0, | |||||
"create": 1, | |||||
"delete": 1, | |||||
"email": 1, | |||||
"export": 1, | |||||
"if_owner": 0, | |||||
"import": 0, | |||||
"permlevel": 0, | |||||
"print": 1, | |||||
"read": 1, | |||||
"report": 1, | |||||
"role": "System Manager", | |||||
"set_user_permissions": 0, | |||||
"share": 1, | |||||
"submit": 0, | |||||
"write": 1 | |||||
} | |||||
], | |||||
"quick_entry": 1, | |||||
"read_only": 0, | |||||
"read_only_onload": 0, | |||||
"show_name_in_global_search": 0, | |||||
"sort_field": "modified", | |||||
"sort_order": "DESC", | |||||
"track_changes": 1, | |||||
"track_seen": 0 | |||||
} |
@@ -1,78 +0,0 @@ | |||||
# Copyright (c) 2021, Frappe Technologies and contributors | |||||
# License: MIT. See LICENSE | |||||
import frappe | |||||
from frappe.custom.doctype.custom_field.custom_field import create_custom_field | |||||
from frappe.model.document import Document | |||||
from frappe.modules import get_module_path, scrub_dt_dn | |||||
from frappe.modules.export_file import create_init_py, export_to_files | |||||
def get_mapping_module(module, mapping_name): | |||||
app_name = frappe.db.get_value("Module Def", module, "app_name") | |||||
mapping_name = frappe.scrub(mapping_name) | |||||
module = frappe.scrub(module) | |||||
try: | |||||
return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") | |||||
except ImportError: | |||||
return None | |||||
class DataMigrationPlan(Document): | |||||
def on_update(self): | |||||
# update custom fields in mappings | |||||
self.make_custom_fields_for_mappings() | |||||
if frappe.flags.in_import or frappe.flags.in_test: | |||||
return | |||||
if frappe.local.conf.get("developer_mode"): | |||||
record_list = [["Data Migration Plan", self.name]] | |||||
for m in self.mappings: | |||||
record_list.append(["Data Migration Mapping", m.mapping]) | |||||
export_to_files(record_list=record_list, record_module=self.module) | |||||
for m in self.mappings: | |||||
dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) | |||||
create_init_py(get_module_path(self.module), dt, dn) | |||||
def make_custom_fields_for_mappings(self): | |||||
frappe.flags.ignore_in_install = True | |||||
label = self.name + " ID" | |||||
fieldname = frappe.scrub(label) | |||||
df = { | |||||
"label": label, | |||||
"fieldname": fieldname, | |||||
"fieldtype": "Data", | |||||
"hidden": 1, | |||||
"read_only": 1, | |||||
"unique": 1, | |||||
"no_copy": 1, | |||||
} | |||||
for m in self.mappings: | |||||
mapping = frappe.get_doc("Data Migration Mapping", m.mapping) | |||||
create_custom_field(mapping.local_doctype, df) | |||||
mapping.migration_id_field = fieldname | |||||
mapping.save() | |||||
# Create custom field in Deleted Document | |||||
create_custom_field("Deleted Document", df) | |||||
frappe.flags.ignore_in_install = False | |||||
def pre_process_doc(self, mapping_name, doc): | |||||
module = get_mapping_module(self.module, mapping_name) | |||||
if module and hasattr(module, "pre_process"): | |||||
return module.pre_process(doc) | |||||
return doc | |||||
def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): | |||||
module = get_mapping_module(self.module, mapping_name) | |||||
if module and hasattr(module, "post_process"): | |||||
return module.post_process(local_doc=local_doc, remote_doc=remote_doc) |
@@ -1,8 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and Contributors | |||||
# License: MIT. See LICENSE | |||||
import unittest | |||||
class TestDataMigrationPlan(unittest.TestCase): | |||||
pass |
@@ -1,103 +0,0 @@ | |||||
{ | |||||
"allow_copy": 0, | |||||
"allow_guest_to_view": 0, | |||||
"allow_import": 0, | |||||
"allow_rename": 0, | |||||
"beta": 1, | |||||
"creation": "2017-08-11 05:15:38.390831", | |||||
"custom": 0, | |||||
"docstatus": 0, | |||||
"doctype": "DocType", | |||||
"document_type": "", | |||||
"editable_grid": 1, | |||||
"engine": "InnoDB", | |||||
"fields": [ | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "mapping", | |||||
"fieldtype": "Link", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Mapping", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Data Migration Mapping", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"default": "1", | |||||
"fieldname": "enabled", | |||||
"fieldtype": "Check", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Enabled", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"unique": 0 | |||||
} | |||||
], | |||||
"has_web_view": 0, | |||||
"hide_heading": 0, | |||||
"hide_toolbar": 0, | |||||
"idx": 0, | |||||
"image_view": 0, | |||||
"in_create": 0, | |||||
"is_submittable": 0, | |||||
"issingle": 0, | |||||
"istable": 1, | |||||
"max_attachments": 0, | |||||
"modified": "2017-09-20 21:43:04.908650", | |||||
"modified_by": "Administrator", | |||||
"module": "Data Migration", | |||||
"name": "Data Migration Plan Mapping", | |||||
"name_case": "", | |||||
"owner": "Administrator", | |||||
"permissions": [], | |||||
"quick_entry": 1, | |||||
"read_only": 0, | |||||
"read_only_onload": 0, | |||||
"show_name_in_global_search": 0, | |||||
"sort_field": "modified", | |||||
"sort_order": "DESC", | |||||
"track_changes": 1, | |||||
"track_seen": 0 | |||||
} |
@@ -1,9 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and contributors | |||||
# License: MIT. See LICENSE | |||||
from frappe.model.document import Document | |||||
class DataMigrationPlanMapping(Document): | |||||
pass |
@@ -1,14 +0,0 @@ | |||||
// Copyright (c) 2017, Frappe Technologies and contributors | |||||
// For license information, please see license.txt | |||||
frappe.ui.form.on('Data Migration Run', { | |||||
refresh: function(frm) { | |||||
if (frm.doc.status !== 'Success') { | |||||
frm.add_custom_button(__('Run'), () => frm.call('run')); | |||||
} | |||||
if (frm.doc.status === 'Started') { | |||||
frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete, | |||||
__('Currently updating {0}', [frm.doc.current_mapping])); | |||||
} | |||||
} | |||||
}); |
@@ -1,838 +0,0 @@ | |||||
{ | |||||
"allow_copy": 0, | |||||
"allow_guest_to_view": 0, | |||||
"allow_import": 0, | |||||
"allow_rename": 0, | |||||
"beta": 0, | |||||
"creation": "2017-09-11 12:55:27.597728", | |||||
"custom": 0, | |||||
"docstatus": 0, | |||||
"doctype": "DocType", | |||||
"document_type": "", | |||||
"editable_grid": 1, | |||||
"engine": "InnoDB", | |||||
"fields": [ | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "data_migration_plan", | |||||
"fieldtype": "Link", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Data Migration Plan", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Data Migration Plan", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "data_migration_connector", | |||||
"fieldtype": "Link", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Data Migration Connector", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Data Migration Connector", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 1, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"default": "Pending", | |||||
"fieldname": "status", | |||||
"fieldtype": "Select", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 1, | |||||
"in_standard_filter": 0, | |||||
"label": "Status", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "start_time", | |||||
"fieldtype": "Datetime", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Start Time", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "end_time", | |||||
"fieldtype": "Datetime", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "End Time", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "remote_id", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Remote ID", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "current_mapping", | |||||
"fieldtype": "Data", | |||||
"hidden": 1, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Current Mapping", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"options": "", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "current_mapping_start", | |||||
"fieldtype": "Int", | |||||
"hidden": 1, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Current Mapping Start", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "current_mapping_delete_start", | |||||
"fieldtype": "Int", | |||||
"hidden": 1, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Current Mapping Delete Start", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "current_mapping_type", | |||||
"fieldtype": "Select", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Current Mapping Type", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"options": "Push\nPull", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"depends_on": "eval:(doc.status !== 'Pending')", | |||||
"fieldname": "current_mapping_action", | |||||
"fieldtype": "Select", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Current Mapping Action", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"options": "Insert\nDelete", | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "total_pages", | |||||
"fieldtype": "Int", | |||||
"hidden": 1, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Total Pages", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "percent_complete", | |||||
"fieldtype": "Percent", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Percent Complete", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "trigger_name", | |||||
"fieldtype": "Data", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Trigger Name", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"depends_on": "eval:(doc.status !== 'Pending')", | |||||
"fieldname": "logs_sb", | |||||
"fieldtype": "Section Break", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Logs", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "push_insert", | |||||
"fieldtype": "Int", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Push Insert", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "push_update", | |||||
"fieldtype": "Int", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Push Update", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "push_delete", | |||||
"fieldtype": "Int", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Push Delete", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "push_failed", | |||||
"fieldtype": "Code", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Push Failed", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "column_break_16", | |||||
"fieldtype": "Column Break", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "pull_insert", | |||||
"fieldtype": "Int", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Pull Insert", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "pull_update", | |||||
"fieldtype": "Int", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Pull Update", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"fieldname": "pull_failed", | |||||
"fieldtype": "Code", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Pull Failed", | |||||
"length": 0, | |||||
"no_copy": 0, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 0, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
}, | |||||
{ | |||||
"allow_bulk_edit": 0, | |||||
"allow_in_quick_entry": 0, | |||||
"allow_on_submit": 0, | |||||
"bold": 0, | |||||
"collapsible": 0, | |||||
"columns": 0, | |||||
"depends_on": "eval:doc.failed_log !== '[]'", | |||||
"fieldname": "log", | |||||
"fieldtype": "Code", | |||||
"hidden": 0, | |||||
"ignore_user_permissions": 0, | |||||
"ignore_xss_filter": 0, | |||||
"in_filter": 0, | |||||
"in_global_search": 0, | |||||
"in_list_view": 0, | |||||
"in_standard_filter": 0, | |||||
"label": "Log", | |||||
"length": 0, | |||||
"no_copy": 1, | |||||
"permlevel": 0, | |||||
"precision": "", | |||||
"print_hide": 0, | |||||
"print_hide_if_no_value": 0, | |||||
"read_only": 1, | |||||
"remember_last_selected_value": 0, | |||||
"report_hide": 0, | |||||
"reqd": 0, | |||||
"search_index": 0, | |||||
"set_only_once": 0, | |||||
"translatable": 0, | |||||
"unique": 0 | |||||
} | |||||
], | |||||
"has_web_view": 0, | |||||
"hide_heading": 0, | |||||
"hide_toolbar": 0, | |||||
"idx": 0, | |||||
"image_view": 0, | |||||
"in_create": 0, | |||||
"is_submittable": 0, | |||||
"issingle": 0, | |||||
"istable": 0, | |||||
"max_attachments": 0, | |||||
"modified": "2020-09-18 17:26:09.703215", | |||||
"modified_by": "Administrator", | |||||
"module": "Data Migration", | |||||
"name": "Data Migration Run", | |||||
"name_case": "", | |||||
"owner": "Administrator", | |||||
"permissions": [ | |||||
{ | |||||
"amend": 0, | |||||
"cancel": 0, | |||||
"create": 1, | |||||
"delete": 1, | |||||
"email": 1, | |||||
"export": 1, | |||||
"if_owner": 0, | |||||
"import": 0, | |||||
"permlevel": 0, | |||||
"print": 1, | |||||
"read": 1, | |||||
"report": 1, | |||||
"role": "System Manager", | |||||
"set_user_permissions": 0, | |||||
"share": 1, | |||||
"submit": 0, | |||||
"write": 1 | |||||
} | |||||
], | |||||
"quick_entry": 0, | |||||
"read_only": 0, | |||||
"read_only_onload": 0, | |||||
"show_name_in_global_search": 0, | |||||
"sort_field": "modified", | |||||
"sort_order": "DESC", | |||||
"track_changes": 1, | |||||
"track_seen": 0 | |||||
} |
@@ -1,514 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and contributors | |||||
# License: MIT. See LICENSE | |||||
import json | |||||
import math | |||||
import frappe | |||||
from frappe import _ | |||||
from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( | |||||
get_source_value, | |||||
) | |||||
from frappe.model.document import Document | |||||
from frappe.utils import cstr | |||||
class DataMigrationRun(Document): | |||||
@frappe.whitelist() | |||||
def run(self): | |||||
self.begin() | |||||
if self.total_pages > 0: | |||||
self.enqueue_next_mapping() | |||||
else: | |||||
self.complete() | |||||
def enqueue_next_mapping(self): | |||||
next_mapping_name = self.get_next_mapping_name() | |||||
if next_mapping_name: | |||||
next_mapping = self.get_mapping(next_mapping_name) | |||||
self.db_set( | |||||
dict( | |||||
current_mapping=next_mapping.name, | |||||
current_mapping_start=0, | |||||
current_mapping_delete_start=0, | |||||
current_mapping_action="Insert", | |||||
), | |||||
notify=True, | |||||
commit=True, | |||||
) | |||||
frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) | |||||
else: | |||||
self.complete() | |||||
def enqueue_next_page(self): | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
percent_complete = self.percent_complete + (100.0 / self.total_pages) | |||||
fields = dict(percent_complete=percent_complete) | |||||
if self.current_mapping_action == "Insert": | |||||
start = self.current_mapping_start + mapping.page_length | |||||
fields["current_mapping_start"] = start | |||||
elif self.current_mapping_action == "Delete": | |||||
delete_start = self.current_mapping_delete_start + mapping.page_length | |||||
fields["current_mapping_delete_start"] = delete_start | |||||
self.db_set(fields, notify=True, commit=True) | |||||
if percent_complete < 100: | |||||
frappe.publish_realtime( | |||||
self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user | |||||
) | |||||
frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) | |||||
def run_current_mapping(self): | |||||
try: | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
if mapping.mapping_type == "Push": | |||||
done = self.push() | |||||
elif mapping.mapping_type == "Pull": | |||||
done = self.pull() | |||||
if done: | |||||
self.enqueue_next_mapping() | |||||
else: | |||||
self.enqueue_next_page() | |||||
except Exception as e: | |||||
self.db_set("status", "Error", notify=True, commit=True) | |||||
print("Data Migration Run failed") | |||||
print(frappe.get_traceback()) | |||||
self.execute_postprocess("Error") | |||||
raise e | |||||
def get_last_modified_condition(self): | |||||
last_run_timestamp = frappe.db.get_value( | |||||
"Data Migration Run", | |||||
dict( | |||||
data_migration_plan=self.data_migration_plan, | |||||
data_migration_connector=self.data_migration_connector, | |||||
name=("!=", self.name), | |||||
), | |||||
"modified", | |||||
) | |||||
if last_run_timestamp: | |||||
condition = dict(modified=(">", last_run_timestamp)) | |||||
else: | |||||
condition = {} | |||||
return condition | |||||
def begin(self): | |||||
plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] | |||||
self.mappings = [ | |||||
frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings | |||||
] | |||||
total_pages = 0 | |||||
for m in [mapping for mapping in self.mappings]: | |||||
if m.mapping_type == "Push": | |||||
count = float(self.get_count(m)) | |||||
page_count = math.ceil(count / m.page_length) | |||||
total_pages += page_count | |||||
if m.mapping_type == "Pull": | |||||
total_pages += 10 | |||||
self.db_set( | |||||
dict( | |||||
status="Started", | |||||
current_mapping=None, | |||||
current_mapping_start=0, | |||||
current_mapping_delete_start=0, | |||||
percent_complete=0, | |||||
current_mapping_action="Insert", | |||||
total_pages=total_pages, | |||||
), | |||||
notify=True, | |||||
commit=True, | |||||
) | |||||
def complete(self): | |||||
fields = dict() | |||||
push_failed = self.get_log("push_failed", []) | |||||
pull_failed = self.get_log("pull_failed", []) | |||||
status = "Partial Success" | |||||
if not push_failed and not pull_failed: | |||||
status = "Success" | |||||
fields["percent_complete"] = 100 | |||||
fields["status"] = status | |||||
self.db_set(fields, notify=True, commit=True) | |||||
self.execute_postprocess(status) | |||||
frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) | |||||
def execute_postprocess(self, status): | |||||
# Execute post process | |||||
postprocess_method_path = self.get_plan().postprocess_method | |||||
if postprocess_method_path: | |||||
frappe.get_attr(postprocess_method_path)( | |||||
{ | |||||
"status": status, | |||||
"stats": { | |||||
"push_insert": self.push_insert, | |||||
"push_update": self.push_update, | |||||
"push_delete": self.push_delete, | |||||
"pull_insert": self.pull_insert, | |||||
"pull_update": self.pull_update, | |||||
}, | |||||
} | |||||
) | |||||
def get_plan(self): | |||||
if not hasattr(self, "plan"): | |||||
self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) | |||||
return self.plan | |||||
def get_mapping(self, mapping_name): | |||||
if hasattr(self, "mappings"): | |||||
for m in self.mappings: | |||||
if m.name == mapping_name: | |||||
return m | |||||
return frappe.get_doc("Data Migration Mapping", mapping_name) | |||||
def get_next_mapping_name(self): | |||||
mappings = [m for m in self.get_plan().mappings if m.enabled] | |||||
if not self.current_mapping: | |||||
# first | |||||
return mappings[0].mapping | |||||
for i, d in enumerate(mappings): | |||||
if i == len(mappings) - 1: | |||||
# last | |||||
return None | |||||
if d.mapping == self.current_mapping: | |||||
return mappings[i + 1].mapping | |||||
raise frappe.ValidationError("Mapping Broken") | |||||
def get_data(self, filters): | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
or_filters = self.get_or_filters(mapping) | |||||
start = self.current_mapping_start | |||||
data = [] | |||||
doclist = frappe.get_all( | |||||
mapping.local_doctype, | |||||
filters=filters, | |||||
or_filters=or_filters, | |||||
start=start, | |||||
page_length=mapping.page_length, | |||||
) | |||||
for d in doclist: | |||||
doc = frappe.get_doc(mapping.local_doctype, d["name"]) | |||||
data.append(doc) | |||||
return data | |||||
def get_new_local_data(self): | |||||
"""Fetch newly inserted local data using `frappe.get_all`. Used during Push""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
filters = mapping.get_filters() or {} | |||||
# new docs dont have migration field set | |||||
filters.update({mapping.migration_id_field: ""}) | |||||
return self.get_data(filters) | |||||
def get_updated_local_data(self): | |||||
"""Fetch local updated data using `frappe.get_all`. Used during Push""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
filters = mapping.get_filters() or {} | |||||
# existing docs must have migration field set | |||||
filters.update({mapping.migration_id_field: ("!=", "")}) | |||||
return self.get_data(filters) | |||||
def get_deleted_local_data(self): | |||||
"""Fetch local deleted data using `frappe.get_all`. Used during Push""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
filters = self.get_last_modified_condition() | |||||
filters.update({"deleted_doctype": mapping.local_doctype}) | |||||
data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) | |||||
_data = [] | |||||
for d in data: | |||||
doc = json.loads(d.data) | |||||
if doc.get(mapping.migration_id_field): | |||||
doc["_deleted_document_name"] = d["name"] | |||||
_data.append(doc) | |||||
return _data | |||||
def get_remote_data(self): | |||||
"""Fetch data from remote using `connection.get`. Used during Pull""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
start = self.current_mapping_start | |||||
filters = mapping.get_filters() or {} | |||||
connection = self.get_connection() | |||||
return connection.get( | |||||
mapping.remote_objectname, | |||||
fields=["*"], | |||||
filters=filters, | |||||
start=start, | |||||
page_length=mapping.page_length, | |||||
) | |||||
def get_count(self, mapping): | |||||
filters = mapping.get_filters() or {} | |||||
or_filters = self.get_or_filters(mapping) | |||||
to_insert = frappe.get_all( | |||||
mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters | |||||
)[0].total | |||||
to_delete = frappe.get_all( | |||||
"Deleted Document", | |||||
["count(name) as total"], | |||||
filters={"deleted_doctype": mapping.local_doctype}, | |||||
or_filters=or_filters, | |||||
)[0].total | |||||
return to_insert + to_delete | |||||
def get_or_filters(self, mapping): | |||||
or_filters = self.get_last_modified_condition() | |||||
# docs whose migration_id_field is not set | |||||
# failed in the previous run, include those too | |||||
or_filters.update({mapping.migration_id_field: ("=", "")}) | |||||
return or_filters | |||||
def get_connection(self): | |||||
if not hasattr(self, "connection"): | |||||
self.connection = frappe.get_doc( | |||||
"Data Migration Connector", self.data_migration_connector | |||||
).get_connection() | |||||
return self.connection | |||||
def push(self): | |||||
self.db_set("current_mapping_type", "Push") | |||||
done = True | |||||
if self.current_mapping_action == "Insert": | |||||
done = self._push_insert() | |||||
elif self.current_mapping_action == "Update": | |||||
done = self._push_update() | |||||
elif self.current_mapping_action == "Delete": | |||||
done = self._push_delete() | |||||
return done | |||||
def _push_insert(self): | |||||
"""Inserts new local docs on remote""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
connection = self.get_connection() | |||||
data = self.get_new_local_data() | |||||
for d in data: | |||||
# pre process before insert | |||||
doc = self.pre_process_doc(d) | |||||
doc = mapping.get_mapped_record(doc) | |||||
try: | |||||
response_doc = connection.insert(mapping.remote_objectname, doc) | |||||
frappe.db.set_value( | |||||
mapping.local_doctype, | |||||
d.name, | |||||
mapping.migration_id_field, | |||||
response_doc[connection.name_field], | |||||
update_modified=False, | |||||
) | |||||
frappe.db.commit() | |||||
self.update_log("push_insert", 1) | |||||
# post process after insert | |||||
self.post_process_doc(local_doc=d, remote_doc=response_doc) | |||||
except Exception as e: | |||||
self.update_log("push_failed", {d.name: cstr(e)}) | |||||
# update page_start | |||||
self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) | |||||
if len(data) < mapping.page_length: | |||||
# done, no more new data to insert | |||||
self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) | |||||
# not done with this mapping | |||||
return False | |||||
def _push_update(self): | |||||
"""Updates local modified docs on remote""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
connection = self.get_connection() | |||||
data = self.get_updated_local_data() | |||||
for d in data: | |||||
migration_id_value = d.get(mapping.migration_id_field) | |||||
# pre process before update | |||||
doc = self.pre_process_doc(d) | |||||
doc = mapping.get_mapped_record(doc) | |||||
try: | |||||
response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) | |||||
self.update_log("push_update", 1) | |||||
# post process after update | |||||
self.post_process_doc(local_doc=d, remote_doc=response_doc) | |||||
except Exception as e: | |||||
self.update_log("push_failed", {d.name: cstr(e)}) | |||||
# update page_start | |||||
self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) | |||||
if len(data) < mapping.page_length: | |||||
# done, no more data to update | |||||
self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) | |||||
# not done with this mapping | |||||
return False | |||||
def _push_delete(self): | |||||
"""Deletes docs deleted from local on remote""" | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
connection = self.get_connection() | |||||
data = self.get_deleted_local_data() | |||||
for d in data: | |||||
# Deleted Document also has a custom field for migration_id | |||||
migration_id_value = d.get(mapping.migration_id_field) | |||||
# pre process before update | |||||
self.pre_process_doc(d) | |||||
try: | |||||
response_doc = connection.delete(mapping.remote_objectname, migration_id_value) | |||||
self.update_log("push_delete", 1) | |||||
# post process only when action is success | |||||
self.post_process_doc(local_doc=d, remote_doc=response_doc) | |||||
except Exception as e: | |||||
self.update_log("push_failed", {d.name: cstr(e)}) | |||||
# update page_start | |||||
self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) | |||||
if len(data) < mapping.page_length: | |||||
# done, no more new data to delete | |||||
# done with this mapping | |||||
return True | |||||
def pull(self): | |||||
self.db_set("current_mapping_type", "Pull") | |||||
connection = self.get_connection() | |||||
mapping = self.get_mapping(self.current_mapping) | |||||
data = self.get_remote_data() | |||||
for d in data: | |||||
migration_id_value = get_source_value(d, connection.name_field) | |||||
doc = self.pre_process_doc(d) | |||||
doc = mapping.get_mapped_record(doc) | |||||
if migration_id_value: | |||||
try: | |||||
if not local_doc_exists(mapping, migration_id_value): | |||||
# insert new local doc | |||||
local_doc = insert_local_doc(mapping, doc) | |||||
self.update_log("pull_insert", 1) | |||||
# set migration id | |||||
frappe.db.set_value( | |||||
mapping.local_doctype, | |||||
local_doc.name, | |||||
mapping.migration_id_field, | |||||
migration_id_value, | |||||
update_modified=False, | |||||
) | |||||
frappe.db.commit() | |||||
else: | |||||
# update doc | |||||
local_doc = update_local_doc(mapping, doc, migration_id_value) | |||||
self.update_log("pull_update", 1) | |||||
# post process doc after success | |||||
self.post_process_doc(remote_doc=d, local_doc=local_doc) | |||||
except Exception as e: | |||||
# failed, append to log | |||||
self.update_log("pull_failed", {migration_id_value: cstr(e)}) | |||||
if len(data) < mapping.page_length: | |||||
# last page, done with pull | |||||
return True | |||||
def pre_process_doc(self, doc): | |||||
plan = self.get_plan() | |||||
doc = plan.pre_process_doc(self.current_mapping, doc) | |||||
return doc | |||||
def post_process_doc(self, local_doc=None, remote_doc=None): | |||||
plan = self.get_plan() | |||||
doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc) | |||||
return doc | |||||
def set_log(self, key, value): | |||||
value = json.dumps(value) if "_failed" in key else value | |||||
self.db_set(key, value) | |||||
def update_log(self, key, value=None): | |||||
""" | |||||
Helper for updating logs, | |||||
push_failed and pull_failed are stored as json, | |||||
other keys are stored as int | |||||
""" | |||||
if "_failed" in key: | |||||
# json | |||||
self.set_log(key, self.get_log(key, []) + [value]) | |||||
else: | |||||
# int | |||||
self.set_log(key, self.get_log(key, 0) + (value or 1)) | |||||
def get_log(self, key, default=None): | |||||
value = self.db_get(key) | |||||
if "_failed" in key: | |||||
if not value: | |||||
value = json.dumps(default) | |||||
value = json.loads(value) | |||||
return value or default | |||||
def insert_local_doc(mapping, doc): | |||||
try: | |||||
# insert new doc | |||||
if not doc.doctype: | |||||
doc.doctype = mapping.local_doctype | |||||
doc = frappe.get_doc(doc).insert() | |||||
return doc | |||||
except Exception: | |||||
print("Data Migration Run failed: Error in Pull insert") | |||||
print(frappe.get_traceback()) | |||||
return None | |||||
def update_local_doc(mapping, remote_doc, migration_id_value): | |||||
try: | |||||
# migration id value is set in migration_id_field in mapping.local_doctype | |||||
docname = frappe.db.get_value( | |||||
mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} | |||||
) | |||||
doc = frappe.get_doc(mapping.local_doctype, docname) | |||||
doc.update(remote_doc) | |||||
doc.save() | |||||
return doc | |||||
except Exception: | |||||
print("Data Migration Run failed: Error in Pull update") | |||||
print(frappe.get_traceback()) | |||||
return None | |||||
def local_doc_exists(mapping, migration_id_value): | |||||
return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) |
@@ -1,128 +0,0 @@ | |||||
# -*- coding: utf-8 -*- | |||||
# Copyright (c) 2017, Frappe Technologies and Contributors | |||||
# License: MIT. See LICENSE | |||||
import unittest | |||||
import frappe | |||||
class TestDataMigrationRun(unittest.TestCase): | |||||
def test_run(self): | |||||
create_plan() | |||||
description = "data migration todo" | |||||
new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() | |||||
event_subject = "data migration event" | |||||
frappe.get_doc( | |||||
dict( | |||||
doctype="Event", | |||||
subject=event_subject, | |||||
repeat_on="Monthly", | |||||
starts_on=frappe.utils.now_datetime(), | |||||
) | |||||
).insert() | |||||
run = frappe.get_doc( | |||||
{ | |||||
"doctype": "Data Migration Run", | |||||
"data_migration_plan": "ToDo Sync", | |||||
"data_migration_connector": "Local Connector", | |||||
} | |||||
).insert() | |||||
run.run() | |||||
self.assertEqual(run.db_get("status"), "Success") | |||||
self.assertEqual(run.db_get("push_insert"), 1) | |||||
self.assertEqual(run.db_get("pull_insert"), 1) | |||||
todo = frappe.get_doc("ToDo", new_todo.name) | |||||
self.assertTrue(todo.todo_sync_id) | |||||
# Pushed Event | |||||
event = frappe.get_doc("Event", todo.todo_sync_id) | |||||
self.assertEqual(event.subject, description) | |||||
# Pulled ToDo | |||||
created_todo = frappe.get_doc("ToDo", {"description": event_subject}) | |||||
self.assertEqual(created_todo.description, event_subject) | |||||
todo_list = frappe.get_list( | |||||
"ToDo", filters={"description": "data migration todo"}, fields=["name"] | |||||
) | |||||
todo_name = todo_list[0].name | |||||
todo = frappe.get_doc("ToDo", todo_name) | |||||
todo.description = "data migration todo updated" | |||||
todo.save() | |||||
run = frappe.get_doc( | |||||
{ | |||||
"doctype": "Data Migration Run", | |||||
"data_migration_plan": "ToDo Sync", | |||||
"data_migration_connector": "Local Connector", | |||||
} | |||||
).insert() | |||||
run.run() | |||||
# Update | |||||
self.assertEqual(run.db_get("status"), "Success") | |||||
self.assertEqual(run.db_get("pull_update"), 1) | |||||
def create_plan(): | |||||
frappe.get_doc( | |||||
{ | |||||
"doctype": "Data Migration Mapping", | |||||
"mapping_name": "Todo to Event", | |||||
"remote_objectname": "Event", | |||||
"remote_primary_key": "name", | |||||
"mapping_type": "Push", | |||||
"local_doctype": "ToDo", | |||||
"fields": [ | |||||
{"remote_fieldname": "subject", "local_fieldname": "description"}, | |||||
{ | |||||
"remote_fieldname": "starts_on", | |||||
"local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", | |||||
}, | |||||
], | |||||
"condition": '{"description": "data migration todo" }', | |||||
} | |||||
).insert(ignore_if_duplicate=True) | |||||
frappe.get_doc( | |||||
{ | |||||
"doctype": "Data Migration Mapping", | |||||
"mapping_name": "Event to ToDo", | |||||
"remote_objectname": "Event", | |||||
"remote_primary_key": "name", | |||||
"local_doctype": "ToDo", | |||||
"local_primary_key": "name", | |||||
"mapping_type": "Pull", | |||||
"condition": '{"subject": "data migration event" }', | |||||
"fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], | |||||
} | |||||
).insert(ignore_if_duplicate=True) | |||||
frappe.get_doc( | |||||
{ | |||||
"doctype": "Data Migration Plan", | |||||
"plan_name": "ToDo Sync", | |||||
"module": "Core", | |||||
"mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], | |||||
} | |||||
).insert(ignore_if_duplicate=True) | |||||
frappe.get_doc( | |||||
{ | |||||
"doctype": "Data Migration Connector", | |||||
"connector_name": "Local Connector", | |||||
"connector_type": "Frappe", | |||||
# connect to same host. | |||||
"hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), | |||||
"username": "Administrator", | |||||
"password": frappe.conf.get("admin_password") or "admin", | |||||
} | |||||
).insert(ignore_if_duplicate=True) |
@@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False): | |||||
os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") | os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") | ||||
) | ) | ||||
for data_migration_module in [ | |||||
"data_migration_mapping_detail", | |||||
"data_migration_mapping", | |||||
"data_migration_plan_mapping", | |||||
"data_migration_plan", | |||||
]: | |||||
files.append( | |||||
os.path.join( | |||||
FRAPPE_PATH, | |||||
"data_migration", | |||||
"doctype", | |||||
data_migration_module, | |||||
f"{data_migration_module}.json", | |||||
) | |||||
) | |||||
for desk_module in [ | for desk_module in [ | ||||
"number_card", | "number_card", | ||||
"dashboard_chart", | "dashboard_chart", | ||||
@@ -124,8 +108,6 @@ def get_doc_files(files, start_path): | |||||
"web_template", | "web_template", | ||||
"notification", | "notification", | ||||
"print_style", | "print_style", | ||||
"data_migration_mapping", | |||||
"data_migration_plan", | |||||
"workspace", | "workspace", | ||||
"onboarding_step", | "onboarding_step", | ||||
"module_onboarding", | "module_onboarding", | ||||
@@ -8,7 +8,6 @@ Desk | |||||
Integrations | Integrations | ||||
Printing | Printing | ||||
Contacts | Contacts | ||||
Data Migration | |||||
Social | Social | ||||
Automation | Automation | ||||
Event Streaming | Event Streaming |
@@ -201,3 +201,4 @@ frappe.patches.v14_0.update_color_names_in_kanban_board_column | |||||
frappe.patches.v14_0.update_is_system_generated_flag | frappe.patches.v14_0.update_is_system_generated_flag | ||||
frappe.patches.v14_0.update_auto_account_deletion_duration | frappe.patches.v14_0.update_auto_account_deletion_duration | ||||
frappe.patches.v14_0.set_document_expiry_default | frappe.patches.v14_0.set_document_expiry_default | ||||
frappe.patches.v14_0.delete_data_migration_tool |
@@ -0,0 +1,12 @@ | |||||
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors | |||||
# MIT License. See license.txt | |||||
import frappe | |||||
def execute(): | |||||
doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name") | |||||
for doctype in doctypes: | |||||
frappe.delete_doc("DocType", doctype, ignore_missing=True) | |||||
frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True) |