diff --git a/frappe/__init__.py b/frappe/__init__.py
index 9e7039acd4..d01866c136 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -1319,6 +1319,20 @@ def enqueue(*args, **kwargs):
import frappe.utils.background_jobs
return frappe.utils.background_jobs.enqueue(*args, **kwargs)
+def enqueue_doc(*args, **kwargs):
+ '''
+ Enqueue method to be executed using a background worker
+
+ :param doctype: DocType of the document on which you want to run the event
+ :param name: Name of the document on which you want to run the event
+ :param method: method string or method object
+ :param queue: (optional) should be either long, default or short
+ :param timeout: (optional) should be set according to the functions
+ :param kwargs: keyword arguments to be passed to the method
+ '''
+ import frappe.utils.background_jobs
+ return frappe.utils.background_jobs.enqueue_doc(*args, **kwargs)
+
def get_doctype_app(doctype):
def _get_doctype_app():
doctype_module = local.db.get_value("DocType", doctype, "module")
@@ -1371,4 +1385,4 @@ def get_system_settings(key):
def get_active_domains():
from frappe.core.doctype.domain_settings.domain_settings import get_active_domains
- return get_active_domains()
\ No newline at end of file
+ return get_active_domains()
diff --git a/frappe/app.py b/frappe/app.py
index 79cfdfe442..b2e19beff0 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -128,7 +128,7 @@ def handle_exception(e):
http_status_code = getattr(e, "http_status_code", 500)
return_as_message = False
- if frappe.local.is_ajax or 'application/json' in frappe.local.request.headers.get('Accept', ''):
+ if frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept'):
# handle ajax responses first
# if the request is ajax, send back the trace or error message
response = frappe.utils.response.report_error(http_status_code)
diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py
index 13270cc352..4fa1183e74 100644
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@@ -42,6 +42,7 @@ class User(Document):
def before_insert(self):
self.flags.in_insert = True
+ throttle_user_creation()
def validate(self):
self.check_demo()
@@ -976,4 +977,10 @@ def reset_otp_secret(user):
enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, async=True, job_name=None, now=False, **email_args)
return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login."))
else:
- return frappe.throw(_("OTP secret can only be reset by the Administrator."))
\ No newline at end of file
+ return frappe.throw(_("OTP secret can only be reset by the Administrator."))
+
+def throttle_user_creation():
+ if frappe.flags.in_import:
+ return
+ if frappe.db.get_creation_count('User', 60) > 60:
+ frappe.throw(_('Throttled'))
\ No newline at end of file
diff --git a/frappe/core/page/data_import_tool/data_import_main.html b/frappe/core/page/data_import_tool/data_import_main.html
index db44a06e70..549bebcdfd 100644
--- a/frappe/core/page/data_import_tool/data_import_main.html
+++ b/frappe/core/page/data_import_tool/data_import_main.html
@@ -93,10 +93,16 @@
{%= __("Ignore encoding errors.") %}
+
+
+
+
diff --git a/frappe/core/page/data_import_tool/data_import_tool.js b/frappe/core/page/data_import_tool/data_import_tool.js
index 78c3bf47c6..9523a79a13 100644
--- a/frappe/core/page/data_import_tool/data_import_tool.js
+++ b/frappe/core/page/data_import_tool/data_import_tool.js
@@ -113,6 +113,7 @@ frappe.DataImportTool = Class.extend({
return {
submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"),
ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
+ skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"),
overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
update_only: me.page.main.find('[name="update_only"]').prop("checked"),
no_email: me.page.main.find('[name="no_email"]').prop("checked"),
diff --git a/frappe/core/page/data_import_tool/importer.py b/frappe/core/page/data_import_tool/importer.py
index d26dcfd2b1..083715b24d 100644
--- a/frappe/core/page/data_import_tool/importer.py
+++ b/frappe/core/page/data_import_tool/importer.py
@@ -21,7 +21,8 @@ from six import text_type, string_types
@frappe.whitelist()
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
- update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No"):
+ update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
+ skip_errors = True):
"""upload data"""
frappe.flags.in_import = True
@@ -341,13 +342,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
doc.submit()
log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
except Exception as e:
- error = True
- if doc:
- frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
- err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
- log('Error for row (#%d) %s : %s' % (row_idx + 1,
- len(row)>1 and row[1] or "", err_msg))
- frappe.errprint(frappe.get_traceback())
+ if not skip_errors:
+ error = True
+ if doc:
+ frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
+ err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
+ log('Error for row (#%d) %s : %s' % (row_idx + 1,
+ len(row)>1 and row[1] or "", err_msg))
+ frappe.errprint(frappe.get_traceback())
finally:
frappe.local.message_log = []
diff --git a/frappe/data_migration/__init__.py b/frappe/data_migration/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/__init__.py b/frappe/data_migration/doctype/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_connector/__init__.py b/frappe/data_migration/doctype/data_migration_connector/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py b/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py
new file mode 100644
index 0000000000..e69c5708f4
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py
@@ -0,0 +1,8 @@
+
+
+class BaseConnection(object):
+ def pull(self):
+ pass
+
+ def push(self):
+ pass
diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py
new file mode 100644
index 0000000000..a0b3d203e4
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py
@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+import frappe
+from frappe.frappeclient import FrappeClient
+from .base import BaseConnection
+
+class FrappeConnection(BaseConnection):
+ def __init__(self, connector):
+ self.connector = connector
+ self.connection = FrappeClient(self.connector.hostname,
+ self.connector.username, self.connector.password)
+ self.name_field = 'name'
+
+ def insert(self, doctype, doc):
+ doc = frappe._dict(doc)
+ doc.doctype = doctype
+ return self.connection.insert(doc)
+
+ def update(self, doctype, doc, migration_id):
+ doc = frappe._dict(doc)
+ doc.doctype = doctype
+ doc.name = migration_id
+ return self.connection.update(doc)
+
+ def delete(self, doctype, migration_id):
+ return self.connection.delete(doctype, migration_id)
+
+ def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20):
+ return self.connection.get_list(doctype, fields=fields, filters=filters,
+ limit_start=start, limit_page_length=page_length)
diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/postgres.py b/frappe/data_migration/doctype/data_migration_connector/connectors/postgres.py
new file mode 100644
index 0000000000..9c3e2af64d
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/connectors/postgres.py
@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+import frappe, psycopg2
+from .base import BaseConnection
+
+class PostGresConnection(BaseConnection):
+ def __init__(self, properties):
+ self.__dict__.update(properties)
+ self._connector = psycopg2.connect("host='{0}' dbname='{1}' user='{2}' password='{3}'".format(self.hostname,
+ self.database_name, self.username, self.password))
+ self.cursor = self._connector.cursor()
+
+ def get_objects(self, object_type, condition, selection):
+ if not condition:
+ condition = ''
+ else:
+ condition = ' WHERE ' + condition
+ self.cursor.execute('SELECT {0} FROM {1}{2}'.format(selection, object_type, condition))
+ raw_data = self.cursor.fetchall()
+ data = []
+ for r in raw_data:
+ row_dict = frappe._dict({})
+ for i, value in enumerate(r):
+ row_dict[self.cursor.description[i][0]] = value
+ data.append(row_dict)
+
+ return data
+
+ def get_join_objects(self, object_type, field, primary_key):
+ """
+ field.formula 's first line will be list of tables that needs to be linked to fetch an item
+ The subsequent lines that follows will contain one to one mapping across tables keys
+ """
+ condition = ""
+ key_mapping = field.formula.split('\n')
+ obj_type = key_mapping[0]
+ selection = field.source_fieldname
+
+ for d in key_mapping[1:]:
+ condition += d + ' AND '
+
+ condition += str(object_type) + ".id=" + str(primary_key)
+
+ return self.get_objects(obj_type, condition, selection)
diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js
new file mode 100644
index 0000000000..f4f0d9f474
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Connector', {
+ refresh: function() {
+
+ }
+});
diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json
new file mode 100644
index 0000000000..3632531873
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json
@@ -0,0 +1,245 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 1,
+ "autoname": "field:connector_name",
+ "beta": 1,
+ "creation": "2017-08-11 05:03:27.091416",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "connector_name",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Connector Name",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "connector_type",
+ "fieldtype": "Select",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Connector Type",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Frappe\nPostgres",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "default": "localhost",
+ "fieldname": "hostname",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Hostname",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "database_name",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Database Name",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "username",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Username",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "password",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Password",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2017-09-07 12:40:15.008483",
+ "modified_by": "faris@erpnext.com",
+ "module": "Data Migration",
+ "name": "Data Migration Connector",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "amend": 0,
+ "apply_user_permissions": 0,
+ "cancel": 0,
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 1,
+ "if_owner": 0,
+ "import": 0,
+ "permlevel": 0,
+ "print": 1,
+ "read": 1,
+ "report": 1,
+ "role": "System Manager",
+ "set_user_permissions": 0,
+ "share": 1,
+ "submit": 0,
+ "write": 1
+ }
+ ],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py
new file mode 100644
index 0000000000..a48842debf
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+from .connectors.postgres import PostGresConnection
+from .connectors.frappe_connection import FrappeConnection
+
+class DataMigrationConnector(Document):
+ def get_connection(self):
+ if self.connector_type == 'Frappe':
+ self.connection = FrappeConnection(self)
+ elif self.connector_type == 'PostGres':
+ self.connection = PostGresConnection(self.as_dict())
+
+ return self.connection
+
+ def get_objects(self, object_type, condition=None, selection="*"):
+ return self.connector.get_objects(object_type, condition, selection)
+
+ def get_join_objects(self, object_type, join_type, primary_key):
+ return self.connector.get_join_objects(object_type, join_type, primary_key)
diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.js
new file mode 100644
index 0000000000..b933deb433
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.js
@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Connector", function (assert) {
+ let done = assert.async();
+
+ // number of asserts
+ assert.expect(1);
+
+ frappe.run_serially([
+ // insert a new Data Migration Connector
+ () => frappe.tests.make('Data Migration Connector', [
+ // values to be set
+ {key: 'value'}
+ ]),
+ () => {
+ assert.equal(cur_frm.doc.key, 'value');
+ },
+ () => done()
+ ]);
+
+});
diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py
new file mode 100644
index 0000000000..a6e30fbe44
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationConnector(unittest.TestCase):
+ pass
diff --git a/frappe/data_migration/doctype/data_migration_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_mapping/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js
new file mode 100644
index 0000000000..6c99b9a54d
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Mapping', {
+ refresh: function() {
+
+ }
+});
diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json
new file mode 100644
index 0000000000..998abdf6ca
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json
@@ -0,0 +1,456 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 1,
+ "autoname": "field:mapping_name",
+ "beta": 1,
+ "creation": "2017-08-11 05:11:49.975801",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "mapping_name",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Mapping Name",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "remote_objectname",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Remote Objectname",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "remote_primary_key",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Remote Primary Key",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "local_doctype",
+ "fieldtype": "Link",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Local DocType",
+ "length": 0,
+ "no_copy": 0,
+ "options": "DocType",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "local_primary_key",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Local Primary Key",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "column_break_5",
+ "fieldtype": "Column Break",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "mapping_type",
+ "fieldtype": "Select",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Mapping Type",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Push\nPull\nSync",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "default": "10",
+ "fieldname": "page_length",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Page Length",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "migration_id_field",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Migration ID Field",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "mapping",
+ "fieldtype": "Section Break",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Mapping",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "fields",
+ "fieldtype": "Table",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Field Maps",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Data Migration Mapping Detail",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 1,
+ "columns": 0,
+ "fieldname": "condition_detail",
+ "fieldtype": "Section Break",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Condition Detail",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "condition",
+ "fieldtype": "Code",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Condition",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2017-09-27 18:06:43.275207",
+ "modified_by": "Administrator",
+ "module": "Data Migration",
+ "name": "Data Migration Mapping",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "amend": 0,
+ "apply_user_permissions": 0,
+ "cancel": 0,
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 1,
+ "if_owner": 0,
+ "import": 0,
+ "permlevel": 0,
+ "print": 1,
+ "read": 1,
+ "report": 1,
+ "role": "System Manager",
+ "set_user_permissions": 0,
+ "share": 1,
+ "submit": 0,
+ "write": 1
+ }
+ ],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py
new file mode 100644
index 0000000000..ca20162f42
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class DataMigrationMapping(Document):
+ def get_filters(self):
+ if self.condition:
+ return frappe.safe_eval(self.condition, dict(frappe=frappe))
+
+ def get_fields(self):
+ fields = []
+ for f in self.fields:
+ if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith('eval:')):
+ fields.append(f.local_fieldname)
+
+ if frappe.db.has_column(self.local_doctype, self.migration_id_field):
+ fields.append(self.migration_id_field)
+
+ if 'name' not in fields:
+ fields.append('name')
+
+ return fields
+
+ def get_mapped_record(self, doc):
+ mapped = frappe._dict()
+
+ key_fieldname = 'remote_fieldname'
+ value_fieldname = 'local_fieldname'
+
+ if self.mapping_type == 'Pull':
+ key_fieldname, value_fieldname = value_fieldname, key_fieldname
+
+ for field_map in self.fields:
+ if not field_map.is_child_table:
+ value = get_value_from_fieldname(field_map, value_fieldname, doc)
+ mapped[field_map.get(key_fieldname)] = value
+ else:
+ mapping_name = field_map.child_table_mapping
+ value = get_mapped_child_records(mapping_name, doc.get(field_map.get(value_fieldname)))
+ mapped[field_map.get(key_fieldname)] = value
+ return mapped
+
+def get_mapped_child_records(mapping_name, child_docs):
+ mapped_child_docs = []
+ mapping = frappe.get_doc('Data Migration Mapping', mapping_name)
+ for child_doc in child_docs:
+ mapped_child_docs.append(mapping.get_mapped_record(child_doc))
+
+ return mapped_child_docs
+
+def get_value_from_fieldname(field_map, fieldname_field, doc):
+ field_name = field_map.get(fieldname_field)
+
+ if field_name.startswith('eval:'):
+ value = frappe.safe_eval(field_name[5:], dict(frappe=frappe))
+ elif field_name[0] in ('"', "'"):
+ value = field_name[1:-1]
+ else:
+ value = doc.get(field_name)
+ return value
diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.js
new file mode 100644
index 0000000000..e6966ef131
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.js
@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Mapping", function (assert) {
+ let done = assert.async();
+
+ // number of asserts
+ assert.expect(1);
+
+ frappe.run_serially([
+ // insert a new Data Migration Mapping
+ () => frappe.tests.make('Data Migration Mapping', [
+ // values to be set
+ {key: 'value'}
+ ]),
+ () => {
+ assert.equal(cur_frm.doc.key, 'value');
+ },
+ () => done()
+ ]);
+
+});
diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py
new file mode 100644
index 0000000000..e6f0ce2796
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationMapping(unittest.TestCase):
+ pass
diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py b/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json
new file mode 100644
index 0000000000..ede9213f14
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json
@@ -0,0 +1,163 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 0,
+ "beta": 0,
+ "creation": "2017-08-11 05:09:10.900237",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "remote_fieldname",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Remote Fieldname",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "local_fieldname",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Local Fieldname",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "is_child_table",
+ "fieldtype": "Check",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Is Child Table",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "depends_on": "is_child_table",
+ "fieldname": "child_table_mapping",
+ "fieldtype": "Link",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Child Table Mapping",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Data Migration Mapping",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 1,
+ "max_attachments": 0,
+ "modified": "2017-09-28 17:13:31.337005",
+ "modified_by": "Administrator",
+ "module": "Data Migration",
+ "name": "Data Migration Mapping Detail",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py
new file mode 100644
index 0000000000..1ccdf76eed
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class DataMigrationMappingDetail(Document):
+ pass
diff --git a/frappe/data_migration/doctype/data_migration_plan/__init__.py b/frappe/data_migration/doctype/data_migration_plan/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js
new file mode 100644
index 0000000000..935a227e79
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js
@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Plan', {
+ refresh: function() {
+
+ }
+});
diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json
new file mode 100644
index 0000000000..8feff93286
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json
@@ -0,0 +1,155 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 0,
+ "autoname": "field:plan_name",
+ "beta": 0,
+ "creation": "2017-08-11 05:15:51.482165",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "plan_name",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Plan Name",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "module",
+ "fieldtype": "Link",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Module",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Module Def",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "mappings",
+ "fieldtype": "Table",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Mappings",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Data Migration Plan Mapping",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2017-09-13 15:47:26.336541",
+ "modified_by": "prateeksha@erpnext.com",
+ "module": "Data Migration",
+ "name": "Data Migration Plan",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "amend": 0,
+ "apply_user_permissions": 0,
+ "cancel": 0,
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 1,
+ "if_owner": 0,
+ "import": 0,
+ "permlevel": 0,
+ "print": 1,
+ "read": 1,
+ "report": 1,
+ "role": "System Manager",
+ "set_user_permissions": 0,
+ "share": 1,
+ "submit": 0,
+ "write": 1
+ }
+ ],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py
new file mode 100644
index 0000000000..45883d0669
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.modules import get_module_path, scrub_dt_dn
+from frappe.modules.export_file import export_to_files, create_init_py
+from frappe.custom.doctype.custom_field.custom_field import create_custom_field
+from frappe.model.document import Document
+
+class DataMigrationPlan(Document):
+ def after_insert(self):
+ self.make_custom_fields_for_mappings()
+
+ def on_update(self):
+ if frappe.flags.in_import or frappe.flags.in_test:
+ return
+
+ if frappe.local.conf.get('developer_mode'):
+ record_list =[['Data Migration Plan', self.name]]
+
+ for m in self.mappings:
+ record_list.append(['Data Migration Mapping', m.mapping])
+
+ export_to_files(record_list=record_list, record_module=self.module)
+
+ for m in self.mappings:
+ dt, dn = scrub_dt_dn('Data Migration Mapping', m.mapping)
+ create_init_py(get_module_path(self.module), dt, dn)
+
+ def make_custom_fields_for_mappings(self):
+ label = self.name + ' ID'
+ fieldname = frappe.scrub(label)
+
+ df = {
+ 'label': label,
+ 'fieldname': fieldname,
+ 'fieldtype': 'Data',
+ 'hidden': 1,
+ 'read_only': 1,
+ 'unique': 1
+ }
+
+ for m in self.mappings:
+ mapping = frappe.get_doc('Data Migration Mapping', m.mapping)
+ create_custom_field(mapping.local_doctype, df)
+ mapping.migration_id_field = fieldname
+ mapping.save()
+
+ # Create custom field in Deleted Document
+ create_custom_field('Deleted Document', df)
+
+ def pre_process_doc(self, mapping_name, doc):
+ module = self.get_mapping_module(mapping_name)
+
+ if module and hasattr(module, 'pre_process'):
+ return module.pre_process(doc)
+ return doc
+
+ def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None):
+ module = self.get_mapping_module(mapping_name)
+
+ if module and hasattr(module, 'post_process'):
+ return module.post_process(local_doc=local_doc, remote_doc=remote_doc)
+
+ def get_mapping_module(self, mapping_name):
+ try:
+ module_def = frappe.get_doc("Module Def", self.module)
+ module = frappe.get_module('{app}.{module}.data_migration_mapping.{mapping_name}'.format(
+ app= module_def.app_name,
+ module=frappe.scrub(self.module),
+ mapping_name=frappe.scrub(mapping_name)
+ ))
+ return module
+ except ImportError:
+ return None
diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.js
new file mode 100644
index 0000000000..9943cd6ec1
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.js
@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Plan", function (assert) {
+ let done = assert.async();
+
+ // number of asserts
+ assert.expect(1);
+
+ frappe.run_serially([
+ // insert a new Data Migration Plan
+ () => frappe.tests.make('Data Migration Plan', [
+ // values to be set
+ {key: 'value'}
+ ]),
+ () => {
+ assert.equal(cur_frm.doc.key, 'value');
+ },
+ () => done()
+ ]);
+
+});
diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py
new file mode 100644
index 0000000000..3a33039c3d
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationPlan(unittest.TestCase):
+ pass
diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json
new file mode 100644
index 0000000000..5acf014715
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json
@@ -0,0 +1,103 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 0,
+ "beta": 1,
+ "creation": "2017-08-11 05:15:38.390831",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "mapping",
+ "fieldtype": "Link",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Mapping",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Data Migration Mapping",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "default": "1",
+ "fieldname": "enabled",
+ "fieldtype": "Check",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Enabled",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 1,
+ "max_attachments": 0,
+ "modified": "2017-09-20 21:43:04.908650",
+ "modified_by": "Administrator",
+ "module": "Data Migration",
+ "name": "Data Migration Plan Mapping",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py
new file mode 100644
index 0000000000..85f879069c
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class DataMigrationPlanMapping(Document):
+ pass
diff --git a/frappe/data_migration/doctype/data_migration_run/__init__.py b/frappe/data_migration/doctype/data_migration_run/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js
new file mode 100644
index 0000000000..82323c62f1
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js
@@ -0,0 +1,14 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Run', {
+ refresh: function(frm) {
+ if (frm.doc.status !== 'Success') {
+ frm.add_custom_button(__('Run'), () => frm.call('run'));
+ }
+ if (frm.doc.status === 'Started') {
+ frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete,
+ __('Currently updating {0}', [frm.doc.current_mapping]));
+ }
+ }
+});
diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json
new file mode 100644
index 0000000000..dad75d0a2a
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json
@@ -0,0 +1,671 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 0,
+ "beta": 0,
+ "creation": "2017-09-11 12:55:27.597728",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "data_migration_plan",
+ "fieldtype": "Link",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Data Migration Plan",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Data Migration Plan",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "data_migration_connector",
+ "fieldtype": "Link",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Data Migration Connector",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Data Migration Connector",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "default": "Pending",
+ "fieldname": "status",
+ "fieldtype": "Select",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Status",
+ "length": 0,
+ "no_copy": 1,
+ "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "current_mapping",
+ "fieldtype": "Data",
+ "hidden": 1,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Current Mapping",
+ "length": 0,
+ "no_copy": 1,
+ "options": "",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "current_mapping_start",
+ "fieldtype": "Int",
+ "hidden": 1,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Current Mapping Start",
+ "length": 0,
+ "no_copy": 1,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "current_mapping_delete_start",
+ "fieldtype": "Int",
+ "hidden": 1,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Current Mapping Delete Start",
+ "length": 0,
+ "no_copy": 1,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "current_mapping_type",
+ "fieldtype": "Select",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Current Mapping Type",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Push\nPull",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "depends_on": "eval:(doc.status !== 'Pending')",
+ "fieldname": "current_mapping_action",
+ "fieldtype": "Select",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Current Mapping Action",
+ "length": 0,
+ "no_copy": 1,
+ "options": "Insert\nDelete",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "total_pages",
+ "fieldtype": "Int",
+ "hidden": 1,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Total Pages",
+ "length": 0,
+ "no_copy": 1,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "percent_complete",
+ "fieldtype": "Percent",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Percent Complete",
+ "length": 0,
+ "no_copy": 1,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "depends_on": "eval:(doc.status !== 'Pending')",
+ "fieldname": "logs_sb",
+ "fieldtype": "Section Break",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Logs",
+ "length": 0,
+ "no_copy": 1,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "push_insert",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Push Insert",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "push_update",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Push Update",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "push_delete",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Push Delete",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "push_failed",
+ "fieldtype": "Code",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Push Failed",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "column_break_16",
+ "fieldtype": "Column Break",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "pull_insert",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Pull Insert",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "pull_update",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Pull Update",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "pull_failed",
+ "fieldtype": "Code",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Pull Failed",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "depends_on": "eval:doc.failed_log !== '[]'",
+ "fieldname": "log",
+ "fieldtype": "Code",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Log",
+ "length": 0,
+ "no_copy": 1,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 1,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2017-10-02 05:12:16.094991",
+ "modified_by": "Administrator",
+ "module": "Data Migration",
+ "name": "Data Migration Run",
+ "name_case": "",
+ "owner": "faris@erpnext.com",
+ "permissions": [
+ {
+ "amend": 0,
+ "apply_user_permissions": 0,
+ "cancel": 0,
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 1,
+ "if_owner": 0,
+ "import": 0,
+ "permlevel": 0,
+ "print": 1,
+ "read": 1,
+ "report": 1,
+ "role": "System Manager",
+ "set_user_permissions": 0,
+ "share": 1,
+ "submit": 0,
+ "write": 1
+ }
+ ],
+ "quick_entry": 0,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py
new file mode 100644
index 0000000000..c39be68d8d
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py
@@ -0,0 +1,467 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe, json, math
+from frappe.model.document import Document
+from frappe import _
+
+class DataMigrationRun(Document):
+
+ def validate(self):
+ exists = frappe.db.exists('Data Migration Run', dict(
+ status=('in', ['Fail', 'Error']),
+ name=('!=', self.name)
+ ))
+ if exists:
+ frappe.throw(_('There are failed runs with the same Data Migration Plan'))
+
+ def run(self):
+ self.begin()
+ if self.total_pages > 0:
+ self.enqueue_next_mapping()
+ else:
+ self.complete()
+
+ def enqueue_next_mapping(self):
+ next_mapping_name = self.get_next_mapping_name()
+ if next_mapping_name:
+ next_mapping = self.get_mapping(next_mapping_name)
+ self.db_set(dict(
+ current_mapping = next_mapping.name,
+ current_mapping_start = 0,
+ current_mapping_delete_start = 0,
+ current_mapping_action = 'Insert'
+ ), notify=True, commit=True)
+ frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test)
+ else:
+ self.complete()
+
+ def enqueue_next_page(self):
+ mapping = self.get_mapping(self.current_mapping)
+ fields = dict(
+ percent_complete = self.percent_complete + (100.0 / self.total_pages)
+ )
+ if self.current_mapping_action == 'Insert':
+ start = self.current_mapping_start + mapping.page_length
+ fields['current_mapping_start'] = start
+ elif self.current_mapping_action == 'Delete':
+ delete_start = self.current_mapping_delete_start + mapping.page_length
+ fields['current_mapping_delete_start'] = delete_start
+
+ self.db_set(fields, notify=True, commit=True)
+ frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test)
+
+ def run_current_mapping(self):
+ try:
+ mapping = self.get_mapping(self.current_mapping)
+
+ if mapping.mapping_type == 'Push':
+ done = self.push()
+ elif mapping.mapping_type == 'Pull':
+ done = self.pull()
+
+ if done:
+ self.enqueue_next_mapping()
+ else:
+ self.enqueue_next_page()
+
+ except Exception as e:
+ self.db_set('status', 'Error', notify=True, commit=True)
+ print('Data Migration Run failed')
+ print(frappe.get_traceback())
+ raise e
+
+ def get_last_modified_condition(self):
+ last_run_timestamp = frappe.db.get_value('Data Migration Run', dict(
+ data_migration_plan=self.data_migration_plan,
+ name=('!=', self.name)
+ ), 'modified')
+ if last_run_timestamp:
+ condition = dict(modified=('>', last_run_timestamp))
+ else:
+ condition = {}
+ return condition
+
+ def begin(self):
+ plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled]
+ self.mappings = [frappe.get_doc(
+ 'Data Migration Mapping', m.mapping) for m in plan_active_mappings]
+
+ total_pages = 0
+ for m in [mapping for mapping in self.mappings if mapping.mapping_type == 'Push']:
+ count = float(self.get_count(m))
+ page_count = math.ceil(count / m.page_length)
+ total_pages += page_count
+
+ total_pages = 10
+
+ self.db_set(dict(
+ status = 'Started',
+ current_mapping = None,
+ current_mapping_start = 0,
+ current_mapping_delete_start = 0,
+ percent_complete = 0,
+ current_mapping_action = 'Insert',
+ total_pages = total_pages
+ ), notify=True, commit=True)
+
+ def complete(self):
+ fields = dict()
+
+ push_failed = self.get_log('push_failed', [])
+ pull_failed = self.get_log('pull_failed', [])
+
+ if push_failed or pull_failed:
+ fields['status'] = 'Partial Success'
+ else:
+ fields['status'] = 'Success'
+ fields['percent_complete'] = 100
+
+ self.db_set(fields, notify=True, commit=True)
+
+ def get_plan(self):
+ if not hasattr(self, 'plan'):
+ self.plan = frappe.get_doc('Data Migration Plan', self.data_migration_plan)
+ return self.plan
+
+ def get_mapping(self, mapping_name):
+ if hasattr(self, 'mappings'):
+ for m in self.mappings:
+ if m.name == mapping_name:
+ return m
+ return frappe.get_doc('Data Migration Mapping', mapping_name)
+
+ def get_next_mapping_name(self):
+ mappings = [m for m in self.get_plan().mappings if m.enabled]
+ if not self.current_mapping:
+ # first
+ return mappings[0].mapping
+ for i, d in enumerate(mappings):
+ if i == len(mappings) - 1:
+ # last
+ return None
+ if d.mapping == self.current_mapping:
+ return mappings[i+1].mapping
+
+ raise frappe.ValidationError('Mapping Broken')
+
+ def get_data(self, filters):
+ mapping = self.get_mapping(self.current_mapping)
+ or_filters = self.get_or_filters(mapping)
+ start = self.current_mapping_start
+
+ data = []
+ doclist = frappe.get_all(mapping.local_doctype,
+ filters=filters, or_filters=or_filters,
+ start=start, page_length=mapping.page_length)
+
+ for d in doclist:
+ doc = frappe.get_doc(mapping.local_doctype, d['name'])
+ data.append(doc)
+ return data
+
+ def get_new_local_data(self):
+ '''Fetch newly inserted local data using `frappe.get_all`. Used during Push'''
+ mapping = self.get_mapping(self.current_mapping)
+ filters = mapping.get_filters() or {}
+
+ # new docs dont have migration field set
+ filters.update({
+ mapping.migration_id_field: ''
+ })
+
+ return self.get_data(filters)
+
+ def get_updated_local_data(self):
+ '''Fetch local updated data using `frappe.get_all`. Used during Push'''
+ mapping = self.get_mapping(self.current_mapping)
+ filters = mapping.get_filters() or {}
+
+ # existing docs must have migration field set
+ filters.update({
+ mapping.migration_id_field: ('!=', '')
+ })
+
+ return self.get_data(filters)
+
+ def get_deleted_local_data(self):
+ '''Fetch local deleted data using `frappe.get_all`. Used during Push'''
+ mapping = self.get_mapping(self.current_mapping)
+ or_filters = self.get_or_filters(mapping)
+ filters = dict(
+ deleted_doctype=mapping.local_doctype
+ )
+
+ data = frappe.get_all('Deleted Document', fields=['data'],
+ filters=filters, or_filters=or_filters)
+
+ _data = []
+ for d in data:
+ doc = json.loads(d.data)
+ if doc.get(mapping.migration_id_field):
+ doc['_deleted_document_name'] = d.name
+ _data.append(doc)
+
+ return _data
+
+ def get_remote_data(self):
+ '''Fetch data from remote using `connection.get`. Used during Pull'''
+ mapping = self.get_mapping(self.current_mapping)
+ start = self.current_mapping_start
+ filters = mapping.get_filters() or {}
+ connection = self.get_connection()
+
+ return connection.get(mapping.remote_objectname,
+ fields=["*"], filters=filters, start=start,
+ page_length=mapping.page_length)
+
+ def get_count(self, mapping):
+ filters = mapping.get_filters() or {}
+ or_filters = self.get_or_filters(mapping)
+
+ to_insert = frappe.get_all(mapping.local_doctype, ['count(name) as total'],
+ filters=filters, or_filters=or_filters)[0].total
+
+ to_delete = frappe.get_all('Deleted Document', ['count(name) as total'],
+ filters={'deleted_doctype': mapping.local_doctype}, or_filters=or_filters)[0].total
+
+ return to_insert + to_delete
+
+ def get_or_filters(self, mapping):
+ or_filters = self.get_last_modified_condition()
+
+ # include docs whose migration_id_field is not set
+ or_filters.update({
+ mapping.migration_id_field: ('=', '')
+ })
+
+ return or_filters
+
+ def get_connection(self):
+ if not hasattr(self, 'connection'):
+ self.connection = frappe.get_doc('Data Migration Connector',
+ self.data_migration_connector).get_connection()
+
+ return self.connection
+
+ def push(self):
+ self.db_set('current_mapping_type', 'Push')
+ done = True
+
+ if self.current_mapping_action == 'Insert':
+ done = self._push_insert()
+
+ elif self.current_mapping_action == 'Update':
+ done = self._push_update()
+
+ elif self.current_mapping_action == 'Delete':
+ done = self._push_delete()
+
+ return done
+
+ def _push_insert(self):
+ '''Inserts new local docs on remote'''
+ mapping = self.get_mapping(self.current_mapping)
+ connection = self.get_connection()
+ data = self.get_new_local_data()
+
+ push_insert = self.get_log('push_insert', 0)
+ push_failed = self.get_log('push_failed', [])
+
+ for d in data:
+ # pre process before insert
+ doc = self.pre_process_doc(d)
+ doc = mapping.get_mapped_record(doc)
+
+ try:
+ response_doc = connection.insert(mapping.remote_objectname, doc)
+ frappe.db.set_value(mapping.local_doctype, d.name,
+ mapping.migration_id_field, response_doc[connection.name_field],
+ update_modified=False)
+ frappe.db.commit()
+ self.set_log('push_insert', push_insert + 1)
+ # post process after insert
+ self.post_process_doc(local_doc=d, remote_doc=response_doc)
+ except Exception:
+ push_failed.append(d.as_json())
+ self.set_log('push_failed', push_failed)
+
+ # update page_start
+ self.db_set('current_mapping_start',
+ self.current_mapping_start + mapping.page_length)
+
+ if len(data) < mapping.page_length:
+ # done, no more new data to insert
+ self.db_set({
+ 'current_mapping_action': 'Update',
+ 'current_mapping_start': 0
+ })
+ # not done with this mapping
+ return False
+
+ def _push_update(self):
+ '''Updates local modified docs on remote'''
+ mapping = self.get_mapping(self.current_mapping)
+ connection = self.get_connection()
+ data = self.get_updated_local_data()
+
+ push_update = self.get_log('push_update', 0)
+ push_failed = self.get_log('push_failed', [])
+
+ for d in data:
+ migration_id_value = d.get(mapping.migration_id_field)
+ # pre process before update
+ doc = self.pre_process_doc(d)
+ doc = mapping.get_mapped_record(doc)
+ try:
+ response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value)
+ self.set_log('push_update', push_update + 1)
+ # post process after update
+ self.post_process_doc(local_doc=d, remote_doc=response_doc)
+ except Exception:
+ push_failed.append(d.as_json())
+ self.set_log('push_failed', push_failed)
+
+ # update page_start
+ self.db_set('current_mapping_start',
+ self.current_mapping_start + mapping.page_length)
+
+ if len(data) < mapping.page_length:
+ # done, no more data to update
+ self.db_set({
+ 'current_mapping_action': 'Delete',
+ 'current_mapping_start': 0
+ })
+ # not done with this mapping
+ return False
+
+ def _push_delete(self):
+ '''Deletes docs deleted from local on remote'''
+ mapping = self.get_mapping(self.current_mapping)
+ connection = self.get_connection()
+ data = self.get_deleted_local_data()
+
+ push_delete = self.get_log('push_delete', 0)
+ push_failed = self.get_log('push_failed', [])
+
+ for d in data:
+ # Deleted Document also has a custom field for migration_id
+ migration_id_value = d.get(mapping.migration_id_field)
+ # pre process before update
+ self.pre_process_doc(d)
+ try:
+ response_doc = connection.delete(mapping.remote_objectname, migration_id_value)
+ self.set_log('push_delete', push_delete + 1)
+ # post process only when action is success
+ self.post_process_doc(local_doc=d, remote_doc=response_doc)
+ except Exception:
+ push_failed.append(d.as_json())
+ self.set_log('push_failed', push_failed)
+
+ # update page_start
+ self.db_set('current_mapping_start',
+ self.current_mapping_start + mapping.page_length)
+
+ if len(data) < mapping.page_length:
+ # done, no more new data to delete
+ # done with this mapping
+ return True
+
+ def pull(self):
+ self.db_set('current_mapping_type', 'Pull')
+
+ connection = self.get_connection()
+ mapping = self.get_mapping(self.current_mapping)
+ data = self.get_remote_data()
+
+ pull_insert = self.get_log('pull_insert', 0)
+ pull_update = self.get_log('pull_update', 0)
+ pull_failed = self.get_log('pull_failed', [])
+
+ for d in data:
+ migration_id_value = d[connection.name_field]
+ doc = self.pre_process_doc(d)
+ doc = mapping.get_mapped_record(doc)
+
+ if migration_id_value:
+ if not local_doc_exists(mapping, migration_id_value):
+ # insert new local doc
+ local_doc = insert_local_doc(mapping, doc)
+
+ self.set_log('pull_insert', pull_insert + 1)
+ # set migration id
+ frappe.db.set_value(mapping.local_doctype, local_doc.name,
+ mapping.migration_id_field, migration_id_value,
+ update_modified=False)
+ frappe.db.commit()
+ else:
+ # update doc
+ local_doc = update_local_doc(mapping, doc, migration_id_value)
+ self.set_log('pull_update', pull_update + 1)
+
+ if local_doc:
+ # post process doc after success
+ self.post_process_doc(remote_doc=d, local_doc=local_doc)
+ else:
+ # failed, append to log
+ pull_failed.append(d)
+ self.set_log('pull_failed', pull_failed)
+
+ if len(data) < mapping.page_length:
+ # last page, done with pull
+ return True
+
+ def pre_process_doc(self, doc):
+ plan = self.get_plan()
+ doc = plan.pre_process_doc(self.current_mapping, doc)
+ return doc
+
+ def post_process_doc(self, local_doc=None, remote_doc=None):
+ plan = self.get_plan()
+ doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc)
+ return doc
+
+ def set_log(self, key, value):
+ value = json.dumps(value) if '_failed' in key else value
+ self.db_set(key, value)
+
+ def get_log(self, key, default=None):
+ value = self.db_get(key)
+ if '_failed' in key:
+ if not value: value = json.dumps(default)
+ value = json.loads(value)
+ return value or default
+
+def insert_local_doc(mapping, doc):
+ try:
+ # insert new doc
+ if not doc.doctype:
+ doc.doctype = mapping.local_doctype
+ doc = frappe.get_doc(doc).insert()
+ return doc
+ except Exception:
+ print('Data Migration Run failed: Error in Pull insert')
+ print(frappe.get_traceback())
+ return None
+
+def update_local_doc(mapping, remote_doc, migration_id_value):
+ try:
+ # migration id value is set in migration_id_field in mapping.local_doctype
+ docname = frappe.db.get_value(mapping.local_doctype,
+ filters={ mapping.migration_id_field: migration_id_value })
+
+ doc = frappe.get_doc(mapping.local_doctype, docname)
+ doc.update(remote_doc)
+ doc.save()
+ return doc
+ except Exception:
+ print('Data Migration Run failed: Error in Pull update')
+ print(frappe.get_traceback())
+ return None
+
+def local_doc_exists(mapping, migration_id_value):
+ return frappe.db.exists(mapping.local_doctype, {
+ mapping.migration_id_field: migration_id_value
+ })
diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.js
new file mode 100644
index 0000000000..04a127f730
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.js
@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Run", function (assert) {
+ let done = assert.async();
+
+ // number of asserts
+ assert.expect(1);
+
+ frappe.run_serially([
+ // insert a new Data Migration Run
+ () => frappe.tests.make('Data Migration Run', [
+ // values to be set
+ {key: 'value'}
+ ]),
+ () => {
+ assert.equal(cur_frm.doc.key, 'value');
+ },
+ () => done()
+ ]);
+
+});
diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py
new file mode 100644
index 0000000000..189b4cc228
--- /dev/null
+++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py
@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import frappe, unittest
+
+class TestDataMigrationRun(unittest.TestCase):
+ def test_run(self):
+ create_plan()
+
+ description = 'Data migration todo'
+ new_todo = frappe.get_doc({
+ 'doctype': 'ToDo',
+ 'description': description
+ }).insert()
+
+ event_subject = 'Data migration event'
+ frappe.get_doc(dict(
+ doctype='Event',
+ subject=event_subject,
+ repeat_on='Every Month',
+ starts_on=frappe.utils.now_datetime()
+ )).insert()
+
+ run = frappe.get_doc({
+ 'doctype': 'Data Migration Run',
+ 'data_migration_plan': 'ToDo Sync',
+ 'data_migration_connector': 'Local Connector'
+ }).insert()
+
+ run.run()
+ self.assertEqual(run.db_get('status'), 'Success')
+
+ self.assertEqual(run.db_get('push_insert'), 1)
+ self.assertEqual(run.db_get('pull_insert'), 1)
+
+ todo = frappe.get_doc('ToDo', new_todo.name)
+ self.assertTrue(todo.todo_sync_id)
+
+ # Pushed Event
+ event = frappe.get_doc('Event', todo.todo_sync_id)
+ self.assertEqual(event.subject, description)
+
+ # Pulled ToDo
+ created_todo = frappe.get_doc('ToDo', {'description': event_subject})
+ self.assertEqual(created_todo.description, event_subject)
+
+ todo_list = frappe.get_list('ToDo', filters={'description': 'Data migration todo'}, fields=['name'])
+ todo_name = todo_list[0].name
+
+ todo = frappe.get_doc('ToDo', todo_name)
+ todo.description = 'Data migration todo updated'
+ todo.save()
+
+ run = frappe.get_doc({
+ 'doctype': 'Data Migration Run',
+ 'data_migration_plan': 'ToDo Sync',
+ 'data_migration_connector': 'Local Connector'
+ }).insert()
+
+ run.run()
+
+ # Update
+ self.assertEqual(run.db_get('status'), 'Success')
+ self.assertEqual(run.db_get('push_update'), 1)
+ self.assertEqual(run.db_get('pull_update'), 1)
+
+def create_plan():
+ frappe.get_doc({
+ 'doctype': 'Data Migration Mapping',
+ 'mapping_name': 'Todo to Event',
+ 'remote_objectname': 'Event',
+ 'remote_primary_key': 'name',
+ 'mapping_type': 'Push',
+ 'local_doctype': 'ToDo',
+ 'fields': [
+ { 'remote_fieldname': 'subject', 'local_fieldname': 'description' },
+ { 'remote_fieldname': 'starts_on', 'local_fieldname': 'eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())' }
+ ]
+ }).insert()
+
+ frappe.get_doc({
+ 'doctype': 'Data Migration Mapping',
+ 'mapping_name': 'Event to ToDo',
+ 'remote_objectname': 'Event',
+ 'remote_primary_key': 'name',
+ 'local_doctype': 'ToDo',
+ 'local_primary_key': 'name',
+ 'mapping_type': 'Pull',
+ 'condition': '{"subject": "Data migration event"}',
+ 'fields': [
+ { 'remote_fieldname': 'subject', 'local_fieldname': 'description' }
+ ]
+ }).insert()
+
+ frappe.get_doc({
+ 'doctype': 'Data Migration Plan',
+ 'plan_name': 'ToDo sync',
+ 'module': 'Core',
+ 'mappings': [
+ { 'mapping': 'Todo to Event' },
+ { 'mapping': 'Event to ToDo' }
+ ]
+ }).insert()
+
+ frappe.get_doc({
+ 'doctype': 'Data Migration Connector',
+ 'connector_name': 'Local Connector',
+ 'connector_type': 'Frappe',
+ 'hostname': 'http://localhost:8000',
+ 'username': 'Administrator',
+ 'password': 'admin'
+ }).insert()
diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py
index 8280f6c162..6e10c670d8 100644
--- a/frappe/desk/form/assign_to.py
+++ b/frappe/desk/form/assign_to.py
@@ -7,6 +7,7 @@ from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.desk.form.load import get_docinfo
+import frappe.share
class DuplicateToDoError(frappe.ValidationError): pass
@@ -62,6 +63,13 @@ def add(args=None):
if frappe.get_meta(args['doctype']).get_field("assigned_to"):
frappe.db.set_value(args['doctype'], args['name'], "assigned_to", args['assign_to'])
+ doc = frappe.get_doc(args['doctype'], args['name'])
+
+ # if assignee does not have permissions, share
+ if not frappe.has_permission(doc=doc, user=args['assign_to']):
+ frappe.share.add(doc.doctype, doc.name, args['assign_to'])
+ frappe.msgprint(_('Shared with user {0} with read access').format(args['assign_to']), alert=True)
+
# notify
notify_assignment(d.assigned_by, d.owner, d.reference_type, d.reference_name, action='ASSIGN',\
description=args.get("description"), notify=args.get('notify'))
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js
index 54be2ca009..f83f1c57ed 100644
--- a/frappe/desk/page/setup_wizard/setup_wizard.js
+++ b/frappe/desk/page/setup_wizard/setup_wizard.js
@@ -70,7 +70,12 @@ frappe.pages['setup-wizard'].on_page_show = function(wrapper) {
frappe.setup.on("before_load", function() {
// load slides
- frappe.setup.slides_settings.map(frappe.setup.add_slide);
+ frappe.setup.slides_settings.forEach((s) => {
+ if(!(s.name==='user' && frappe.boot.developer_mode)) {
+ // if not user slide with developer mode
+ frappe.setup.add_slide(s);
+ }
+ });
});
frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
@@ -232,6 +237,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
this.working_state_message = this.get_message(
__("Setting Up"),
__("Sit tight while your system is being setup. This may take a few moments."),
+ 'orange',
true
).appendTo(this.parent);
@@ -239,7 +245,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
this.current_slide = null;
this.completed_state_message = this.get_message(
__("Setup Complete"),
- __("You're all set!")
+ __("You're all set!"),
+ 'green'
);
}
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py
index db9f29fc80..9f7d194d22 100755
--- a/frappe/desk/page/setup_wizard/setup_wizard.py
+++ b/frappe/desk/page/setup_wizard/setup_wizard.py
@@ -82,7 +82,7 @@ def update_system_settings(args):
system_settings.save()
def update_user_name(args):
- first_name, last_name = args.get('full_name'), ''
+ first_name, last_name = args.get('full_name', ''), ''
if ' ' in first_name:
first_name, last_name = first_name.split(' ', 1)
@@ -106,7 +106,7 @@ def update_user_name(args):
frappe.flags.mute_emails = _mute_emails
update_password(args.get("email"), args.get("password"))
- else:
+ elif first_name:
args.update({
"name": frappe.session.user,
"first_name": first_name,
@@ -123,7 +123,8 @@ def update_user_name(args):
fileurl = save_file(filename, content, "User", args.get("name"), decode=True).file_url
frappe.db.set_value("User", args.get("name"), "user_image", fileurl)
- add_all_roles_to(args.get("name"))
+ if args.get('name'):
+ add_all_roles_to(args.get("name"))
def process_args(args):
if not args:
diff --git a/frappe/docs/assets/img/data-migration/add-connector-type.png b/frappe/docs/assets/img/data-migration/add-connector-type.png
new file mode 100644
index 0000000000..f24dd79bd9
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/add-connector-type.png differ
diff --git a/frappe/docs/assets/img/data-migration/atlas-connection-py.png b/frappe/docs/assets/img/data-migration/atlas-connection-py.png
new file mode 100644
index 0000000000..3c7e191aff
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/atlas-connection-py.png differ
diff --git a/frappe/docs/assets/img/data-migration/atlas-connector.png b/frappe/docs/assets/img/data-migration/atlas-connector.png
new file mode 100644
index 0000000000..74e9c03373
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/atlas-connector.png differ
diff --git a/frappe/docs/assets/img/data-migration/atlas-sync-plan.png b/frappe/docs/assets/img/data-migration/atlas-sync-plan.png
new file mode 100644
index 0000000000..104b7bdb85
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/atlas-sync-plan.png differ
diff --git a/frappe/docs/assets/img/data-migration/data-migration-run.png b/frappe/docs/assets/img/data-migration/data-migration-run.png
new file mode 100644
index 0000000000..584dd930d4
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/data-migration-run.png differ
diff --git a/frappe/docs/assets/img/data-migration/edit-connector-py.png b/frappe/docs/assets/img/data-migration/edit-connector-py.png
new file mode 100644
index 0000000000..e8d6cbe024
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/edit-connector-py.png differ
diff --git a/frappe/docs/assets/img/data-migration/mapping-init-py.png b/frappe/docs/assets/img/data-migration/mapping-init-py.png
new file mode 100644
index 0000000000..1cbe941fb9
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/mapping-init-py.png differ
diff --git a/frappe/docs/assets/img/data-migration/mapping-pre-and-post-process.png b/frappe/docs/assets/img/data-migration/mapping-pre-and-post-process.png
new file mode 100644
index 0000000000..b465bf3107
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/mapping-pre-and-post-process.png differ
diff --git a/frappe/docs/assets/img/data-migration/new-connector.png b/frappe/docs/assets/img/data-migration/new-connector.png
new file mode 100644
index 0000000000..a0b6d473ab
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/new-connector.png differ
diff --git a/frappe/docs/assets/img/data-migration/new-data-migration-mapping-fields.png b/frappe/docs/assets/img/data-migration/new-data-migration-mapping-fields.png
new file mode 100644
index 0000000000..5eaa5a1b3b
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/new-data-migration-mapping-fields.png differ
diff --git a/frappe/docs/assets/img/data-migration/new-data-migration-mapping.png b/frappe/docs/assets/img/data-migration/new-data-migration-mapping.png
new file mode 100644
index 0000000000..783bac8a50
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/new-data-migration-mapping.png differ
diff --git a/frappe/docs/assets/img/data-migration/new-data-migration-plan.png b/frappe/docs/assets/img/data-migration/new-data-migration-plan.png
new file mode 100644
index 0000000000..9a14dc9c91
Binary files /dev/null and b/frappe/docs/assets/img/data-migration/new-data-migration-plan.png differ
diff --git a/frappe/docs/user/en/guides/data/index.txt b/frappe/docs/user/en/guides/data/index.txt
index 4794a002bc..620bafa4e3 100755
--- a/frappe/docs/user/en/guides/data/index.txt
+++ b/frappe/docs/user/en/guides/data/index.txt
@@ -1 +1,2 @@
import-large-csv-file
+using-data-migration-tool
diff --git a/frappe/docs/user/en/guides/data/using-data-migration-tool.md b/frappe/docs/user/en/guides/data/using-data-migration-tool.md
new file mode 100644
index 0000000000..a7a57cb8c5
--- /dev/null
+++ b/frappe/docs/user/en/guides/data/using-data-migration-tool.md
@@ -0,0 +1,99 @@
+# Using the Data Migration Tool
+
+> Data Migration Tool was introduced in Frappé Framework version 9.
+
+The Data Migration Tool was built to abstract all the syncing of data between a remote source and a DocType. This is a middleware layer between your Frappé based website and a remote data source.
+
+To understand this tool, let's make a connector to push ERPNext Items to an imaginary service called Atlas.
+
+### Data Migration Plan
+A Data Migration Plan encapsulates a set of mappings.
+
+Let's make a new *Data Migration Plan*. Set the plan name as 'Atlas Sync'. We also need to add mappings in the mappings child table.
+
+
@@ -53,30 +52,15 @@ frappe.ui.Page = Class.extend({
${title}
${message}
-
+
+ ${primary_action}
+
`);
- $empty_state.find('.btn-primary').on('click', () => {
- primary_action.on_click();
- });
-
return $empty_state;
},
- setup_render: function() {
- var lib_exists = (typeof this.required_libs === 'string' && this.required_libs)
- || ($.isArray(this.required_libs) && this.required_libs.length);
-
- if (lib_exists) {
- this.load_lib(() => {
- this.add_main_section();
- });
- } else {
- this.add_main_section();
- }
- },
-
load_lib: function (callback) {
frappe.require(this.required_libs, callback);
},
diff --git a/frappe/public/js/frappe/ui/sidebar.js b/frappe/public/js/frappe/ui/sidebar.js
new file mode 100644
index 0000000000..9ea6323067
--- /dev/null
+++ b/frappe/public/js/frappe/ui/sidebar.js
@@ -0,0 +1,56 @@
+frappe.provide('frappe.ui');
+
+frappe.ui.Sidebar = class Sidebar {
+ constructor({ wrapper, css_class }) {
+ this.wrapper = wrapper;
+ this.css_class = css_class;
+ this.make_dom();
+ }
+
+ make_dom() {
+ this.wrapper.html(`
+
+ `);
+
+ this.$sidebar = this.wrapper.find(this.css_class);
+ }
+
+ add_item(item, section) {
+ let $section;
+ if(!section && this.wrapper.find('.sidebar-menu').length === 0) {
+ // if no section, add section with no heading
+ $section = this.get_section();
+ } else {
+ $section = this.get_section(section);
+ }
+
+ const $li_item = $(`
+ ${item.label} ${section_heading} ` : '';
+
+ $section = $(`
+
+ `);
+
+ this.$sidebar.append($section);
+ return $section;
+ }
+};
diff --git a/frappe/public/less/desktop.less b/frappe/public/less/desktop.less
index f257b4d97a..55bc98a4dd 100644
--- a/frappe/public/less/desktop.less
+++ b/frappe/public/less/desktop.less
@@ -70,7 +70,7 @@ body[data-route=""] .navbar-default, body[data-route="desktop"] .navbar-default
}
.app-icon path {
- fill: @icon-color;
+ // fill: @icon-color;
transition: 0.2s;
-webkit-transition: 0.2s;
}
@@ -94,7 +94,7 @@ body[data-route=""] .navbar-default, body[data-route="desktop"] .navbar-default
}
.app-icon:hover path {
- fill: @icon-hover;
+ // fill: @icon-hover;
}
.app-icon:hover i,
diff --git a/frappe/public/less/page.less b/frappe/public/less/page.less
index 52294214d8..b9222ec835 100644
--- a/frappe/public/less/page.less
+++ b/frappe/public/less/page.less
@@ -339,6 +339,10 @@ select.input-sm {
background-color: #f5f7fa;
}
+.page-container .page-card-container {
+ background-color: #fff;
+}
+
.page-card-container {
padding: 70px;
}
diff --git a/frappe/tests/ui/setup_wizard.js b/frappe/tests/ui/setup_wizard.js
deleted file mode 100644
index add10883c3..0000000000
--- a/frappe/tests/ui/setup_wizard.js
+++ /dev/null
@@ -1,43 +0,0 @@
-var login = require("./login.js")['Login'];
-
-module.exports = {
- before: browser => {
- browser
- .url(browser.launch_url + '/login')
- .waitForElementVisible('body', 5000);
- },
- 'Login': login,
- 'Welcome': browser => {
- let slide_selector = '[data-slide-name="welcome"]';
- browser
- .assert.title('Frappe Desk')
- .pause(5000)
- .assert.visible(slide_selector, 'Check if welcome slide is visible')
- .assert.value('select[data-fieldname="language"]', 'English')
- .click(slide_selector + ' .next-btn');
- },
- 'Region': browser => {
- let slide_selector = '[data-slide-name="region"]';
- browser
- .waitForElementVisible(slide_selector , 2000)
- .pause(6000)
- .setValue('select[data-fieldname="language"]', "India")
- .pause(4000)
- .assert.containsText('div[data-fieldname="timezone"]', 'India Time - Asia/Kolkata')
- .click(slide_selector + ' .next-btn');
- },
- 'User': browser => {
- let slide_selector = '[data-slide-name="user"]';
- browser
- .waitForElementVisible(slide_selector, 2000)
- .pause(3000)
- .setValue('input[data-fieldname="full_name"]', "John Doe")
- .setValue('input[data-fieldname="email"]', "john@example.com")
- .setValue('input[data-fieldname="password"]', "vbjwearghu")
- .click(slide_selector + ' .next-btn');
- },
-
- after: browser => {
- browser.end();
- },
-};
\ No newline at end of file
diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py
index 234ba3b929..02a841336e 100755
--- a/frappe/utils/background_jobs.py
+++ b/frappe/utils/background_jobs.py
@@ -49,6 +49,15 @@ def enqueue(method, queue='default', timeout=300, event=None,
"kwargs": kwargs
})
+def enqueue_doc(doctype, name=None, method=None, queue='default', timeout=300,
+ now=False, **kwargs):
+ '''Enqueue a method to be run on a document'''
+ enqueue('frappe.utils.background_jobs.run_doc_method', doctype=doctype, name=name,
+ doc_method=method, queue=queue, timeout=timeout, now=now, **kwargs)
+
+def run_doc_method(doctype, name, doc_method, **kwargs):
+ getattr(frappe.get_doc(doctype, name), doc_method)(**kwargs)
+
def execute_job(site, method, event, job_name, kwargs, user=None, async=True, retry=0):
'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
from frappe.utils.scheduler import log
diff --git a/requirements.txt b/requirements.txt
index 0216f85690..57fae9d6b3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -48,4 +48,5 @@ pyotp
pyqrcode
pypng
premailer
+psycopg2
+
+
+### Data Migration Mapping
+A Data Migration Mapping is a set of rules that specify field-to-field mapping.
+
+Make a new *Data Migration Mapping*. Call it 'Item to Atlas Item'.
+
+To define a mapping, we need to put in some values that define the structure of local and remote data.
+
+1. Remote Objectname: A name that identifies the remote object e.g Atlas Item
+1. Remote primary key: This is the name of the primary key for Atlas Item e.g id
+1. Local DocType: The DocType which will be used for syncing e.g Item
+1. Mapping Type: A Mapping can be of type 'Push' or 'Pull', depending on whether the data is to be mapped remotely or locally. It can also be 'Sync', which will perform both push and pull operations in a single cycle.
+1. Page Length: This defines the batch size of the sync.
+
+
+
+#### Specifying field mappings:
+
+The most basic form of a field mapping would be to specify fieldnames of the remote and local object. However, if the mapping is one-way (push or pull), the source field name can also take literal values in quotes (for e.g `"GadgetTech"`) and eval statements (for e.g `"eval:frappe.db.get_value('Company', 'Gadget Tech', 'default_currency')"`). For example, in the case of a push mapping, the local fieldname can be set to a string in quotes or an `eval` expression, instead of a field name from the local doctype. (This is not possible with a sync mapping, where both local and remote fieldnames serve as a target destination at a some point, and thus cannot be a literal value).
+
+Let's add the field mappings and save:
+
+
+
+We can now add the 'Item to Atlas Item' mapping to our Data Migration Plan and save it.
+
+
+
+#### Additional layer of control with pre and post process:
+
+Migrating data frequently involves more steps in addition to one-to-one mapping. For a Data Migration Mapping that is added to a Plan, a mapping module is generated in the module specified in that plan.
+
+In our case, an `item_to_atlas_item` module is created under the `data_migration_mapping` directory in `Integrations` (module for the 'Atlas Sync' plan).
+
+
+
+You can implement the `pre_process` (receives the source doc) and `post_process` (receives both source and target docs, as well as any additional arguments) methods, to extend the mapping process. Here's what some operations could look like:
+
+
+
+### Data Migration Connector
+Now, to connect to the remote source, we need to create a *Data Migration Connector*.
+
+
+
+We only have two connector types right now, let's add another Connector Type in the Data Migration Connector DocType.
+
+
+
+Now, let's create a new Data Migration Connector.
+
+
+
+As you can see we chose the Connector Type as Atlas. We also added the hostname, username and password for our Atlas instance so that we can authenticate.
+
+Now, we need to write the code for our connector so that we can actually push data.
+
+Create a new file called `atlas_connection.py` in `frappe/data_migration/doctype/data_migration_connector/connectors/` directory. Other connectors also live here.
+
+We just have to implement the `insert`, `update` and `delete` methods for our atlas connector. We also need to write the code to connect to our Atlas instance in the `__init__` method. Just see `frappe_connection.py` for reference.
+
+
+
+After creating the Atlas Connector, we also need to import it into `data_migration_connector.py`
+
+
+
+### Data Migration Run
+Now that we have our connector, the last thing to do is to create a new *Data Migration Run*.
+
+A Data Migration Run takes a Data Migration Plan and Data Migration Connector and execute the plan according to our configuration. It takes care of queueing, batching, delta updates and more.
+
+
+
+Just click Run. It will now push our Items to the remote Atlas instance and you can see the progress which updates in realtime.
+
+After a run is executed successfully, you cannot run it again. You will have to create another run and execute it.
+
+Data Migration Run will try to be as efficient as possible, so the next time you execute it, it will only push those items which were changed or failed in the last run.
+
+
+> Note: Data Migration Tool is still in beta. If you find any issues please report them [here](https://github.com/frappe/erpnext/issues)
+
+
\ No newline at end of file
diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py
index d619e0d6ee..e323ffb7c6 100644
--- a/frappe/frappeclient.py
+++ b/frappe/frappeclient.py
@@ -16,6 +16,7 @@ class FrappeException(Exception):
class FrappeClient(object):
def __init__(self, url, username, password, verify=True):
+ self.headers = dict(Accept='application/json')
self.verify = verify
self.session = requests.session()
self.url = url
@@ -33,7 +34,7 @@ class FrappeClient(object):
'cmd': 'login',
'usr': username,
'pwd': password
- }, verify=self.verify)
+ }, verify=self.verify, headers=self.headers)
if r.status_code==200 and r.json().get('message') == "Logged In":
return r.json()
@@ -45,7 +46,7 @@ class FrappeClient(object):
'''Logout session'''
self.session.get(self.url, params={
'cmd': 'logout',
- }, verify=self.verify)
+ }, verify=self.verify, headers=self.headers)
def get_list(self, doctype, fields='"*"', filters=None, limit_start=0, limit_page_length=0):
"""Returns list of records of a particular type"""
@@ -59,7 +60,7 @@ class FrappeClient(object):
if limit_page_length:
params["limit_start"] = limit_start
params["limit_page_length"] = limit_page_length
- res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify)
+ res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify, headers=self.headers)
return self.post_process(res)
def insert(self, doc):
@@ -67,7 +68,7 @@ class FrappeClient(object):
:param doc: A dict or Document object to be inserted remotely'''
res = self.session.post(self.url + "/api/resource/" + doc.get("doctype"),
- data={"data":frappe.as_json(doc)}, verify=self.verify)
+ data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers)
return self.post_process(res)
def insert_many(self, docs):
@@ -84,7 +85,7 @@ class FrappeClient(object):
:param doc: dict or Document object to be updated remotely. `name` is mandatory for this'''
url = self.url + "/api/resource/" + doc.get("doctype") + "/" + doc.get("name")
- res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify)
+ res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers)
return self.post_process(res)
def bulk_update(self, docs):
@@ -169,7 +170,7 @@ class FrappeClient(object):
params["fields"] = json.dumps(fields)
res = self.session.get(self.url + "/api/resource/" + doctype + "/" + name,
- params=params, verify=self.verify)
+ params=params, verify=self.verify, headers=self.headers)
return self.post_process(res)
@@ -251,21 +252,21 @@ class FrappeClient(object):
def get_api(self, method, params={}):
res = self.session.get(self.url + "/api/method/" + method + "/",
- params=params, verify=self.verify)
+ params=params, verify=self.verify, headers=self.headers)
return self.post_process(res)
def post_api(self, method, params={}):
res = self.session.post(self.url + "/api/method/" + method + "/",
- params=params, verify=self.verify)
+ params=params, verify=self.verify, headers=self.headers)
return self.post_process(res)
def get_request(self, params):
- res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify)
+ res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify, headers=self.headers)
res = self.post_process(res)
return res
def post_request(self, data):
- res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify)
+ res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify, headers=self.headers)
res = self.post_process(res)
return res
diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py
index 4a5568b238..b3d0bdc277 100644
--- a/frappe/model/base_document.py
+++ b/frappe/model/base_document.py
@@ -362,39 +362,6 @@ class BaseDocument(object):
# this is used to preserve traceback
raise frappe.UniqueValidationError(self.doctype, self.name, e)
- def db_set(self, fieldname, value=None, update_modified=True):
- '''Set a value in the document object, update the timestamp and update the database.
-
- WARNING: This method does not trigger controller validations and should
- be used very carefully.
-
- :param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
- :param value: value of the property to be updated
- :param update_modified: default True. updates the `modified` and `modified_by` properties
- '''
- if isinstance(fieldname, dict):
- self.update(fieldname)
- else:
- self.set(fieldname, value)
-
- if update_modified and (self.doctype, self.name) not in frappe.flags.currently_saving:
- # don't update modified timestamp if called from post save methods
- # like on_update or on_submit
- self.set("modified", now())
- self.set("modified_by", frappe.session.user)
-
- # to trigger email alert on value change
- self.run_method('before_change')
-
- frappe.db.set_value(self.doctype, self.name, fieldname, value,
- self.modified, self.modified_by, update_modified=update_modified)
-
- self.run_method('on_change')
-
- def db_get(self, fieldname):
- '''get database vale for this fieldname'''
- return frappe.db.get_value(self.doctype, self.name, fieldname)
-
def update_modified(self):
'''Update modified timestamp'''
self.set("modified", now())
diff --git a/frappe/model/document.py b/frappe/model/document.py
index 56a14cff8c..6d11a7607d 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -867,6 +867,46 @@ class Document(BaseDocument):
not self.meta.get("istable"):
frappe.publish_realtime("list_update", {"doctype": self.doctype}, after_commit=True)
+ def db_set(self, fieldname, value=None, update_modified=True, notify=False, commit=False):
+ '''Set a value in the document object, update the timestamp and update the database.
+
+ WARNING: This method does not trigger controller validations and should
+ be used very carefully.
+
+ :param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
+ :param value: value of the property to be updated
+ :param update_modified: default True. updates the `modified` and `modified_by` properties
+ :param notify: default False. run doc.notify_updated() to send updates via socketio
+ :param commit: default False. run frappe.db.commit()
+ '''
+ if isinstance(fieldname, dict):
+ self.update(fieldname)
+ else:
+ self.set(fieldname, value)
+
+ if update_modified and (self.doctype, self.name) not in frappe.flags.currently_saving:
+ # don't update modified timestamp if called from post save methods
+ # like on_update or on_submit
+ self.set("modified", now())
+ self.set("modified_by", frappe.session.user)
+
+ # to trigger email alert on value change
+ self.run_method('before_change')
+
+ frappe.db.set_value(self.doctype, self.name, fieldname, value,
+ self.modified, self.modified_by, update_modified=update_modified)
+
+ self.run_method('on_change')
+
+ if notify:
+ self.notify_update()
+
+ if commit:
+ frappe.db.commit()
+
+ def db_get(self, fieldname):
+ '''get database vale for this fieldname'''
+ return frappe.db.get_value(self.doctype, self.name, fieldname)
def check_no_back_links_exist(self):
"""Check if document links to any active document before Cancel."""
diff --git a/frappe/model/meta.py b/frappe/model/meta.py
index 74cce521a5..ba6508c637 100644
--- a/frappe/model/meta.py
+++ b/frappe/model/meta.py
@@ -93,6 +93,9 @@ class Meta(Document):
return self.get("fields", {"fieldtype": "Select", "options":["not in",
["[Select]", "Loading..."]]})
+ def get_image_fields(self):
+ return self.get("fields", {"fieldtype": "Attach Image"})
+
def get_table_fields(self):
if not hasattr(self, "_table_fields"):
if self.name!="DocType":
diff --git a/frappe/model/sync.py b/frappe/model/sync.py
index 9c80a946a1..de2bca903c 100644
--- a/frappe/model/sync.py
+++ b/frappe/model/sync.py
@@ -56,8 +56,10 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe
def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=False):
"""walk and sync all doctypes and pages"""
+ # load in sequence - warning for devs
document_types = ['doctype', 'page', 'report', 'print_format',
- 'website_theme', 'web_form', 'email_alert', 'print_style']
+ 'website_theme', 'web_form', 'email_alert', 'print_style',
+ 'data_migration_mapping', 'data_migration_plan']
for doctype in document_types:
doctype_path = os.path.join(start_path, doctype)
if os.path.exists(doctype_path):
diff --git a/frappe/modules.txt b/frappe/modules.txt
index 0d2e91b35f..a4ceff3d39 100644
--- a/frappe/modules.txt
+++ b/frappe/modules.txt
@@ -8,3 +8,4 @@ Desk
Integrations
Printing
Contacts
+Data Migration
\ No newline at end of file
diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py
index 2b7a6cab3b..275db10b13 100644
--- a/frappe/modules/export_file.py
+++ b/frappe/modules/export_file.py
@@ -5,7 +5,7 @@ from __future__ import unicode_literals
import frappe, os, json
import frappe.model
-from frappe.modules import scrub, get_module_path, lower_case_files_for, scrub_dt_dn
+from frappe.modules import scrub, get_module_path, scrub_dt_dn
def export_doc(doc):
export_to_files([[doc.doctype, doc.name]])
@@ -21,7 +21,7 @@ def export_to_files(record_list=None, record_module=None, verbose=0, create_init
for record in record_list:
write_document_file(frappe.get_doc(record[0], record[1]), record_module, create_init=create_init)
-def write_document_file(doc, record_module=None, create_init=None):
+def write_document_file(doc, record_module=None, create_init=True):
newdoc = doc.as_dict(no_nulls=True)
# strip out default fields from children
@@ -32,14 +32,12 @@ def write_document_file(doc, record_module=None, create_init=None):
del d[fieldname]
module = record_module or get_module_name(doc)
- if create_init is None:
- create_init = doc.doctype in lower_case_files_for
# create folder
folder = create_folder(module, doc.doctype, doc.name, create_init)
# write the data file
- fname = (doc.doctype in lower_case_files_for and scrub(doc.name)) or doc.name
+ fname = scrub(doc.name)
with open(os.path.join(folder, fname +".json"),'w+') as txtfile:
txtfile.write(frappe.as_json(newdoc))
diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py
index 9e43a28c6f..83cb863f2d 100644
--- a/frappe/modules/utils.py
+++ b/frappe/modules/utils.py
@@ -9,11 +9,6 @@ import frappe, os, json
import frappe.utils
from frappe import _
-lower_case_files_for = ['DocType', 'Page', 'Report',
- "Workflow", 'Module Def', 'Desktop Item', 'Workflow State',
- 'Workflow Action', 'Print Format', "Website Theme", 'Web Form',
- 'Email Alert', 'Print Style']
-
def export_module_json(doc, is_standard, module):
"""Make a folder for the given doc and add its json file (make it a standard
object that will be synced)"""
@@ -134,11 +129,7 @@ def scrub(txt):
def scrub_dt_dn(dt, dn):
"""Returns in lowercase and code friendly names of doctype and name for certain types"""
- ndt, ndn = dt, dn
- if dt in lower_case_files_for:
- ndt, ndn = scrub(dt), scrub(dn)
-
- return ndt, ndn
+ return scrub(dt), scrub(dn)
def get_module_path(module):
"""Returns path of the given module"""
diff --git a/frappe/public/build.json b/frappe/public/build.json
index 8d10760cf1..e8d76e5e5c 100755
--- a/frappe/public/build.json
+++ b/frappe/public/build.json
@@ -150,6 +150,7 @@
"public/js/frappe/ui/keyboard.js",
"public/js/frappe/ui/emoji.js",
"public/js/frappe/ui/colors.js",
+ "public/js/frappe/ui/sidebar.js",
"public/js/frappe/request.js",
"public/js/frappe/socketio_client.js",
@@ -269,7 +270,7 @@
"public/js/frappe/form/linked_with.js",
"public/js/frappe/form/workflow.js",
"public/js/frappe/form/print.js",
- "public/js/frappe/form/sidebar.js",
+ "public/js/frappe/form/form_sidebar.js",
"public/js/frappe/form/user_image.js",
"public/js/frappe/form/share.js",
"public/js/frappe/form/form_viewers.js",
diff --git a/frappe/public/css/desktop.css b/frappe/public/css/desktop.css
index 32bbd19c47..8b918ca8fe 100644
--- a/frappe/public/css/desktop.css
+++ b/frappe/public/css/desktop.css
@@ -59,7 +59,6 @@ body[data-route="desktop"] .navbar-default {
width: 32px;
}
.app-icon path {
- fill: #fafbfc;
transition: 0.2s;
-webkit-transition: 0.2s;
}
@@ -80,9 +79,6 @@ body[data-route="desktop"] .navbar-default {
letter-spacing: normal;
cursor: pointer;
}
-.app-icon:hover path {
- fill: #fff;
-}
.app-icon:hover i,
.app-icon:hover {
color: #fff;
diff --git a/frappe/public/css/page.css b/frappe/public/css/page.css
index 5dae59687c..d856222fbc 100644
--- a/frappe/public/css/page.css
+++ b/frappe/public/css/page.css
@@ -281,6 +281,9 @@ select.input-sm {
.setup-state {
background-color: #f5f7fa;
}
+.page-container .page-card-container {
+ background-color: #fff;
+}
.page-card-container {
padding: 70px;
}
diff --git a/frappe/public/js/frappe/form/sidebar.js b/frappe/public/js/frappe/form/form_sidebar.js
similarity index 100%
rename from frappe/public/js/frappe/form/sidebar.js
rename to frappe/public/js/frappe/form/form_sidebar.js
diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js
index bde19db1f3..1d4bfdddfc 100644
--- a/frappe/public/js/frappe/request.js
+++ b/frappe/public/js/frappe/request.js
@@ -8,10 +8,16 @@ frappe.request.url = '/';
frappe.request.ajax_count = 0;
frappe.request.waiting_for_ajax = [];
-
-
// generic server call (call page, object)
frappe.call = function(opts) {
+ if (typeof arguments[0]==='string') {
+ opts = {
+ method: arguments[0],
+ args: arguments[1],
+ callback: arguments[2]
+ }
+ }
+
if(opts.quiet) {
opts.no_spinner = true;
}
diff --git a/frappe/public/js/frappe/ui/page.js b/frappe/public/js/frappe/ui/page.js
index 659d007e6a..21fcc9e46b 100644
--- a/frappe/public/js/frappe/ui/page.js
+++ b/frappe/public/js/frappe/ui/page.js
@@ -9,7 +9,6 @@
* @param {string} opts.parent [HTMLElement] Parent element
* @param {boolean} opts.single_column Whether to include sidebar
* @param {string} [opts.title] Page title
- * @param {Object} [opts.required_libs] resources to load
* @param {Object} [opts.make_page]
*
* @returns {frappe.ui.Page}
@@ -42,10 +41,10 @@ frappe.ui.Page = Class.extend({
make: function() {
this.wrapper = $(this.parent);
- this.setup_render();
+ this.add_main_section();
},
- get_empty_state: function({title, message, primary_action}) {
+ get_empty_state: function(title, message, primary_action) {
let $empty_state = $(`