Data Migration Tool (for hub) (#4144)

* migration tool

* custom field for primary key added

* foreign key and multiple linking F_key issue resolved

* refined code

* many-to-one mapping temp fix

* added support for pre-process + cleaned up code

* [various] fixes to setup wizard for developer mode, frappe.enqueue_doc, share with assign

* Refactor data migration module

* added migration for hub

* Add "Skip errors" in data import tool

* move db_set to document.py

* Add Data Migration Run

* Dynamic Migration ID

* move run() from Mapping to Run

* Push Deleted Documents

* fixes

* [migration] doc operation counts

* insert and update instead of push in connection

* fix count and total_pages, skip sync if total_pages is 0

* [migration] child tables

* fix complete()

* [page] remove required libs

* Add sidebar.js, rename old sidebar.js to form_sidebar.js

* [minor] get_empty_state fixes

* svg in icon

* remove image check

* fix codacy

* fix is_child_table check

* [connector] add get_list()

* Add test for Data Migration Run

* fix test

* truncate tabNote

* fix test

* sync todo with event to fix test

* fix db count

* [mapping] export Mapping to json

* Add docs for Data Migration Tool

* [migration] pull data as list, test case

* [hub] remove mapping export to files

* Pull refactor

* [test]

* Add comments

* [mapping] exec in mapping formula

* fix codacy

* fix codacy

* Remove exec for pre-process and post-process

* Add pre and post process for Push

* Remove formula

* fixes

* [refactor] add failed_log to pull, handle error in pull

* [test] Push, pull, update

* Fix codacy, fix insert_doc for pull

* Set migration id on successful insert

* fix update_doc

* fix update_doc

* method is a function

* child table mapping

* Refactor logging

* fix update_doc again

* fix hostname, password

* update docs, minors

* Remove assign_if_none

* Remove error handling from connection methods

* [refactor] Data migration run

* Break push stages into methods

* Migration run refactor

- fix test
- add separate fields for logging

* fix codacy

* fix hostname password

* fix test

frappe/__init__.py

@@ -1319,6 +1319,20 @@ def enqueue(*args, **kwargs):
 	import frappe.utils.background_jobs
 	return frappe.utils.background_jobs.enqueue(*args, **kwargs)
 
+def enqueue_doc(*args, **kwargs):
+	'''
+		Enqueue method to be executed using a background worker
+
+		:param doctype: DocType of the document on which you want to run the event
+		:param name: Name of the document on which you want to run the event
+		:param method: method string or method object
+		:param queue: (optional) should be either long, default or short
+		:param timeout: (optional) should be set according to the functions
+		:param kwargs: keyword arguments to be passed to the method
+	'''
+	import frappe.utils.background_jobs
+	return frappe.utils.background_jobs.enqueue_doc(*args, **kwargs)
+
 def get_doctype_app(doctype):
 	def _get_doctype_app():
 		doctype_module = local.db.get_value("DocType", doctype, "module")
@@ -1371,4 +1385,4 @@ def get_system_settings(key):
 
 def get_active_domains():
 	from frappe.core.doctype.domain_settings.domain_settings import get_active_domains
-	return get_active_domains()
+	return get_active_domains()

frappe/app.py

@@ -128,7 +128,7 @@ def handle_exception(e):
 	http_status_code = getattr(e, "http_status_code", 500)
 	return_as_message = False
 
-	if frappe.local.is_ajax or 'application/json' in frappe.local.request.headers.get('Accept', ''):
+	if frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept'):
 		# handle ajax responses first
 		# if the request is ajax, send back the trace or error message
 		response = frappe.utils.response.report_error(http_status_code)

frappe/core/doctype/user/user.py

@@ -42,6 +42,7 @@ class User(Document):
 
 	def before_insert(self):
 		self.flags.in_insert = True
+		throttle_user_creation()
 
 	def validate(self):
 		self.check_demo()
@@ -976,4 +977,10 @@ def reset_otp_secret(user):
 		enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, async=True, job_name=None, now=False, **email_args)
 		return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login."))
 	else:
-		return frappe.throw(_("OTP secret can only be reset by the Administrator."))
+		return frappe.throw(_("OTP secret can only be reset by the Administrator."))
+
+def throttle_user_creation():
+	if frappe.flags.in_import:
+		return
+	if frappe.db.get_creation_count('User', 60) > 60:
+		frappe.throw(_('Throttled'))

frappe/core/page/data_import_tool/data_import_main.html

@@ -93,10 +93,16 @@
                             {%= __("Ignore encoding errors.") %}
                     </label>
                 </div>
+                <div class="checkbox">
+                    <label>
+                        <input type="checkbox" name="skip_errors">
+                            {%= __("Skip rows with errors.") %}
+                    </label>
+                </div>
                 <div class="checkbox">
                     <label>
                         <input type="checkbox" name="no_email" checked>
-                            {%= __("Do not send Emails.") %}
+                            {%= __("Do not send emails.") %}
                     </label>
                 </div>
                 <p>

frappe/core/page/data_import_tool/data_import_tool.js

@@ -113,6 +113,7 @@ frappe.DataImportTool = Class.extend({
 				return {
 					submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"),
 					ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
+					skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"),
 					overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
 					update_only: me.page.main.find('[name="update_only"]').prop("checked"),
 					no_email: me.page.main.find('[name="no_email"]').prop("checked"),

frappe/core/page/data_import_tool/importer.py

@@ -21,7 +21,8 @@ from six import text_type, string_types
 
 @frappe.whitelist()
 def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
-	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No"):
+	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
+	skip_errors = True):
 	"""upload data"""
 
 	frappe.flags.in_import = True
@@ -341,13 +342,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 					doc.submit()
 					log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
 		except Exception as e:
-			error = True
-			if doc:
-				frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
-			err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
-			log('Error for row (#%d) %s : %s' % (row_idx + 1,
-				len(row)>1 and row[1] or "", err_msg))
-			frappe.errprint(frappe.get_traceback())
+			if not skip_errors:
+				error = True
+				if doc:
+					frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
+				err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
+				log('Error for row (#%d) %s : %s' % (row_idx + 1,
+					len(row)>1 and row[1] or "", err_msg))
+				frappe.errprint(frappe.get_traceback())
 		finally:
 			frappe.local.message_log = []
 

frappe/data_migration/doctype/data_migration_connector/connectors/base.py

@@ -0,0 +1,8 @@
+
+
+class BaseConnection(object):
+	def pull(self):
+		pass
+
+	def push(self):
+		pass

frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py

@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+import frappe
+from frappe.frappeclient import FrappeClient
+from .base import BaseConnection
+
+class FrappeConnection(BaseConnection):
+	def __init__(self, connector):
+		self.connector = connector
+		self.connection = FrappeClient(self.connector.hostname,
+			self.connector.username, self.connector.password)
+		self.name_field = 'name'
+
+	def insert(self, doctype, doc):
+		doc = frappe._dict(doc)
+		doc.doctype = doctype
+		return self.connection.insert(doc)
+
+	def update(self, doctype, doc, migration_id):
+		doc = frappe._dict(doc)
+		doc.doctype = doctype
+		doc.name = migration_id
+		return self.connection.update(doc)
+
+	def delete(self, doctype, migration_id):
+		return self.connection.delete(doctype, migration_id)
+
+	def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20):
+		return self.connection.get_list(doctype, fields=fields, filters=filters,
+				limit_start=start, limit_page_length=page_length)

frappe/data_migration/doctype/data_migration_connector/connectors/postgres.py

@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+import frappe, psycopg2
+from .base import BaseConnection
+
+class PostGresConnection(BaseConnection):
+	def __init__(self, properties):
+		self.__dict__.update(properties)
+		self._connector = psycopg2.connect("host='{0}' dbname='{1}' user='{2}' password='{3}'".format(self.hostname,
+				self.database_name, self.username, self.password))
+		self.cursor = self._connector.cursor()
+
+	def get_objects(self, object_type, condition, selection):
+		if not condition:
+			condition = ''
+		else:
+			condition = ' WHERE ' + condition
+		self.cursor.execute('SELECT {0} FROM {1}{2}'.format(selection, object_type, condition))
+		raw_data = self.cursor.fetchall()
+		data = []
+		for r in raw_data:
+			row_dict = frappe._dict({})
+			for i, value in enumerate(r):
+				row_dict[self.cursor.description[i][0]] = value
+			data.append(row_dict)
+
+		return data
+
+	def get_join_objects(self, object_type, field, primary_key):
+		"""
+		field.formula 's first line will be list of tables that needs to be linked to fetch an item
+		The subsequent lines that follows will contain one to one mapping across tables keys
+		"""
+		condition = ""
+		key_mapping = field.formula.split('\n')
+		obj_type = key_mapping[0]
+		selection = field.source_fieldname
+
+		for d in key_mapping[1:]:
+			condition += d + ' AND '
+
+		condition += str(object_type) + ".id=" + str(primary_key)
+
+		return self.get_objects(obj_type, condition, selection)

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Connector', {
+	refresh: function() {
+
+	}
+});

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json

@@ -0,0 +1,245 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 1, 
+ "autoname": "field:connector_name", 
+ "beta": 1, 
+ "creation": "2017-08-11 05:03:27.091416", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "connector_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Connector Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "connector_type", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Connector Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Frappe\nPostgres", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "localhost", 
+   "fieldname": "hostname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Hostname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "database_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Database Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "username", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Username", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "password", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Password", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-09-07 12:40:15.008483", 
+ "modified_by": "faris@erpnext.com", 
+ "module": "Data Migration", 
+ "name": "Data Migration Connector", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+from .connectors.postgres import PostGresConnection
+from .connectors.frappe_connection import FrappeConnection
+
+class DataMigrationConnector(Document):
+	def get_connection(self):
+		if self.connector_type == 'Frappe':
+			self.connection = FrappeConnection(self)
+		elif self.connector_type == 'PostGres':
+			self.connection = PostGresConnection(self.as_dict())
+
+		return self.connection
+
+	def get_objects(self, object_type, condition=None, selection="*"):
+		return self.connector.get_objects(object_type, condition, selection)
+
+	def get_join_objects(self, object_type, join_type, primary_key):
+		return self.connector.get_join_objects(object_type, join_type, primary_key)

frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Connector", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Connector
+		() => frappe.tests.make('Data Migration Connector', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationConnector(unittest.TestCase):
+	pass

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Mapping', {
+	refresh: function() {
+
+	}
+});

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json

@@ -0,0 +1,456 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 1, 
+ "autoname": "field:mapping_name", 
+ "beta": 1, 
+ "creation": "2017-08-11 05:11:49.975801", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Mapping Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "remote_objectname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Remote Objectname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "remote_primary_key", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Remote Primary Key", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "local_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Local DocType", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "local_primary_key", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Local Primary Key", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_5", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping_type", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Mapping Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Push\nPull\nSync", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "10", 
+   "fieldname": "page_length", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Page Length", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "migration_id_field", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Migration ID Field", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Mapping", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "fields", 
+   "fieldtype": "Table", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Field Maps", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Mapping Detail", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 1, 
+   "columns": 0, 
+   "fieldname": "condition_detail", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Condition Detail", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "condition", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Condition", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-09-27 18:06:43.275207", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Mapping", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class DataMigrationMapping(Document):
+	def get_filters(self):
+		if self.condition:
+			return frappe.safe_eval(self.condition, dict(frappe=frappe))
+
+	def get_fields(self):
+		fields = []
+		for f in self.fields:
+			if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith('eval:')):
+				fields.append(f.local_fieldname)
+
+		if frappe.db.has_column(self.local_doctype, self.migration_id_field):
+			fields.append(self.migration_id_field)
+
+		if 'name' not in fields:
+			fields.append('name')
+
+		return fields
+
+	def get_mapped_record(self, doc):
+		mapped = frappe._dict()
+
+		key_fieldname = 'remote_fieldname'
+		value_fieldname = 'local_fieldname'
+
+		if self.mapping_type == 'Pull':
+			key_fieldname, value_fieldname = value_fieldname, key_fieldname
+
+		for field_map in self.fields:
+			if not field_map.is_child_table:
+				value = get_value_from_fieldname(field_map, value_fieldname, doc)
+				mapped[field_map.get(key_fieldname)] = value
+			else:
+				mapping_name = field_map.child_table_mapping
+				value = get_mapped_child_records(mapping_name, doc.get(field_map.get(value_fieldname)))
+				mapped[field_map.get(key_fieldname)] = value
+		return mapped
+
+def get_mapped_child_records(mapping_name, child_docs):
+	mapped_child_docs = []
+	mapping = frappe.get_doc('Data Migration Mapping', mapping_name)
+	for child_doc in child_docs:
+		mapped_child_docs.append(mapping.get_mapped_record(child_doc))
+
+	return mapped_child_docs
+
+def get_value_from_fieldname(field_map, fieldname_field, doc):
+	field_name = field_map.get(fieldname_field)
+
+	if field_name.startswith('eval:'):
+		value = frappe.safe_eval(field_name[5:], dict(frappe=frappe))
+	elif field_name[0] in ('"', "'"):
+		value = field_name[1:-1]
+	else:
+		value = doc.get(field_name)
+	return value

frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Mapping", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Mapping
+		() => frappe.tests.make('Data Migration Mapping', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationMapping(unittest.TestCase):
+	pass

frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json

@@ -0,0 +1,163 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2017-08-11 05:09:10.900237", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "remote_fieldname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Remote Fieldname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "local_fieldname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Local Fieldname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "is_child_table", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Is Child Table", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "is_child_table", 
+   "fieldname": "child_table_mapping", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Child Table Mapping", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Mapping", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 1, 
+ "max_attachments": 0, 
+ "modified": "2017-09-28 17:13:31.337005", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Mapping Detail", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class DataMigrationMappingDetail(Document):
+	pass

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Plan', {
+	refresh: function() {
+
+	}
+});

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json

@@ -0,0 +1,155 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "field:plan_name", 
+ "beta": 0, 
+ "creation": "2017-08-11 05:15:51.482165", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "plan_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Plan Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "module", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Module", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Module Def", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mappings", 
+   "fieldtype": "Table", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Mappings", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Plan Mapping", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-09-13 15:47:26.336541", 
+ "modified_by": "prateeksha@erpnext.com", 
+ "module": "Data Migration", 
+ "name": "Data Migration Plan", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.modules import get_module_path, scrub_dt_dn
+from frappe.modules.export_file import export_to_files, create_init_py
+from frappe.custom.doctype.custom_field.custom_field import create_custom_field
+from frappe.model.document import Document
+
+class DataMigrationPlan(Document):
+	def after_insert(self):
+		self.make_custom_fields_for_mappings()
+
+	def on_update(self):
+		if frappe.flags.in_import or frappe.flags.in_test:
+			return
+
+		if frappe.local.conf.get('developer_mode'):
+			record_list =[['Data Migration Plan', self.name]]
+
+			for m in self.mappings:
+				record_list.append(['Data Migration Mapping', m.mapping])
+
+			export_to_files(record_list=record_list, record_module=self.module)
+
+			for m in self.mappings:
+				dt, dn = scrub_dt_dn('Data Migration Mapping', m.mapping)
+				create_init_py(get_module_path(self.module), dt, dn)
+
+	def make_custom_fields_for_mappings(self):
+		label = self.name + ' ID'
+		fieldname = frappe.scrub(label)
+
+		df = {
+			'label': label,
+			'fieldname': fieldname,
+			'fieldtype': 'Data',
+			'hidden': 1,
+			'read_only': 1,
+			'unique': 1
+		}
+
+		for m in self.mappings:
+			mapping = frappe.get_doc('Data Migration Mapping', m.mapping)
+			create_custom_field(mapping.local_doctype, df)
+			mapping.migration_id_field = fieldname
+			mapping.save()
+
+		# Create custom field in Deleted Document
+		create_custom_field('Deleted Document', df)
+
+	def pre_process_doc(self, mapping_name, doc):
+		module = self.get_mapping_module(mapping_name)
+
+		if module and hasattr(module, 'pre_process'):
+			return module.pre_process(doc)
+		return doc
+
+	def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None):
+		module = self.get_mapping_module(mapping_name)
+
+		if module and hasattr(module, 'post_process'):
+			return module.post_process(local_doc=local_doc, remote_doc=remote_doc)
+
+	def get_mapping_module(self, mapping_name):
+		try:
+			module_def = frappe.get_doc("Module Def", self.module)
+			module = frappe.get_module('{app}.{module}.data_migration_mapping.{mapping_name}'.format(
+				app= module_def.app_name,
+				module=frappe.scrub(self.module),
+				mapping_name=frappe.scrub(mapping_name)
+			))
+			return module
+		except ImportError:
+			return None

frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Plan", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Plan
+		() => frappe.tests.make('Data Migration Plan', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationPlan(unittest.TestCase):
+	pass

frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json

@@ -0,0 +1,103 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 1, 
+ "creation": "2017-08-11 05:15:38.390831", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Mapping", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Mapping", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "1", 
+   "fieldname": "enabled", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Enabled", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 1, 
+ "max_attachments": 0, 
+ "modified": "2017-09-20 21:43:04.908650", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Plan Mapping", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class DataMigrationPlanMapping(Document):
+	pass

frappe/data_migration/doctype/data_migration_run/data_migration_run.js

@@ -0,0 +1,14 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Run', {
+	refresh: function(frm) {
+		if (frm.doc.status !== 'Success') {
+			frm.add_custom_button(__('Run'), () => frm.call('run'));
+		}
+		if (frm.doc.status === 'Started') {
+			frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete,
+				__('Currently updating {0}', [frm.doc.current_mapping]));
+		}
+	}
+});

frappe/data_migration/doctype/data_migration_run/data_migration_run.json

@@ -0,0 +1,671 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2017-09-11 12:55:27.597728", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "data_migration_plan", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Data Migration Plan", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Plan", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "data_migration_connector", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Data Migration Connector", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Connector", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "Pending", 
+   "fieldname": "status", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Status", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping", 
+   "fieldtype": "Data", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping_start", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Start", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping_delete_start", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Delete Start", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping_type", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Push\nPull", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:(doc.status !== 'Pending')", 
+   "fieldname": "current_mapping_action", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Action", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "Insert\nDelete", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "total_pages", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Total Pages", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "percent_complete", 
+   "fieldtype": "Percent", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Percent Complete", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:(doc.status !== 'Pending')", 
+   "fieldname": "logs_sb", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Logs", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_insert", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Insert", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_update", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Update", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_delete", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Delete", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_failed", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Failed", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_16", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "pull_insert", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Pull Insert", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "pull_update", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Pull Update", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "pull_failed", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Pull Failed", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:doc.failed_log !== '[]'", 
+   "fieldname": "log", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Log", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-10-02 05:12:16.094991", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Run", 
+ "name_case": "", 
+ "owner": "faris@erpnext.com", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_run/data_migration_run.py

@@ -0,0 +1,467 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe, json, math
+from frappe.model.document import Document
+from frappe import _
+
+class DataMigrationRun(Document):
+
+	def validate(self):
+		exists = frappe.db.exists('Data Migration Run', dict(
+			status=('in', ['Fail', 'Error']),
+			name=('!=', self.name)
+		))
+		if exists:
+			frappe.throw(_('There are failed runs with the same Data Migration Plan'))
+
+	def run(self):
+		self.begin()
+		if self.total_pages > 0:
+			self.enqueue_next_mapping()
+		else:
+			self.complete()
+
+	def enqueue_next_mapping(self):
+		next_mapping_name = self.get_next_mapping_name()
+		if next_mapping_name:
+			next_mapping = self.get_mapping(next_mapping_name)
+			self.db_set(dict(
+				current_mapping = next_mapping.name,
+				current_mapping_start = 0,
+				current_mapping_delete_start = 0,
+				current_mapping_action = 'Insert'
+			), notify=True, commit=True)
+			frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test)
+		else:
+			self.complete()
+
+	def enqueue_next_page(self):
+		mapping = self.get_mapping(self.current_mapping)
+		fields = dict(
+			percent_complete = self.percent_complete + (100.0 / self.total_pages)
+		)
+		if self.current_mapping_action == 'Insert':
+			start = self.current_mapping_start + mapping.page_length
+			fields['current_mapping_start'] = start
+		elif self.current_mapping_action == 'Delete':
+			delete_start = self.current_mapping_delete_start + mapping.page_length
+			fields['current_mapping_delete_start'] = delete_start
+
+		self.db_set(fields, notify=True, commit=True)
+		frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test)
+
+	def run_current_mapping(self):
+		try:
+			mapping = self.get_mapping(self.current_mapping)
+
+			if mapping.mapping_type == 'Push':
+				done = self.push()
+			elif mapping.mapping_type == 'Pull':
+				done = self.pull()
+
+			if done:
+				self.enqueue_next_mapping()
+			else:
+				self.enqueue_next_page()
+
+		except Exception as e:
+			self.db_set('status', 'Error', notify=True, commit=True)
+			print('Data Migration Run failed')
+			print(frappe.get_traceback())
+			raise e
+
+	def get_last_modified_condition(self):
+		last_run_timestamp = frappe.db.get_value('Data Migration Run', dict(
+			data_migration_plan=self.data_migration_plan,
+			name=('!=', self.name)
+		), 'modified')
+		if last_run_timestamp:
+			condition = dict(modified=('>', last_run_timestamp))
+		else:
+			condition = {}
+		return condition
+
+	def begin(self):
+		plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled]
+		self.mappings = [frappe.get_doc(
+			'Data Migration Mapping', m.mapping) for m in plan_active_mappings]
+
+		total_pages = 0
+		for m in [mapping for mapping in self.mappings if mapping.mapping_type == 'Push']:
+			count = float(self.get_count(m))
+			page_count = math.ceil(count / m.page_length)
+			total_pages += page_count
+
+		total_pages = 10
+
+		self.db_set(dict(
+			status = 'Started',
+			current_mapping = None,
+			current_mapping_start = 0,
+			current_mapping_delete_start = 0,
+			percent_complete = 0,
+			current_mapping_action = 'Insert',
+			total_pages = total_pages
+		), notify=True, commit=True)
+
+	def complete(self):
+		fields = dict()
+
+		push_failed = self.get_log('push_failed', [])
+		pull_failed = self.get_log('pull_failed', [])
+
+		if push_failed or pull_failed:
+			fields['status'] = 'Partial Success'
+		else:
+			fields['status'] = 'Success'
+			fields['percent_complete'] = 100
+
+		self.db_set(fields, notify=True, commit=True)
+
+	def get_plan(self):
+		if not hasattr(self, 'plan'):
+			self.plan = frappe.get_doc('Data Migration Plan', self.data_migration_plan)
+		return self.plan
+
+	def get_mapping(self, mapping_name):
+		if hasattr(self, 'mappings'):
+			for m in self.mappings:
+				if m.name == mapping_name:
+					return m
+		return frappe.get_doc('Data Migration Mapping', mapping_name)
+
+	def get_next_mapping_name(self):
+		mappings = [m for m in self.get_plan().mappings if m.enabled]
+		if not self.current_mapping:
+			# first
+			return mappings[0].mapping
+		for i, d in enumerate(mappings):
+			if i == len(mappings) - 1:
+				# last
+				return None
+			if d.mapping == self.current_mapping:
+				return mappings[i+1].mapping
+
+		raise frappe.ValidationError('Mapping Broken')
+
+	def get_data(self, filters):
+		mapping = self.get_mapping(self.current_mapping)
+		or_filters = self.get_or_filters(mapping)
+		start = self.current_mapping_start
+
+		data = []
+		doclist = frappe.get_all(mapping.local_doctype,
+			filters=filters, or_filters=or_filters,
+			start=start, page_length=mapping.page_length)
+
+		for d in doclist:
+			doc = frappe.get_doc(mapping.local_doctype, d['name'])
+			data.append(doc)
+		return data
+
+	def get_new_local_data(self):
+		'''Fetch newly inserted local data using `frappe.get_all`. Used during Push'''
+		mapping = self.get_mapping(self.current_mapping)
+		filters = mapping.get_filters() or {}
+
+		# new docs dont have migration field set
+		filters.update({
+			mapping.migration_id_field: ''
+		})
+
+		return self.get_data(filters)
+
+	def get_updated_local_data(self):
+		'''Fetch local updated data using `frappe.get_all`. Used during Push'''
+		mapping = self.get_mapping(self.current_mapping)
+		filters = mapping.get_filters() or {}
+
+		# existing docs must have migration field set
+		filters.update({
+			mapping.migration_id_field: ('!=', '')
+		})
+
+		return self.get_data(filters)
+
+	def get_deleted_local_data(self):
+		'''Fetch local deleted data using `frappe.get_all`. Used during Push'''
+		mapping = self.get_mapping(self.current_mapping)
+		or_filters = self.get_or_filters(mapping)
+		filters = dict(
+			deleted_doctype=mapping.local_doctype
+		)
+
+		data = frappe.get_all('Deleted Document', fields=['data'],
+			filters=filters, or_filters=or_filters)
+
+		_data = []
+		for d in data:
+			doc = json.loads(d.data)
+			if doc.get(mapping.migration_id_field):
+				doc['_deleted_document_name'] = d.name
+				_data.append(doc)
+
+		return _data
+
+	def get_remote_data(self):
+		'''Fetch data from remote using `connection.get`. Used during Pull'''
+		mapping = self.get_mapping(self.current_mapping)
+		start = self.current_mapping_start
+		filters = mapping.get_filters() or {}
+		connection = self.get_connection()
+
+		return connection.get(mapping.remote_objectname,
+			fields=["*"], filters=filters, start=start,
+			page_length=mapping.page_length)
+
+	def get_count(self, mapping):
+		filters = mapping.get_filters() or {}
+		or_filters = self.get_or_filters(mapping)
+
+		to_insert = frappe.get_all(mapping.local_doctype, ['count(name) as total'],
+			filters=filters, or_filters=or_filters)[0].total
+
+		to_delete = frappe.get_all('Deleted Document', ['count(name) as total'],
+			filters={'deleted_doctype': mapping.local_doctype}, or_filters=or_filters)[0].total
+
+		return to_insert + to_delete
+
+	def get_or_filters(self, mapping):
+		or_filters = self.get_last_modified_condition()
+
+		# include docs whose migration_id_field is not set
+		or_filters.update({
+			mapping.migration_id_field: ('=', '')
+		})
+
+		return or_filters
+
+	def get_connection(self):
+		if not hasattr(self, 'connection'):
+			self.connection = frappe.get_doc('Data Migration Connector',
+				self.data_migration_connector).get_connection()
+
+		return self.connection
+
+	def push(self):
+		self.db_set('current_mapping_type', 'Push')
+		done = True
+
+		if self.current_mapping_action == 'Insert':
+			done = self._push_insert()
+
+		elif self.current_mapping_action == 'Update':
+			done = self._push_update()
+
+		elif self.current_mapping_action == 'Delete':
+			done = self._push_delete()
+
+		return done
+
+	def _push_insert(self):
+		'''Inserts new local docs on remote'''
+		mapping = self.get_mapping(self.current_mapping)
+		connection = self.get_connection()
+		data = self.get_new_local_data()
+
+		push_insert = self.get_log('push_insert', 0)
+		push_failed = self.get_log('push_failed', [])
+
+		for d in data:
+			# pre process before insert
+			doc = self.pre_process_doc(d)
+			doc = mapping.get_mapped_record(doc)
+
+			try:
+				response_doc = connection.insert(mapping.remote_objectname, doc)
+				frappe.db.set_value(mapping.local_doctype, d.name,
+						mapping.migration_id_field, response_doc[connection.name_field],
+						update_modified=False)
+				frappe.db.commit()
+				self.set_log('push_insert', push_insert + 1)
+				# post process after insert
+				self.post_process_doc(local_doc=d, remote_doc=response_doc)
+			except Exception:
+				push_failed.append(d.as_json())
+				self.set_log('push_failed', push_failed)
+
+		# update page_start
+		self.db_set('current_mapping_start',
+			self.current_mapping_start + mapping.page_length)
+
+		if len(data) < mapping.page_length:
+			# done, no more new data to insert
+			self.db_set({
+				'current_mapping_action': 'Update',
+				'current_mapping_start': 0
+			})
+			# not done with this mapping
+			return False
+
+	def _push_update(self):
+		'''Updates local modified docs on remote'''
+		mapping = self.get_mapping(self.current_mapping)
+		connection = self.get_connection()
+		data = self.get_updated_local_data()
+
+		push_update = self.get_log('push_update', 0)
+		push_failed = self.get_log('push_failed', [])
+
+		for d in data:
+			migration_id_value = d.get(mapping.migration_id_field)
+			# pre process before update
+			doc = self.pre_process_doc(d)
+			doc = mapping.get_mapped_record(doc)
+			try:
+				response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value)
+				self.set_log('push_update', push_update + 1)
+				# post process after update
+				self.post_process_doc(local_doc=d, remote_doc=response_doc)
+			except Exception:
+				push_failed.append(d.as_json())
+				self.set_log('push_failed', push_failed)
+
+		# update page_start
+		self.db_set('current_mapping_start',
+			self.current_mapping_start + mapping.page_length)
+
+		if len(data) < mapping.page_length:
+			# done, no more data to update
+			self.db_set({
+				'current_mapping_action': 'Delete',
+				'current_mapping_start': 0
+			})
+			# not done with this mapping
+			return False
+
+	def _push_delete(self):
+		'''Deletes docs deleted from local on remote'''
+		mapping = self.get_mapping(self.current_mapping)
+		connection = self.get_connection()
+		data = self.get_deleted_local_data()
+
+		push_delete = self.get_log('push_delete', 0)
+		push_failed = self.get_log('push_failed', [])
+
+		for d in data:
+			# Deleted Document also has a custom field for migration_id
+			migration_id_value = d.get(mapping.migration_id_field)
+			# pre process before update
+			self.pre_process_doc(d)
+			try:
+				response_doc = connection.delete(mapping.remote_objectname, migration_id_value)
+				self.set_log('push_delete', push_delete + 1)
+				# post process only when action is success
+				self.post_process_doc(local_doc=d, remote_doc=response_doc)
+			except Exception:
+				push_failed.append(d.as_json())
+				self.set_log('push_failed', push_failed)
+
+		# update page_start
+		self.db_set('current_mapping_start',
+			self.current_mapping_start + mapping.page_length)
+
+		if len(data) < mapping.page_length:
+			# done, no more new data to delete
+			# done with this mapping
+			return True
+
+	def pull(self):
+		self.db_set('current_mapping_type', 'Pull')
+
+		connection = self.get_connection()
+		mapping = self.get_mapping(self.current_mapping)
+		data = self.get_remote_data()
+
+		pull_insert = self.get_log('pull_insert', 0)
+		pull_update = self.get_log('pull_update', 0)
+		pull_failed = self.get_log('pull_failed', [])
+
+		for d in data:
+			migration_id_value = d[connection.name_field]
+			doc = self.pre_process_doc(d)
+			doc = mapping.get_mapped_record(doc)
+
+			if migration_id_value:
+				if not local_doc_exists(mapping, migration_id_value):
+					# insert new local doc
+					local_doc = insert_local_doc(mapping, doc)
+
+					self.set_log('pull_insert', pull_insert + 1)
+					# set migration id
+					frappe.db.set_value(mapping.local_doctype, local_doc.name,
+						mapping.migration_id_field, migration_id_value,
+						update_modified=False)
+					frappe.db.commit()
+				else:
+					# update doc
+					local_doc = update_local_doc(mapping, doc, migration_id_value)
+					self.set_log('pull_update', pull_update + 1)
+
+			if local_doc:
+				# post process doc after success
+				self.post_process_doc(remote_doc=d, local_doc=local_doc)
+			else:
+				# failed, append to log
+				pull_failed.append(d)
+				self.set_log('pull_failed', pull_failed)
+
+		if len(data) < mapping.page_length:
+			# last page, done with pull
+			return True
+
+	def pre_process_doc(self, doc):
+		plan = self.get_plan()
+		doc = plan.pre_process_doc(self.current_mapping, doc)
+		return doc
+
+	def post_process_doc(self, local_doc=None, remote_doc=None):
+		plan = self.get_plan()
+		doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc)
+		return doc
+
+	def set_log(self, key, value):
+		value = json.dumps(value) if '_failed' in key else value
+		self.db_set(key, value)
+
+	def get_log(self, key, default=None):
+		value = self.db_get(key)
+		if '_failed' in key:
+			if not value: value = json.dumps(default)
+			value = json.loads(value)
+		return value or default
+
+def insert_local_doc(mapping, doc):
+	try:
+		# insert new doc
+		if not doc.doctype:
+			doc.doctype = mapping.local_doctype
+		doc = frappe.get_doc(doc).insert()
+		return doc
+	except Exception:
+		print('Data Migration Run failed: Error in Pull insert')
+		print(frappe.get_traceback())
+		return None
+
+def update_local_doc(mapping, remote_doc, migration_id_value):
+	try:
+		# migration id value is set in migration_id_field in mapping.local_doctype
+		docname = frappe.db.get_value(mapping.local_doctype,
+			filters={ mapping.migration_id_field: migration_id_value })
+
+		doc = frappe.get_doc(mapping.local_doctype, docname)
+		doc.update(remote_doc)
+		doc.save()
+		return doc
+	except Exception:
+		print('Data Migration Run failed: Error in Pull update')
+		print(frappe.get_traceback())
+		return None
+
+def local_doc_exists(mapping, migration_id_value):
+	return frappe.db.exists(mapping.local_doctype, {
+		mapping.migration_id_field: migration_id_value
+	})

frappe/data_migration/doctype/data_migration_run/test_data_migration_run.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Run", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Run
+		() => frappe.tests.make('Data Migration Run', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import frappe, unittest
+
+class TestDataMigrationRun(unittest.TestCase):
+	def test_run(self):
+		create_plan()
+
+		description = 'Data migration todo'
+		new_todo = frappe.get_doc({
+			'doctype': 'ToDo',
+			'description': description
+		}).insert()
+
+		event_subject = 'Data migration event'
+		frappe.get_doc(dict(
+			doctype='Event',
+			subject=event_subject,
+			repeat_on='Every Month',
+			starts_on=frappe.utils.now_datetime()
+		)).insert()
+
+		run = frappe.get_doc({
+			'doctype': 'Data Migration Run',
+			'data_migration_plan': 'ToDo Sync',
+			'data_migration_connector': 'Local Connector'
+		}).insert()
+
+		run.run()
+		self.assertEqual(run.db_get('status'), 'Success')
+
+		self.assertEqual(run.db_get('push_insert'), 1)
+		self.assertEqual(run.db_get('pull_insert'), 1)
+
+		todo = frappe.get_doc('ToDo', new_todo.name)
+		self.assertTrue(todo.todo_sync_id)
+
+		# Pushed Event
+		event = frappe.get_doc('Event', todo.todo_sync_id)
+		self.assertEqual(event.subject, description)
+
+		# Pulled ToDo
+		created_todo = frappe.get_doc('ToDo', {'description': event_subject})
+		self.assertEqual(created_todo.description, event_subject)
+
+		todo_list = frappe.get_list('ToDo', filters={'description': 'Data migration todo'}, fields=['name'])
+		todo_name = todo_list[0].name
+
+		todo = frappe.get_doc('ToDo', todo_name)
+		todo.description = 'Data migration todo updated'
+		todo.save()
+
+		run = frappe.get_doc({
+			'doctype': 'Data Migration Run',
+			'data_migration_plan': 'ToDo Sync',
+			'data_migration_connector': 'Local Connector'
+		}).insert()
+
+		run.run()
+
+		# Update
+		self.assertEqual(run.db_get('status'), 'Success')
+		self.assertEqual(run.db_get('push_update'), 1)
+		self.assertEqual(run.db_get('pull_update'), 1)
+
+def create_plan():
+	frappe.get_doc({
+		'doctype': 'Data Migration Mapping',
+		'mapping_name': 'Todo to Event',
+		'remote_objectname': 'Event',
+		'remote_primary_key': 'name',
+		'mapping_type': 'Push',
+		'local_doctype': 'ToDo',
+		'fields': [
+			{ 'remote_fieldname': 'subject', 'local_fieldname': 'description' },
+			{ 'remote_fieldname': 'starts_on', 'local_fieldname': 'eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())' }
+		]
+	}).insert()
+
+	frappe.get_doc({
+		'doctype': 'Data Migration Mapping',
+		'mapping_name': 'Event to ToDo',
+		'remote_objectname': 'Event',
+		'remote_primary_key': 'name',
+		'local_doctype': 'ToDo',
+		'local_primary_key': 'name',
+		'mapping_type': 'Pull',
+		'condition': '{"subject": "Data migration event"}',
+		'fields': [
+			{ 'remote_fieldname': 'subject', 'local_fieldname': 'description' }
+		]
+	}).insert()
+
+	frappe.get_doc({
+		'doctype': 'Data Migration Plan',
+		'plan_name': 'ToDo sync',
+		'module': 'Core',
+		'mappings': [
+			{ 'mapping': 'Todo to Event' },
+			{ 'mapping': 'Event to ToDo' }
+		]
+	}).insert()
+
+	frappe.get_doc({
+		'doctype': 'Data Migration Connector',
+		'connector_name': 'Local Connector',
+		'connector_type': 'Frappe',
+		'hostname': 'http://localhost:8000',
+		'username': 'Administrator',
+		'password': 'admin'
+	}).insert()

frappe/desk/form/assign_to.py

@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 from frappe.desk.form.load import get_docinfo
+import frappe.share
 
 class DuplicateToDoError(frappe.ValidationError): pass
 
@@ -62,6 +63,13 @@ def add(args=None):
 		if frappe.get_meta(args['doctype']).get_field("assigned_to"):
 			frappe.db.set_value(args['doctype'], args['name'], "assigned_to", args['assign_to'])
 
+		doc = frappe.get_doc(args['doctype'], args['name'])
+
+		# if assignee does not have permissions, share
+		if not frappe.has_permission(doc=doc, user=args['assign_to']):
+			frappe.share.add(doc.doctype, doc.name, args['assign_to'])
+			frappe.msgprint(_('Shared with user {0} with read access').format(args['assign_to']), alert=True)
+
 	# notify
 	notify_assignment(d.assigned_by, d.owner, d.reference_type, d.reference_name, action='ASSIGN',\
 			 description=args.get("description"), notify=args.get('notify'))

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -70,7 +70,12 @@ frappe.pages['setup-wizard'].on_page_show = function(wrapper) {
 
 frappe.setup.on("before_load", function() {
 	// load slides
-	frappe.setup.slides_settings.map(frappe.setup.add_slide);
+	frappe.setup.slides_settings.forEach((s) => {
+		if(!(s.name==='user' && frappe.boot.developer_mode)) {
+			// if not user slide with developer mode
+			frappe.setup.add_slide(s);
+		}
+	});
 });
 
 frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
@@ -232,6 +237,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		this.working_state_message = this.get_message(
 			__("Setting Up"),
 			__("Sit tight while your system is being setup. This may take a few moments."),
+			'orange',
 			true
 		).appendTo(this.parent);
 
@@ -239,7 +245,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		this.current_slide = null;
 		this.completed_state_message = this.get_message(
 			__("Setup Complete"),
-			__("You're all set!")
+			__("You're all set!"),
+			'green'
 		);
 	}
 

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -82,7 +82,7 @@ def update_system_settings(args):
 	system_settings.save()
 
 def update_user_name(args):
-	first_name, last_name = args.get('full_name'), ''
+	first_name, last_name = args.get('full_name', ''), ''
 	if ' ' in first_name:
 		first_name, last_name = first_name.split(' ', 1)
 
@@ -106,7 +106,7 @@ def update_user_name(args):
 		frappe.flags.mute_emails = _mute_emails
 		update_password(args.get("email"), args.get("password"))
 
-	else:
+	elif first_name:
 		args.update({
 			"name": frappe.session.user,
 			"first_name": first_name,
@@ -123,7 +123,8 @@ def update_user_name(args):
 			fileurl = save_file(filename, content, "User", args.get("name"), decode=True).file_url
 			frappe.db.set_value("User", args.get("name"), "user_image", fileurl)
 
-	add_all_roles_to(args.get("name"))
+	if args.get('name'):
+		add_all_roles_to(args.get("name"))
 
 def process_args(args):
 	if not args:

frappe/docs/user/en/guides/data/index.txt

@@ -1 +1,2 @@
 import-large-csv-file
+using-data-migration-tool

frappe/docs/user/en/guides/data/using-data-migration-tool.md

@@ -0,0 +1,99 @@
+# Using the Data Migration Tool
+
+> Data Migration Tool was introduced in Frappé Framework version 9.
+
+The Data Migration Tool was built to abstract all the syncing of data between a remote source and a DocType. This is a middleware layer between your Frappé based website and a remote data source.
+
+To understand this tool, let's make a connector to push ERPNext Items to an imaginary service called Atlas.
+
+### Data Migration Plan
+A Data Migration Plan encapsulates a set of mappings.
+
+Let's make a new *Data Migration Plan*. Set the plan name as 'Atlas Sync'. We also need to add mappings in the mappings child table.
+
+<img class="screenshot" alt="New Data Migration Plan" src="/docs/assets/img/data-migration/new-data-migration-plan.png">
+
+
+### Data Migration Mapping
+A Data Migration Mapping is a set of rules that specify field-to-field mapping.
+
+Make a new *Data Migration Mapping*. Call it 'Item to Atlas Item'.
+
+To define a mapping, we need to put in some values that define the structure of local and remote data.
+
+1. Remote Objectname: A name that identifies the remote object e.g Atlas Item
+1. Remote primary key: This is the name of the primary key for Atlas Item e.g id
+1. Local DocType: The DocType which will be used for syncing e.g Item
+1. Mapping Type: A Mapping can be of type 'Push' or 'Pull', depending on whether the data is to be mapped remotely or locally. It can also be 'Sync', which will perform both push and pull operations in a single cycle.
+1. Page Length: This defines the batch size of the sync.
+
+<img class="screenshot" alt="New Data Migration Mapping" src="/docs/assets/img/data-migration/new-data-migration-mapping.png">
+
+#### Specifying field mappings:
+
+The most basic form of a field mapping would be to specify fieldnames of the remote and local object. However, if the mapping is one-way (push or pull), the source field name can also take literal values in quotes (for e.g `"GadgetTech"`) and eval statements (for e.g `"eval:frappe.db.get_value('Company', 'Gadget Tech', 'default_currency')"`). For example, in the case of a push mapping, the local fieldname can be set to a string in quotes or an `eval` expression, instead of a field name from the local doctype. (This is not possible with a sync mapping, where both local and remote fieldnames serve as a target destination at a some point, and thus cannot be a literal value).
+
+Let's add the field mappings and save:
+
+<img class="screenshot" alt="Add fields in Data Migration Mapping" src="/docs/assets/img/data-migration/new-data-migration-mapping-fields.png">
+
+We can now add the 'Item to Atlas Item' mapping to our Data Migration Plan and save it.
+
+<img class="screenshot" alt="Save Atlas Sync Plan" src="/docs/assets/img/data-migration/atlas-sync-plan.png">
+
+#### Additional layer of control with pre and post process:
+
+Migrating data frequently involves more steps in addition to one-to-one mapping. For a Data Migration Mapping that is added to a Plan, a mapping module is generated in the module specified in that plan.
+
+In our case, an `item_to_atlas_item` module is created under the `data_migration_mapping` directory in `Integrations` (module for the 'Atlas Sync' plan).
+
+<img class="screenshot" alt="Mapping __init__.py" src="/docs/assets/img/data-migration/mapping-init-py.png">
+
+You can implement the `pre_process` (receives the source doc) and `post_process` (receives both source and target docs, as well as any additional arguments) methods, to extend the mapping process. Here's what some operations could look like:
+
+<img class="screenshot" alt="Pre and Post Process" src="/docs/assets/img/data-migration/mapping-pre-and-post-process.png">
+
+### Data Migration Connector
+Now, to connect to the remote source, we need to create a *Data Migration Connector*.
+
+<img class="screenshot" alt="New Data Migration Connector" src="/docs/assets/img/data-migration/new-connector.png">
+
+We only have two connector types right now, let's add another Connector Type in the Data Migration Connector DocType.
+
+<img class="screenshot" alt="Add Connector Type in Data Migration Connector" src="/docs/assets/img/data-migration/add-connector-type.png">
+
+Now, let's create a new Data Migration Connector.
+
+<img class="screenshot" alt="Atlas Connector" src="/docs/assets/img/data-migration/atlas-connector.png">
+
+As you can see we chose the Connector Type as Atlas. We also added the hostname, username and password for our Atlas instance so that we can authenticate.
+
+Now, we need to write the code for our connector so that we can actually push data.
+
+Create a new file called `atlas_connection.py` in `frappe/data_migration/doctype/data_migration_connector/connectors/` directory. Other connectors also live here.
+
+We just have to implement the `insert`, `update` and `delete` methods for our atlas connector. We also need to write the code to connect to our Atlas instance in the `__init__` method. Just see `frappe_connection.py` for reference.
+
+<img class="screenshot" alt="Atlas Connection file" src="/docs/assets/img/data-migration/atlas-connection-py.png">
+
+After creating the Atlas Connector, we also need to import it into `data_migration_connector.py`
+
+<img class="screenshot" alt="Edit Connector file" src="/docs/assets/img/data-migration/edit-connector-py.png">
+
+### Data Migration Run
+Now that we have our connector, the last thing to do is to create a new *Data Migration Run*.
+
+A Data Migration Run takes a Data Migration Plan and Data Migration Connector and execute the plan according to our configuration. It takes care of queueing, batching, delta updates and more.
+
+<img class="screenshot" alt="Data Migration Run" src="/docs/assets/img/data-migration/data-migration-run.png">
+
+Just click Run. It will now push our Items to the remote Atlas instance and you can see the progress which updates in realtime.
+
+After a run is executed successfully, you cannot run it again. You will have to create another run and execute it.
+
+Data Migration Run will try to be as efficient as possible, so the next time you execute it, it will only push those items which were changed or failed in the last run.
+
+
+> Note: Data Migration Tool is still in beta. If you find any issues please report them [here](https://github.com/frappe/erpnext/issues)
+
+<!-- markdown -->

frappe/frappeclient.py

@@ -16,6 +16,7 @@ class FrappeException(Exception):
 
 class FrappeClient(object):
 	def __init__(self, url, username, password, verify=True):
+		self.headers = dict(Accept='application/json')
 		self.verify = verify
 		self.session = requests.session()
 		self.url = url
@@ -33,7 +34,7 @@ class FrappeClient(object):
 			'cmd': 'login',
 			'usr': username,
 			'pwd': password
-		}, verify=self.verify)
+		}, verify=self.verify, headers=self.headers)
 
 		if r.status_code==200 and r.json().get('message') == "Logged In":
 			return r.json()
@@ -45,7 +46,7 @@ class FrappeClient(object):
 		'''Logout session'''
 		self.session.get(self.url, params={
 			'cmd': 'logout',
-		}, verify=self.verify)
+		}, verify=self.verify, headers=self.headers)
 
 	def get_list(self, doctype, fields='"*"', filters=None, limit_start=0, limit_page_length=0):
 		"""Returns list of records of a particular type"""
@@ -59,7 +60,7 @@ class FrappeClient(object):
 		if limit_page_length:
 			params["limit_start"] = limit_start
 			params["limit_page_length"] = limit_page_length
-		res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify)
+		res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def insert(self, doc):
@@ -67,7 +68,7 @@ class FrappeClient(object):
 
 		:param doc: A dict or Document object to be inserted remotely'''
 		res = self.session.post(self.url + "/api/resource/" + doc.get("doctype"),
-			data={"data":frappe.as_json(doc)}, verify=self.verify)
+			data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def insert_many(self, docs):
@@ -84,7 +85,7 @@ class FrappeClient(object):
 
 		:param doc: dict or Document object to be updated remotely. `name` is mandatory for this'''
 		url = self.url + "/api/resource/" + doc.get("doctype") + "/" + doc.get("name")
-		res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify)
+		res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def bulk_update(self, docs):
@@ -169,7 +170,7 @@ class FrappeClient(object):
 			params["fields"] = json.dumps(fields)
 
 		res = self.session.get(self.url + "/api/resource/" + doctype + "/" + name,
-			params=params, verify=self.verify)
+			params=params, verify=self.verify, headers=self.headers)
 
 		return self.post_process(res)
 
@@ -251,21 +252,21 @@ class FrappeClient(object):
 
 	def get_api(self, method, params={}):
 		res = self.session.get(self.url + "/api/method/" + method + "/",
-			params=params, verify=self.verify)
+			params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def post_api(self, method, params={}):
 		res = self.session.post(self.url + "/api/method/" + method + "/",
-			params=params, verify=self.verify)
+			params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def get_request(self, params):
-		res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify)
+		res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify, headers=self.headers)
 		res = self.post_process(res)
 		return res
 
 	def post_request(self, data):
-		res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify)
+		res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify, headers=self.headers)
 		res = self.post_process(res)
 		return res
 

frappe/model/base_document.py

@@ -362,39 +362,6 @@ class BaseDocument(object):
 		# this is used to preserve traceback
 		raise frappe.UniqueValidationError(self.doctype, self.name, e)
 
-	def db_set(self, fieldname, value=None, update_modified=True):
-		'''Set a value in the document object, update the timestamp and update the database.
-
-		WARNING: This method does not trigger controller validations and should
-		be used very carefully.
-
-		:param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
-		:param value: value of the property to be updated
-		:param update_modified: default True. updates the `modified` and `modified_by` properties
-		'''
-		if isinstance(fieldname, dict):
-			self.update(fieldname)
-		else:
-			self.set(fieldname, value)
-
-		if update_modified and (self.doctype, self.name) not in frappe.flags.currently_saving:
-			# don't update modified timestamp if called from post save methods
-			# like on_update or on_submit
-			self.set("modified", now())
-			self.set("modified_by", frappe.session.user)
-
-		# to trigger email alert on value change
-		self.run_method('before_change')
-
-		frappe.db.set_value(self.doctype, self.name, fieldname, value,
-			self.modified, self.modified_by, update_modified=update_modified)
-
-		self.run_method('on_change')
-
-	def db_get(self, fieldname):
-		'''get database vale for this fieldname'''
-		return frappe.db.get_value(self.doctype, self.name, fieldname)
-
 	def update_modified(self):
 		'''Update modified timestamp'''
 		self.set("modified", now())

frappe/model/document.py

@@ -867,6 +867,46 @@ class Document(BaseDocument):
 			not self.meta.get("istable"):
 			frappe.publish_realtime("list_update", {"doctype": self.doctype}, after_commit=True)
 
+	def db_set(self, fieldname, value=None, update_modified=True, notify=False, commit=False):
+		'''Set a value in the document object, update the timestamp and update the database.
+
+		WARNING: This method does not trigger controller validations and should
+		be used very carefully.
+
+		:param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
+		:param value: value of the property to be updated
+		:param update_modified: default True. updates the `modified` and `modified_by` properties
+		:param notify: default False. run doc.notify_updated() to send updates via socketio
+		:param commit: default False. run frappe.db.commit()
+		'''
+		if isinstance(fieldname, dict):
+			self.update(fieldname)
+		else:
+			self.set(fieldname, value)
+
+		if update_modified and (self.doctype, self.name) not in frappe.flags.currently_saving:
+			# don't update modified timestamp if called from post save methods
+			# like on_update or on_submit
+			self.set("modified", now())
+			self.set("modified_by", frappe.session.user)
+
+		# to trigger email alert on value change
+		self.run_method('before_change')
+
+		frappe.db.set_value(self.doctype, self.name, fieldname, value,
+			self.modified, self.modified_by, update_modified=update_modified)
+
+		self.run_method('on_change')
+
+		if notify:
+			self.notify_update()
+
+		if commit:
+			frappe.db.commit()
+
+	def db_get(self, fieldname):
+		'''get database vale for this fieldname'''
+		return frappe.db.get_value(self.doctype, self.name, fieldname)
 
 	def check_no_back_links_exist(self):
 		"""Check if document links to any active document before Cancel."""

frappe/model/meta.py

@@ -93,6 +93,9 @@ class Meta(Document):
 		return self.get("fields", {"fieldtype": "Select", "options":["not in",
 			["[Select]", "Loading..."]]})
 
+	def get_image_fields(self):
+		return self.get("fields", {"fieldtype": "Attach Image"})
+
 	def get_table_fields(self):
 		if not hasattr(self, "_table_fields"):
 			if self.name!="DocType":

frappe/model/sync.py

@@ -56,8 +56,10 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe
 def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=False):
 	"""walk and sync all doctypes and pages"""
 
+	# load in sequence - warning for devs
 	document_types = ['doctype', 'page', 'report', 'print_format',
-		'website_theme', 'web_form', 'email_alert', 'print_style']
+		'website_theme', 'web_form', 'email_alert', 'print_style',
+		 'data_migration_mapping', 'data_migration_plan']
 	for doctype in document_types:
 		doctype_path = os.path.join(start_path, doctype)
 		if os.path.exists(doctype_path):

frappe/modules.txt

@@ -8,3 +8,4 @@ Desk
 Integrations
 Printing
 Contacts
+Data Migration

frappe/modules/export_file.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 
 import frappe, os, json
 import frappe.model
-from frappe.modules import scrub, get_module_path, lower_case_files_for, scrub_dt_dn
+from frappe.modules import scrub, get_module_path, scrub_dt_dn
 
 def export_doc(doc):
 	export_to_files([[doc.doctype, doc.name]])
@@ -21,7 +21,7 @@ def export_to_files(record_list=None, record_module=None, verbose=0, create_init
 		for record in record_list:
 			write_document_file(frappe.get_doc(record[0], record[1]), record_module, create_init=create_init)
 
-def write_document_file(doc, record_module=None, create_init=None):
+def write_document_file(doc, record_module=None, create_init=True):
 	newdoc = doc.as_dict(no_nulls=True)
 
 	# strip out default fields from children
@@ -32,14 +32,12 @@ def write_document_file(doc, record_module=None, create_init=None):
 					del d[fieldname]
 
 	module = record_module or get_module_name(doc)
-	if create_init is None:
-		create_init = doc.doctype in lower_case_files_for
 
 	# create folder
 	folder = create_folder(module, doc.doctype, doc.name, create_init)
 
 	# write the data file
-	fname = (doc.doctype in lower_case_files_for and scrub(doc.name)) or doc.name
+	fname = scrub(doc.name)
 	with open(os.path.join(folder, fname +".json"),'w+') as txtfile:
 		txtfile.write(frappe.as_json(newdoc))
 

frappe/modules/utils.py

@@ -9,11 +9,6 @@ import frappe, os, json
 import frappe.utils
 from frappe import _
 
-lower_case_files_for = ['DocType', 'Page', 'Report',
-	"Workflow", 'Module Def', 'Desktop Item', 'Workflow State',
-	'Workflow Action', 'Print Format', "Website Theme", 'Web Form',
-	'Email Alert', 'Print Style']
-
 def export_module_json(doc, is_standard, module):
 	"""Make a folder for the given doc and add its json file (make it a standard
 		object that will be synced)"""
@@ -134,11 +129,7 @@ def scrub(txt):
 
 def scrub_dt_dn(dt, dn):
 	"""Returns in lowercase and code friendly names of doctype and name for certain types"""
-	ndt, ndn = dt, dn
-	if dt in lower_case_files_for:
-		ndt, ndn = scrub(dt), scrub(dn)
-
-	return ndt, ndn
+	return scrub(dt), scrub(dn)
 
 def get_module_path(module):
 	"""Returns path of the given module"""

frappe/public/build.json

@@ -150,6 +150,7 @@
 		"public/js/frappe/ui/keyboard.js",
 		"public/js/frappe/ui/emoji.js",
 		"public/js/frappe/ui/colors.js",
+		"public/js/frappe/ui/sidebar.js",
 
 		"public/js/frappe/request.js",
 		"public/js/frappe/socketio_client.js",
@@ -269,7 +270,7 @@
 		"public/js/frappe/form/linked_with.js",
 		"public/js/frappe/form/workflow.js",
 		"public/js/frappe/form/print.js",
-		"public/js/frappe/form/sidebar.js",
+		"public/js/frappe/form/form_sidebar.js",
 		"public/js/frappe/form/user_image.js",
 		"public/js/frappe/form/share.js",
 		"public/js/frappe/form/form_viewers.js",

frappe/public/css/desktop.css

@@ -59,7 +59,6 @@ body[data-route="desktop"] .navbar-default {
   width: 32px;
 }
 .app-icon path {
-  fill: #fafbfc;
   transition: 0.2s;
   -webkit-transition: 0.2s;
 }
@@ -80,9 +79,6 @@ body[data-route="desktop"] .navbar-default {
   letter-spacing: normal;
   cursor: pointer;
 }
-.app-icon:hover path {
-  fill: #fff;
-}
 .app-icon:hover i,
 .app-icon:hover {
   color: #fff;

frappe/public/css/page.css

@@ -281,6 +281,9 @@ select.input-sm {
 .setup-state {
   background-color: #f5f7fa;
 }
+.page-container .page-card-container {
+  background-color: #fff;
+}
 .page-card-container {
   padding: 70px;
 }

frappe/public/js/frappe/request.js

@@ -8,10 +8,16 @@ frappe.request.url = '/';
 frappe.request.ajax_count = 0;
 frappe.request.waiting_for_ajax = [];
 
-
-
 // generic server call (call page, object)
 frappe.call = function(opts) {
+	if (typeof arguments[0]==='string') {
+		opts = {
+			method: arguments[0],
+			args: arguments[1],
+			callback: arguments[2]
+		}
+	}
+
 	if(opts.quiet) {
 		opts.no_spinner = true;
 	}

frappe/public/js/frappe/ui/page.js

@@ -9,7 +9,6 @@
  * @param {string} opts.parent [HTMLElement] Parent element
  * @param {boolean} opts.single_column Whether to include sidebar
  * @param {string} [opts.title] Page title
- * @param {Object} [opts.required_libs] resources to load
  * @param {Object} [opts.make_page]
  *
  * @returns {frappe.ui.Page}
@@ -42,10 +41,10 @@ frappe.ui.Page = Class.extend({
 
 	make: function() {
 		this.wrapper = $(this.parent);
-		this.setup_render();
+		this.add_main_section();
 	},
 
-	get_empty_state: function({title, message, primary_action}) {
+	get_empty_state: function(title, message, primary_action) {
 		let $empty_state = $(`<div class="page-card-container">
 			<div class="page-card">
 				<div class="page-card-head">
@@ -53,30 +52,15 @@ frappe.ui.Page = Class.extend({
 						${title}</span>
 				</div>
 				<p>${message}</p>
-				<div><a href="/login" class="btn btn-primary btn-sm">${primary_action.label}</a></div>
+				<div>
+					<button class="btn btn-primary btn-sm">${primary_action}</button>
+				</div>
 			</div>
 		</div>`);
 
-		$empty_state.find('.btn-primary').on('click', () => {
-			primary_action.on_click();
-		});
-
 		return $empty_state;
 	},
 
-	setup_render: function() {
-		var lib_exists = (typeof this.required_libs === 'string' && this.required_libs)
-			|| ($.isArray(this.required_libs) && this.required_libs.length);
-
-		if (lib_exists) {
-			this.load_lib(() => {
-				this.add_main_section();
-			});
-		} else {
-			this.add_main_section();
-		}
-	},
-
 	load_lib: function (callback) {
 		frappe.require(this.required_libs, callback);
 	},

frappe/public/js/frappe/ui/sidebar.js

@@ -0,0 +1,56 @@
+frappe.provide('frappe.ui');
+
+frappe.ui.Sidebar = class Sidebar {
+	constructor({ wrapper, css_class }) {
+		this.wrapper = wrapper;
+		this.css_class = css_class;
+		this.make_dom();
+	}
+
+	make_dom() {
+		this.wrapper.html(`
+			<div class="${this.css_class} overlay-sidebar hidden-xs hidden-sm">
+			</div>
+		`);
+
+		this.$sidebar = this.wrapper.find(this.css_class);
+	}
+
+	add_item(item, section) {
+		let $section;
+		if(!section && this.wrapper.find('.sidebar-menu').length === 0) {
+			// if no section, add section with no heading
+			$section = this.get_section();
+		} else {
+			$section = this.get_section(section);
+		}
+
+		const $li_item = $(`
+			<li><a ${item.href ? `href="${item.href}"` : ''}>${item.label}</a></li>
+		`).click(
+			() => item.on_click && item.on_click()
+		);
+
+		$section.append($li_item);
+	}
+
+	get_section(section_heading="") {
+		let $section = $(this.wrapper.find(
+			`[data-section-heading="${section_heading}"]`));
+		if($section.length) {
+			return $section;
+		}
+
+		const $section_heading = section_heading ?
+			`<li class="h6">${section_heading}</li>` : '';
+
+		$section = $(`
+			<ul class="list-unstyled sidebar-menu" data-section-heading="${section_heading || 'default'}">
+				${$section_heading}
+			</ul>
+		`);
+
+		this.$sidebar.append($section);
+		return $section;
+	}
+};

frappe/public/less/desktop.less

@@ -70,7 +70,7 @@ body[data-route=""] .navbar-default, body[data-route="desktop"] .navbar-default
 }
 
 .app-icon path {
-	fill: @icon-color;
+	// fill: @icon-color;
 	transition: 0.2s;
 	-webkit-transition: 0.2s;
 }
@@ -94,7 +94,7 @@ body[data-route=""] .navbar-default, body[data-route="desktop"] .navbar-default
 }
 
 .app-icon:hover path {
-	fill: @icon-hover;
+	// fill: @icon-hover;
 }
 
 .app-icon:hover i,

frappe/public/less/page.less

@@ -339,6 +339,10 @@ select.input-sm {
 	background-color: #f5f7fa;
 }
 
+.page-container .page-card-container {
+	background-color: #fff;
+}
+
 .page-card-container {
 	padding: 70px;
 }

frappe/tests/ui/setup_wizard.js

@@ -1,43 +0,0 @@
-var login = require("./login.js")['Login'];
-
-module.exports = {
-	before: browser => {
-		browser
-			.url(browser.launch_url + '/login')
-			.waitForElementVisible('body', 5000);
-	},
-	'Login': login,
-	'Welcome': browser => {
-		let slide_selector = '[data-slide-name="welcome"]';
-		browser
-			.assert.title('Frappe Desk')
-			.pause(5000)
-			.assert.visible(slide_selector, 'Check if welcome slide is visible')
-			.assert.value('select[data-fieldname="language"]', 'English')
-			.click(slide_selector + ' .next-btn');
-	},
-	'Region': browser => {
-		let slide_selector = '[data-slide-name="region"]';
-		browser
-			.waitForElementVisible(slide_selector , 2000)
-			.pause(6000)
-			.setValue('select[data-fieldname="language"]', "India")
-			.pause(4000)
-			.assert.containsText('div[data-fieldname="timezone"]', 'India Time - Asia/Kolkata')
-			.click(slide_selector + ' .next-btn');
-	},
-	'User': browser => {
-		let slide_selector = '[data-slide-name="user"]';
-		browser
-			.waitForElementVisible(slide_selector, 2000)
-			.pause(3000)
-			.setValue('input[data-fieldname="full_name"]', "John Doe")
-			.setValue('input[data-fieldname="email"]', "john@example.com")
-			.setValue('input[data-fieldname="password"]', "vbjwearghu")
-			.click(slide_selector + ' .next-btn');
-	},
-
-	after: browser => {
-		browser.end();
-	},
-};

frappe/utils/background_jobs.py

@@ -49,6 +49,15 @@ def enqueue(method, queue='default', timeout=300, event=None,
 			"kwargs": kwargs
 		})
 
+def enqueue_doc(doctype, name=None, method=None, queue='default', timeout=300,
+	now=False, **kwargs):
+	'''Enqueue a method to be run on a document'''
+	enqueue('frappe.utils.background_jobs.run_doc_method', doctype=doctype, name=name,
+		doc_method=method, queue=queue, timeout=timeout, now=now, **kwargs)
+
+def run_doc_method(doctype, name, doc_method, **kwargs):
+	getattr(frappe.get_doc(doctype, name), doc_method)(**kwargs)
+
 def execute_job(site, method, event, job_name, kwargs, user=None, async=True, retry=0):
 	'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
 	from frappe.utils.scheduler import log

requirements.txt

@@ -48,4 +48,5 @@ pyotp
 pyqrcode
 pypng
 premailer
+psycopg2