Merge branch 'staging'

.eslintrc

@@ -119,7 +119,6 @@
 		"getCookies": true,
 		"get_url_arg": true,
 		"QUnit": true,
-		"Snap": true,
-		"mina": true
+		"JsBarcode": true
 	}
 }

.travis.yml

@@ -21,6 +21,7 @@ install:
   - sudo apt-get purge -y mysql-common mysql-server mysql-client
   - nvm install v7.10.0
   - wget https://raw.githubusercontent.com/frappe/bench/master/playbooks/install.py
+    
   - sudo python install.py --develop --user travis --without-bench-setup
   - sudo pip install -e ~/bench
 
@@ -42,6 +43,7 @@ before_script:
   - echo "USE mysql;\nCREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe';\nFLUSH PRIVILEGES;\n" | mysql -u root -ptravis
   - echo "USE mysql;\nGRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost';\n" | mysql -u root -ptravis
 
+
   - cd ~/frappe-bench
   - bench use test_site
   - bench reinstall --yes

frappe/__init__.py

@@ -6,7 +6,7 @@ globals attached to frappe module
 """
 from __future__ import unicode_literals, print_function
 
-from six import iteritems, text_type, string_types
+from six import iteritems, binary_type, text_type, string_types
 from werkzeug.local import Local, release_local
 import os, sys, importlib, inspect, json
 
@@ -14,7 +14,7 @@ import os, sys, importlib, inspect, json
 from .exceptions import *
 from .utils.jinja import get_jenv, get_template, render_template, get_email_from_template
 
-__version__ = '9.1.11'
+__version__ = '9.2.0'
 __title__ = "Frappe Framework"
 
 local = Local()
@@ -61,7 +61,7 @@ def as_unicode(text, encoding='utf-8'):
 		return text
 	elif text==None:
 		return ''
-	elif isinstance(text, string_types):
+	elif isinstance(text, binary_type):
 		return text_type(text, encoding)
 	else:
 		return text_type(text)
@@ -475,13 +475,26 @@ def only_for(roles):
 	if not roles.intersection(myroles):
 		raise PermissionError
 
+def get_domain_data(module):
+	try:
+		domain_data = get_hooks('domains')
+		if module in domain_data:
+			return _dict(get_attr(get_hooks('domains')[module][0] + '.data'))
+		else:
+			return _dict()
+	except ImportError:
+		if local.flags.in_test:
+			return _dict()
+		else:
+			raise
+
+
 def clear_cache(user=None, doctype=None):
 	"""Clear **User**, **DocType** or global cache.
 
 	:param user: If user is given, only user cache is cleared.
 	:param doctype: If doctype is given, only DocType cache is cleared."""
 	import frappe.sessions
-	from frappe.core.doctype.domain_settings.domain_settings import clear_domain_cache
 	if doctype:
 		import frappe.model.meta
 		frappe.model.meta.clear_cache(doctype)
@@ -493,7 +506,6 @@ def clear_cache(user=None, doctype=None):
 		frappe.sessions.clear_cache()
 		translate.clear_cache()
 		reset_metadata_version()
-		clear_domain_cache()
 		local.cache = {}
 		local.new_doc_templates = {}
 
@@ -1319,6 +1331,20 @@ def enqueue(*args, **kwargs):
 	import frappe.utils.background_jobs
 	return frappe.utils.background_jobs.enqueue(*args, **kwargs)
 
+def enqueue_doc(*args, **kwargs):
+	'''
+		Enqueue method to be executed using a background worker
+
+		:param doctype: DocType of the document on which you want to run the event
+		:param name: Name of the document on which you want to run the event
+		:param method: method string or method object
+		:param queue: (optional) should be either long, default or short
+		:param timeout: (optional) should be set according to the functions
+		:param kwargs: keyword arguments to be passed to the method
+	'''
+	import frappe.utils.background_jobs
+	return frappe.utils.background_jobs.enqueue_doc(*args, **kwargs)
+
 def get_doctype_app(doctype):
 	def _get_doctype_app():
 		doctype_module = local.db.get_value("DocType", doctype, "module")
@@ -1371,4 +1397,4 @@ def get_system_settings(key):
 
 def get_active_domains():
 	from frappe.core.doctype.domain_settings.domain_settings import get_active_domains
-	return get_active_domains()
+	return get_active_domains()

frappe/app.py

@@ -128,7 +128,7 @@ def handle_exception(e):
 	http_status_code = getattr(e, "http_status_code", 500)
 	return_as_message = False
 
-	if frappe.local.is_ajax or 'application/json' in frappe.local.request.headers.get('Accept', ''):
+	if frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept'):
 		# handle ajax responses first
 		# if the request is ajax, send back the trace or error message
 		response = frappe.utils.response.report_error(http_status_code)

frappe/build.js

@@ -60,11 +60,11 @@ function watch() {
 				io.emit('reload_css', filename);
 			}
 		});
-		watch_js(function (filename) {
-			if(socket_connection) {
-				io.emit('reload_js', filename);
-			}
-		});
+		// watch_js(function (filename) {
+		// 	if(socket_connection) {
+		// 		io.emit('reload_js', filename);
+		// 	}
+		// });
 		watch_build_json();
 	});
 

frappe/config/integrations.py

@@ -32,6 +32,11 @@ def get_data():
 					"name": "Dropbox Settings",
 					"description": _("Dropbox backup settings"),
 				},
+				{
+					"type": "doctype",
+					"name": "S3 Backup Settings",
+					"description": _("S3 Backup Settings"),
+				},
 			]
 		},
 		{

frappe/core/doctype/communication/comment.py

@@ -81,7 +81,17 @@ def notify_mentions(doc):
 			return
 
 		sender_fullname = get_fullname(frappe.session.user)
-		parent_doc_label = "{0} {1}".format(_(doc.reference_doctype), doc.reference_name)
+		title_field = frappe.get_meta(doc.reference_doctype).get_title_field()
+		title = doc.reference_name if title_field == "name" else \
+			frappe.db.get_value(doc.reference_doctype, doc.reference_name, title_field)
+
+		if title != doc.reference_name:
+			parent_doc_label = "{0}: {1} (#{2})".format(_(doc.reference_doctype),
+				title, doc.reference_name)
+		else:
+			parent_doc_label = "{0}: {1}".format(_(doc.reference_doctype),
+				doc.reference_name)
+
 		subject = _("{0} mentioned you in a comment").format(sender_fullname)
 
 		recipients = [frappe.db.get_value("User", {"enabled": 1, "username": username, "user_type": "System User"})

frappe/core/doctype/communication/communication.py

@@ -9,7 +9,7 @@ from frappe.utils import validate_email_add, get_fullname, strip_html, cstr
 from frappe.core.doctype.communication.comment import (notify_mentions,
 	update_comment_in_doc, on_trash)
 from frappe.core.doctype.communication.email import (validate_email,
-	notify, _notify, update_parent_status)
+	notify, _notify, update_parent_mins_to_first_response)
 from frappe.utils.bot import BotReply
 from frappe.utils import parse_addr
 
@@ -95,7 +95,7 @@ class Communication(Document):
 	def on_update(self):
 		"""Update parent status as `Open` or `Replied`."""
 		if self.comment_type != 'Updated':
-			update_parent_status(self)
+			update_parent_mins_to_first_response(self)
 			update_comment_in_doc(self)
 			self.bot_reply()
 

frappe/core/doctype/communication/email.py

@@ -164,32 +164,30 @@ def _notify(doc, print_html=None, print_format=None, attachments=None,
 		is_notification=True if doc.sent_or_received =="Received" else False
 	)
 
-def update_parent_status(doc):
-	"""Update status of parent document based on who is replying."""
+def update_parent_mins_to_first_response(doc):
+	"""Update mins_to_first_communication of parent document based on who is replying."""
 	parent = doc.get_parent_doc()
 	if not parent:
 		return
 
-	# update parent status only if we create the Email communication
+	# update parent mins_to_first_communication only if we create the Email communication
 	# ignore in case of only Comment is added
 	if doc.communication_type == "Comment":
 		return
 
 	status_field = parent.meta.get_field("status")
-
 	if status_field:
 		options = (status_field.options or '').splitlines()
 
-		# if status has a "Replied" option, then update the status
-		if 'Replied' in options:
-			to_status = "Open" if doc.sent_or_received=="Received" else "Replied"
-
-			if to_status in options:
-				parent.db_set("status", to_status)
+		# if status has a "Replied" option, then update the status for received communication
+		if ('Replied' in options) and doc.sent_or_received=="Received":
+			parent.db_set("status", "Open")
+		else:
+			# update the modified date for document
+			parent.update_modified()
 
 	update_mins_to_first_communication(parent, doc)
 	parent.run_method('notify_communication', doc)
-
 	parent.notify_update()
 
 def get_recipients_and_cc(doc, recipients, cc, fetched_from_email_account=False):

frappe/core/doctype/docfield/docfield.json

@@ -96,7 +96,7 @@
    "no_copy": 0, 
    "oldfieldname": "fieldtype", 
    "oldfieldtype": "Select", 
-   "options": "Attach\nAttach Image\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nHeading\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nText\nText Editor\nTime\nSignature", 
+   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nHeading\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nText\nText Editor\nTime\nSignature", 
    "permlevel": 0, 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
@@ -1364,7 +1364,7 @@
  "issingle": 0, 
  "istable": 1, 
  "max_attachments": 0, 
- "modified": "2017-08-29 15:30:55.489568", 
+ "modified": "2017-10-07 19:20:15.888708", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "DocField", 

frappe/core/doctype/doctype/doctype.py

@@ -321,7 +321,7 @@ class DocType(Document):
 	def export_doc(self):
 		"""Export to standard folder `[module]/doctype/[name]/[name].json`."""
 		from frappe.modules.export_file import export_to_files
-		export_to_files(record_list=[['DocType', self.name]])
+		export_to_files(record_list=[['DocType', self.name]], create_init=True)
 
 	def import_doc(self):
 		"""Import from standard folder `[module]/doctype/[name]/[name].json`."""

frappe/core/doctype/domain/domain.py

@@ -4,7 +4,89 @@
 
 from __future__ import unicode_literals
 import frappe
+
 from frappe.model.document import Document
+from frappe.custom.doctype.custom_field.custom_field import create_custom_fields
 
 class Domain(Document):
-	pass
+	'''Domain documents are created automatically when DocTypes
+	with "Restricted" domains are imported during
+	installation or migration'''
+	def setup_domain(self):
+		'''Setup domain icons, permissions, custom fields etc.'''
+		self.setup_data()
+		self.setup_roles()
+		self.setup_properties()
+		self.set_values()
+		if not int(frappe.db.get_single_value('System Settings', 'setup_complete') or 0):
+			# if setup not complete, setup desktop etc.
+			self.setup_sidebar_items()
+			self.setup_desktop_icons()
+			self.set_default_portal_role()
+
+		if self.data.custom_fields:
+			create_custom_fields(self.data.custom_fields)
+
+		if self.data.on_setup:
+			# custom on_setup method
+			frappe.get_attr(self.data.on_setup)()
+
+
+	def setup_roles(self):
+		'''Enable roles that are restricted to this domain'''
+		if self.data.restricted_roles:
+			for role_name in self.data.restricted_roles:
+				role = frappe.get_doc('Role', role_name)
+				role.disabled = 0
+				role.save()
+
+	def setup_data(self, domain=None):
+		'''Load domain info via hooks'''
+		self.data = frappe.get_domain_data(self.name)
+
+	def get_domain_data(self, module):
+		return frappe.get_attr(frappe.get_hooks('domains')[self.name] + '.data')
+
+	def set_default_portal_role(self):
+		'''Set default portal role based on domain'''
+		if self.data.get('default_portal_role'):
+			frappe.db.set_value('Portal Settings', None, 'default_role',
+				self.data.get('default_portal_role'))
+
+	def setup_desktop_icons(self):
+		'''set desktop icons form `data.desktop_icons`'''
+		from frappe.desk.doctype.desktop_icon.desktop_icon import set_desktop_icons
+		if self.data.desktop_icons:
+			set_desktop_icons(self.data.desktop_icons)
+
+	def setup_properties(self):
+		if self.data.properties:
+			for args in self.data.properties:
+				frappe.make_property_setter(args)
+
+
+	def set_values(self):
+		'''set values based on `data.set_value`'''
+		if self.data.set_value:
+			for args in self.data.set_value:
+				doc = frappe.get_doc(args[0], args[1] or args[0])
+				doc.set(args[2], args[3])
+				doc.save()
+
+	def setup_sidebar_items(self):
+		'''Enable / disable sidebar items'''
+		if self.data.allow_sidebar_items:
+			# disable all
+			frappe.db.sql('update `tabPortal Menu Item` set enabled=0')
+
+			# enable
+			frappe.db.sql('''update `tabPortal Menu Item` set enabled=1
+				where route in ({0})'''.format(', '.join(['"{0}"'.format(d) for d in self.data.allow_sidebar_items])))
+
+		if self.data.remove_sidebar_items:
+			# disable all
+			frappe.db.sql('update `tabPortal Menu Item` set enabled=1')
+
+			# enable
+			frappe.db.sql('''update `tabPortal Menu Item` set enabled=0
+				where route in ({0})'''.format(', '.join(['"{0}"'.format(d) for d in self.data.remove_sidebar_items])))

frappe/core/doctype/domain_settings/domain_settings.py

@@ -7,8 +7,46 @@ import frappe
 from frappe.model.document import Document
 
 class DomainSettings(Document):
+	def set_active_domains(self, domains):
+		self.active_domains = []
+		for d in domains:
+			self.append('active_domains', dict(domain=d))
+		self.save()
+
 	def on_update(self):
-		clear_domain_cache()
+		for d in self.active_domains:
+			domain = frappe.get_doc('Domain', d.domain)
+			domain.setup_domain()
+
+		self.restrict_roles_and_modules()
+		frappe.clear_cache()
+
+	def restrict_roles_and_modules(self):
+		'''Disable all restricted roles and set `restrict_to_domain` property in Module Def'''
+		active_domains = frappe.get_active_domains()
+		all_domains = (frappe.get_hooks('domains') or {}).keys()
+
+		def remove_role(role):
+			frappe.db.sql('delete from `tabHas Role` where role=%s', role)
+			frappe.set_value('Role', role, 'disabled', 1)
+
+		for domain in all_domains:
+			data = frappe.get_domain_data(domain)
+			if not frappe.db.get_value('Domain', domain):
+				frappe.get_doc(dict(doctype='Domain', domain=domain)).insert()
+			if 'modules' in data:
+				for module in data.get('modules'):
+					frappe.db.set_value('Module Def', module, 'restrict_to_domain', domain)
+
+			if 'restricted_roles' in data:
+				for role in data['restricted_roles']:
+					if not frappe.db.get_value('Role', role):
+						frappe.get_doc(dict(doctype='Role', role_name=role)).insert()
+					frappe.db.set_value('Role', role, 'restrict_to_domain', domain)
+
+					if domain not in active_domains:
+						remove_role(role)
+
 
 def get_active_domains():
 	""" get the domains set in the Domain Settings as active domain """
@@ -33,6 +71,3 @@ def get_active_modules():
 		return active_modules
 
 	return frappe.cache().get_value('active_modules', _get_active_modules)
-
-def clear_domain_cache():
-	frappe.cache().delete_key(['active_domains', 'active_modules'])

frappe/core/doctype/feedback_trigger/test_feedback_trigger.py

@@ -72,6 +72,7 @@ class TestFeedbackTrigger(unittest.TestCase):
 		}).insert(ignore_permissions=True)
 
 		# check if feedback mail alert is triggered
+		todo.reload()
 		todo.status = "Closed"
 		todo.save(ignore_permissions=True)
 
@@ -112,6 +113,7 @@ class TestFeedbackTrigger(unittest.TestCase):
 			reference_doctype="ToDo", reference_name=todo.name, feedback="Thank You !!", rating=4, fullname="Test User")
 
 		# auto feedback request should trigger only once
+		todo.reload()
 		todo.save(ignore_permissions=True)
 		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where
 			reference_doctype='ToDo' and reference_name='{0}'""".format(todo.name))
@@ -125,11 +127,10 @@ class TestFeedbackTrigger(unittest.TestCase):
 			"communication_type": "Feedback"
 		})
 		self.assertFalse(communications)
-		
+
 		feedback_requests = frappe.get_all("Feedback Request", {
 			"reference_doctype": "ToDo",
 			"reference_name": todo.name,
 			"is_feedback_submitted": 0
 		})
 		self.assertFalse(feedback_requests)
-		

frappe/core/doctype/report/report.py

@@ -84,7 +84,7 @@ class Report(Document):
 
 		if self.is_standard == 'Yes' and (frappe.local.conf.get('developer_mode') or 0) == 1:
 			export_to_files(record_list=[['Report', self.name]],
-				record_module=self.module)
+				record_module=self.module, create_init=True)
 
 			self.create_report_py()
 

frappe/core/doctype/sms_parameter/sms_parameter.json

@@ -71,6 +71,36 @@
    "set_only_once": 0, 
    "unique": 0, 
    "width": "150px"
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "header", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Header", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
   }
  ], 
  "has_web_view": 0, 
@@ -83,8 +113,8 @@
  "issingle": 0, 
  "istable": 1, 
  "max_attachments": 0, 
- "modified": "2017-07-22 22:52:53.309396", 
- "modified_by": "chude.osiegbu@manqala.com", 
+ "modified": "2017-10-13 16:48:00.518463", 
+ "modified_by": "Administrator", 
  "module": "Core", 
  "name": "SMS Parameter", 
  "owner": "Administrator", 

frappe/core/doctype/sms_settings/sms_settings.json

@@ -159,6 +159,36 @@
    "search_index": 0, 
    "set_only_once": 0, 
    "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "use_post", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Use POST", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
   }
  ], 
  "has_web_view": 0, 

frappe/core/doctype/sms_settings/sms_settings.py

@@ -65,13 +65,17 @@ def send_sms(receiver_list, msg, sender_name = '', success_msg = True):
 def send_via_gateway(arg):
 	ss = frappe.get_doc('SMS Settings', 'SMS Settings')
 	args = {ss.message_parameter: arg.get('message')}
+	headers={'Accept': "text/plain, text/html, */*"}
 	for d in ss.get("parameters"):
+		if d.header == 1:
+			headers.update({d.parameter: d.value})
+			continue
 		args[d.parameter] = d.value
 
 	success_list = []
 	for d in arg.get('receiver_list'):
 		args[ss.receiver_parameter] = d
-		status = send_request(ss.sms_gateway_url, args)
+		status = send_request(ss.sms_gateway_url, headers, args, ss.use_post)
 
 		if 200 <= status < 300:
 			success_list.append(d)
@@ -83,9 +87,12 @@ def send_via_gateway(arg):
 			frappe.msgprint(_("SMS sent to following numbers: {0}").format("\n" + "\n".join(success_list)))
 
 
-def send_request(gateway_url, params):
+def send_request(gateway_url, headers, params, use_post=False):
 	import requests
-	response = requests.get(gateway_url, params = params, headers={'Accept': "text/plain, text/html, */*"})
+	if use_post:
+		response = requests.post(gateway_url, headers=headers, data=params)
+	else:
+		response = requests.get(gateway_url, headers=headers, params=params)
 	response.raise_for_status()
 	return response.status_code
 

frappe/core/doctype/system_settings/system_settings.json

@@ -160,37 +160,37 @@
    "unique": 0
   }, 
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fieldname": "is_first_startup",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Is First Startup",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "is_first_startup", 
+   "fieldtype": "Check", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Is First Startup", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
    "unique": 0
-  },
+  }, 
   {
-   "allow_bulk_edit": 0,
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1019,40 +1019,40 @@
    "unique": 0
   }, 
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "default": "0",
-   "depends_on": "enable_two_factor_auth",
-   "fieldname": "bypass_2fa_for_retricted_ip_users",
-   "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Bypass Two Factor Auth for users who login from restricted IP Address",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "enable_two_factor_auth", 
+   "fieldname": "bypass_2fa_for_retricted_ip_users", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Bypass Two Factor Auth for users who login from restricted IP Address", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
    "unique": 0
-  },
+  }, 
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
@@ -1268,6 +1268,36 @@
    "search_index": 0, 
    "set_only_once": 0, 
    "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "hide_footer_in_auto_email_reports", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Hide footer in auto email reports", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
   }
  ], 
  "has_web_view": 0, 
@@ -1281,8 +1311,8 @@
  "issingle": 1, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-09-13 13:26:11.045262", 
- "modified_by": "shri@zerodha.com", 
+ "modified": "2017-10-15 20:29:46.700707", 
+ "modified_by": "Administrator", 
  "module": "Core", 
  "name": "System Settings", 
  "name_case": "", 

frappe/core/doctype/system_settings/system_settings.py

@@ -14,7 +14,7 @@ from frappe.twofactor import toggle_two_factor_auth
 class SystemSettings(Document):
 	def validate(self):
 		enable_password_policy = cint(self.enable_password_policy) and True or False
-		minimum_password_score = cint(self.minimum_password_score) or 0
+		minimum_password_score = cint(getattr(self, 'minimum_password_score', 0)) or 0
 		if enable_password_policy and minimum_password_score <= 0:
 			frappe.throw(_("Please select Minimum Password Score"))
 		elif not enable_password_policy:

frappe/core/doctype/system_settings/test_system_settings.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import frappe
+import unittest
+
+class TestSystemSettings(unittest.TestCase):
+	pass

frappe/core/doctype/user/user.py

@@ -42,6 +42,7 @@ class User(Document):
 
 	def before_insert(self):
 		self.flags.in_insert = True
+		throttle_user_creation()
 
 	def validate(self):
 		self.check_demo()
@@ -976,4 +977,10 @@ def reset_otp_secret(user):
 		enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, async=True, job_name=None, now=False, **email_args)
 		return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login."))
 	else:
-		return frappe.throw(_("OTP secret can only be reset by the Administrator."))
+		return frappe.throw(_("OTP secret can only be reset by the Administrator."))
+
+def throttle_user_creation():
+	if frappe.flags.in_import:
+		return
+	if frappe.db.get_creation_count('User', 60) > 60:
+		frappe.throw(_('Throttled'))

frappe/core/page/data_import_tool/data_import_main.html

@@ -93,10 +93,16 @@
                             {%= __("Ignore encoding errors.") %}
                     </label>
                 </div>
+                <div class="checkbox">
+                    <label>
+                        <input type="checkbox" name="skip_errors">
+                            {%= __("Skip rows with errors.") %}
+                    </label>
+                </div>
                 <div class="checkbox">
                     <label>
                         <input type="checkbox" name="no_email" checked>
-                            {%= __("Do not send Emails.") %}
+                            {%= __("Do not send emails.") %}
                     </label>
                 </div>
                 <p>

frappe/core/page/data_import_tool/data_import_tool.js

@@ -114,6 +114,7 @@ frappe.DataImportTool = Class.extend({
 				return {
 					submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"),
 					ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
+					skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"),
 					overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
 					update_only: me.page.main.find('[name="update_only"]').prop("checked"),
 					no_email: me.page.main.find('[name="no_email"]').prop("checked"),

frappe/core/page/data_import_tool/importer.py

@@ -21,7 +21,8 @@ from six import text_type, string_types
 
 @frappe.whitelist()
 def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
-	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No"):
+	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
+	skip_errors = True):
 	"""upload data"""
 
 	frappe.flags.in_import = True
@@ -341,13 +342,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 					doc.submit()
 					log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
 		except Exception as e:
-			error = True
-			if doc:
-				frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
-			err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
-			log('Error for row (#%d) %s : %s' % (row_idx + 1,
-				len(row)>1 and row[1] or "", err_msg))
-			frappe.errprint(frappe.get_traceback())
+			if not skip_errors:
+				error = True
+				if doc:
+					frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
+				err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
+				log('Error for row (#%d) %s : %s' % (row_idx + 1,
+					len(row)>1 and row[1] or "", err_msg))
+				frappe.errprint(frappe.get_traceback())
 		finally:
 			frappe.local.message_log = []
 

frappe/core/page/usage_info/usage_info.html

@@ -1,114 +1,75 @@
-<div class="padding" style="max-width: 800px;">
+<div>
 	{% if limits.expiry %}
-		<h3>{{ __("Expires in {0} days", [days_to_expiry]) }}</h3>
-		{{ __("Renew before: {0}", [expires_on]) }}
-		<br><br>
+	<div class="upgrade-message padding" style="border-bottom: 1px solid #d0d8dc;">
+		<h4>{{ __("You have {0} days left in your trial", [days_to_expiry]) }}</h4>
+
+		{% if limits.upgrade_url %}
+		<p>Upgrade to a premium plan with more users, storage and priority support.</p>
+		<button class="btn btn-primary btn-sm primary-action">Upgrade</button>
+		{% endif %}
+	</div>
 	{% endif %}
 
 	{% if limits.users %}
 	    {% var users_percent = ((enabled_users / limits.users) * 100); %}
-	    <h3>{{ __("Users") }}</h3>
+	<div class="usage-info-section" style="margin: 30px;">
+		<h4>{{ __("Users") }}</h4>
 
-	    <div class="progress">
-	        <div class="progress-bar progress-bar-{%= (users_percent < 75 ? "success" : "warning") %}" style="width: {{ users_percent }}%">
-	        </div>
-	    </div>
+		<div class="progress" style="margin-bottom: 0;">
+			<div class="progress-bar progress-bar-{%= (users_percent < 75 ? "success" : "warning") %}" style="width: {{ users_percent }}%">
+			</div>
+		</div>
 
-	    <table class="table table-bordered">
-	        <thead>
-	            <tr>
-	                <th style="width: 33%">{{ __("Current Users") }}</th>
-	                <th style="width: 33%">{{ __("Max Users") }}</th>
-	                <th style="width: 33%">{{ __("Remaining") }}</th>
-	            </tr>
-	        </thead>
-	        <tbody>
-	            <tr>
-	                <td>{%= enabled_users %}</td>
-	                <td>{%= limits.users %}</td>
-	                <td class="{%= users_percent < 75 ? "" : "text-warning" %}">{%= limits.users - enabled_users %}</td>
-	            </tr>
-	        </tbody>
-	    </table>
-	    <br>
+		<p>{%= enabled_users %} out of {%= limits.users %} enabled</p>
+	</div>
 	{% endif %}
 
 	{% if limits.emails %}
-	    <h3>{{ __("Emails sent this month") }}</h3>
+	<div class="usage-info-section" style="margin: 30px;">
+	    <h4>{{ __("Emails") }}</h4>
 
 	    {% var email_percent = (( emails_sent / limits.emails ) * 100); %}
 	    {% var emails_remaining = (limits.emails - emails_sent) %}
 
-	    <div class="progress">
+	    <div class="progress" style="margin-bottom: 0;">
 	        <div class="progress-bar progress-bar-{%= (email_percent < 75 ? "success" : "warning") %}" style="width: {{ email_percent }}%">
 	        </div>
 	    </div>
 
-	    <table class="table table-bordered">
-	        <thead>
-	            <tr>
-	                <th style="width: 33%">{{ __("Emails Sent") }}</th>
-	                <th style="width: 33%">{{ __("Max Emails") }}</th>
-	                <th style="width: 33%">{{ __("Remaining") }}</th>
-	            </tr>
-	        </thead>
-	        <tbody>
-	            <tr>
-	                <td>{%= emails_sent %}</td>
-	                <td>{%= limits.emails %}</td>
-	                <td class="{%= (email_percent < 75) ? "" : "text-warning" %}">{%= emails_remaining %}</td>
-	            </tr>
-	        </tbody>
-	    </table>
-	    <br>
+		<p>{%= emails_sent %} out of {%= limits.emails %} sent this month</p>
+	</div>
 	{% endif %}
 
 	{% if limits.space %}
-	    <h3>{{ __("Space usage") }}</h3>
+	<div class="usage-info-section" style="margin: 30px;">
+	    <h4>{{ __("Space") }}</h4>
 
 	    {% var database_percent = ((limits.space_usage.database_size / limits.space) * 100); %}
 	    {% var files_percent = ((limits.space_usage.files_size / limits.space) * 100); %}
 	    {% var backup_percent = ((limits.space_usage.backup_size / limits.space) * 100); %}
 
-	    <div class="progress">
-	        <div class="progress-bar progress-bar-success" style="width: {%= database_percent %}%">
-	        </div>
-	        <div class="progress-bar progress-bar-info" style="width: {%= files_percent %}%">
-	        </div>
-	        <div class="progress-bar progress-bar-warning" style="width: {%= backup_percent %}%">
-	        </div>
+	    <div class="progress" style="margin-bottom: 0;">
+	        <div class="progress-bar" style="width: {%= database_percent %}%; background-color: #5e64ff"></div>
+	        <div class="progress-bar" style="width: {%= files_percent %}%; background-color: #743ee2"></div>
+	        <div class="progress-bar" style="width: {%= backup_percent %}%; background-color: #7CD6FD"></div>
 	    </div>
 
-	    <table class="table table-bordered">
-	        <thead>
-	            <tr>
-	                <th style="width: 50%">{{ __("Type") }} </th>
-	                <th style="width: 50%">{{ __("Size (MB)") }}</th>
-	            </tr>
-	        </thead>
-	        <tbody>
-	            <tr>
-	                <td><span class="indicator-right green">{{ __("Database Size") }}</span></td>
-	                <td>{%= limits.space_usage.database_size %} MB</td>
-	            </tr>
-	            <tr>
-	                <td><span class="indicator-right purple">{{ __("Files Size") }}</span></td>
-	                <td>{%= limits.space_usage.files_size %} MB</td>
-	            </tr>
-	            <tr>
-	                <td><span class="indicator-right orange">{{ __("Backup Size") }}</span></td>
-	                <td>{%= limits.space_usage.backup_size %} MB</td>
-	            </tr>
-	            <tr>
-	                <td><b>{{ __("Total") }}</b></td>
-	                <td><b>{%= limits.space_usage.total %} MB</b></td>
-	            </tr>
-	            <tr>
-	                <td><b>{{ __("Remaining") }}</b></td>
-	                <td class="{%= ((limits.space - limits.space_usage.total) > 50) ? "" : "text-warning" %}">
-	                    <b>{%= flt(limits.space - limits.space_usage.total, 2) %} MB</b></td>
-	            </tr>
-	        </tbody>
-	    </table>
+		<span class="indicator blue" style="margin-right: 20px;">
+			{{ __("Database Size:") }} {%= limits.space_usage.files_size %} MB
+		</span>
+		<span class="indicator purple" style="margin-right: 20px;">
+			{{ __("Files Size:") }} {%= limits.space_usage.files_size %} MB
+		</span>
+		<span class="indicator lightblue" style="margin-right: 20px;">
+			{{ __("Backup Size:") }} {%= limits.space_usage.backup_size %} MB
+		</span>
+
+		<p>
+			<span class="{%= ((limits.space - limits.space_usage.total) > 50) ? "" : "text-warning" %}">
+	                    <b>{%= flt(limits.space - limits.space_usage.total, 2) %} MB</b></span>
+			available out of
+			<span><b>{%= limits.space %} MB</b></span>
+		</p>
+	</div>
 	{% endif %}
 </div>

frappe/core/page/usage_info/usage_info.js

@@ -18,12 +18,9 @@ frappe.pages['usage-info'].on_page_load = function(wrapper) {
 			$(frappe.render_template("usage_info", usage_info)).appendTo(page.main);
 
 			var btn_text = usage_info.limits.users == 1 ? __("Upgrade") : __("Renew / Upgrade");
-
-			if(usage_info.upgrade_url) {
-				page.set_primary_action(btn_text, function() {
-					window.open(usage_info.upgrade_url);
-				});
-			}
+			$(page.main).find('.btn-primary').html(btn_text).on('click', () => {
+				window.open(usage_info.upgrade_url);
+			});
 		}
 	});
 

frappe/custom/doctype/custom_field/custom_field.py

@@ -111,6 +111,10 @@ def create_custom_fields(custom_fields):
 
 	:param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`'''
 	for doctype, fields in custom_fields.items():
+		if isinstance(fields, dict):
+			# only one field
+			fields = [fields]
+
 		for df in fields:
 			field = frappe.db.get_value("Custom Field", {"dt": doctype, "fieldname": df["fieldname"]})
 			if not field:

frappe/custom/doctype/customize_form/customize_form.py

@@ -66,7 +66,7 @@ docfield_properties = {
 
 allowed_fieldtype_change = (('Currency', 'Float', 'Percent'), ('Small Text', 'Data'),
 	('Text', 'Data'), ('Text', 'Text Editor', 'Code', 'Signature'), ('Data', 'Select'),
-	('Text', 'Small Text'))
+	('Text', 'Small Text'), ('Text', 'Data', 'Barcode'))
 
 allowed_fieldtype_for_options_change = ('Read Only', 'HTML', 'Select',)
 

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -94,7 +94,7 @@
    "no_copy": 0, 
    "oldfieldname": "fieldtype", 
    "oldfieldtype": "Select", 
-   "options": "Attach\nAttach Image\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nHeading\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nText\nText Editor\nTime", 
+   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nHeading\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nText\nText Editor\nTime", 
    "permlevel": 0, 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
@@ -1202,7 +1202,7 @@
  "issingle": 0, 
  "istable": 1, 
  "max_attachments": 0, 
- "modified": "2017-07-06 17:24:03.665171", 
+ "modified": "2017-10-11 06:45:20.172291", 
  "modified_by": "Administrator", 
  "module": "Custom", 
  "name": "Customize Form Field", 

frappe/data_migration/doctype/data_migration_connector/connectors/base.py

@@ -0,0 +1,24 @@
+from six import with_metaclass
+from abc import ABCMeta, abstractmethod
+from frappe.utils.password import get_decrypted_password
+
+class BaseConnection(with_metaclass(ABCMeta)):
+
+	@abstractmethod
+	def get(self):
+		pass
+
+	@abstractmethod
+	def insert(self):
+		pass
+
+	@abstractmethod
+	def update(self):
+		pass
+
+	@abstractmethod
+	def delete(self):
+		pass
+
+	def get_password(self):
+		return get_decrypted_password('Data Migration Connector', self.connector.name)

frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py

@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+import frappe
+from frappe.frappeclient import FrappeClient
+from .base import BaseConnection
+
+class FrappeConnection(BaseConnection):
+	def __init__(self, connector):
+		self.connector = connector
+		self.connection = FrappeClient(self.connector.hostname,
+			self.connector.username, self.get_password())
+		self.name_field = 'name'
+
+	def insert(self, doctype, doc):
+		doc = frappe._dict(doc)
+		doc.doctype = doctype
+		return self.connection.insert(doc)
+
+	def update(self, doctype, doc, migration_id):
+		doc = frappe._dict(doc)
+		doc.doctype = doctype
+		doc.name = migration_id
+		return self.connection.update(doc)
+
+	def delete(self, doctype, migration_id):
+		return self.connection.delete(doctype, migration_id)
+
+	def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20):
+		return self.connection.get_list(doctype, fields=fields, filters=filters,
+				limit_start=start, limit_page_length=page_length)

frappe/data_migration/doctype/data_migration_connector/connectors/postgres.py

@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+import frappe, psycopg2
+from .base import BaseConnection
+
+class PostGresConnection(BaseConnection):
+	def __init__(self, properties):
+		self.__dict__.update(properties)
+		self._connector = psycopg2.connect("host='{0}' dbname='{1}' user='{2}' password='{3}'".format(self.hostname,
+				self.database_name, self.username, self.password))
+		self.cursor = self._connector.cursor()
+
+	def get_objects(self, object_type, condition, selection):
+		if not condition:
+			condition = ''
+		else:
+			condition = ' WHERE ' + condition
+		self.cursor.execute('SELECT {0} FROM {1}{2}'.format(selection, object_type, condition))
+		raw_data = self.cursor.fetchall()
+		data = []
+		for r in raw_data:
+			row_dict = frappe._dict({})
+			for i, value in enumerate(r):
+				row_dict[self.cursor.description[i][0]] = value
+			data.append(row_dict)
+
+		return data
+
+	def get_join_objects(self, object_type, field, primary_key):
+		"""
+		field.formula 's first line will be list of tables that needs to be linked to fetch an item
+		The subsequent lines that follows will contain one to one mapping across tables keys
+		"""
+		condition = ""
+		key_mapping = field.formula.split('\n')
+		obj_type = key_mapping[0]
+		selection = field.source_fieldname
+
+		for d in key_mapping[1:]:
+			condition += d + ' AND '
+
+		condition += str(object_type) + ".id=" + str(primary_key)
+
+		return self.get_objects(obj_type, condition, selection)

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Connector', {
+	refresh: function() {
+
+	}
+});

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json

@@ -0,0 +1,275 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 1, 
+ "autoname": "field:connector_name", 
+ "beta": 1, 
+ "creation": "2017-08-11 05:03:27.091416", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "connector_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Connector Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "connector_type", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Connector Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Frappe\nPostgres", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "python_module", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Python Module", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "localhost", 
+   "fieldname": "hostname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Hostname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "database_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Database Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "username", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Username", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "password", 
+   "fieldtype": "Password", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Password", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-10-08 14:34:30.603690", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Connector", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+from frappe import _
+from .connectors.postgres import PostGresConnection
+from .connectors.frappe_connection import FrappeConnection
+
+class DataMigrationConnector(Document):
+	def validate(self):
+		if not (self.python_module or self.connector_type):
+			frappe.throw(_('Enter python module or select connector type'))
+
+		if self.python_module:
+			try:
+				frappe.get_module(self.python_module)
+			except:
+				frappe.throw(frappe._('Invalid module path'))
+
+	def get_connection(self):
+		if self.python_module:
+			module = frappe.get_module(self.python_module)
+			return module.get_connection(self)
+		else:
+			if self.connector_type == 'Frappe':
+				self.connection = FrappeConnection(self)
+			elif self.connector_type == 'PostGres':
+				self.connection = PostGresConnection(self.as_dict())
+
+		return self.connection
+
+	def get_objects(self, object_type, condition=None, selection="*"):
+		return self.connector.get_objects(object_type, condition, selection)
+
+	def get_join_objects(self, object_type, join_type, primary_key):
+		return self.connector.get_join_objects(object_type, join_type, primary_key)

frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Connector", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Connector
+		() => frappe.tests.make('Data Migration Connector', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationConnector(unittest.TestCase):
+	pass

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Mapping', {
+	refresh: function() {
+
+	}
+});

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json

@@ -0,0 +1,456 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 1, 
+ "autoname": "field:mapping_name", 
+ "beta": 1, 
+ "creation": "2017-08-11 05:11:49.975801", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Mapping Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "remote_objectname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Remote Objectname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "remote_primary_key", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Remote Primary Key", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "local_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Local DocType", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "local_primary_key", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Local Primary Key", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_5", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping_type", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Mapping Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Push\nPull\nSync", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "10", 
+   "fieldname": "page_length", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Page Length", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "migration_id_field", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Migration ID Field", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Mapping", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "fields", 
+   "fieldtype": "Table", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Field Maps", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Mapping Detail", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 1, 
+   "columns": 0, 
+   "fieldname": "condition_detail", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Condition Detail", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "condition", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Condition", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-09-27 18:06:43.275207", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Mapping", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class DataMigrationMapping(Document):
+	def get_filters(self):
+		if self.condition:
+			return frappe.safe_eval(self.condition, dict(frappe=frappe))
+
+	def get_fields(self):
+		fields = []
+		for f in self.fields:
+			if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith('eval:')):
+				fields.append(f.local_fieldname)
+
+		if frappe.db.has_column(self.local_doctype, self.migration_id_field):
+			fields.append(self.migration_id_field)
+
+		if 'name' not in fields:
+			fields.append('name')
+
+		return fields
+
+	def get_mapped_record(self, doc):
+		mapped = frappe._dict()
+
+		key_fieldname = 'remote_fieldname'
+		value_fieldname = 'local_fieldname'
+
+		if self.mapping_type == 'Pull':
+			key_fieldname, value_fieldname = value_fieldname, key_fieldname
+
+		for field_map in self.fields:
+			if not field_map.is_child_table:
+				value = get_value_from_fieldname(field_map, value_fieldname, doc)
+				mapped[field_map.get(key_fieldname)] = value
+			else:
+				mapping_name = field_map.child_table_mapping
+				value = get_mapped_child_records(mapping_name, doc.get(field_map.get(value_fieldname)))
+				mapped[field_map.get(key_fieldname)] = value
+		return mapped
+
+def get_mapped_child_records(mapping_name, child_docs):
+	mapped_child_docs = []
+	mapping = frappe.get_doc('Data Migration Mapping', mapping_name)
+	for child_doc in child_docs:
+		mapped_child_docs.append(mapping.get_mapped_record(child_doc))
+
+	return mapped_child_docs
+
+def get_value_from_fieldname(field_map, fieldname_field, doc):
+	field_name = field_map.get(fieldname_field)
+
+	if field_name.startswith('eval:'):
+		value = frappe.safe_eval(field_name[5:], dict(frappe=frappe))
+	elif field_name[0] in ('"', "'"):
+		value = field_name[1:-1]
+	else:
+		value = doc.get(field_name)
+	return value

frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Mapping", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Mapping
+		() => frappe.tests.make('Data Migration Mapping', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationMapping(unittest.TestCase):
+	pass

frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json

@@ -0,0 +1,163 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2017-08-11 05:09:10.900237", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "remote_fieldname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Remote Fieldname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "local_fieldname", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Local Fieldname", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "is_child_table", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Is Child Table", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "is_child_table", 
+   "fieldname": "child_table_mapping", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Child Table Mapping", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Mapping", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 1, 
+ "max_attachments": 0, 
+ "modified": "2017-09-28 17:13:31.337005", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Mapping Detail", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class DataMigrationMappingDetail(Document):
+	pass

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Plan', {
+	refresh: function() {
+
+	}
+});

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json

@@ -0,0 +1,155 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "field:plan_name", 
+ "beta": 0, 
+ "creation": "2017-08-11 05:15:51.482165", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "plan_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Plan Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "module", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Module", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Module Def", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mappings", 
+   "fieldtype": "Table", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Mappings", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Plan Mapping", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-09-13 15:47:26.336541", 
+ "modified_by": "prateeksha@erpnext.com", 
+ "module": "Data Migration", 
+ "name": "Data Migration Plan", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.modules import get_module_path, scrub_dt_dn
+from frappe.modules.export_file import export_to_files, create_init_py
+from frappe.custom.doctype.custom_field.custom_field import create_custom_field
+from frappe.model.document import Document
+
+class DataMigrationPlan(Document):
+
+	def on_update(self):
+		# update custom fields in mappings
+		self.make_custom_fields_for_mappings()
+
+		if frappe.flags.in_import or frappe.flags.in_test:
+			return
+
+		if frappe.local.conf.get('developer_mode'):
+			record_list =[['Data Migration Plan', self.name]]
+
+			for m in self.mappings:
+				record_list.append(['Data Migration Mapping', m.mapping])
+
+			export_to_files(record_list=record_list, record_module=self.module)
+
+			for m in self.mappings:
+				dt, dn = scrub_dt_dn('Data Migration Mapping', m.mapping)
+				create_init_py(get_module_path(self.module), dt, dn)
+
+	def make_custom_fields_for_mappings(self):
+		label = self.name + ' ID'
+		fieldname = frappe.scrub(label)
+
+		df = {
+			'label': label,
+			'fieldname': fieldname,
+			'fieldtype': 'Data',
+			'hidden': 1,
+			'read_only': 1,
+			'unique': 1
+		}
+
+		for m in self.mappings:
+			mapping = frappe.get_doc('Data Migration Mapping', m.mapping)
+			create_custom_field(mapping.local_doctype, df)
+			mapping.migration_id_field = fieldname
+			mapping.save()
+
+		# Create custom field in Deleted Document
+		create_custom_field('Deleted Document', df)
+
+	def pre_process_doc(self, mapping_name, doc):
+		module = self.get_mapping_module(mapping_name)
+
+		if module and hasattr(module, 'pre_process'):
+			return module.pre_process(doc)
+		return doc
+
+	def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None):
+		module = self.get_mapping_module(mapping_name)
+
+		if module and hasattr(module, 'post_process'):
+			return module.post_process(local_doc=local_doc, remote_doc=remote_doc)
+
+	def get_mapping_module(self, mapping_name):
+		try:
+			module_def = frappe.get_doc("Module Def", self.module)
+			module = frappe.get_module('{app}.{module}.data_migration_mapping.{mapping_name}'.format(
+				app= module_def.app_name,
+				module=frappe.scrub(self.module),
+				mapping_name=frappe.scrub(mapping_name)
+			))
+			return module
+		except ImportError:
+			return None

frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Plan", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Plan
+		() => frappe.tests.make('Data Migration Plan', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import unittest
+
+class TestDataMigrationPlan(unittest.TestCase):
+	pass

frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json

@@ -0,0 +1,103 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 1, 
+ "creation": "2017-08-11 05:15:38.390831", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "mapping", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Mapping", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Mapping", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "1", 
+   "fieldname": "enabled", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Enabled", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 1, 
+ "max_attachments": 0, 
+ "modified": "2017-09-20 21:43:04.908650", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Plan Mapping", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class DataMigrationPlanMapping(Document):
+	pass

frappe/data_migration/doctype/data_migration_run/data_migration_run.js

@@ -0,0 +1,14 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Migration Run', {
+	refresh: function(frm) {
+		if (frm.doc.status !== 'Success') {
+			frm.add_custom_button(__('Run'), () => frm.call('run'));
+		}
+		if (frm.doc.status === 'Started') {
+			frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete,
+				__('Currently updating {0}', [frm.doc.current_mapping]));
+		}
+	}
+});

frappe/data_migration/doctype/data_migration_run/data_migration_run.json

@@ -0,0 +1,671 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2017-09-11 12:55:27.597728", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "data_migration_plan", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Data Migration Plan", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Plan", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "data_migration_connector", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Data Migration Connector", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Data Migration Connector", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "Pending", 
+   "fieldname": "status", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Status", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping", 
+   "fieldtype": "Data", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping_start", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Start", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping_delete_start", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Delete Start", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "current_mapping_type", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Push\nPull", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:(doc.status !== 'Pending')", 
+   "fieldname": "current_mapping_action", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Current Mapping Action", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "Insert\nDelete", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "total_pages", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Total Pages", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "percent_complete", 
+   "fieldtype": "Percent", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Percent Complete", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:(doc.status !== 'Pending')", 
+   "fieldname": "logs_sb", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Logs", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_insert", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Insert", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_update", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Update", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_delete", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Delete", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "push_failed", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Push Failed", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_16", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "pull_insert", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Pull Insert", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "pull_update", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Pull Update", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "pull_failed", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Pull Failed", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:doc.failed_log !== '[]'", 
+   "fieldname": "log", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Log", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-10-02 05:12:16.094991", 
+ "modified_by": "Administrator", 
+ "module": "Data Migration", 
+ "name": "Data Migration Run", 
+ "name_case": "", 
+ "owner": "faris@erpnext.com", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/data_migration/doctype/data_migration_run/data_migration_run.py

@@ -0,0 +1,476 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe, json, math
+from frappe.model.document import Document
+from frappe import _
+
+class DataMigrationRun(Document):
+
+	def validate(self):
+		exists = frappe.db.exists('Data Migration Run', dict(
+			status=('in', ['Fail', 'Error']),
+			name=('!=', self.name)
+		))
+		if exists:
+			frappe.throw(_('There are failed runs with the same Data Migration Plan'))
+
+	def run(self):
+		self.begin()
+		if self.total_pages > 0:
+			self.enqueue_next_mapping()
+		else:
+			self.complete()
+
+	def enqueue_next_mapping(self):
+		next_mapping_name = self.get_next_mapping_name()
+		if next_mapping_name:
+			next_mapping = self.get_mapping(next_mapping_name)
+			self.db_set(dict(
+				current_mapping = next_mapping.name,
+				current_mapping_start = 0,
+				current_mapping_delete_start = 0,
+				current_mapping_action = 'Insert'
+			), notify=True, commit=True)
+			frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test)
+		else:
+			self.complete()
+
+	def enqueue_next_page(self):
+		mapping = self.get_mapping(self.current_mapping)
+		fields = dict(
+			percent_complete = self.percent_complete + (100.0 / self.total_pages)
+		)
+		if self.current_mapping_action == 'Insert':
+			start = self.current_mapping_start + mapping.page_length
+			fields['current_mapping_start'] = start
+		elif self.current_mapping_action == 'Delete':
+			delete_start = self.current_mapping_delete_start + mapping.page_length
+			fields['current_mapping_delete_start'] = delete_start
+
+		self.db_set(fields, notify=True, commit=True)
+		frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test)
+
+	def run_current_mapping(self):
+		try:
+			mapping = self.get_mapping(self.current_mapping)
+
+			if mapping.mapping_type == 'Push':
+				done = self.push()
+			elif mapping.mapping_type == 'Pull':
+				done = self.pull()
+
+			if done:
+				self.enqueue_next_mapping()
+			else:
+				self.enqueue_next_page()
+
+		except Exception as e:
+			self.db_set('status', 'Error', notify=True, commit=True)
+			print('Data Migration Run failed')
+			print(frappe.get_traceback())
+			raise e
+
+	def get_last_modified_condition(self):
+		last_run_timestamp = frappe.db.get_value('Data Migration Run', dict(
+			data_migration_plan=self.data_migration_plan,
+			name=('!=', self.name)
+		), 'modified')
+		if last_run_timestamp:
+			condition = dict(modified=('>', last_run_timestamp))
+		else:
+			condition = {}
+		return condition
+
+	def begin(self):
+		plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled]
+		self.mappings = [frappe.get_doc(
+			'Data Migration Mapping', m.mapping) for m in plan_active_mappings]
+
+		total_pages = 0
+		for m in [mapping for mapping in self.mappings]:
+			if m.mapping_type == 'Push':
+				count = float(self.get_count(m))
+				page_count = math.ceil(count / m.page_length)
+				total_pages += page_count
+			if m.mapping_type == 'Pull':
+				total_pages += 10
+
+		self.db_set(dict(
+			status = 'Started',
+			current_mapping = None,
+			current_mapping_start = 0,
+			current_mapping_delete_start = 0,
+			percent_complete = 0,
+			current_mapping_action = 'Insert',
+			total_pages = total_pages
+		), notify=True, commit=True)
+
+	def complete(self):
+		fields = dict()
+
+		push_failed = self.get_log('push_failed', [])
+		pull_failed = self.get_log('pull_failed', [])
+
+		if push_failed or pull_failed:
+			fields['status'] = 'Partial Success'
+		else:
+			fields['status'] = 'Success'
+			fields['percent_complete'] = 100
+
+		self.db_set(fields, notify=True, commit=True)
+
+	def get_plan(self):
+		if not hasattr(self, 'plan'):
+			self.plan = frappe.get_doc('Data Migration Plan', self.data_migration_plan)
+		return self.plan
+
+	def get_mapping(self, mapping_name):
+		if hasattr(self, 'mappings'):
+			for m in self.mappings:
+				if m.name == mapping_name:
+					return m
+		return frappe.get_doc('Data Migration Mapping', mapping_name)
+
+	def get_next_mapping_name(self):
+		mappings = [m for m in self.get_plan().mappings if m.enabled]
+		if not self.current_mapping:
+			# first
+			return mappings[0].mapping
+		for i, d in enumerate(mappings):
+			if i == len(mappings) - 1:
+				# last
+				return None
+			if d.mapping == self.current_mapping:
+				return mappings[i+1].mapping
+
+		raise frappe.ValidationError('Mapping Broken')
+
+	def get_data(self, filters):
+		mapping = self.get_mapping(self.current_mapping)
+		or_filters = self.get_or_filters(mapping)
+		start = self.current_mapping_start
+
+		data = []
+		doclist = frappe.get_all(mapping.local_doctype,
+			filters=filters, or_filters=or_filters,
+			start=start, page_length=mapping.page_length)
+
+		for d in doclist:
+			doc = frappe.get_doc(mapping.local_doctype, d['name'])
+			data.append(doc)
+		return data
+
+	def get_new_local_data(self):
+		'''Fetch newly inserted local data using `frappe.get_all`. Used during Push'''
+		mapping = self.get_mapping(self.current_mapping)
+		filters = mapping.get_filters() or {}
+
+		# new docs dont have migration field set
+		filters.update({
+			mapping.migration_id_field: ''
+		})
+
+		return self.get_data(filters)
+
+	def get_updated_local_data(self):
+		'''Fetch local updated data using `frappe.get_all`. Used during Push'''
+		mapping = self.get_mapping(self.current_mapping)
+		filters = mapping.get_filters() or {}
+
+		# existing docs must have migration field set
+		filters.update({
+			mapping.migration_id_field: ('!=', '')
+		})
+
+		return self.get_data(filters)
+
+	def get_deleted_local_data(self):
+		'''Fetch local deleted data using `frappe.get_all`. Used during Push'''
+		mapping = self.get_mapping(self.current_mapping)
+		or_filters = self.get_or_filters(mapping)
+		filters = dict(
+			deleted_doctype=mapping.local_doctype
+		)
+
+		data = frappe.get_all('Deleted Document', fields=['data'],
+			filters=filters, or_filters=or_filters)
+
+		_data = []
+		for d in data:
+			doc = json.loads(d.data)
+			if doc.get(mapping.migration_id_field):
+				doc['_deleted_document_name'] = d.name
+				_data.append(doc)
+
+		return _data
+
+	def get_remote_data(self):
+		'''Fetch data from remote using `connection.get`. Used during Pull'''
+		mapping = self.get_mapping(self.current_mapping)
+		start = self.current_mapping_start
+		filters = mapping.get_filters() or {}
+		connection = self.get_connection()
+
+		return connection.get(mapping.remote_objectname,
+			fields=["*"], filters=filters, start=start,
+			page_length=mapping.page_length)
+
+	def get_count(self, mapping):
+		filters = mapping.get_filters() or {}
+		or_filters = self.get_or_filters(mapping)
+
+		to_insert = frappe.get_all(mapping.local_doctype, ['count(name) as total'],
+			filters=filters, or_filters=or_filters)[0].total
+
+		to_delete = frappe.get_all('Deleted Document', ['count(name) as total'],
+			filters={'deleted_doctype': mapping.local_doctype}, or_filters=or_filters)[0].total
+
+		return to_insert + to_delete
+
+	def get_or_filters(self, mapping):
+		or_filters = self.get_last_modified_condition()
+
+		# include docs whose migration_id_field is not set
+		or_filters.update({
+			mapping.migration_id_field: ('=', '')
+		})
+
+		return or_filters
+
+	def get_connection(self):
+		if not hasattr(self, 'connection'):
+			self.connection = frappe.get_doc('Data Migration Connector',
+				self.data_migration_connector).get_connection()
+
+		return self.connection
+
+	def push(self):
+		self.db_set('current_mapping_type', 'Push')
+		done = True
+
+		if self.current_mapping_action == 'Insert':
+			done = self._push_insert()
+
+		elif self.current_mapping_action == 'Update':
+			done = self._push_update()
+
+		elif self.current_mapping_action == 'Delete':
+			done = self._push_delete()
+
+		return done
+
+	def _push_insert(self):
+		'''Inserts new local docs on remote'''
+		mapping = self.get_mapping(self.current_mapping)
+		connection = self.get_connection()
+		data = self.get_new_local_data()
+
+		push_insert = self.get_log('push_insert', 0)
+		push_failed = self.get_log('push_failed', [])
+
+		for d in data:
+			# pre process before insert
+			doc = self.pre_process_doc(d)
+			doc = mapping.get_mapped_record(doc)
+
+			try:
+				response_doc = connection.insert(mapping.remote_objectname, doc)
+				frappe.db.set_value(mapping.local_doctype, d.name,
+						mapping.migration_id_field, response_doc[connection.name_field],
+						update_modified=False)
+				frappe.db.commit()
+				self.set_log('push_insert', push_insert + 1)
+				# post process after insert
+				self.post_process_doc(local_doc=d, remote_doc=response_doc)
+			except Exception:
+				push_failed.append(d.as_json())
+				self.set_log('push_failed', push_failed)
+
+		# update page_start
+		self.db_set('current_mapping_start',
+			self.current_mapping_start + mapping.page_length)
+
+		if len(data) < mapping.page_length:
+			# done, no more new data to insert
+			self.db_set({
+				'current_mapping_action': 'Update',
+				'current_mapping_start': 0
+			})
+			# not done with this mapping
+			return False
+
+	def _push_update(self):
+		'''Updates local modified docs on remote'''
+		mapping = self.get_mapping(self.current_mapping)
+		connection = self.get_connection()
+		data = self.get_updated_local_data()
+
+		push_update = self.get_log('push_update', 0)
+		push_failed = self.get_log('push_failed', [])
+
+		for d in data:
+			migration_id_value = d.get(mapping.migration_id_field)
+			# pre process before update
+			doc = self.pre_process_doc(d)
+			doc = mapping.get_mapped_record(doc)
+			try:
+				response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value)
+				self.set_log('push_update', push_update + 1)
+				# post process after update
+				self.post_process_doc(local_doc=d, remote_doc=response_doc)
+			except Exception:
+				push_failed.append(d.as_json())
+				self.set_log('push_failed', push_failed)
+
+		# update page_start
+		self.db_set('current_mapping_start',
+			self.current_mapping_start + mapping.page_length)
+
+		if len(data) < mapping.page_length:
+			# done, no more data to update
+			self.db_set({
+				'current_mapping_action': 'Delete',
+				'current_mapping_start': 0
+			})
+			# not done with this mapping
+			return False
+
+	def _push_delete(self):
+		'''Deletes docs deleted from local on remote'''
+		mapping = self.get_mapping(self.current_mapping)
+		connection = self.get_connection()
+		data = self.get_deleted_local_data()
+
+		push_delete = self.get_log('push_delete', 0)
+		push_failed = self.get_log('push_failed', [])
+
+		for d in data:
+			# Deleted Document also has a custom field for migration_id
+			migration_id_value = d.get(mapping.migration_id_field)
+			# pre process before update
+			self.pre_process_doc(d)
+			try:
+				response_doc = connection.delete(mapping.remote_objectname, migration_id_value)
+				self.set_log('push_delete', push_delete + 1)
+				# post process only when action is success
+				self.post_process_doc(local_doc=d, remote_doc=response_doc)
+			except Exception:
+				push_failed.append(d.as_json())
+				self.set_log('push_failed', push_failed)
+
+		# update page_start
+		self.db_set('current_mapping_start',
+			self.current_mapping_start + mapping.page_length)
+
+		if len(data) < mapping.page_length:
+			# done, no more new data to delete
+			# done with this mapping
+			return True
+
+	def pull(self):
+		self.db_set('current_mapping_type', 'Pull')
+
+		connection = self.get_connection()
+		mapping = self.get_mapping(self.current_mapping)
+		data = self.get_remote_data()
+
+		pull_insert = self.get_log('pull_insert', 0)
+		pull_update = self.get_log('pull_update', 0)
+		pull_failed = self.get_log('pull_failed', [])
+
+		def get_migration_id_value(source, key):
+			value = None
+			try:
+				value = source[key]
+			except:
+				value = getattr(source, key)
+			return value
+
+		for d in data:
+			migration_id_value = get_migration_id_value(d, connection.name_field)
+			doc = self.pre_process_doc(d)
+			doc = mapping.get_mapped_record(doc)
+
+			if migration_id_value:
+				if not local_doc_exists(mapping, migration_id_value):
+					# insert new local doc
+					local_doc = insert_local_doc(mapping, doc)
+
+					self.set_log('pull_insert', pull_insert + 1)
+					# set migration id
+					frappe.db.set_value(mapping.local_doctype, local_doc.name,
+						mapping.migration_id_field, migration_id_value,
+						update_modified=False)
+					frappe.db.commit()
+				else:
+					# update doc
+					local_doc = update_local_doc(mapping, doc, migration_id_value)
+					self.set_log('pull_update', pull_update + 1)
+
+			if local_doc:
+				# post process doc after success
+				self.post_process_doc(remote_doc=d, local_doc=local_doc)
+			else:
+				# failed, append to log
+				pull_failed.append(d)
+				self.set_log('pull_failed', pull_failed)
+
+		if len(data) < mapping.page_length:
+			# last page, done with pull
+			return True
+
+	def pre_process_doc(self, doc):
+		plan = self.get_plan()
+		doc = plan.pre_process_doc(self.current_mapping, doc)
+		return doc
+
+	def post_process_doc(self, local_doc=None, remote_doc=None):
+		plan = self.get_plan()
+		doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc)
+		return doc
+
+	def set_log(self, key, value):
+		value = json.dumps(value) if '_failed' in key else value
+		self.db_set(key, value)
+
+	def get_log(self, key, default=None):
+		value = self.db_get(key)
+		if '_failed' in key:
+			if not value: value = json.dumps(default)
+			value = json.loads(value)
+		return value or default
+
+def insert_local_doc(mapping, doc):
+	try:
+		# insert new doc
+		if not doc.doctype:
+			doc.doctype = mapping.local_doctype
+		doc = frappe.get_doc(doc).insert()
+		return doc
+	except Exception:
+		print('Data Migration Run failed: Error in Pull insert')
+		print(frappe.get_traceback())
+		return None
+
+def update_local_doc(mapping, remote_doc, migration_id_value):
+	try:
+		# migration id value is set in migration_id_field in mapping.local_doctype
+		docname = frappe.db.get_value(mapping.local_doctype,
+			filters={ mapping.migration_id_field: migration_id_value })
+
+		doc = frappe.get_doc(mapping.local_doctype, docname)
+		doc.update(remote_doc)
+		doc.save()
+		return doc
+	except Exception:
+		print('Data Migration Run failed: Error in Pull update')
+		print(frappe.get_traceback())
+		return None
+
+def local_doc_exists(mapping, migration_id_value):
+	return frappe.db.exists(mapping.local_doctype, {
+		mapping.migration_id_field: migration_id_value
+	})

frappe/data_migration/doctype/data_migration_run/test_data_migration_run.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Data Migration Run", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Data Migration Run
+		() => frappe.tests.make('Data Migration Run', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import frappe, unittest
+
+class TestDataMigrationRun(unittest.TestCase):
+	def test_run(self):
+		create_plan()
+
+		description = 'Data migration todo'
+		new_todo = frappe.get_doc({
+			'doctype': 'ToDo',
+			'description': description
+		}).insert()
+
+		event_subject = 'Data migration event'
+		frappe.get_doc(dict(
+			doctype='Event',
+			subject=event_subject,
+			repeat_on='Every Month',
+			starts_on=frappe.utils.now_datetime()
+		)).insert()
+
+		run = frappe.get_doc({
+			'doctype': 'Data Migration Run',
+			'data_migration_plan': 'ToDo Sync',
+			'data_migration_connector': 'Local Connector'
+		}).insert()
+
+		run.run()
+		self.assertEqual(run.db_get('status'), 'Success')
+
+		self.assertEqual(run.db_get('push_insert'), 1)
+		self.assertEqual(run.db_get('pull_insert'), 1)
+
+		todo = frappe.get_doc('ToDo', new_todo.name)
+		self.assertTrue(todo.todo_sync_id)
+
+		# Pushed Event
+		event = frappe.get_doc('Event', todo.todo_sync_id)
+		self.assertEqual(event.subject, description)
+
+		# Pulled ToDo
+		created_todo = frappe.get_doc('ToDo', {'description': event_subject})
+		self.assertEqual(created_todo.description, event_subject)
+
+		todo_list = frappe.get_list('ToDo', filters={'description': 'Data migration todo'}, fields=['name'])
+		todo_name = todo_list[0].name
+
+		todo = frappe.get_doc('ToDo', todo_name)
+		todo.description = 'Data migration todo updated'
+		todo.save()
+
+		run = frappe.get_doc({
+			'doctype': 'Data Migration Run',
+			'data_migration_plan': 'ToDo Sync',
+			'data_migration_connector': 'Local Connector'
+		}).insert()
+
+		run.run()
+
+		# Update
+		self.assertEqual(run.db_get('status'), 'Success')
+		self.assertEqual(run.db_get('push_update'), 1)
+		self.assertEqual(run.db_get('pull_update'), 1)
+
+def create_plan():
+	frappe.get_doc({
+		'doctype': 'Data Migration Mapping',
+		'mapping_name': 'Todo to Event',
+		'remote_objectname': 'Event',
+		'remote_primary_key': 'name',
+		'mapping_type': 'Push',
+		'local_doctype': 'ToDo',
+		'fields': [
+			{ 'remote_fieldname': 'subject', 'local_fieldname': 'description' },
+			{ 'remote_fieldname': 'starts_on', 'local_fieldname': 'eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())' }
+		]
+	}).insert()
+
+	frappe.get_doc({
+		'doctype': 'Data Migration Mapping',
+		'mapping_name': 'Event to ToDo',
+		'remote_objectname': 'Event',
+		'remote_primary_key': 'name',
+		'local_doctype': 'ToDo',
+		'local_primary_key': 'name',
+		'mapping_type': 'Pull',
+		'condition': '{"subject": "Data migration event"}',
+		'fields': [
+			{ 'remote_fieldname': 'subject', 'local_fieldname': 'description' }
+		]
+	}).insert()
+
+	frappe.get_doc({
+		'doctype': 'Data Migration Plan',
+		'plan_name': 'ToDo sync',
+		'module': 'Core',
+		'mappings': [
+			{ 'mapping': 'Todo to Event' },
+			{ 'mapping': 'Event to ToDo' }
+		]
+	}).insert()
+
+	frappe.get_doc({
+		'doctype': 'Data Migration Connector',
+		'connector_name': 'Local Connector',
+		'connector_type': 'Frappe',
+		'hostname': 'http://localhost:8000',
+		'username': 'Administrator',
+		'password': 'admin'
+	}).insert()

frappe/database.py

@@ -14,15 +14,13 @@ import frappe
 import frappe.defaults
 import frappe.async
 import re
-import redis
 import frappe.model.meta
 from frappe.utils import now, get_datetime, cstr
 from frappe import _
 from six import text_type, binary_type, string_types, integer_types
-from frappe.utils.global_search import sync_global_search
 from frappe.model.utils.link_count import flush_local_link_count
 from six import iteritems, text_type
-
+from frappe.utils.background_jobs import execute_job, get_queue
 
 class Database:
 	"""
@@ -740,20 +738,9 @@ class Database:
 		self.sql("commit")
 		frappe.local.rollback_observers = []
 		self.flush_realtime_log()
-		self.enqueue_global_search()
+		enqueue_jobs_after_commit()
 		flush_local_link_count()
 
-	def enqueue_global_search(self):
-		if frappe.flags.update_global_search:
-			try:
-				frappe.enqueue('frappe.utils.global_search.sync_global_search',
-					now=frappe.flags.in_test or frappe.flags.in_install or frappe.flags.in_migrate,
-					flags=frappe.flags.update_global_search)
-			except redis.exceptions.ConnectionError:
-				sync_global_search()
-
-			frappe.flags.update_global_search = []
-
 	def flush_realtime_log(self):
 		for args in frappe.local.realtime_log:
 			frappe.async.emit_via_redis(*args)
@@ -895,3 +882,11 @@ class Database:
 			s = s.replace("%", "%%")
 
 		return s
+
+def enqueue_jobs_after_commit():
+	if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0:
+		for job in frappe.flags.enqueue_after_commit:
+			q = get_queue(job.get("queue"), async=job.get("async"))
+			q.enqueue_call(execute_job, timeout=job.get("timeout"),
+							kwargs=job.get("queue_args"))
+		frappe.flags.enqueue_after_commit = []

frappe/desk/doctype/event/test_event.py

@@ -117,13 +117,13 @@ class TestEvent(unittest.TestCase):
 		ev.insert()
 
 		ev_list = get_events("2014-02-01", "2014-02-01", "Administrator", for_reminder=True)
-		self.assertTrue(filter(lambda e: e.name==ev.name, ev_list))
+		self.assertTrue(list(filter(lambda e: e.name==ev.name, ev_list)))
 
 		ev_list1 = get_events("2015-01-20", "2015-01-20", "Administrator", for_reminder=True)
-		self.assertFalse(filter(lambda e: e.name==ev.name, ev_list1))
+		self.assertFalse(list(filter(lambda e: e.name==ev.name, ev_list1)))
 
 		ev_list2 = get_events("2014-02-20", "2014-02-20", "Administrator", for_reminder=True)
-		self.assertFalse(filter(lambda e: e.name==ev.name, ev_list2))
+		self.assertFalse(list(filter(lambda e: e.name==ev.name, ev_list2)))
 
 		ev_list3 = get_events("2015-02-01", "2015-02-01", "Administrator", for_reminder=True)
-		self.assertTrue(filter(lambda e: e.name==ev.name, ev_list3))
+		self.assertTrue(list(filter(lambda e: e.name==ev.name, ev_list3)))

frappe/desk/form/assign_to.py

@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 from frappe.desk.form.load import get_docinfo
+import frappe.share
 
 class DuplicateToDoError(frappe.ValidationError): pass
 
@@ -62,6 +63,13 @@ def add(args=None):
 		if frappe.get_meta(args['doctype']).get_field("assigned_to"):
 			frappe.db.set_value(args['doctype'], args['name'], "assigned_to", args['assign_to'])
 
+		doc = frappe.get_doc(args['doctype'], args['name'])
+
+		# if assignee does not have permissions, share
+		if not frappe.has_permission(doc=doc, user=args['assign_to']):
+			frappe.share.add(doc.doctype, doc.name, args['assign_to'])
+			frappe.msgprint(_('Shared with user {0} with read access').format(args['assign_to']), alert=True)
+
 	# notify
 	notify_assignment(d.assigned_by, d.owner, d.reference_type, d.reference_name, action='ASSIGN',\
 			 description=args.get("description"), notify=args.get('notify'))

frappe/desk/page/chat/chat_row.html

@@ -29,7 +29,7 @@
 		{% if (data.owner==user) { %}
 		<div>
 			<a class="delete text-extra-muted" data-name="{%= data.name %}"
-				onclick="frappe.desk.pages.messages.delete(this)">Delete</a>
+				onclick="frappe.pages.chat.chat.delete(this)">Delete</a>
 		</div>
 		{% } %}
 	</div>

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -70,7 +70,12 @@ frappe.pages['setup-wizard'].on_page_show = function(wrapper) {
 
 frappe.setup.on("before_load", function() {
 	// load slides
-	frappe.setup.slides_settings.map(frappe.setup.add_slide);
+	frappe.setup.slides_settings.forEach((s) => {
+		if(!(s.name==='user' && frappe.boot.developer_mode)) {
+			// if not user slide with developer mode
+			frappe.setup.add_slide(s);
+		}
+	});
 });
 
 frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
@@ -232,6 +237,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		this.working_state_message = this.get_message(
 			__("Setting Up"),
 			__("Sit tight while your system is being setup. This may take a few moments."),
+			'orange',
 			true
 		).appendTo(this.parent);
 
@@ -239,7 +245,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		this.current_slide = null;
 		this.completed_state_message = this.get_message(
 			__("Setup Complete"),
-			__("You're all set!")
+			__("You're all set!"),
+			'green'
 		);
 	}
 
@@ -501,19 +508,22 @@ frappe.setup.utils = {
 
 	bind_language_events: function(slide) {
 		slide.get_input("language").unbind("change").on("change", function() {
-			var lang = $(this).val() || "English";
-			frappe._messages = {};
-			frappe.call({
-				method: "frappe.desk.page.setup_wizard.setup_wizard.load_messages",
-				freeze: true,
-				args: {
-					language: lang
-				},
-				callback: function(r) {
-					frappe.setup._from_load_messages = true;
-					frappe.wizard.refresh_slides();
-				}
-			});
+			clearTimeout (slide.language_call_timeout);
+			slide.language_call_timeout = setTimeout (() => {
+				var lang = $(this).val() || "English";
+				frappe._messages = {};
+				frappe.call({
+					method: "frappe.desk.page.setup_wizard.setup_wizard.load_messages",
+					freeze: true,
+					args: {
+						language: lang
+					},
+					callback: function(r) {
+						frappe.setup._from_load_messages = true;
+						frappe.wizard.refresh_slides();
+					}
+				});
+			}, 500);
 		});
 	},
 

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -82,7 +82,7 @@ def update_system_settings(args):
 	system_settings.save()
 
 def update_user_name(args):
-	first_name, last_name = args.get('full_name'), ''
+	first_name, last_name = args.get('full_name', ''), ''
 	if ' ' in first_name:
 		first_name, last_name = first_name.split(' ', 1)
 
@@ -106,7 +106,7 @@ def update_user_name(args):
 		frappe.flags.mute_emails = _mute_emails
 		update_password(args.get("email"), args.get("password"))
 
-	else:
+	elif first_name:
 		args.update({
 			"name": frappe.session.user,
 			"first_name": first_name,
@@ -123,7 +123,8 @@ def update_user_name(args):
 			fileurl = save_file(filename, content, "User", args.get("name"), decode=True).file_url
 			frappe.db.set_value("User", args.get("name"), "user_image", fileurl)
 
-	add_all_roles_to(args.get("name"))
+	if args.get('name'):
+		add_all_roles_to(args.get("name"))
 
 def process_args(args):
 	if not args:

frappe/desk/reportview.py

@@ -352,7 +352,7 @@ def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with
 			for f in filters:
 				if isinstance(f[1], string_types) and f[1][0] == '!':
 					flt.append([doctype, f[0], '!=', f[1][1:]])
-				elif isinstance(f[1], list) and \
+				elif isinstance(f[1], (list, tuple)) and \
 					f[1][0] in (">", "<", ">=", "<=", "like", "not like", "in", "not in", "between"):
 
 					flt.append([doctype, f[0], f[1][0], f[1][1]])

frappe/desk/treeview.py

@@ -59,7 +59,7 @@ def make_tree_args(**kwarg):
 	
 	doctype = kwarg['doctype']
 	parent_field = 'parent_' + doctype.lower().replace(' ', '_')
-	name_field = doctype.lower().replace(' ', '_') + '_name'
+	name_field = kwarg.get('name_field', doctype.lower().replace(' ', '_') + '_name')
 	
 	kwarg.update({
 		name_field: kwarg[name_field],

frappe/docs/user/en/guides/data/index.txt

@@ -1 +1,2 @@
 import-large-csv-file
+using-data-migration-tool

frappe/docs/user/en/guides/data/using-data-migration-tool.md

@@ -0,0 +1,99 @@
+# Using the Data Migration Tool
+
+> Data Migration Tool was introduced in Frappé Framework version 9.
+
+The Data Migration Tool was built to abstract all the syncing of data between a remote source and a DocType. This is a middleware layer between your Frappé based website and a remote data source.
+
+To understand this tool, let's make a connector to push ERPNext Items to an imaginary service called Atlas.
+
+### Data Migration Plan
+A Data Migration Plan encapsulates a set of mappings.
+
+Let's make a new *Data Migration Plan*. Set the plan name as 'Atlas Sync'. We also need to add mappings in the mappings child table.
+
+<img class="screenshot" alt="New Data Migration Plan" src="/docs/assets/img/data-migration/new-data-migration-plan.png">
+
+
+### Data Migration Mapping
+A Data Migration Mapping is a set of rules that specify field-to-field mapping.
+
+Make a new *Data Migration Mapping*. Call it 'Item to Atlas Item'.
+
+To define a mapping, we need to put in some values that define the structure of local and remote data.
+
+1. Remote Objectname: A name that identifies the remote object e.g Atlas Item
+1. Remote primary key: This is the name of the primary key for Atlas Item e.g id
+1. Local DocType: The DocType which will be used for syncing e.g Item
+1. Mapping Type: A Mapping can be of type 'Push' or 'Pull', depending on whether the data is to be mapped remotely or locally. It can also be 'Sync', which will perform both push and pull operations in a single cycle.
+1. Page Length: This defines the batch size of the sync.
+
+<img class="screenshot" alt="New Data Migration Mapping" src="/docs/assets/img/data-migration/new-data-migration-mapping.png">
+
+#### Specifying field mappings:
+
+The most basic form of a field mapping would be to specify fieldnames of the remote and local object. However, if the mapping is one-way (push or pull), the source field name can also take literal values in quotes (for e.g `"GadgetTech"`) and eval statements (for e.g `"eval:frappe.db.get_value('Company', 'Gadget Tech', 'default_currency')"`). For example, in the case of a push mapping, the local fieldname can be set to a string in quotes or an `eval` expression, instead of a field name from the local doctype. (This is not possible with a sync mapping, where both local and remote fieldnames serve as a target destination at a some point, and thus cannot be a literal value).
+
+Let's add the field mappings and save:
+
+<img class="screenshot" alt="Add fields in Data Migration Mapping" src="/docs/assets/img/data-migration/new-data-migration-mapping-fields.png">
+
+We can now add the 'Item to Atlas Item' mapping to our Data Migration Plan and save it.
+
+<img class="screenshot" alt="Save Atlas Sync Plan" src="/docs/assets/img/data-migration/atlas-sync-plan.png">
+
+#### Additional layer of control with pre and post process:
+
+Migrating data frequently involves more steps in addition to one-to-one mapping. For a Data Migration Mapping that is added to a Plan, a mapping module is generated in the module specified in that plan.
+
+In our case, an `item_to_atlas_item` module is created under the `data_migration_mapping` directory in `Integrations` (module for the 'Atlas Sync' plan).
+
+<img class="screenshot" alt="Mapping __init__.py" src="/docs/assets/img/data-migration/mapping-init-py.png">
+
+You can implement the `pre_process` (receives the source doc) and `post_process` (receives both source and target docs, as well as any additional arguments) methods, to extend the mapping process. Here's what some operations could look like:
+
+<img class="screenshot" alt="Pre and Post Process" src="/docs/assets/img/data-migration/mapping-pre-and-post-process.png">
+
+### Data Migration Connector
+Now, to connect to the remote source, we need to create a *Data Migration Connector*.
+
+<img class="screenshot" alt="New Data Migration Connector" src="/docs/assets/img/data-migration/new-connector.png">
+
+We only have two connector types right now, let's add another Connector Type in the Data Migration Connector DocType.
+
+<img class="screenshot" alt="Add Connector Type in Data Migration Connector" src="/docs/assets/img/data-migration/add-connector-type.png">
+
+Now, let's create a new Data Migration Connector.
+
+<img class="screenshot" alt="Atlas Connector" src="/docs/assets/img/data-migration/atlas-connector.png">
+
+As you can see we chose the Connector Type as Atlas. We also added the hostname, username and password for our Atlas instance so that we can authenticate.
+
+Now, we need to write the code for our connector so that we can actually push data.
+
+Create a new file called `atlas_connection.py` in `frappe/data_migration/doctype/data_migration_connector/connectors/` directory. Other connectors also live here.
+
+We just have to implement the `insert`, `update` and `delete` methods for our atlas connector. We also need to write the code to connect to our Atlas instance in the `__init__` method. Just see `frappe_connection.py` for reference.
+
+<img class="screenshot" alt="Atlas Connection file" src="/docs/assets/img/data-migration/atlas-connection-py.png">
+
+After creating the Atlas Connector, we also need to import it into `data_migration_connector.py`
+
+<img class="screenshot" alt="Edit Connector file" src="/docs/assets/img/data-migration/edit-connector-py.png">
+
+### Data Migration Run
+Now that we have our connector, the last thing to do is to create a new *Data Migration Run*.
+
+A Data Migration Run takes a Data Migration Plan and Data Migration Connector and execute the plan according to our configuration. It takes care of queueing, batching, delta updates and more.
+
+<img class="screenshot" alt="Data Migration Run" src="/docs/assets/img/data-migration/data-migration-run.png">
+
+Just click Run. It will now push our Items to the remote Atlas instance and you can see the progress which updates in realtime.
+
+After a run is executed successfully, you cannot run it again. You will have to create another run and execute it.
+
+Data Migration Run will try to be as efficient as possible, so the next time you execute it, it will only push those items which were changed or failed in the last run.
+
+
+> Note: Data Migration Tool is still in beta. If you find any issues please report them [here](https://github.com/frappe/erpnext/issues)
+
+<!-- markdown -->

frappe/email/doctype/email_account/test_email_account.py

@@ -111,7 +111,7 @@ class TestEmailAccount(unittest.TestCase):
 		frappe.sendmail(sender="test_sender@example.com", recipients="test_recipient@example.com",
 			content="test mail 001", subject="test-mail-001", delayed=False)
 
-		sent_mail = email.message_from_string(frappe.flags.sent_mail)
+		sent_mail = email.message_from_string(frappe.flags.sent_mail.decode())
 		self.assertTrue("test-mail-001" in sent_mail.get("Subject"))
 
 	def test_print_format(self):

frappe/email/email_body.py

@@ -10,6 +10,7 @@ from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint
 import email.utils
 from six import iteritems, text_type, string_types
 from email.mime.multipart import MIMEMultipart
+from email.header import Header
 
 
 def get_email(recipients, sender='', msg='', subject='[No Subject]',
@@ -183,7 +184,7 @@ class EMail:
 		if cint(self.email_account.always_use_account_email_id_as_sender):
 			self.set_header('X-Original-From', self.sender)
 			sender_name, sender_email = parse_addr(self.sender)
-			self.sender = email.utils.formataddr((sender_name or self.email_account.name, self.email_account.email_id))
+			self.sender = email.utils.formataddr((str(Header(sender_name or self.email_account.name, 'utf-8')), self.email_account.email_id))
 
 	def set_message_id(self, message_id, is_notification=False):
 		if message_id:
@@ -321,9 +322,9 @@ def add_attachment(fname, fcontent, content_type=None,
 	# Set the filename parameter
 	if fname:
 		attachment_type = 'inline' if inline else 'attachment'
-		part.add_header(b'Content-Disposition', attachment_type, filename=text_type(fname))
+		part.add_header('Content-Disposition', attachment_type, filename=text_type(fname))
 	if content_id:
-		part.add_header(b'Content-ID', '<{0}>'.format(content_id))
+		part.add_header('Content-ID', '<{0}>'.format(content_id))
 
 	parent.attach(part)
 
@@ -414,7 +415,7 @@ def get_filecontent_from_path(path):
 		full_path = path
 
 	if os.path.exists(full_path):
-		with open(full_path) as f:
+		with open(full_path, 'rb') as f:
 			filecontent = f.read()
 
 		return filecontent

frappe/email/test_email_body.py

@@ -21,9 +21,9 @@ This is the text version of this email
 '''
 
 		img_path = os.path.abspath('assets/frappe/images/favicon.png')
-		with open(img_path) as f:
+		with open(img_path, 'rb') as f:
 			img_content = f.read()
-			img_base64 = base64.b64encode(img_content)
+			img_base64 = base64.b64encode(img_content).decode()
 
 		# email body keeps 76 characters on one line
 		self.img_base64 = fixed_column_width(img_base64, 76)

frappe/frappeclient.py

@@ -16,6 +16,7 @@ class FrappeException(Exception):
 
 class FrappeClient(object):
 	def __init__(self, url, username, password, verify=True):
+		self.headers = dict(Accept='application/json')
 		self.verify = verify
 		self.session = requests.session()
 		self.url = url
@@ -33,7 +34,7 @@ class FrappeClient(object):
 			'cmd': 'login',
 			'usr': username,
 			'pwd': password
-		}, verify=self.verify)
+		}, verify=self.verify, headers=self.headers)
 
 		if r.status_code==200 and r.json().get('message') == "Logged In":
 			return r.json()
@@ -45,7 +46,7 @@ class FrappeClient(object):
 		'''Logout session'''
 		self.session.get(self.url, params={
 			'cmd': 'logout',
-		}, verify=self.verify)
+		}, verify=self.verify, headers=self.headers)
 
 	def get_list(self, doctype, fields='"*"', filters=None, limit_start=0, limit_page_length=0):
 		"""Returns list of records of a particular type"""
@@ -59,7 +60,7 @@ class FrappeClient(object):
 		if limit_page_length:
 			params["limit_start"] = limit_start
 			params["limit_page_length"] = limit_page_length
-		res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify)
+		res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def insert(self, doc):
@@ -67,7 +68,7 @@ class FrappeClient(object):
 
 		:param doc: A dict or Document object to be inserted remotely'''
 		res = self.session.post(self.url + "/api/resource/" + doc.get("doctype"),
-			data={"data":frappe.as_json(doc)}, verify=self.verify)
+			data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def insert_many(self, docs):
@@ -84,7 +85,7 @@ class FrappeClient(object):
 
 		:param doc: dict or Document object to be updated remotely. `name` is mandatory for this'''
 		url = self.url + "/api/resource/" + doc.get("doctype") + "/" + doc.get("name")
-		res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify)
+		res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def bulk_update(self, docs):
@@ -169,7 +170,7 @@ class FrappeClient(object):
 			params["fields"] = json.dumps(fields)
 
 		res = self.session.get(self.url + "/api/resource/" + doctype + "/" + name,
-			params=params, verify=self.verify)
+			params=params, verify=self.verify, headers=self.headers)
 
 		return self.post_process(res)
 
@@ -251,21 +252,21 @@ class FrappeClient(object):
 
 	def get_api(self, method, params={}):
 		res = self.session.get(self.url + "/api/method/" + method + "/",
-			params=params, verify=self.verify)
+			params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def post_api(self, method, params={}):
 		res = self.session.post(self.url + "/api/method/" + method + "/",
-			params=params, verify=self.verify)
+			params=params, verify=self.verify, headers=self.headers)
 		return self.post_process(res)
 
 	def get_request(self, params):
-		res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify)
+		res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify, headers=self.headers)
 		res = self.post_process(res)
 		return res
 
 	def post_request(self, data):
-		res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify)
+		res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify, headers=self.headers)
 		res = self.post_process(res)
 		return res
 

frappe/hooks.py

@@ -30,7 +30,6 @@ app_include_js = [
 	"assets/js/form.min.js",
 	"assets/js/control.min.js",
 	"assets/js/report.min.js",
-	"assets/js/d3.min.js",
 	"assets/frappe/js/frappe/toolbar.js"
 ]
 app_include_css = [
@@ -157,15 +156,19 @@ scheduler_events = {
 		"frappe.core.doctype.authentication_log.authentication_log.clear_authentication_logs"
 	],
 	"daily_long": [
-		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily"
+		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
+		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily"
 	],
 	"weekly_long": [
-		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly"
+		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
+		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly"
 	],
 	"monthly": [
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_monthly"
+	],
+	"monthly_long": [
+		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_monthly"
 	]
-
 }
 
 get_translated_dict = {

frappe/integrations/doctype/oauth_client/oauth_client.json

@@ -1,5 +1,6 @@
 {
  "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
  "allow_import": 0, 
  "allow_rename": 0, 
  "autoname": "", 
@@ -13,6 +14,7 @@
  "engine": "InnoDB", 
  "fields": [
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -24,6 +26,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "App Client ID", 
@@ -41,6 +44,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -51,6 +55,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "App Name", 
@@ -69,6 +74,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -79,6 +85,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "User", 
@@ -98,6 +105,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -108,6 +116,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "length": 0, 
@@ -125,6 +134,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -135,6 +145,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "App Client Secret", 
@@ -153,6 +164,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -164,6 +176,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "Skip Authorization", 
@@ -182,6 +195,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -193,6 +207,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "", 
@@ -211,6 +226,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -223,6 +239,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "Scopes", 
@@ -240,6 +257,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -250,6 +268,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "length": 0, 
@@ -267,6 +286,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -278,6 +298,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "Redirect URIs", 
@@ -295,6 +316,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -305,6 +327,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "Default Redirect URI", 
@@ -323,6 +346,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -334,6 +358,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": " Advanced Settings", 
@@ -352,6 +377,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -362,12 +388,13 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 1, 
    "in_standard_filter": 1, 
    "label": "Grant Type", 
    "length": 0, 
    "no_copy": 0, 
-   "options": "Authorization Code\nImplicit\nResource Owner Password Credentials\nClient Credentials", 
+   "options": "Authorization Code\nImplicit", 
    "permlevel": 0, 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
@@ -380,6 +407,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -390,6 +418,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
    "length": 0, 
@@ -407,6 +436,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -418,6 +448,7 @@
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
+   "in_global_search": 0, 
    "in_list_view": 1, 
    "in_standard_filter": 1, 
    "label": "Response Type", 
@@ -436,17 +467,17 @@
    "unique": 0
   }
  ], 
+ "has_web_view": 0, 
  "hide_heading": 0, 
  "hide_toolbar": 0, 
  "idx": 0, 
  "image_view": 0, 
  "in_create": 0, 
- "in_dialog": 0, 
  "is_submittable": 0, 
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-03-08 14:40:03.031779", 
+ "modified": "2017-10-05 21:07:39.476360", 
  "modified_by": "Administrator", 
  "module": "Integrations", 
  "name": "OAuth Client", 
@@ -463,7 +494,6 @@
    "export": 1, 
    "if_owner": 0, 
    "import": 0, 
-   "is_custom": 0, 
    "permlevel": 0, 
    "print": 1, 
    "read": 1, 
@@ -478,6 +508,7 @@
  "quick_entry": 0, 
  "read_only": 0, 
  "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
  "sort_field": "modified", 
  "sort_order": "DESC", 
  "title_field": "app_name", 

frappe/integrations/doctype/oauth_client/oauth_client.py

@@ -4,6 +4,7 @@
 
 from __future__ import unicode_literals
 import frappe
+from frappe import _
 from frappe.model.document import Document
 
 class OAuthClient(Document):
@@ -11,3 +12,8 @@ class OAuthClient(Document):
 		self.client_id = self.name
 		if not self.client_secret:
 			self.client_secret = frappe.generate_hash(length=10)
+		self.validate_grant_and_response()
+	def validate_grant_and_response(self):
+		if self.grant_type == "Authorization Code" and self.response_type != "Code" or \
+			self.grant_type == "Implicit" and self.response_type != "Token":
+			frappe.throw(_("Combination of Grant Type (<code>{0}</code>) and Response Type (<code>{1}</code>) not allowed".format(self.grant_type, self.response_type)))

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js

@@ -0,0 +1,26 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('S3 Backup Settings', {
+	refresh: function(frm) {
+		frm.clear_custom_buttons();
+		frm.events.take_backup(frm);
+	},
+
+	take_backup: function(frm) {
+		if (frm.doc.access_key_id && frm.doc.secret_access_key) {
+			frm.add_custom_button(__("Take Backup Now"), function(){
+				frm.dashboard.set_headline_alert("S3 Backup Started!");
+				frappe.call({
+					method: "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
+					callback: function(r) {
+						if(!r.exc) {
+							frappe.msgprint(__("S3 Backup complete!"));
+							frm.dashboard.clear_headline();
+						}
+					}
+				});
+			}).addClass("btn-primary");
+		}
+	}
+});

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json

@@ -0,0 +1,273 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2017-09-04 20:57:20.129205", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "enabled", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Enable Automatic Backup", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "notify_email", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Send Notifications To", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "frequency", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Backup Frequency", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Daily\nWeekly\nMonthly\nNone", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "access_key_id", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Access Key ID", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "secret_access_key", 
+   "fieldtype": "Password", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Secret Access Key", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "bucket", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Bucket", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "backup_limit", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Backup Limit", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 1, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 1, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-10-06 18:27:09.022674", 
+ "modified_by": "Administrator", 
+ "module": "Integrations", 
+ "name": "S3 Backup Settings", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 0, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import os
+import os.path
+import frappe
+import boto3
+from frappe import _
+from frappe.model.document import Document
+from frappe.utils import cint, split_emails
+from frappe.utils.background_jobs import enqueue
+from botocore.exceptions import ClientError
+
+class S3BackupSettings(Document):
+
+	def validate(self):
+		conn = boto3.client(
+			's3',
+			aws_access_key_id=self.access_key_id,
+			aws_secret_access_key=self.get_password('secret_access_key'),
+		)
+
+		bucket_lower = str(self.bucket).lower()
+
+		try:
+			conn.list_buckets()
+
+		except ClientError:
+			frappe.throw(_("Invalid Access Key ID or Secret Access Key."))
+
+		try:
+			conn.create_bucket(Bucket=bucket_lower)
+		except ClientError:
+			frappe.throw(_("Unable to create bucket: {0}. Change it to a more unique name.").format(bucket_lower))
+
+
+@frappe.whitelist()
+def take_backup():
+	"Enqueue longjob for taking backup to s3"
+	enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500)
+	frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
+
+
+def take_backups_daily():
+	take_backups_if("Daily")
+
+
+def take_backups_weekly():
+	take_backups_if("Weekly")
+
+
+def take_backups_monthly():
+	take_backups_if("Monthly")
+
+
+def take_backups_if(freq):
+	if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
+		if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
+			take_backups_s3()
+
+
+@frappe.whitelist()
+def take_backups_s3():
+	try:
+		backup_to_s3()
+		send_email(True, "S3 Backup Settings")
+	except Exception:
+		error_message = frappe.get_traceback()
+		frappe.errprint(error_message)
+		send_email(False, "S3 Backup Settings", error_message)
+
+
+def send_email(success, service_name, error_status=None):
+	if success:
+		subject = "Backup Upload Successful"
+		message = """<h3>Backup Uploaded Successfully! </h3><p>Hi there, this is just to inform you
+		that your backup was successfully uploaded to your Amazon S3 bucket. So relax!</p> """
+
+	else:
+		subject = "[Warning] Backup Upload Failed"
+		message = """<h3>Backup Upload Failed! </h3><p>Oops, your automated backup to Amazon S3 failed.
+		</p> <p>Error message: %s</p> <p>Please contact your system manager
+		for more information.</p>""" % error_status
+
+	if not frappe.db:
+		frappe.connect()
+
+	if frappe.db.get_value("S3 Backup Settings", None, "notification_email"):
+		recipients = split_emails(frappe.db.get_value("S3 Backup Settings", None, "notification_email"))
+		frappe.sendmail(recipients=recipients, subject=subject, message=message)
+
+
+def backup_to_s3():
+	from frappe.utils.backups import new_backup
+	from frappe.utils import get_backups_path
+
+	doc = frappe.get_single("S3 Backup Settings")
+	bucket = doc.bucket
+
+	conn = boto3.client(
+			's3',
+			aws_access_key_id=doc.access_key_id,
+			aws_secret_access_key=doc.get_password('secret_access_key'),
+			)
+
+	backup = new_backup(ignore_files=False, backup_path_db=None,
+						backup_path_files=None, backup_path_private_files=None, force=True)
+	db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
+	files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
+	private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))
+	folder = os.path.basename(db_filename)[:15] + '/'
+	# for adding datetime to folder name
+
+	upload_file_to_s3(db_filename, folder, conn, bucket)
+	upload_file_to_s3(private_files, folder, conn, bucket)
+	upload_file_to_s3(files_filename, folder, conn, bucket)
+	delete_old_backups(doc.backup_limit, bucket)
+
+def upload_file_to_s3(filename, folder, conn, bucket):
+
+	destpath = os.path.join(folder, os.path.basename(filename))
+	try:
+		print "Uploading file:", filename
+		conn.upload_file(filename, bucket, destpath)
+
+	except Exception as e:
+		print "Error uploading: %s" % (e)
+
+
+def delete_old_backups(limit, bucket):
+	all_backups = list()
+	doc = frappe.get_single("S3 Backup Settings")
+	backup_limit = int(limit)
+
+	s3 = boto3.resource(
+			's3',
+			aws_access_key_id=doc.access_key_id,
+			aws_secret_access_key=doc.get_password('secret_access_key'),
+			)
+	bucket = s3.Bucket(bucket)
+	objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter='/')
+	for obj in objects.get('CommonPrefixes'):
+		all_backups.append(obj.get('Prefix'))
+
+	oldest_backup = sorted(all_backups)[0]
+
+	if len(all_backups) > backup_limit:
+		print "Deleting Backup: {0}".format(oldest_backup)
+		for obj in bucket.objects.filter(Prefix=oldest_backup):
+			# delete all keys that are inside the oldest_backup
+			s3.Object(bucket.name, obj.key).delete()

frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: S3 Backup Settings", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new S3 Backup Settings
+		() => frappe.tests.make('S3 Backup Settings', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});