Merge branch 'develop'

.eslintrc

@@ -59,7 +59,6 @@
 		"PhotoSwipeUI_Default": true,
 		"fluxify": true,
 		"io": true,
-		"c3": true,
 		"__": true,
 		"_p": true,
 		"_f": true,
@@ -119,6 +118,8 @@
 		"getCookies": true,
 		"get_url_arg": true,
 		"QUnit": true,
-		"JsBarcode": true
+		"JsBarcode": true,
+		"L": true,
+		"Chart": true
 	}
 }

.gitignore

@@ -10,4 +10,8 @@ dist/
 build/
 frappe/docs/current
 .vscode
-node_modules
+node_modules
+
+
+# Not Recommended, but will remove once webpack ready
+package-lock.json

.travis.yml

@@ -30,7 +30,7 @@ install:
   - cp -r $TRAVIS_BUILD_DIR/test_sites/test_site ~/frappe-bench/sites/
 
 before_script:
-  - wget http://chromedriver.storage.googleapis.com/2.27/chromedriver_linux64.zip
+  - wget http://chromedriver.storage.googleapis.com/2.33/chromedriver_linux64.zip
   - unzip chromedriver_linux64.zip
   - sudo apt-get install libnss3
   - sudo apt-get --only-upgrade install google-chrome-stable

README.md

@@ -40,7 +40,6 @@ Full-stack web application framework that uses Python and MariaDB on the server
 ### Website
 
 For details and documentation, see the website
-
 [https://frappe.io](https://frappe.io)
 
 ### License

frappe/__init__.py

@@ -14,7 +14,7 @@ import os, sys, importlib, inspect, json
 from .exceptions import *
 from .utils.jinja import get_jenv, get_template, render_template, get_email_from_template
 
-__version__ = '9.2.25'
+__version__ = '10.0.0'
 __title__ = "Frappe Framework"
 
 local = Local()
@@ -311,6 +311,10 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, indicator=None,
 def clear_messages():
 	local.message_log = []
 
+def clear_last_message():
+	if len(local.message_log) > 0:
+		local.message_log = local.message_log[:-1]
+
 def throw(msg, exc=ValidationError, title=None):
 	"""Throw execption and show message (`msgprint`).
 
@@ -378,7 +382,7 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 		as_markdown=False, delayed=True, reference_doctype=None, reference_name=None,
 		unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None,
 		attachments=None, content=None, doctype=None, name=None, reply_to=None,
-		cc=[], message_id=None, in_reply_to=None, send_after=None, expose_recipients=None,
+		cc=[], bcc=[], message_id=None, in_reply_to=None, send_after=None, expose_recipients=None,
 		send_priority=1, communication=None, retry=1, now=None, read_receipt=None, is_notification=False,
 		inline_images=None, template=None, args=None, header=None):
 	"""Send email using user's default **Email Account** or global default **Email Account**.
@@ -426,7 +430,7 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 		subject=subject, message=message, text_content=text_content,
 		reference_doctype = doctype or reference_doctype, reference_name = name or reference_name,
 		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message,
-		attachments=attachments, reply_to=reply_to, cc=cc, message_id=message_id, in_reply_to=in_reply_to,
+		attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to,
 		send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority,
 		communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification,
 		inline_images=inline_images, header=header)
@@ -972,9 +976,9 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp
 		ps.insert()
 
 def import_doc(path, ignore_links=False, ignore_insert=False, insert=False):
-	"""Import a file using Data Import Tool."""
-	from frappe.core.page.data_import_tool import data_import_tool
-	data_import_tool.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
+	"""Import a file using Data Import."""
+	from frappe.core.doctype.data_import import data_import
+	data_import.import_doc(path, ignore_links=ignore_links, ignore_insert=ignore_insert, insert=insert)
 
 def copy_doc(doc, ignore_no_copy=True):
 	""" No_copy fields also get copied."""
@@ -1362,7 +1366,7 @@ def logger(module=None, with_more_info=True):
 
 def log_error(message=None, title=None):
 	'''Log error to Error Log'''
-	get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
+	return get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
 		method=title)).insert(ignore_permissions=True)
 
 def get_desk_link(doctype, name):

frappe/app.py

@@ -4,7 +4,6 @@
 from __future__ import unicode_literals
 
 import os
-import MySQLdb
 from six import iteritems
 import logging
 
@@ -27,6 +26,12 @@ from frappe.utils.error import make_error_snapshot
 from frappe.core.doctype.communication.comment import update_comments_in_parent_after_request
 from frappe import _
 
+# imports - third-party imports
+import pymysql
+from pymysql.constants import ER
+
+# imports - module imports
+
 local_manager = LocalManager([frappe.local])
 
 _site = None
@@ -116,8 +121,15 @@ def init_request(request):
 	frappe.local.http_request = frappe.auth.HTTPRequest()
 
 def make_form_dict(request):
+	import json
+
+	if request.content_type == 'application/json':
+		args = json.loads(request.data)
+	else:
+		args = request.form or request.args
+
 	frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
-		for k, v in iteritems(request.form or request.args) })
+		for k, v in iteritems(args) })
 
 	if "_" in frappe.local.form_dict:
 		# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
@@ -134,11 +146,8 @@ def handle_exception(e):
 		response = frappe.utils.response.report_error(http_status_code)
 
 	elif (http_status_code==500
-		and isinstance(e, MySQLdb.OperationalError)
-		and e.args[0] in (1205, 1213)):
-			# 1205 = lock wait timeout
-			# 1213 = deadlock
-			# code 409 represents conflict
+		and isinstance(e, pymysql.InternalError)
+		and e.args[0] in (ER.LOCK_WAIT_TIMEOUT, ER.LOCK_DEADLOCK)):
 			http_status_code = 508
 
 	elif http_status_code==401:

frappe/auth.py

@@ -15,7 +15,7 @@ from frappe.sessions import Session, clear_sessions, delete_session
 from frappe.modules.patch_handler import check_session_stopped
 from frappe.translate import get_lang_code
 from frappe.utils.password import check_password
-from frappe.core.doctype.authentication_log.authentication_log import add_authentication_log
+from frappe.core.doctype.activity_log.activity_log import add_authentication_log
 from frappe.utils.background_jobs import enqueue
 from frappe.twofactor import (should_run_2fa, authenticate_for_2factor,
 	confirm_otp_token, get_cached_user_pass)

frappe/build.js

@@ -20,6 +20,7 @@ const apps = apps_contents.split('\n');
 const app_paths = apps.map(app => path_join(apps_path, app, app)) // base_path of each app
 const assets_path = path_join(sites_path, 'assets');
 let build_map = make_build_map();
+let compiled_js_cache = {}; // cache each js file after it is compiled
 const file_watcher_port = get_conf().file_watcher_port;
 
 // command line args
@@ -65,11 +66,12 @@ function watch() {
 				io.emit('reload_css', filename);
 			}
 		});
-		watch_js(/*function (filename) {
-			if(socket_connection) {
-				io.emit('reload_js', filename);
-			}
-		}*/);
+		watch_js(//function (filename) {
+			// if(socket_connection) {
+			// 	io.emit('reload_js', filename);
+			// }
+		//}
+		);
 		watch_build_json();
 	});
 
@@ -82,9 +84,7 @@ function watch() {
 	});
 }
 
-function pack(output_path, inputs, minify) {
-	const output_type = output_path.split('.').pop();
-
+function pack(output_path, inputs, minify, file_changed) {
 	let output_txt = '';
 	for (const file of inputs) {
 
@@ -93,25 +93,18 @@ function pack(output_path, inputs, minify) {
 			continue;
 		}
 
-		let file_content = fs.readFileSync(file, 'utf-8');
-
-		if (file.endsWith('.html') && output_type === 'js') {
-			file_content = html_to_js_template(file, file_content);
+		let force_compile = false;
+		if (file_changed) {
+			// if file_changed is passed and is equal to file, force_compile it
+			force_compile = file_changed === file;
 		}
 
-		if(file.endsWith('class.js')) {
-			file_content = minify_js(file_content, file);
-		}
-
-		if (file.endsWith('.js') && !file.includes('/lib/') && output_type === 'js' && !file.endsWith('class.js')) {
-			file_content = babelify(file_content, file, minify);
-		}
+		let file_content = get_compiled_file(file, output_path, minify, force_compile);
 
 		if(!minify) {
 			output_txt += `\n/*\n *\t${file}\n */\n`
 		}
 		output_txt += file_content;
-
 		output_txt = output_txt.replace(/['"]use strict['"];/, '');
 	}
 
@@ -127,13 +120,47 @@ function pack(output_path, inputs, minify) {
 	}
 }
 
+function get_compiled_file(file, output_path, minify, force_compile) {
+	const output_type = output_path.split('.').pop();
+
+	let file_content;
+
+	if (force_compile === false) {
+		// force compile is false
+		// attempt to get from cache
+		file_content = compiled_js_cache[file];
+		if (file_content) {
+			return file_content;
+		}
+	}
+
+	file_content = fs.readFileSync(file, 'utf-8');
+
+	if (file.endsWith('.html') && output_type === 'js') {
+		file_content = html_to_js_template(file, file_content);
+	}
+
+	if(file.endsWith('class.js')) {
+		file_content = minify_js(file_content, file);
+	}
+
+	if (minify && file.endsWith('.js') && !file.includes('/lib/') && output_type === 'js' && !file.endsWith('class.js')) {
+		file_content = babelify(file_content, file, minify);
+	}
+
+	compiled_js_cache[file] = file_content;
+	return file_content;
+}
+
 function babelify(content, path, minify) {
 	let presets = ['env'];
+	var plugins = ['transform-object-rest-spread']
 	// Minification doesn't work when loading Frappe Desk
 	// Avoid for now, trace the error and come back.
 	try {
 		return babel.transform(content, {
 			presets: presets,
+			plugins: plugins,
 			comments: false
 		}).code;
 	} catch (e) {
@@ -258,16 +285,16 @@ function watch_less(ondirty) {
 }
 
 function watch_js(ondirty) {
-	const js_paths = app_paths.map(path => path_join(path, 'public', 'js'));
-
-	const to_watch = filter_valid_paths(js_paths);
-	chokidar.watch(to_watch).on('change', (filename, stats) => {
-		console.log(filename, 'dirty');
+	chokidar.watch([
+		path_join(apps_path, '**', '*.js'),
+		path_join(apps_path, '**', '*.html')
+	]).on('change', (filename) => {
 		// build the target js file for which this js/html file is input
 		for (const target in build_map) {
 			const sources = build_map[target];
 			if (sources.includes(filename)) {
-				pack(target, sources);
+				console.log(filename, 'dirty');
+				pack(target, sources, null, filename);
 				ondirty && ondirty(target);
 				// break;
 			}

frappe/change_log/v10_0_0.md

@@ -0,0 +1,10 @@
+- Enhanced Data Import Tool
+	- Data Import Tool is now a normal form, you can maintain records for each Data Import.
+	- Better error handling
+	- Background processing for large files
+
+- Frappé now has a github connector
+
+- Any doctype can have a calendar view
+
+- Frappé has a new simple, responsive, modern SVG [charts library](https://github.com/frappe/charts), developed by us

frappe/commands/site.py

@@ -3,7 +3,6 @@ import click
 import hashlib, os, sys, compileall
 import frappe
 from frappe import _
-from _mysql_exceptions import ProgrammingError
 from frappe.commands import pass_context, get_site
 from frappe.commands.scheduler import _is_scheduler_enabled
 from frappe.limits import update_limits, get_limits
@@ -11,6 +10,12 @@ from frappe.installer import update_site_config
 from frappe.utils import touch_file, get_site_path
 from six import text_type
 
+# imports - third-party imports
+from pymysql.constants import ER
+
+# imports - module imports
+from frappe.exceptions import SQLError
+
 @click.command('new-site')
 @click.argument('site')
 @click.option('--db-name', help='Database name')
@@ -348,8 +353,8 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
 
 	try:
 		scheduled_backup(ignore_files=False, force=True)
-	except ProgrammingError as err:
-		if err[0] == 1146:
+	except SQLError as err:
+		if err[0] == ER.NO_SUCH_TABLE:
 			if force:
 				pass
 			else:
@@ -400,8 +405,9 @@ def move(dest_dir, site):
 
 @click.command('set-admin-password')
 @click.argument('admin-password')
+@click.option('--logout-all-sessions', help='Logout from all sessions', is_flag=True, default=False)
 @pass_context
-def set_admin_password(context, admin_password):
+def set_admin_password(context, admin_password, logout_all_sessions=False):
 	"Set Administrator password for a site"
 	import getpass
 	from frappe.utils.password import update_password
@@ -414,7 +420,7 @@ def set_admin_password(context, admin_password):
 				admin_password = getpass.getpass("Administrator's password for {0}: ".format(site))
 
 			frappe.connect()
-			update_password('Administrator', admin_password)
+			update_password(user='Administrator', pwd=admin_password, logout_all_sessions=logout_all_sessions)
 			frappe.db.commit()
 			admin_password = None
 		finally:

frappe/commands/utils.py

@@ -15,7 +15,9 @@ def build(make_copy=False, restore = False, verbose=False):
 	import frappe.build
 	import frappe
 	frappe.init('')
-	frappe.build.bundle(False, make_copy=make_copy, restore = restore, verbose=verbose)
+	# don't minify in developer_mode for faster builds
+	no_compress = frappe.local.conf.developer_mode or False
+	frappe.build.bundle(no_compress, make_copy=make_copy, restore = restore, verbose=verbose)
 
 @click.command('watch')
 def watch():
@@ -162,12 +164,12 @@ def export_doc(context, doctype, docname):
 @pass_context
 def export_json(context, doctype, path, name=None):
 	"Export doclist as json to the given path, use '-' as name for Singles."
-	from frappe.core.page.data_import_tool import data_import_tool
+	from frappe.core.doctype.data_import import data_import
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			data_import_tool.export_json(doctype, path, name=name)
+			data_import.export_json(doctype, path, name=name)
 		finally:
 			frappe.destroy()
 
@@ -177,12 +179,12 @@ def export_json(context, doctype, path, name=None):
 @pass_context
 def export_csv(context, doctype, path):
 	"Export data import template with data for DocType"
-	from frappe.core.page.data_import_tool import data_import_tool
+	from frappe.core.doctype.data_import import data_import
 	for site in context.sites:
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			data_import_tool.export_csv(doctype, path)
+			data_import.export_csv(doctype, path)
 		finally:
 			frappe.destroy()
 
@@ -204,7 +206,7 @@ def export_fixtures(context):
 @pass_context
 def import_doc(context, path, force=False):
 	"Import (insert/update) doclist. If the argument is a directory, all files ending with .json are imported"
-	from frappe.core.page.data_import_tool import data_import_tool
+	from frappe.core.doctype.data_import import data_import
 
 	if not os.path.exists(path):
 		path = os.path.join('..', path)
@@ -216,7 +218,7 @@ def import_doc(context, path, force=False):
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			data_import_tool.import_doc(path, overwrite=context.force)
+			data_import.import_doc(path, overwrite=context.force)
 		finally:
 			frappe.destroy()
 
@@ -229,8 +231,8 @@ def import_doc(context, path, force=False):
 
 @pass_context
 def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
-	"Import CSV using data import tool"
-	from frappe.core.page.data_import_tool import importer
+	"Import CSV using data import"
+	from frappe.core.doctype.data_import import importer
 	from frappe.utils.csvutils import read_csv_content
 	site = get_site(context)
 
@@ -300,6 +302,7 @@ def console(context):
 @click.command('run-tests')
 @click.option('--app', help="For App")
 @click.option('--doctype', help="For DocType")
+@click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt")
 @click.option('--test', multiple=True, help="Specific test")
 @click.option('--driver', help="For Travis")
 @click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests")
@@ -308,7 +311,7 @@ def console(context):
 @click.option('--junit-xml-output', help="Destination file path for junit xml report")
 @pass_context
 def run_tests(context, app=None, module=None, doctype=None, test=(),
-	driver=None, profile=False, junit_xml_output=False, ui_tests = False):
+	driver=None, profile=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None):
 	"Run tests"
 	import frappe.test_runner
 	tests = test
@@ -318,7 +321,7 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
 
 	ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
 		force=context.force, profile=profile, junit_xml_output=junit_xml_output,
-		ui_tests = ui_tests)
+		ui_tests = ui_tests, doctype_list_path = doctype_list_path)
 	if len(ret.failures) == 0 and len(ret.errors) == 0:
 		ret = 0
 
@@ -327,10 +330,11 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
 
 @click.command('run-ui-tests')
 @click.option('--app', help="App to run tests on, leave blank for all apps")
-@click.option('--test', help="File name of the test you want to run")
+@click.option('--test', help="Path to the specific test you want to run")
+@click.option('--test-list', help="Path to the txt file with the list of test cases")
 @click.option('--profile', is_flag=True, default=False)
 @pass_context
-def run_ui_tests(context, app=None, test=False, profile=False):
+def run_ui_tests(context, app=None, test=False, test_list=False, profile=False):
 	"Run UI tests"
 	import frappe.test_runner
 
@@ -338,7 +342,7 @@ def run_ui_tests(context, app=None, test=False, profile=False):
 	frappe.init(site=site)
 	frappe.connect()
 
-	ret = frappe.test_runner.run_ui_tests(app=app, test=test, verbose=context.verbose,
+	ret = frappe.test_runner.run_ui_tests(app=app, test=test, test_list=test_list, verbose=context.verbose,
 		profile=profile)
 	if len(ret.failures) == 0 and len(ret.errors) == 0:
 		ret = 0

frappe/config/desktop.py

@@ -70,5 +70,5 @@ def get_data():
 			"icon": "octicon octicon-book",
 			"color": '#FFAEDB',
 			"hidden": 1,
-		},
+		}
 	]

frappe/config/integrations.py

@@ -82,7 +82,16 @@ def get_data():
 					"name": "Webhook",
 					"description": _("Webhooks calling API requests into web apps"),
 				}
-
+			]
+		},
+		{
+			"label": _("Maps"),
+			"items": [
+				{
+					"type": "doctype",
+					"name": "Google Maps",
+					"description": _("Google Maps integration"),
+				}
 			]
 		}
 	]

frappe/config/setup.py

@@ -17,6 +17,11 @@ def get_data():
 					"type": "doctype",
 					"name": "Role",
 					"description": _("User Roles")
+				},
+				{
+					"type": "doctype",
+					"name": "Role Profile",
+					"description": _("Role Profile")
 				}
 			]
 		},
@@ -81,6 +86,13 @@ def get_data():
 					"name": "Error Snapshot",
 					"description": _("Log of error during requests.")
 				},
+				{
+					"type": "doctype",
+					"name": "Domain Settings",
+					"label": _("Domain Settings"),
+					"description": _("Enable / Disable Domains"),
+					"hide_count": True
+				},
 			]
 		},
 		{
@@ -88,11 +100,11 @@ def get_data():
 			"icon": "fa fa-th",
 			"items": [
 				{
-					"type": "page",
-					"name": "data-import-tool",
+					"type": "doctype",
+					"name": "Data Import",
 					"label": _("Import / Export Data"),
-					"icon": "fa fa-upload",
-					"description": _("Import / Export Data from .csv files.")
+					"icon": "octicon octicon-cloud-upload",
+					"description": _("Import / Export Data from CSV and Excel files.")
 				},
 				{
 					"type": "doctype",

frappe/contacts/address_and_contact.py

@@ -26,18 +26,17 @@ def load_address_and_contact(doc, key=None):
 	doc.set_onload('addr_list', address_list)
 
 	contact_list = []
-	if doc.doctype != "Lead":
-		filters = [
-			["Dynamic Link", "link_doctype", "=", doc.doctype],
-			["Dynamic Link", "link_name", "=", doc.name],
-			["Dynamic Link", "parenttype", "=", "Contact"],
-		]
-		contact_list = frappe.get_all("Contact", filters=filters, fields=["*"])
-
-		contact_list = sorted(contact_list,
-			lambda a, b:
-				(int(a.is_primary_contact - b.is_primary_contact)) or
-				(1 if a.modified - b.modified else 0), reverse=True)
+	filters = [
+		["Dynamic Link", "link_doctype", "=", doc.doctype],
+		["Dynamic Link", "link_name", "=", doc.name],
+		["Dynamic Link", "parenttype", "=", "Contact"],
+	]
+	contact_list = frappe.get_all("Contact", filters=filters, fields=["*"])
+
+	contact_list = sorted(contact_list,
+		lambda a, b:
+			(int(a.is_primary_contact - b.is_primary_contact)) or
+			(1 if a.modified - b.modified else 0), reverse=True)
 
 	doc.set_onload('contact_list', contact_list)
 
@@ -147,4 +146,4 @@ def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, fil
 		order_by="dt asc", as_list=True)
 
 	all_doctypes = doctypes + _doctypes
-	return sorted(all_doctypes, key=lambda item: item[0])
+	return sorted(all_doctypes, key=lambda item: item[0])

frappe/contacts/doctype/contact/contact.py

@@ -46,6 +46,11 @@ class Contact(Document):
 
 		return None
 
+	def has_link(self, doctype, name):
+		for link in self.links:
+			if link.link_doctype==doctype and link.link_name== name:
+				return True
+
 	def has_common_link(self, doc):
 		reference_links = [(link.link_doctype, link.link_name) for link in doc.links]
 		for link in self.links:

frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js

@@ -15,15 +15,14 @@ frappe.query_reports["Addresses And Contacts"] = {
 						"name": ["in","Customer,Supplier,Sales Partner"],
 					}
 				}
-			},
-			"default": "Customer"
+			}
 		},
 		{
 			"fieldname":"party_name",
 			"label": __("Party Name"),
 			"fieldtype": "Dynamic Link",
 			"get_options": function() {
-				var party_type = frappe.query_report_filters_by_name.party_type.get_value();
+				let party_type = frappe.query_report_filters_by_name.party_type.get_value();
 				if(!party_type) {
 					frappe.throw(__("Please select Party Type first"));
 				}

frappe/core/doctype/activity_log/activity_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Activity Log', {
+	refresh: function() {
+
+	}
+});

frappe/core/doctype/activity_log/activity_log.json

@@ -0,0 +1,717 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 1, 
+ "allow_rename": 0, 
+ "autoname": "", 
+ "beta": 0, 
+ "creation": "2017-10-05 11:10:38.780133", 
+ "custom": 0, 
+ "description": "Keep track of all update feeds", 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "Setup", 
+ "editable_grid": 0, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "subject", 
+   "fieldtype": "Small Text", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 1, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Subject", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "section_break_8", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "content", 
+   "fieldtype": "Text Editor", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Message", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0, 
+   "width": "400"
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_5", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 1, 
+   "columns": 0, 
+   "fieldname": "additional_info", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "More Information", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "Now", 
+   "fieldname": "communication_date", 
+   "fieldtype": "Datetime", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Date", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_7", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "operation", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Operation", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "\nLogin\nLogout", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "status", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Status", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "\nSuccess\nFailed\nLinked\nClosed", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 1, 
+   "columns": 0, 
+   "fieldname": "reference_section", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Reference", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "reference_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Reference DocType", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "reference_name", 
+   "fieldtype": "Dynamic Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Reference Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "reference_doctype", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "reference_owner", 
+   "fieldtype": "Read Only", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Reference Owner", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "reference_name.owner", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 1, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_14", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "timeline_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Timeline DocType", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "timeline_name", 
+   "fieldtype": "Dynamic Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Timeline Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "timeline_doctype", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "link_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Link DocType", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "link_name", 
+   "fieldtype": "Dynamic Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Link Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "link_doctype", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "__user", 
+   "fieldname": "user", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 1, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "User", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "User", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "full_name", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Full Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "icon": "fa fa-comment", 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-11-21 12:39:23.659308", 
+ "modified_by": "Administrator", 
+ "module": "Core", 
+ "name": "Activity Log", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 0, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 0
+  }, 
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 0, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 1, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 0
+  }, 
+  {
+   "amend": 0, 
+   "apply_user_permissions": 1, 
+   "cancel": 0, 
+   "create": 0, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 1, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 0, 
+   "read": 1, 
+   "report": 0, 
+   "role": "All", 
+   "set_user_permissions": 0, 
+   "share": 0, 
+   "submit": 0, 
+   "user_permission_doctypes": "[\"Email Account\"]", 
+   "write": 0
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "search_fields": "subject", 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "title_field": "subject", 
+ "track_changes": 1, 
+ "track_seen": 1
+}

frappe/core/doctype/activity_log/activity_log.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe import _
+from frappe.utils import get_fullname, now
+from frappe.model.document import Document
+from frappe.core.utils import get_parent_doc, set_timeline_doc
+import frappe
+
+class ActivityLog(Document):
+	def before_insert(self):
+		self.full_name = get_fullname(self.user)
+		self.date = now()
+
+	def validate(self):
+		self.set_status()
+		set_timeline_doc(self)
+
+	def set_status(self):
+		if not self.is_new():
+			return
+
+		if self.reference_doctype and self.reference_name:
+			self.status = "Linked"
+
+	def on_trash(self): # pylint: disable=no-self-use
+		frappe.throw(_("Sorry! You cannot delete auto-generated comments"))
+
+def on_doctype_update():
+	"""Add indexes in `tabActivity Log`"""
+	frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"])
+	frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"])
+	frappe.db.add_index("Activity Log", ["link_doctype", "link_name"])
+
+def add_authentication_log(subject, user, operation="Login", status="Success"):
+	frappe.get_doc({
+		"doctype": "Activity Log",
+		"user": user,
+		"status": status,
+		"subject": subject,
+		"operation": operation,
+	}).insert(ignore_permissions=True)
+
+def clear_authentication_logs():
+	"""clear 100 day old authentication logs"""
+	frappe.db.sql("""delete from `tabActivity Log` where \
+	creation<DATE_SUB(NOW(), INTERVAL 100 DAY)""")

frappe/core/doctype/authentication_log/authentication_log_list.js → frappe/core/doctype/activity_log/activity_log_list.js

@@ -1,4 +1,4 @@
-frappe.listview_settings['Authentication Log'] = {
+frappe.listview_settings['Activity Log'] = {
 	get_indicator: function(doc) {
 		if(doc.operation == "Login" && doc.status == "Success")
 			return [__(doc.status), "green"];

frappe/core/doctype/communication/feed.py → frappe/core/doctype/activity_log/feed.py

@@ -4,22 +4,19 @@
 from __future__ import unicode_literals
 import frappe
 import frappe.permissions
-from frappe.model.document import Document
 from frappe.utils import get_fullname
 from frappe import _
-from frappe.core.doctype.communication.comment import add_info_comment
-from frappe.core.doctype.authentication_log.authentication_log import add_authentication_log
+from frappe.core.doctype.activity_log.activity_log import add_authentication_log
 from six import string_types
 
 def update_feed(doc, method=None):
-	"adds a new communication with comment_type='Updated'"
 	if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import:
 		return
 
 	if doc._action!="save" or doc.flags.ignore_feed:
 		return
 
-	if doc.doctype == "Communication" or doc.meta.issingle:
+	if doc.doctype == "Activity Log" or doc.meta.issingle:
 		return
 
 	if hasattr(doc, "get_feed"):
@@ -34,16 +31,12 @@ def update_feed(doc, method=None):
 			name = feed.name or doc.name
 
 			# delete earlier feed
-			frappe.db.sql("""delete from `tabCommunication`
+			frappe.db.sql("""delete from `tabActivity Log`
 				where
 					reference_doctype=%s and reference_name=%s
-					and communication_type='Comment'
-					and comment_type='Updated'""", (doctype, name))
-
+					and link_doctype=%s""", (doctype, name,feed.link_doctype))
 			frappe.get_doc({
-				"doctype": "Communication",
-				"communication_type": "Comment",
-				"comment_type": "Updated",
+				"doctype": "Activity Log",
 				"reference_doctype": doctype,
 				"reference_name": name,
 				"subject": feed.subject,
@@ -75,9 +68,9 @@ def get_feed_match_conditions(user=None, force=True):
 		list(set(can_read) - set(user_permissions.keys()))]
 
 	if can_read_doctypes:
-		conditions += ["""(tabCommunication.reference_doctype is null
-			or tabCommunication.reference_doctype = ''
-			or tabCommunication.reference_doctype in ({}))""".format(", ".join(can_read_doctypes))]
+		conditions += ["""(`tabCommunication`.reference_doctype is null
+			or `tabCommunication`.reference_doctype = ''
+			or `tabCommunication`.reference_doctype in ({}))""".format(", ".join(can_read_doctypes))]
 
 		if user_permissions:
 			can_read_docs = []
@@ -86,7 +79,7 @@ def get_feed_match_conditions(user=None, force=True):
 					can_read_docs.append('"{}|{}"'.format(doctype, frappe.db.escape(n)))
 
 			if can_read_docs:
-				conditions.append("concat_ws('|', tabCommunication.reference_doctype, tabCommunication.reference_name) in ({})".format(
+				conditions.append("concat_ws('|', `tabCommunication`.reference_doctype, `tabCommunication`.reference_name) in ({})".format(
 					", ".join(can_read_docs)))
 
 	return "(" + " or ".join(conditions) + ")"

frappe/core/doctype/authentication_log/test_authentication_log.py → frappe/core/doctype/activity_log/test_activity_log.py

@@ -6,10 +6,8 @@ from __future__ import unicode_literals
 import frappe
 import unittest
 
-# test_records = frappe.get_test_records('Authentication Log')
-
-class TestAuthenticationLog(unittest.TestCase):
-	def test_authentication_log(self):
+class TestActivityLog(unittest.TestCase):
+	def test_activity_log(self):
 		from frappe.auth import LoginManager, CookieManager
 
 		# test user login log
@@ -40,10 +38,10 @@ class TestAuthenticationLog(unittest.TestCase):
 		frappe.local.form_dict = frappe._dict()
 
 	def get_auth_log(self, operation='Login'):
-		names = frappe.db.sql_list("""select name from `tabAuthentication Log`
+		names = frappe.db.sql_list("""select name from `tabActivity Log`
 					where user='Administrator' and operation='{operation}' order by
 					creation desc""".format(operation=operation))
 
 		name = names[0]
-		auth_log = frappe.get_doc('Authentication Log', name)
+		auth_log = frappe.get_doc('Activity Log', name)
 		return auth_log

frappe/core/doctype/authentication_log/authentication_log.js

@@ -1,8 +0,0 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Authentication Log', {
-	refresh: function(frm) {
-
-	}
-});

frappe/core/doctype/authentication_log/authentication_log.json

@@ -1,341 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-01-23 16:56:25.875531", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "user_details", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "User Details", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "user", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "User", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "full_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Full Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_8", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "date", 
-   "fieldtype": "Datetime", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Date", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "section_break_6", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "subject", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Subject", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_2", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "operation", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Operation", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "\nLogin\nLogout", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "status", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Status", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "\nSuccess\nFailed", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
- "in_dialog": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-01-24 14:51:13.726113", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "Authentication Log", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "All", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 0
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "title_field": "subject", 
- "track_seen": 0
-}

frappe/core/doctype/authentication_log/authentication_log.py

@@ -1,27 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
-# For license information, please see license.txt
-
-from __future__ import unicode_literals
-import frappe
-from frappe.utils import get_fullname, now
-from frappe.model.document import Document
-
-class AuthenticationLog(Document):
-	def before_insert(self):
-		self.full_name = get_fullname(self.user)
-		self.date = now()
-
-def add_authentication_log(subject, user, operation="Login", status="Success"):
-	frappe.get_doc({
-		"doctype": "Authentication Log",
-		"user": user,
-		"status": status,
-		"subject": subject,
-		"operation": operation,
-	}).insert(ignore_permissions=True)
-
-def clear_authentication_logs():
-	"""clear 100 day old authentication logs"""
-	frappe.db.sql("""delete from `tabAuthentication Log` where 
-		creation<DATE_SUB(NOW(), INTERVAL 100 DAY)""")

frappe/core/doctype/communication/communication.json

@@ -15,6 +15,7 @@
  "engine": "InnoDB", 
  "fields": [
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -43,6 +44,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -73,6 +75,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -104,6 +107,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -134,6 +138,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -162,6 +167,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -192,6 +198,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -223,11 +230,44 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "depends_on": "eval:in_list([\"Phone\", \"SMS\"], doc.communication_medium)", 
+   "depends_on": "eval:doc.communication_medium===\"Email\"", 
+   "fieldname": "bcc", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "BCC", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Email", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval:in_list([\"Phone\",\"SMS\"],doc.communication_medium)", 
    "fieldname": "phone_no", 
    "fieldtype": "Data", 
    "hidden": 0, 
@@ -252,6 +292,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -283,6 +324,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -311,6 +353,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -340,6 +383,7 @@
    "width": "400"
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -369,6 +413,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -398,6 +443,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -429,6 +475,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -440,12 +487,12 @@
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
    "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 1, 
    "label": "Comment Type", 
    "length": 0, 
    "no_copy": 0, 
-   "options": "\nComment\nLike\nInfo\nLabel\nWorkflow\nCreated\nUpdated\nSubmitted\nCancelled\nDeleted\nAssigned\nAssignment Completed\nAttachment\nAttachment Removed\nShared\nUnshared\nBot\nRelinked", 
+   "options": "\nComment\nLike\nInfo\nLabel\nWorkflow\nCreated\nSubmitted\nCancelled\nUpdated\nDeleted\nAssigned\nAssignment Completed\nAttachment\nAttachment Removed\nShared\nUnshared\nBot\nRelinked", 
    "permlevel": 0, 
    "precision": "", 
    "print_hide": 0, 
@@ -459,6 +506,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -487,6 +535,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -518,6 +567,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -548,6 +598,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -576,6 +627,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -605,6 +657,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -634,6 +687,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -662,6 +716,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -691,6 +746,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -720,6 +776,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -750,6 +807,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -780,6 +838,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -810,6 +869,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -841,6 +901,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -871,6 +932,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -901,6 +963,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -929,6 +992,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -959,6 +1023,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -989,6 +1054,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1019,6 +1085,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1049,6 +1116,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1079,6 +1147,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1109,6 +1178,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1138,6 +1208,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1166,6 +1237,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -1195,6 +1267,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1224,6 +1297,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -1253,6 +1327,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1283,6 +1358,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1312,6 +1388,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 1, 
@@ -1342,6 +1419,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1371,6 +1449,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -1411,7 +1490,7 @@
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-03-29 23:06:16.469149", 
+ "modified": "2017-11-13 12:00:44.238575", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "Communication", 
@@ -1477,26 +1556,6 @@
    "submit": 0, 
    "user_permission_doctypes": "[\"Email Account\"]", 
    "write": 0
-  }, 
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Super Email User", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 0
   }
  ], 
  "quick_entry": 0, 

frappe/core/doctype/communication/communication.py

@@ -64,7 +64,7 @@ class Communication(Document):
 		self.set_status()
 		self.set_sender_full_name()
 		validate_email(self)
-		self.set_timeline_doc()
+		set_timeline_doc(self)
 
 	def after_insert(self):
 		if not (self.reference_doctype and self.reference_name):
@@ -149,35 +149,6 @@ class Communication(Document):
 				self.sender = sender_email
 				self.sender_full_name = sender_name or get_fullname(frappe.session.user) if frappe.session.user!='Administrator' else None
 
-	def get_parent_doc(self):
-		"""Returns document of `reference_doctype`, `reference_doctype`"""
-		if not hasattr(self, "parent_doc"):
-			if self.reference_doctype and self.reference_name:
-				self.parent_doc = frappe.get_doc(self.reference_doctype, self.reference_name)
-			else:
-				self.parent_doc = None
-		return self.parent_doc
-
-	def set_timeline_doc(self):
-		"""Set timeline_doctype and timeline_name"""
-		parent_doc = self.get_parent_doc()
-		if (self.timeline_doctype and self.timeline_name) or not parent_doc:
-			return
-
-		timeline_field = parent_doc.meta.timeline_field
-		if not timeline_field:
-			return
-
-		doctype = parent_doc.meta.get_link_doctype(timeline_field)
-		name = parent_doc.get(timeline_field)
-
-		if doctype and name:
-			self.timeline_doctype = doctype
-			self.timeline_name = name
-
-		else:
-			return
-
 	def send(self, print_html=None, print_format=None, attachments=None,
 		send_me_a_copy=False, recipients=None):
 		"""Send communication via Email.
@@ -189,7 +160,7 @@ class Communication(Document):
 		self.notify(print_html, print_format, attachments, recipients)
 
 	def notify(self, print_html=None, print_format=None, attachments=None,
-		recipients=None, cc=None, fetched_from_email_account=False):
+		recipients=None, cc=None, bcc=None,fetched_from_email_account=False):
 		"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
 
 		:param print_html: Send given value as HTML attachment
@@ -200,13 +171,13 @@ class Communication(Document):
 		:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
 
 		"""
-		notify(self, print_html, print_format, attachments, recipients, cc,
+		notify(self, print_html, print_format, attachments, recipients, cc, bcc,
 			fetched_from_email_account)
 
 	def _notify(self, print_html=None, print_format=None, attachments=None,
-		recipients=None, cc=None):
+		recipients=None, cc=None, bcc=None):
 
-		_notify(self, print_html, print_format, attachments, recipients, cc)
+		_notify(self, print_html, print_format, attachments, recipients, cc, bcc)
 
 	def bot_reply(self):
 		if self.comment_type == 'Bot' and self.communication_type == 'Chat':
@@ -253,6 +224,35 @@ class Communication(Document):
 			if commit:
 				frappe.db.commit()
 
+def get_parent_doc(doc):
+	"""Returns document of `reference_doctype`, `reference_doctype`"""
+	if not hasattr(doc, "parent_doc"):
+		if doc.reference_doctype and doc.reference_name:
+			doc.parent_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
+		else:
+			doc.parent_doc = None
+	return doc.parent_doc
+
+def set_timeline_doc(doc):
+	"""Set timeline_doctype and timeline_name"""
+	parent_doc = get_parent_doc(doc)
+	if (doc.timeline_doctype and doc.timeline_name) or not parent_doc:
+		return
+
+	timeline_field = parent_doc.meta.timeline_field
+	if not timeline_field:
+		return
+
+	doctype = parent_doc.meta.get_link_doctype(timeline_field)
+	name = parent_doc.get(timeline_field)
+
+	if doctype and name:
+		doc.timeline_doctype = doctype
+		doc.timeline_name = name
+
+	else:
+		return
+
 def on_doctype_update():
 	"""Add indexes in `tabCommunication`"""
 	frappe.db.add_index("Communication", ["reference_doctype", "reference_name"])

frappe/core/doctype/communication/email.py

@@ -7,6 +7,7 @@ from six import string_types
 import frappe
 import json
 from email.utils import formataddr
+from frappe.core.utils import get_parent_doc
 from frappe.utils import (get_url, get_formatted_email, cint,
 	validate_email_add, split_emails, time_diff_in_seconds, parse_addr)
 from frappe.utils.file_manager import get_file
@@ -14,15 +15,18 @@ from frappe.email.queue import check_email_limit
 from frappe.utils.scheduler import log
 from frappe.email.email_body import get_message_id
 import frappe.email.smtp
-import MySQLdb
 import time
 from frappe import _
 from frappe.utils.background_jobs import enqueue
 
+# imports - third-party imports
+import pymysql
+from pymysql.constants import ER
+
 @frappe.whitelist()
 def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
 	sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False,
-	print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, flags=None,read_receipt=None):
+	print_html=None, print_format=None, attachments='[]', send_me_a_copy=False, cc=None, bcc=None, flags=None,read_receipt=None):
 	"""Make a new communication.
 
 	:param doctype: Reference DocType.
@@ -58,6 +62,7 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
 		"sender_full_name":sender_full_name,
 		"recipients": recipients,
 		"cc": cc or None,
+		"bcc": bcc or None,
 		"communication_medium": communication_medium,
 		"sent_or_received": sent_or_received,
 		"reference_doctype": doctype,
@@ -102,10 +107,13 @@ def validate_email(doc):
 	for email in split_emails(doc.cc):
 		validate_email_add(email, throw=True)
 
+	for email in split_emails(doc.bcc):
+		validate_email_add(email, throw=True)
+
 	# validate sender
 
 def notify(doc, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, fetched_from_email_account=False):
+	recipients=None, cc=None, bcc=None, fetched_from_email_account=False):
 	"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
 
 	:param print_html: Send given value as HTML attachment
@@ -113,10 +121,11 @@ def notify(doc, print_html=None, print_format=None, attachments=None,
 	:param attachments: A list of filenames that should be attached when sending this email
 	:param recipients: Email recipients
 	:param cc: Send email as CC to
+	:param bcc: Send email as BCC to
 	:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
 
 	"""
-	recipients, cc = get_recipients_and_cc(doc, recipients, cc,
+	recipients, cc, bcc = get_recipients_cc_and_bcc(doc, recipients, cc, bcc,
 		fetched_from_email_account=fetched_from_email_account)
 
 	if not recipients:
@@ -127,16 +136,16 @@ def notify(doc, print_html=None, print_format=None, attachments=None,
 	if frappe.flags.in_test:
 		# for test cases, run synchronously
 		doc._notify(print_html=print_html, print_format=print_format, attachments=attachments,
-			recipients=recipients, cc=cc)
+			recipients=recipients, cc=cc, bcc=None)
 	else:
 		check_email_limit(list(set(doc.sent_email_addresses)))
 		enqueue(sendmail, queue="default", timeout=300, event="sendmail",
 			communication_name=doc.name,
 			print_html=print_html, print_format=print_format, attachments=attachments,
-			recipients=recipients, cc=cc, lang=frappe.local.lang, session=frappe.local.session)
+			recipients=recipients, cc=cc, bcc=bcc, lang=frappe.local.lang, session=frappe.local.session)
 
 def _notify(doc, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None):
+	recipients=None, cc=None, bcc=None):
 
 	prepare_to_notify(doc, print_html, print_format, attachments)
 
@@ -148,6 +157,7 @@ def _notify(doc, print_html=None, print_format=None, attachments=None,
 	frappe.sendmail(
 		recipients=(recipients or []),
 		cc=(cc or []),
+		bcc=(bcc or []),
 		expose_recipients="header",
 		sender=doc.sender,
 		reply_to=doc.incoming_email_account,
@@ -166,7 +176,8 @@ def _notify(doc, print_html=None, print_format=None, attachments=None,
 
 def update_parent_mins_to_first_response(doc):
 	"""Update mins_to_first_communication of parent document based on who is replying."""
-	parent = doc.get_parent_doc()
+
+	parent = get_parent_doc(doc)
 	if not parent:
 		return
 
@@ -190,7 +201,7 @@ def update_parent_mins_to_first_response(doc):
 	parent.run_method('notify_communication', doc)
 	parent.notify_update()
 
-def get_recipients_and_cc(doc, recipients, cc, fetched_from_email_account=False):
+def get_recipients_cc_and_bcc(doc, recipients, cc, bcc, fetched_from_email_account=False):
 	doc.all_email_addresses = []
 	doc.sent_email_addresses = []
 	doc.previous_email_sender = None
@@ -201,6 +212,9 @@ def get_recipients_and_cc(doc, recipients, cc, fetched_from_email_account=False)
 	if not cc:
 		cc = get_cc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
 
+	if not bcc:
+		bcc = get_bcc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
+
 	if fetched_from_email_account:
 		# email was already sent to the original recipient by the sender's email service
 		original_recipients, recipients = recipients, []
@@ -216,10 +230,13 @@ def get_recipients_and_cc(doc, recipients, cc, fetched_from_email_account=False)
 		# don't cc to people who already received the mail from sender's email service
 		cc = list(set(cc) - set(original_cc) - set(original_recipients))
 
+		original_bcc = split_emails(doc.bcc)
+		bcc = list(set(bcc) - set(original_bcc) - set(original_recipients))
+
 	if 'Administrator' in recipients:
 		recipients.remove('Administrator')
 
-	return recipients, cc
+	return recipients, cc, bcc
 
 def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None):
 	"""Prepare to make multipart MIME Email
@@ -247,8 +264,8 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None)
 	doc.attachments = []
 
 	if print_html or print_format:
-		doc.attachments.append(frappe.attach_print(doc.reference_doctype, doc.reference_name,
-			print_format=print_format, html=print_html))
+		doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype,
+			"name":doc.reference_name, "print_format":print_format, "html":print_html})
 
 	if attachments:
 		if isinstance(attachments, string_types):
@@ -258,8 +275,11 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None)
 			if isinstance(a, string_types):
 				# is it a filename?
 				try:
+					# keep this for error handling
 					file = get_file(a)
-					doc.attachments.append({"fname": file[0], "fcontent": file[1]})
+					# these attachments will be attached on-demand
+					# and won't be stored in the message
+					doc.attachments.append({"fid": a})
 				except IOError:
 					frappe.throw(_("Unable to find attachment {0}").format(a))
 			else:
@@ -345,6 +365,34 @@ def get_cc(doc, recipients=None, fetched_from_email_account=False):
 
 	return cc
 
+def get_bcc(doc, recipients=None, fetched_from_email_account=False):
+	"""Build a list of email addresses for BCC"""
+	bcc = split_emails(doc.bcc)
+
+	if doc.reference_doctype and doc.reference_name:
+		if fetched_from_email_account:
+			bcc.append(get_owner_email(doc))
+			bcc += get_assignees(doc)
+
+	if getattr(doc, "send_me_a_copy", False) and doc.sender not in bcc:
+		bcc.append(doc.sender)
+
+	if bcc:
+		exclude = []
+		exclude += [d[0] for d in frappe.db.get_all("User", ["name"], {"thread_notify": 0}, as_list=True)]
+		exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
+
+		if fetched_from_email_account:
+			# exclude sender when pulling email
+			exclude += [parse_addr(doc.sender)[1]]
+
+		if doc.reference_doctype and doc.reference_name:
+			exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
+				{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
+
+		bcc = filter_email_list(doc, bcc, exclude, is_bcc=True)
+
+	return bcc
 
 def add_attachments(name, attachments):
 	'''Add attachments to the given Communiction'''
@@ -360,7 +408,7 @@ def add_attachments(name, attachments):
 			save_url(attach.file_url, attach.file_name, "Communication", name,
 				"Home/Attachments", attach.is_private)
 
-def filter_email_list(doc, email_list, exclude, is_cc=False):
+def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False):
 	# temp variables
 	filtered = []
 	email_address_list = []
@@ -382,6 +430,11 @@ def filter_email_list(doc, email_list, exclude, is_cc=False):
 				# don't send to disabled users
 				continue
 
+		if is_bcc:
+			is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
+			if is_user_enabled==0:
+				continue
+
 		# make sure of case-insensitive uniqueness of email address
 		if email_address not in email_address_list:
 			# append the full email i.e. "Human <human@example.com>"
@@ -393,7 +446,7 @@ def filter_email_list(doc, email_list, exclude, is_cc=False):
 	return filtered
 
 def get_owner_email(doc):
-	owner = doc.get_parent_doc().owner
+	owner = get_parent_doc(doc).owner
 	return get_formatted_email(owner) or owner
 
 def get_assignees(doc):
@@ -412,11 +465,11 @@ def get_attach_link(doc, print_format):
 		"doctype": doc.reference_doctype,
 		"name": doc.reference_name,
 		"print_format": print_format,
-		"key": doc.get_parent_doc().get_signature()
+		"key": get_parent_doc(doc).get_signature()
 	})
 
 def sendmail(communication_name, print_html=None, print_format=None, attachments=None,
-	recipients=None, cc=None, lang=None, session=None):
+	recipients=None, cc=None, bcc=None, lang=None, session=None):
 	try:
 
 		if lang:
@@ -432,11 +485,11 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments
 			try:
 				communication = frappe.get_doc("Communication", communication_name)
 				communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
-					recipients=recipients, cc=cc)
+					recipients=recipients, cc=cc, bcc=bcc)
 
-			except MySQLdb.OperationalError as e:
+			except pymysql.InternalError as e:
 				# deadlock, try again
-				if e.args[0]==1213:
+				if e.args[0] == ER.LOCK_DEADLOCK:
 					frappe.db.rollback()
 					time.sleep(1)
 					continue
@@ -453,6 +506,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments
 			"attachments": attachments,
 			"recipients": recipients,
 			"cc": cc,
+			"bcc": bcc,
 			"lang": lang
 		}))
 		frappe.logger(__name__).error(traceback)

frappe/core/doctype/communication/test_communication.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Communication", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Communication
+		() => frappe.tests.make('Communication', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/core/doctype/data_import/data_import.js

@@ -0,0 +1,261 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Data Import', {
+	onload: function(frm) {
+		frm.set_query("reference_doctype", function() {
+			return {
+				"filters": {
+					"issingle": 0,
+					"istable": 0,
+					"name": ['in', frappe.boot.user.can_import]
+				}
+			};
+		});
+
+		frappe.realtime.on("data_import_progress", function(data) {
+			if (data.data_import === frm.doc.name) {
+				if (data.reload && data.reload === true) {
+					frm.reload_doc();
+				}
+				if (data.progress) {
+					let progress_bar = $(frm.dashboard.progress_area).find(".progress-bar");
+					if (progress_bar) {
+						$(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped");
+						$(progress_bar).css("width", data.progress+"%");
+					}
+				}
+			}
+		});
+	},
+
+	refresh: function(frm) {
+		frm.disable_save();
+		frm.dashboard.clear_headline();
+		if (frm.doc.reference_doctype && !frm.doc.import_file) {
+			frm.dashboard.add_comment(__('Please attach a file to import'));
+		} else {
+			if (frm.doc.import_status) {
+				frm.dashboard.add_comment(frm.doc.import_status);
+
+				if (frm.doc.import_status==="In Progress") {
+					frm.dashboard.add_progress("Data Import Progress", "0");
+					frm.set_read_only();
+					frm.refresh_fields();
+				}
+			}
+		}
+
+		if (frm.doc.reference_doctype) {
+			frappe.model.with_doctype(frm.doc.reference_doctype);
+		}
+
+		frm.add_custom_button(__("Help"), function() {
+			frappe.help.show_video("6wiriRKPhmg");
+		});
+
+		if(frm.doc.reference_doctype && frm.doc.docstatus === 0) {
+			frm.add_custom_button(__("Download template"), function() {
+				frappe.data_import.download_dialog(frm).show();
+			});
+		}
+
+		if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows  && frm.doc.docstatus === 0) {
+			frm.page.set_primary_action(__("Start Import"), function() {
+				frappe.call({
+					method: "frappe.core.doctype.data_import.data_import.import_data",
+					args: {
+						data_import: frm.doc.name
+					}
+				});
+			}).addClass('btn btn-primary');
+		}
+
+		if (frm.doc.log_details) {
+			frm.events.create_log_table(frm);
+		} else {
+			$(frm.fields_dict.import_log.wrapper).empty();
+		}
+	},
+
+	reference_doctype: function(frm) {
+		if (frm.doc.reference_doctype) {
+			frm.save();
+		}
+	},
+
+	// import_file: function(frm) {
+	// 	frm.save();
+	// },
+
+	overwrite: function(frm) {
+		if (frm.doc.overwrite === 0) {
+			frm.doc.only_update = 0;
+		}
+		frm.save();
+	},
+
+	only_update: function(frm) {
+		frm.save();
+	},
+
+	submit_after_import: function(frm) {
+		frm.save();
+	},
+
+	skip_errors: function(frm) {
+		frm.save();
+	},
+
+	ignore_encoding_errors: function(frm) {
+		frm.save();
+	},
+
+	no_email: function(frm) {
+		frm.save();
+	},
+
+	show_only_errors: function(frm) {
+		frm.events.create_log_table(frm);
+	},
+
+	create_log_table: function(frm) {
+		let msg = JSON.parse(frm.doc.log_details);
+		var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty();
+		$(frappe.render_template("log_details", {data: msg.messages, show_only_errors: frm.doc.show_only_errors,
+			import_status: frm.doc.import_status})).appendTo($log_wrapper);
+	}
+});
+
+frappe.provide('frappe.data_import');
+frappe.data_import.download_dialog = function(frm) {
+	var dialog;
+	const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden;
+	const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields);
+
+	const get_doctypes = parentdt => {
+		return [parentdt].concat(
+			frappe.meta.get_table_fields(parentdt).map(df => df.options)
+		);
+	};
+
+	const get_doctype_checkbox_fields = () => {
+		return dialog.fields.filter(df => df.fieldname.endsWith('_fields'))
+			.map(df => dialog.fields_dict[df.fieldname]);
+	}
+
+	const doctype_fields = get_fields(frm.doc.reference_doctype)
+		.map(df => ({
+			label: df.label,
+			value: df.fieldname,
+			checked: 1
+		}));
+
+	let fields = [
+		{
+			"label": __("Select Columns"),
+			"fieldname": "select_columns",
+			"fieldtype": "Select",
+			"options": "All\nMandatory\nManually",
+			"reqd": 1,
+			"onchange": function() {
+				const fields = get_doctype_checkbox_fields();
+				fields.map(f => f.toggle(this.value === 'Manually'));
+			}
+		},
+		{
+			"label": __("File Type"),
+			"fieldname": "file_type",
+			"fieldtype": "Select",
+			"options": "Excel\nCSV",
+			"default": "Excel"
+		},
+		{
+			"label": __("Download with Data"),
+			"fieldname": "with_data",
+			"fieldtype": "Check"
+		},
+		{
+			"label": frm.doc.reference_doctype,
+			"fieldname": "doctype_fields",
+			"fieldtype": "MultiCheck",
+			"options": doctype_fields,
+			"columns": 2,
+			"hidden": 1
+		}
+	];
+
+	const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype)
+		.map(df => {
+			return {
+				"label": df.options,
+				"fieldname": df.fieldname + '_fields',
+				"fieldtype": "MultiCheck",
+				"options": frappe.meta.get_docfields(df.options)
+					.filter(filter_fields)
+					.map(df => ({
+						label: df.label,
+						value: df.fieldname,
+						checked: 1
+					})),
+				"columns": 2,
+				"hidden": 1
+			};
+		});
+
+	fields = fields.concat(child_table_fields);
+
+	dialog = new frappe.ui.Dialog({
+		title: __('Download Template'),
+		fields: fields,
+		primary_action: function(values) {
+			var data = values;
+			if (frm.doc.reference_doctype) {
+				var export_params = () => {
+					let columns = {};
+					if (values.select_columns === 'All') {
+						columns = get_doctypes(frm.doc.reference_doctype).reduce((columns, doctype) => {
+							columns[doctype] = get_fields(doctype).map(df => df.fieldname);
+							return columns;
+						}, {});
+					} else if (values.select_columns === 'Mandatory') {
+						// only reqd child tables
+						const doctypes = [frm.doc.reference_doctype].concat(
+							frappe.meta.get_table_fields(frm.doc.reference_doctype)
+								.filter(df => df.reqd).map(df => df.options)
+						);
+
+						columns = doctypes.reduce((columns, doctype) => {
+							columns[doctype] = get_fields(doctype).filter(df => df.reqd).map(df => df.fieldname);
+							return columns;
+						}, {});
+					} else if (values.select_columns === 'Manually') {
+						columns = get_doctype_checkbox_fields().reduce((columns, field) => {
+							const options = field.get_checked_options();
+							columns[field.df.label] = options;
+							return columns;
+						}, {});
+					}
+
+					return {
+						doctype: frm.doc.reference_doctype,
+						parent_doctype: frm.doc.reference_doctype,
+						select_columns: JSON.stringify(columns),
+						with_data: data.with_data ? 'Yes' : 'No',
+						all_doctypes: 'Yes',
+						from_data_import: 'Yes',
+						excel_format: data.file_type === 'Excel' ? 'Yes' : 'No'
+					};
+				};
+				let get_template_url = '/api/method/frappe.core.doctype.data_import.exporter.get_template';
+				open_url_post(get_template_url, export_params());
+			} else {
+				frappe.msgprint(__("Please select the Document Type."));
+			}
+			dialog.hide();
+		},
+		primary_action_label: __('Download')
+	});
+
+	return dialog;
+};

frappe/core/doctype/data_import/data_import.json

@@ -0,0 +1,661 @@
+{
+ "allow_copy": 1, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "", 
+ "beta": 0, 
+ "creation": "2016-12-09 14:27:32.720061", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "Document", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "reference_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 1, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Document Type", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "collapsible_depends_on": "", 
+   "columns": 0, 
+   "depends_on": "eval:(!doc.__islocal)", 
+   "fieldname": "section_break_4", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "import_file", 
+   "fieldtype": "Attach", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Attach file for Import", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "column_break_4", 
+   "fieldtype": "Column Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "eval: doc.import_status == \"Partially Successful\"", 
+   "description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.", 
+   "fieldname": "error_file", 
+   "fieldtype": "Attach", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Generated File", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "collapsible_depends_on": "", 
+   "columns": 0, 
+   "depends_on": "eval:(!doc.__islocal)", 
+   "fieldname": "section_break_6", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "", 
+   "description": "If you are updating/overwriting already created records.", 
+   "fieldname": "overwrite", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Update records", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "overwrite", 
+   "description": "If you don't want to create any new records while updating the older records.", 
+   "fieldname": "only_update", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Don't create new records", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.\n", 
+   "fieldname": "skip_errors", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Skip rows with errors", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "", 
+   "fieldname": "submit_after_import", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Submit after importing", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "depends_on": "", 
+   "fieldname": "ignore_encoding_errors", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Ignore encoding errors", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "1", 
+   "depends_on": "", 
+   "fieldname": "no_email", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Do not send Emails", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 1, 
+   "collapsible_depends_on": "eval: doc.import_status == \"Failed\"", 
+   "columns": 0, 
+   "depends_on": "import_status", 
+   "fieldname": "import_detail", 
+   "fieldtype": "Section Break", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Import Log", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "import_status", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Import Status", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 1, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "1", 
+   "fieldname": "show_only_errors", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Show only errors", 
+   "length": 0, 
+   "no_copy": 1, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 1, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 1, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "", 
+   "depends_on": "import_status", 
+   "fieldname": "import_log", 
+   "fieldtype": "HTML", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Import Log", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 1, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "depends_on": "", 
+   "fieldname": "log_details", 
+   "fieldtype": "Code", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Log Details", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "amended_from", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Amended From", 
+   "length": 0, 
+   "no_copy": 1, 
+   "options": "Data Import", 
+   "permlevel": 0, 
+   "print_hide": 1, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "total_rows", 
+   "fieldtype": "Int", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Total Rows", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 1, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 1, 
+ "modified": "2017-12-15 14:49:24.622128", 
+ "modified_by": "Administrator", 
+ "module": "Core", 
+ "name": "Data Import", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 0, 
+   "read": 1, 
+   "report": 0, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 1, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "title_field": "", 
+ "track_changes": 1, 
+ "track_seen": 1
+}

frappe/core/page/data_import_tool/data_import_tool.py → frappe/core/doctype/data_import/data_import.py

@@ -1,44 +1,67 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals, print_function
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
 
+from __future__ import unicode_literals
 import frappe, os
 from frappe import _
 import frappe.modules.import_file
+from frappe.model.document import Document
+from frappe.utils.data import format_datetime
+from frappe.core.doctype.data_import.importer import upload
+from frappe.utils.background_jobs import enqueue
 
-@frappe.whitelist()
-def get_data_keys():
-    return frappe._dict({
-        "data_separator": _('Start entering data below this line'),
-        "main_table": _("Table") + ":",
-        "parent_table": _("Parent Table") + ":",
-        "columns": _("Column Name") + ":",
-        "doctype": _("DocType") + ":"
-    })
 
-@frappe.whitelist()
-def get_doctypes():
-	return frappe.get_user()._get("can_import")
+class DataImport(Document):
+	def autoname(self):
+		self.name = "Import on "+ format_datetime(self.creation)
+
+	def validate(self):
+		if not self.import_file:
+			self.db_set("total_rows", 0)
+		if self.import_status == "In Progress":
+			frappe.throw(_("Can't save the form as data import is in progress."))
+
+		# validate the template just after the upload
+		# if there is total_rows in the doc, it means that the template is already validated and error free
+		if self.import_file and not self.total_rows:
+			upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
+
 
 @frappe.whitelist()
-def get_doctype_options():
-	doctype = frappe.form_dict['doctype']
-	return [doctype] + [d.options for d in frappe.get_meta(doctype).get_table_fields()]
+def import_data(data_import):
+	frappe.db.set_value("Data Import", data_import, "import_status", "In Progress", update_modified=False)
+	frappe.publish_realtime("data_import_progress", {"progress": "0",
+		"data_import": data_import, "reload": True}, user=frappe.session.user)
+	enqueue(upload, queue='default', timeout=6000, event='data_import',
+		data_import_doc=data_import, from_data_import="Yes", validate_template=False)
+
+
+def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
+	insert=False, submit=False, pre_process=None):
+	if os.path.isdir(path):
+		files = [os.path.join(path, f) for f in os.listdir(path)]
+	else:
+		files = [path]
+
+	for f in files:
+		if f.endswith(".json"):
+			frappe.flags.mute_emails = True
+			frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
+			frappe.flags.mute_emails = False
+			frappe.db.commit()
+		elif f.endswith(".csv"):
+			import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
+			frappe.db.commit()
+
 
 def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True):
 	from frappe.utils.csvutils import read_csv_content
-	from frappe.core.page.data_import_tool.importer import upload
 	print("Importing " + path)
 	with open(path, "r") as infile:
 		upload(rows = read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
-            submit_after_import=submit, pre_process=pre_process)
+			submit_after_import=submit, pre_process=pre_process)
 
-def export_csv(doctype, path):
-	from frappe.core.page.data_import_tool.exporter import get_template
-	with open(path, "wb") as csvfile:
-		get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
-		csvfile.write(frappe.response.result.encode("utf-8"))
 
 def export_json(doctype, path, filters=None, or_filters=None, name=None):
 	def post_process(out):
@@ -71,6 +94,14 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None):
 	with open(path, "w") as outfile:
 		outfile.write(frappe.as_json(out))
 
+
+def export_csv(doctype, path):
+	from frappe.core.doctype.data_import.exporter import get_template
+	with open(path, "wb") as csvfile:
+		get_template(doctype=doctype, all_doctypes="Yes", with_data="Yes")
+		csvfile.write(frappe.response.result.encode("utf-8"))
+
+
 @frappe.whitelist()
 def export_fixture(doctype, app):
 	if frappe.session.user != "Administrator":
@@ -80,21 +111,3 @@ def export_fixture(doctype, app):
 		os.mkdir(frappe.get_app_path(app, "fixtures"))
 
 	export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"))
-
-
-def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
-    insert=False, submit=False, pre_process=None):
-	if os.path.isdir(path):
-		files = [os.path.join(path, f) for f in os.listdir(path)]
-	else:
-		files = [path]
-
-	for f in files:
-		if f.endswith(".json"):
-			frappe.flags.mute_emails = True
-			frappe.modules.import_file.import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
-			frappe.flags.mute_emails = False
-			frappe.db.commit()
-		elif f.endswith(".csv"):
-			import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
-			frappe.db.commit()

frappe/core/doctype/data_import/data_import_list.js

@@ -0,0 +1,24 @@
+frappe.listview_settings['Data Import'] = {
+	add_fields: ["import_status"],
+	has_indicator_for_draft: 1,
+	get_indicator: function(doc) {
+
+		let status = {
+			'Successful': [__("Success"), "green", "import_status,=,Successful"],
+			'Partially Successful': [__("Partial Success"), "blue", "import_status,=,Partially Successful"],
+			'In Progress': [__("In Progress"), "orange", "import_status,=,In Progress"],
+			'Failed': [__("Failed"), "red", "import_status,=,Failed"],
+			'Pending': [__("Pending"), "orange", "import_status,=,"]
+		}
+
+		if (doc.import_status) {
+			return status[doc.import_status];
+		}
+
+		if (doc.docstatus == 0) {
+			return status['Pending'];
+		}
+
+		return status['Pending'];
+	}
+};

frappe/core/page/data_import_tool/exporter.py → frappe/core/doctype/data_import/exporter.py

@@ -9,7 +9,7 @@ import frappe.permissions
 import re, csv, os
 from frappe.utils.csvutils import UnicodeWriter
 from frappe.utils import cstr, formatdate, format_datetime
-from  frappe.core.page.data_import_tool.data_import_tool import get_data_keys
+from frappe.core.doctype.data_import.importer import get_data_keys
 from six import string_types
 
 reflags = {
@@ -165,6 +165,8 @@ def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data
 			return 'Integer'
 		elif docfield.fieldtype == "Check":
 			return "0 or 1"
+		elif docfield.fieldtype in ["Date", "Datetime"]:
+			return cstr(frappe.defaults.get_defaults().date_format)
 		elif hasattr(docfield, "info"):
 			return docfield.info
 		else:

frappe/core/page/data_import_tool/importer.py → frappe/core/doctype/data_import/importer.py

@@ -15,35 +15,54 @@ from frappe.utils.csvutils import getlink
 from frappe.utils.dateutils import parse_date
 from frappe.utils.file_manager import save_url
 
-from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url
-from frappe.core.page.data_import_tool.data_import_tool import get_data_keys
+from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_url_to_form
 from six import text_type, string_types
 
+
 @frappe.whitelist()
-def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
-	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
-	skip_errors = True):
-	"""upload data"""
+def get_data_keys():
+	return frappe._dict({
+		"data_separator": _('Start entering data below this line'),
+		"main_table": _("Table") + ":",
+		"parent_table": _("Parent Table") + ":",
+		"columns": _("Column Name") + ":",
+		"doctype": _("DocType") + ":"
+	})
 
-	frappe.flags.in_import = True
 
-	# extra input params
-	params = json.loads(frappe.form_dict.get("params") or '{}')
 
+@frappe.whitelist()
+def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
+	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
+	skip_errors = True, data_import_doc=None, validate_template=False):
+	"""upload data"""
 
-	if params.get("submit_after_import"):
-		submit_after_import = True
-	if params.get("ignore_encoding_errors"):
-		ignore_encoding_errors = True
-	if not params.get("no_email"):
-		no_email = False
-	if params.get('update_only'):
-		update_only = True
-	if params.get('from_data_import'):
-		from_data_import = params.get('from_data_import')
-	if not params.get('skip_errors'):
-		skip_errors = params.get('skip_errors')
+	if data_import_doc and isinstance(data_import_doc, string_types):
+		data_import_doc = frappe.get_doc("Data Import", data_import_doc)
+	if data_import_doc and from_data_import == "Yes":
+		no_email = data_import_doc.no_email
+		ignore_encoding_errors = data_import_doc.ignore_encoding_errors
+		update_only = data_import_doc.only_update
+		submit_after_import = data_import_doc.submit_after_import
+		overwrite = data_import_doc.overwrite
+		skip_errors = data_import_doc.skip_errors
+	else:
+		# extra input params
+		params = json.loads(frappe.form_dict.get("params") or '{}')
+		if params.get("submit_after_import"):
+			submit_after_import = True
+		if params.get("ignore_encoding_errors"):
+			ignore_encoding_errors = True
+		if not params.get("no_email"):
+			no_email = False
+		if params.get('update_only'):
+			update_only = True
+		if params.get('from_data_import'):
+			from_data_import = params.get('from_data_import')
+		if not params.get('skip_errors'):
+			skip_errors = params.get('skip_errors')
 
+	frappe.flags.in_import = True
 	frappe.flags.mute_emails = no_email
 
 	def get_data_keys_definition():
@@ -55,9 +74,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	def check_data_length():
 		max_rows = 5000
 		if not data:
-			frappe.throw(_("No data found"))
-		elif not via_console and len(data) > max_rows:
-			frappe.throw(_("Only allowed {0} rows in one import").format(max_rows))
+			frappe.throw(_("No data found in the file. Please reattach the new file with data."))
 
 	def get_start_row():
 		for i, row in enumerate(rows):
@@ -110,7 +127,10 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	def get_doc(start_idx):
 		if doctypes:
 			doc = {}
+			attachments = []
+			last_error_row_idx = None
 			for idx in range(start_idx, len(rows)):
+				last_error_row_idx = idx	# pylint: disable=W0612
 				if (not doc) or main_doc_empty(rows[idx]):
 					for dt, parentfield in doctypes:
 						d = {}
@@ -119,6 +139,9 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 								fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
 								fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
 
+								if not fieldname or not rows[idx][column_idx]:
+									continue
+
 								d[fieldname] = rows[idx][column_idx]
 								if fieldtype in ("Int", "Check"):
 									d[fieldname] = cint(d[fieldname])
@@ -158,7 +181,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 							if dt == doctype:
 								doc.update(d)
 							else:
-								if not overwrite:
+								if not overwrite and doc.get("name"):
 									d['parent'] = doc["name"]
 								d['parenttype'] = doctype
 								d['parentfield'] = parentfield
@@ -166,7 +189,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 				else:
 					break
 
-			return doc
+			return doc, attachments, last_error_row_idx
 		else:
 			doc = frappe._dict(zip(columns, rows[start_idx][1:]))
 			doc['doctype'] = doctype
@@ -175,6 +198,22 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	def main_doc_empty(row):
 		return not (row and ((len(row) > 1 and row[1]) or (len(row) > 2 and row[2])))
 
+	def validate_naming(doc):
+		autoname = frappe.get_meta(doctype).autoname
+
+		if ".#" in autoname or "hash" in autoname:
+			autoname = ""
+		elif autoname[0:5] == 'field':
+			autoname = autoname[6:]
+		elif autoname=='naming_series:':
+			autoname = 'naming_series'
+		else:
+			return True
+
+		if (autoname and autoname not in doc) or (autoname and not doc[autoname]):
+			frappe.throw(_("{0} is a mandatory field".format(autoname)))
+		return True
+
 	users = frappe.db.sql_list("select name from tabUser")
 	def prepare_for_insert(doc):
 		# don't block data import if user is not set
@@ -219,19 +258,18 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		save_url(file_url, None, doctype, docname, "Home/Attachments", 0)
 
 	# header
+	filename, file_extension = ['','']
 	if not rows:
-		from frappe.utils.file_manager import get_file_doc
-		file_doc = get_file_doc(dt='', dn="Data Import", folder='Home', is_private=1)
-		filename, file_extension = os.path.splitext(file_doc.file_name)
+		from frappe.utils.file_manager import get_file # get_file_doc
+		fname, fcontent = get_file(data_import_doc.import_file)
+		filename, file_extension = os.path.splitext(fname)
 
 		if file_extension == '.xlsx' and from_data_import == 'Yes':
 			from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
-			rows = read_xlsx_file_from_attached_file(file_id=file_doc.name)
+			rows = read_xlsx_file_from_attached_file(file_id=data_import_doc.import_file)
 
 		elif file_extension == '.csv':
-			from frappe.utils.file_manager import get_file
 			from frappe.utils.csvutils import read_csv_content
-			fname, fcontent = get_file(file_doc.name)
 			rows = read_csv_content(fcontent, ignore_encoding_errors)
 
 		else:
@@ -245,7 +283,9 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	doctypes = []
 	column_idx_to_fieldname = {}
 	column_idx_to_fieldtype = {}
-	attachments = []
+
+	if skip_errors:
+		data_rows_with_error = header
 
 	if submit_after_import and not cint(frappe.db.get_value("DocType",
 			doctype, "is_submittable")):
@@ -261,14 +301,19 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		frappe.flags.mute_emails = False
 		return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
 
-	# allow limit rows to be uploaded
+	# Throw expception in case of the empty data file
 	check_data_length()
 	make_column_map()
+	total = len(data)
+
+	if validate_template:
+		if total:
+			data_import_doc.total_rows = total
+		return True
 
 	if overwrite==None:
 		overwrite = params.get('overwrite')
 
-
 	# delete child rows (if parenttype)
 	parentfield = None
 	if parenttype:
@@ -277,13 +322,12 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		if overwrite:
 			delete_child_rows(data, doctype)
 
-	ret = []
-
-	def log(msg):
+	import_log = []
+	def log(**kwargs):
 		if via_console:
-			print(msg.encode('utf-8'))
+			print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
 		else:
-			ret.append(msg)
+			import_log.append(kwargs)
 
 	def as_link(doctype, name):
 		if via_console:
@@ -291,8 +335,14 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		else:
 			return getlink(doctype, name)
 
-	error = False
-	total = len(data)
+	# publish realtime task update
+	def publish_progress(achieved, reload=False):
+		if data_import_doc:
+			frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
+				"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
+
+
+	error_flag = rollback_flag = False
 	for i, row in enumerate(data):
 		# bypass empty rows
 		if main_doc_empty(row):
@@ -301,23 +351,21 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		row_idx = i + start_row
 		doc = None
 
-		# publish task_update
-		frappe.publish_realtime("data_import_progress", {"progress": [i, total]},
-			user=frappe.session.user)
+		publish_progress(i)
 
 		try:
-			doc = get_doc(row_idx)
+			doc, attachments, last_error_row_idx = get_doc(row_idx)
+			validate_naming(doc)
 			if pre_process:
 				pre_process(doc)
 
+			original = None
 			if parentfield:
 				parent = frappe.get_doc(parenttype, doc["parent"])
 				doc = parent.append(parentfield, doc)
 				parent.save()
-				log('Inserted row for %s at #%s' % (as_link(parenttype,
-					doc.parent),text_type(doc.idx)))
 			else:
-				if overwrite and doc["name"] and frappe.db.exists(doctype, doc["name"]):
+				if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]):
 					original = frappe.get_doc(doctype, doc["name"])
 					original_name = original.name
 					original.update(doc)
@@ -325,7 +373,6 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 					original.name = original_name
 					original.flags.ignore_links = ignore_links
 					original.save()
-					log('Updated row (#%d) %s' % (row_idx + 1, as_link(original.doctype, original.name)))
 					doc = original
 				else:
 					if not update_only:
@@ -333,29 +380,53 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 						prepare_for_insert(doc)
 						doc.flags.ignore_links = ignore_links
 						doc.insert()
-						log('Inserted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
-					else:
-						log('Ignored row (#%d) %s' % (row_idx + 1, row[1]))
 				if attachments:
 					# check file url and create a File document
 					for file_url in attachments:
 						attach_file_to_doc(doc.doctype, doc.name, file_url)
 				if submit_after_import:
 					doc.submit()
-					log('Submitted row (#%d) %s' % (row_idx + 1, as_link(doc.doctype, doc.name)))
+
+			# log errors
+			if parentfield:
+				log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
+					"link": get_url_to_form(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
+			elif submit_after_import:
+				log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
+					"message": "Document successfully submitted", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "blue"})
+			elif original:
+				log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
+					"message": "Document successfully updated", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
+			elif not update_only:
+				log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
+					"message": "Document successfully saved", "link": get_url_to_form(doc.doctype, doc.name), "indicator": "green"})
+			else:
+				log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
+					"message": "Document updation ignored", "indicator": "orange"})
+
 		except Exception as e:
-			if not skip_errors:
-				error = True
-				if doc:
-					frappe.errprint(doc if isinstance(doc, dict) else doc.as_dict())
-				err_msg = frappe.local.message_log and "\n\n".join(frappe.local.message_log) or cstr(e)
-				log('Error for row (#%d) %s : %s' % (row_idx + 1,
-					len(row)>1 and row[1] or "", err_msg))
-				frappe.errprint(frappe.get_traceback())
+			error_flag = True
+			err_msg = frappe.local.message_log and "\n".join([json.loads(msg).get('message') for msg in frappe.local.message_log]) or cstr(e)
+			error_trace = frappe.get_traceback()
+			if error_trace:
+				error_log_doc = frappe.log_error(error_trace)
+				error_link = get_url_to_form("Error Log", error_log_doc.name)
+			else:
+				error_link = None
+			log(**{"row": row_idx + 1, "title":'Error for row %s' % (len(row)>1 and row[1] or ""), "message": err_msg,
+				"indicator": "red", "link":error_link})
+			# data with error to create a new file
+			# include the errored data in the last row as last_error_row_idx will not be updated for the last row
+			if skip_errors:
+				if last_error_row_idx == len(rows)-1:
+					last_error_row_idx = len(rows)
+				data_rows_with_error += rows[row_idx:last_error_row_idx]
+			else:
+				rollback_flag = True
 		finally:
 			frappe.local.message_log = []
 
-	if error:
+	if rollback_flag:
 		frappe.db.rollback()
 	else:
 		frappe.db.commit()
@@ -363,7 +434,42 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 	frappe.flags.mute_emails = False
 	frappe.flags.in_import = False
 
-	return {"messages": ret, "error": error}
+	log_message = {"messages": import_log, "error": error_flag}
+	if data_import_doc:
+		data_import_doc.log_details = json.dumps(log_message)
+
+		import_status = None
+		if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
+			import_status = "Partially Successful"
+			# write the file with the faulty row
+			from frappe.utils.file_manager import save_file
+			file_name = 'error_' + filename + file_extension
+			if file_extension == '.xlsx':
+				from frappe.utils.xlsxutils import make_xlsx
+				xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
+				file_data = xlsx_file.getvalue()
+			else:
+				from frappe.utils.csvutils import to_csv
+				file_data = to_csv(data_rows_with_error)
+			error_data_file = save_file(file_name, file_data, "Data Import",
+				data_import_doc.name,  "Home/Attachments")
+			data_import_doc.error_file = error_data_file.file_url
+
+		elif error_flag:
+			import_status = "Failed"
+		else:
+			import_status = "Successful"
+
+		data_import_doc.import_status = import_status
+		data_import_doc.save()
+		if data_import_doc.import_status in ["Successful", "Partially Successful"]:
+			data_import_doc.submit()
+			publish_progress(100, True)
+		else:
+			publish_progress(0, True)
+		frappe.db.commit()
+	else:
+		return log_message
 
 def get_parent_field(doctype, parenttype):
 	parentfield = None

frappe/core/doctype/data_import/log_details.html

@@ -0,0 +1,38 @@
+<div>
+	<div class="table-responsive">
+		<table class="table table-bordered table-hover log-details-table">
+				<tr>
+					<th style="width:10%"> {{ __("Row No") }} </th>
+					<th style="width:40%"> {{ __("Row Status") }} </th>
+					<th style="width:50%"> {{ __("Message") }} </th>
+				</tr>
+			
+			{% for row in data %}
+				{% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %}
+				<tr>
+					<td>
+						<span>{{ row.row }} </span> 
+					</td>
+					<td>
+						<span class="indicator {{ row.indicator }}"> {{ row.title }} </span>
+					</td>
+					<td>
+						{% if (import_status != "Failed" || (row.indicator == "red")) { %}
+							<span> {{ row.message }} </span>
+							{% if row.link %}
+								<span style="width: 10%; float:right;">
+									<a class="btn-open no-decoration" title="Open Link" href="{{ row.link }}">
+										<i class="octicon octicon-arrow-right"></i>
+									</a>
+								</span>
+							{% endif %}
+						{% } else { %}
+							<span> {{ __("Document can't saved.") }} </span>
+						{% } %}
+					</td>
+				</tr>
+				{% endif %}
+			{% endfor %}
+		</table>
+	</div>
+</div>

frappe/core/doctype/data_import/test_data_import.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import unittest
+
+class TestDataImport(unittest.TestCase):
+	pass

frappe/core/doctype/deleted_document/deleted_document.py

@@ -17,7 +17,9 @@ def restore(name):
 	try:
 		doc.insert()
 	except frappe.DocstatusTransitionError:
-		frappe.throw(_("Cannot restore Cancelled Document"))
+		frappe.msgprint(_("Cancelled Document restored as Draft"))
+        doc.docstatus = 0
+        doc.insert()
 
 	doc.add_comment('Edit', _('restored {0} as {1}').format(deleted.deleted_name, doc.name))
 

frappe/core/doctype/doctype/doctype.py

@@ -4,7 +4,6 @@
 from __future__ import unicode_literals
 
 import re, copy, os
-import MySQLdb
 import frappe
 from frappe import _
 
@@ -17,6 +16,10 @@ from frappe.modules import make_boilerplate
 from frappe.model.db_schema import validate_column_name, validate_column_length
 import frappe.website.render
 
+# imports - third-party imports
+import pymysql
+from pymysql.constants import ER
+
 class InvalidFieldNameError(frappe.ValidationError): pass
 
 form_grid_templates = {
@@ -291,6 +294,8 @@ class DocType(Document):
 		`doctype` property for Single type."""
 		if self.issingle:
 			frappe.db.sql("""update tabSingles set doctype=%s where doctype=%s""", (new, old))
+			frappe.db.sql("""update tabSingles set value=%s
+				where doctype=%s and field='name' and value = %s""", (new, new, old))
 		else:
 			frappe.db.sql("rename table `tab%s` to `tab%s`" % (old, new))
 
@@ -482,8 +487,8 @@ def validate_fields(meta):
 						group by `{fieldname}` having count(*) > 1 limit 1""".format(
 						doctype=d.parent, fieldname=d.fieldname))
 
-				except MySQLdb.OperationalError as e:
-					if e.args and e.args[0]==1054:
+				except pymysql.InternalError as e:
+					if e.args and e.args[0] == ER.BAD_FIELD_ERROR:
 						# ignore if missing column, else raise
 						# this happens in case of Custom Field
 						pass
@@ -764,7 +769,8 @@ def validate_permissions(doctype, for_remove=False):
 def make_module_and_roles(doc, perm_fieldname="permissions"):
 	"""Make `Module Def` and `Role` records if already not made. Called while installing."""
 	try:
-		if doc.restrict_to_domain and not frappe.db.exists('Domain', doc.restrict_to_domain):
+		if hasattr(doc,'restrict_to_domain') and doc.restrict_to_domain and \
+			not frappe.db.exists('Domain', doc.restrict_to_domain):
 			frappe.get_doc(dict(doctype='Domain', domain=doc.restrict_to_domain)).insert()
 
 		if not frappe.db.exists("Module Def", doc.module):

frappe/core/doctype/domain/domain.py

@@ -18,10 +18,11 @@ class Domain(Document):
 		self.setup_roles()
 		self.setup_properties()
 		self.set_values()
-		if not int(frappe.db.get_single_value('System Settings', 'setup_complete') or 0):
+		# always set the desktop icons while changing the domain settings
+		self.setup_desktop_icons()
+		if not int(frappe.defaults.get_defaults().setup_complete or 0):
 			# if setup not complete, setup desktop etc.
 			self.setup_sidebar_items()
-			self.setup_desktop_icons()
 			self.set_default_portal_role()
 
 		if self.data.custom_fields:
@@ -59,7 +60,9 @@ class Domain(Document):
 	def setup_roles(self):
 		'''Enable roles that are restricted to this domain'''
 		if self.data.restricted_roles:
+			user = frappe.get_doc("User", frappe.session.user)
 			for role_name in self.data.restricted_roles:
+				user.append("roles", {"role": role_name})
 				if not frappe.db.get_value('Role', role_name):
 					frappe.get_doc(dict(doctype='Role', role_name=role_name)).insert()
 					continue
@@ -67,6 +70,7 @@ class Domain(Document):
 				role = frappe.get_doc('Role', role_name)
 				role.disabled = 0
 				role.save()
+			user.save()
 
 	def setup_data(self, domain=None):
 		'''Load domain info via hooks'''
@@ -97,6 +101,7 @@ class Domain(Document):
 		'''set values based on `data.set_value`'''
 		if self.data.set_value:
 			for args in self.data.set_value:
+				frappe.reload_doctype(args[0])
 				doc = frappe.get_doc(args[0], args[1] or args[0])
 				doc.set(args[2], args[3])
 				doc.save()

frappe/core/doctype/domain_settings/domain_settings.js

@@ -2,58 +2,62 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Domain Settings', {
-	onload: function(frm) {
-		let domains = $('<div class="domain-editor">')
-			.appendTo(frm.fields_dict.domains_html.wrapper);
-
-		if(!frm.domain_editor) {
-			frm.domain_editor = new frappe.DomainsEditor(domains, frm);
+	before_load: function(frm) {
+		if(!frm.domains_multicheck) {
+			frm.domains_multicheck = frappe.ui.form.make_control({
+				parent: frm.fields_dict.domains_html.$wrapper,
+				df: {
+					fieldname: "domains_multicheck",
+					fieldtype: "MultiCheck",
+					get_data: () => {
+						let active_domains = (frm.doc.active_domains || []).map(row => row.domain);
+						return frappe.boot.all_domains.map(domain => {
+							return {
+								label: domain,
+								value: domain,
+								checked: active_domains.includes(domain)
+							};
+						});
+					}
+				},
+				render_input: true
+			});
+			frm.domains_multicheck.refresh_input();
 		}
-
-		frm.domain_editor.show();
 	},
 
 	validate: function(frm) {
-		if(frm.domain_editor) {
-			frm.domain_editor.set_items_in_table();
-		}
+		frm.trigger('set_options_in_table');
 	},
-});
 
-frappe.DomainsEditor = frappe.CheckboxEditor.extend({
-	init: function(wrapper, frm) {
-		var opts = {};
-		$.extend(opts, {
-			wrapper: wrapper,
-			frm: frm,
-			field_mapper: {
-				child_table_field: "active_domains",
-				item_field: "domain",
-				cdt: "Has Domain"
-			},
-			attribute: 'data-domain',
-			checkbox_selector: false,
-			get_items: this.get_all_domains,
-			editor_template: this.get_template()
+	set_options_in_table: function(frm) {
+		let selected_options = frm.domains_multicheck.get_value();
+		let unselected_options = frm.domains_multicheck.options
+			.map(option => option.value)
+			.filter(value => {
+				return !selected_options.includes(value);
+			});
+
+		let map = {}, list = [];
+		(frm.doc.active_domains || []).map(row => {
+			map[row.domain] = row.name;
+			list.push(row.domain);
 		});
 
-		this._super(opts);
-	},
+		unselected_options.map(option => {
+			if(list.includes(option)) {
+				frappe.model.clear_doc("Has Domain", map[option]);
+			}
+		});
 
-	get_template: function() {
-		return `
-			<div class="checkbox" data-domain="{{item}}">
-				<label>
-				<input type="checkbox">
-				<span class="label-area small">{{ __(item) }}</span>
-				</label>
-			</div>
-		`;
-	},
+		selected_options.map(option => {
+			if(!list.includes(option)) {
+				frappe.model.clear_doc("Has Domain", map[option]);
+				let row = frappe.model.add_child(frm.doc, "Has Domain", "active_domains");
+				row.domain = option;
+			}
+		});
 
-	get_all_domains: function() {
-		// return all the domains available in the system
-		this.items = frappe.boot.all_domains;
-		this.render_items();
-	},
-});
+		refresh_field('active_domains');
+	}
+});

frappe/core/doctype/domain_settings/domain_settings.json

@@ -18,7 +18,7 @@
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "fieldname": "domains", 
+   "fieldname": "active_domains_sb", 
    "fieldtype": "Section Break", 
    "hidden": 0, 
    "ignore_user_permissions": 0, 
@@ -27,7 +27,7 @@
    "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
-   "label": "Domains", 
+   "label": "Active Domains", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
@@ -57,7 +57,7 @@
    "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
-   "label": "Domains", 
+   "label": "Domains HTML", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
@@ -95,7 +95,7 @@
    "precision": "", 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
-   "read_only": 0, 
+   "read_only": 1, 
    "remember_last_selected_value": 0, 
    "report_hide": 0, 
    "reqd": 0, 
@@ -114,7 +114,7 @@
  "issingle": 1, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-05-12 17:01:18.615000", 
+ "modified": "2017-12-05 17:36:46.842134", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "Domain Settings", 

frappe/core/doctype/domain_settings/domain_settings.py

@@ -15,7 +15,10 @@ class DomainSettings(Document):
 		self.save()
 
 	def on_update(self):
-		for d in self.active_domains:
+		for i, d in enumerate(self.active_domains):
+			# set the flag to update the the desktop icons of all domains
+			if i >= 1:
+				frappe.flags.keep_desktop_icons = True
 			domain = frappe.get_doc('Domain', d.domain)
 			domain.setup_domain()
 

frappe/core/doctype/file/file.json

@@ -1,5 +1,6 @@
 {
  "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
  "allow_import": 1, 
  "allow_rename": 0, 
  "autoname": "", 
@@ -11,6 +12,7 @@
  "editable_grid": 0, 
  "fields": [
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -41,6 +43,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -71,6 +74,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -100,6 +104,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -129,6 +134,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -157,6 +163,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -187,6 +194,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -216,6 +224,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -244,6 +253,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -272,6 +282,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -301,6 +312,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -330,6 +342,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -360,6 +373,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -389,6 +403,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -418,6 +433,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -447,6 +463,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -475,6 +492,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -503,12 +521,13 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "fieldname": "content_hash", 
-   "fieldtype": "Data", 
+   "fieldname": "lft", 
+   "fieldtype": "Int", 
    "hidden": 1, 
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
@@ -516,26 +535,28 @@
    "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
-   "label": "Content Hash", 
+   "label": "lft", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
+   "precision": "", 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
    "read_only": 0, 
    "remember_last_selected_value": 0, 
    "report_hide": 0, 
    "reqd": 0, 
-   "search_index": 1, 
+   "search_index": 0, 
    "set_only_once": 0, 
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "fieldname": "lft", 
+   "fieldname": "rgt", 
    "fieldtype": "Int", 
    "hidden": 1, 
    "ignore_user_permissions": 0, 
@@ -544,7 +565,7 @@
    "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
-   "label": "lft", 
+   "label": "rgt", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
@@ -560,12 +581,13 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "fieldname": "rgt", 
-   "fieldtype": "Int", 
+   "fieldname": "old_parent", 
+   "fieldtype": "Data", 
    "hidden": 1, 
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
@@ -573,7 +595,7 @@
    "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
-   "label": "rgt", 
+   "label": "old_parent", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
@@ -589,20 +611,21 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "fieldname": "old_parent", 
+   "fieldname": "content_hash", 
    "fieldtype": "Data", 
-   "hidden": 1, 
+   "hidden": 0, 
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
    "in_global_search": 0, 
    "in_list_view": 0, 
    "in_standard_filter": 0, 
-   "label": "old_parent", 
+   "label": "Content Hash", 
    "length": 0, 
    "no_copy": 0, 
    "permlevel": 0, 
@@ -618,19 +641,19 @@
    "unique": 0
   }
  ], 
+ "has_web_view": 0, 
  "hide_heading": 0, 
  "hide_toolbar": 0, 
  "icon": "fa fa-file", 
  "idx": 1, 
  "image_view": 0, 
  "in_create": 0, 
- "in_dialog": 0, 
  "is_submittable": 0, 
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
  "menu_index": 0, 
- "modified": "2017-02-17 16:42:36.092962", 
+ "modified": "2017-10-27 13:27:43.882914", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "File", 

frappe/core/doctype/file/file.py

@@ -423,3 +423,24 @@ def unzip_file(name):
 	'''Unzip the given file and make file records for each of the extracted files'''
 	file_obj = frappe.get_doc('File', name)
 	file_obj.unzip()
+
+@frappe.whitelist()
+def get_attached_images(doctype, names):
+	'''get list of image urls attached in form
+	returns {name: ['image.jpg', 'image.png']}'''
+
+	if isinstance(names, string_types):
+		names = json.loads(names)
+
+	img_urls = frappe.db.get_list('File', filters={
+		'attached_to_doctype': doctype,
+		'attached_to_name': ('in', names),
+		'is_folder': 0
+	}, fields=['file_url', 'attached_to_name as docname'])
+
+	out = frappe._dict()
+	for i in img_urls:
+		out[i.docname] = out.get(i.docname, [])
+		out[i.docname].append(i.file_url)
+
+	return out

frappe/core/doctype/role_profile/role_profile.js

@@ -0,0 +1,23 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Role Profile', {
+	setup: function(frm) {
+		if(has_common(frappe.user_roles, ["Administrator", "System Manager"])) {
+			if(!frm.roles_editor) {
+				var role_area = $('<div style="min-height: 300px">')
+					.appendTo(frm.fields_dict.roles_html.wrapper);
+				frm.roles_editor = new frappe.RoleEditor(role_area, frm);
+				frm.roles_editor.show();
+			} else {
+				frm.roles_editor.show();
+			}
+		}
+	},
+
+	validate: function(frm) {
+		if(frm.roles_editor) {
+			frm.roles_editor.set_roles_in_table();
+		}
+	}
+});

frappe/core/doctype/role_profile/role_profile.json

@@ -0,0 +1,175 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "role_profile", 
+ "beta": 0, 
+ "creation": "2017-08-31 04:16:38.764465", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "role_profile", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Role Name", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 1
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "roles_html", 
+   "fieldtype": "HTML", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Roles HTML", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "roles", 
+   "fieldtype": "Table", 
+   "hidden": 1, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Roles Assigned", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Has Role", 
+   "permlevel": 1, 
+   "precision": "", 
+   "print_hide": 1, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-10-17 11:05:11.183066", 
+ "modified_by": "Administrator", 
+ "module": "Core", 
+ "name": "Role Profile", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }, 
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 0, 
+   "delete": 0, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 1, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "title_field": "role_profile", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/core/doctype/role_profile/role_profile.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class RoleProfile(Document):
+	def autoname(self):
+		"""set name as Role Profile name"""
+		self.name = self.role_profile
+
+	def on_update(self):
+		""" Changes in role_profile reflected across all its user """
+		from frappe.core.doctype.user.user import update_roles
+		update_roles(self.name)

frappe/core/doctype/role_profile/test_role_profile.js

@@ -0,0 +1,33 @@
+QUnit.module('Core');
+
+QUnit.test("test: Role Profile", function (assert) {
+	let done = assert.async();
+
+	assert.expect(3);
+
+	frappe.run_serially([
+		// insert a new user
+		() => frappe.tests.make('Role Profile', [
+			{role_profile: 'Test 2'}
+		]),
+
+		() => {
+			$('input.box')[0].checked = true;
+			$('input.box')[2].checked = true;
+			$('input.box')[4].checked = true;
+			cur_frm.save();
+		},
+
+		() => frappe.timeout(1),
+		() => cur_frm.refresh(),
+		() => frappe.timeout(2),
+		() => {
+			assert.equal($('input.box')[0].checked, true);
+			assert.equal($('input.box')[2].checked, true);
+			assert.equal($('input.box')[4].checked, true);
+		},
+
+		() => done()
+	]);
+
+});

frappe/core/doctype/role_profile/test_role_profile.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+import frappe
+import unittest
+
+class TestRoleProfile(unittest.TestCase):
+	def test_make_new_role_profile(self):
+		new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
+
+		self.assertEquals(new_role_profile.role_profile, 'Test 1')
+
+		# add role
+		new_role_profile.append("roles", {
+			"role": '_Test Role 2'
+		})
+		new_role_profile.save()
+		self.assertEquals(new_role_profile.roles[0].role, '_Test Role 2')
+
+		# clear roles
+		new_role_profile.roles = []
+		new_role_profile.save()
+		self.assertEquals(new_role_profile.roles, [])

frappe/core/doctype/user/test_user.js

@@ -20,4 +20,4 @@ QUnit.test("test: User", function (assert) {
 		() => done()
 	]);
 
-});
+});

frappe/core/doctype/user/test_user_with_role_profile.js

@@ -0,0 +1,35 @@
+QUnit.module('Core');
+
+QUnit.test("test: Set role profile in user", function (assert) {
+	let done = assert.async();
+
+	assert.expect(3);
+
+	frappe.run_serially([
+
+		// Insert a new user
+		() => frappe.tests.make('User', [
+			{email: 'test@test2.com'},
+			{first_name: 'Test 2'},
+			{send_welcome_email: 0}
+		]),
+
+		() => frappe.timeout(2),
+		() => {
+			return frappe.tests.set_form_values(cur_frm, [
+				{role_profile_name:'Test 2'}
+			]);
+		},
+
+		() => cur_frm.save(),
+		() => frappe.timeout(2),
+
+		() => {
+			assert.equal($('input.box')[0].checked, true);
+			assert.equal($('input.box')[2].checked, true);
+			assert.equal($('input.box')[4].checked, true);
+		},
+		() => done()
+	]);
+
+});

frappe/core/doctype/user/user.js

@@ -17,8 +17,30 @@ frappe.ui.form.on('User', {
 		}
 
 	},
+
+	role_profile_name: function(frm) {
+		if(frm.doc.role_profile_name) {
+			frappe.call({
+				"method": "frappe.core.doctype.user.user.get_role_profile",
+				args: {
+					role_profile: frm.doc.role_profile_name
+				},
+				callback: function (data) {
+					frm.set_value("roles", []);
+					$.each(data.message || [], function(i, v){
+						var d = frm.add_child("roles");
+						d.role = v.role;
+					});
+					frm.roles_editor.show();
+				}
+			});
+		}
+	},
+
 	onload: function(frm) {
-		if(has_common(frappe.user_roles, ["Administrator", "System Manager"]) && !frm.doc.__islocal) {
+		frm.can_edit_roles = has_common(frappe.user_roles, ["Administrator", "System Manager"]);
+
+		if(frm.can_edit_roles && !frm.is_new()) {
 			if(!frm.roles_editor) {
 				var role_area = $('<div style="min-height: 300px">')
 					.appendTo(frm.fields_dict.roles_html.wrapper);
@@ -34,7 +56,10 @@ frappe.ui.form.on('User', {
 	},
 	refresh: function(frm) {
 		var doc = frm.doc;
-
+		if(!frm.is_new() && !frm.roles_editor && frm.can_edit_roles) {
+			frm.reload_doc();
+			return;
+		}
 		if(doc.name===frappe.session.user && !doc.__unsaved
 			&& frappe.all_timezones
 			&& (doc.language || frappe.boot.user.language)
@@ -45,7 +70,7 @@ frappe.ui.form.on('User', {
 
 		frm.toggle_display(['sb1', 'sb3', 'modules_access'], false);
 
-		if(!doc.__islocal){
+		if(!frm.is_new()) {
 			frm.add_custom_button(__("Set Desktop Icons"), function() {
 				frappe.route_options = {
 					"user": doc.name
@@ -89,8 +114,8 @@ frappe.ui.form.on('User', {
 
 			frm.trigger('enabled');
 
-			if (frm.roles_editor) {
-				frm.roles_editor.disabled = frm.doc.role_profile_name ? 1 : 0;
+			if (frm.roles_editor && frm.can_edit_roles) {
+				frm.roles_editor.disable = frm.doc.role_profile_name ? 1 : 0;
 				frm.roles_editor.show();
 			}
 
@@ -133,13 +158,13 @@ frappe.ui.form.on('User', {
 	},
 	enabled: function(frm) {
 		var doc = frm.doc;
-		if(!doc.__islocal && has_common(frappe.user_roles, ["Administrator", "System Manager"])) {
+		if(!frm.is_new() && has_common(frappe.user_roles, ["Administrator", "System Manager"])) {
 			frm.toggle_display(['sb1', 'sb3', 'modules_access'], doc.enabled);
 			frm.set_df_property('enabled', 'read_only', 0);
 		}
 
 		if(frappe.session.user!=="Administrator") {
-			frm.toggle_enable('email', doc.__islocal);
+			frm.toggle_enable('email', frm.is_new());
 		}
 	},
 	create_user_email:function(frm) {

frappe/core/doctype/user/user.json

@@ -503,6 +503,37 @@
    "set_only_once": 0, 
    "unique": 0
   }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "role_profile_name", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Role Profile", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Role Profile", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
   {
    "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
@@ -922,6 +953,37 @@
    "set_only_once": 0, 
    "unique": 0
   }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "0", 
+   "fieldname": "logout_all_sessions", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Logout from all devices while changing Password", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
   {
    "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
@@ -1213,7 +1275,6 @@
    "label": "Background Image", 
    "length": 0, 
    "no_copy": 0, 
-   "options": "image", 
    "permlevel": 0, 
    "precision": "", 
    "print_hide": 0, 
@@ -1389,7 +1450,7 @@
    "bold": 0, 
    "collapsible": 0, 
    "columns": 0, 
-   "description": "Enter default value fields (keys) and values. If you add multiple values for a field, the first one will be picked. These defaults are also used to set \"match\" permission rules. To see list of fields, go to \"Customize Form\".", 
+   "description": "Enter default value fields (keys) and values. If you add multiple values for a field,the first one will be picked. These defaults are also used to set \"match\" permission rules. To see list of fields,go to \"Customize Form\".", 
    "fieldname": "defaults", 
    "fieldtype": "Table", 
    "hidden": 1, 
@@ -1483,7 +1544,7 @@
    "collapsible": 0, 
    "columns": 0, 
    "default": "System User", 
-   "description": "If the user has any role checked, then the user becomes a \"System User\". \"System User\" has access to the desktop", 
+   "description": "If the user has any role checked,then the user becomes a \"System User\". \"System User\" has access to the desktop", 
    "fieldname": "user_type", 
    "fieldtype": "Select", 
    "hidden": 0, 
@@ -2002,8 +2063,8 @@
  "istable": 0, 
  "max_attachments": 5, 
  "menu_index": 0, 
- "modified": "2017-10-09 15:33:43.818915", 
- "modified_by": "Administrator", 
+ "modified": "2017-11-01 09:04:51.151347", 
+ "modified_by": "manas@erpnext.com", 
  "module": "Core", 
  "name": "User", 
  "owner": "Administrator", 

frappe/core/doctype/user/user.py

@@ -67,6 +67,7 @@ class User(Document):
 		self.remove_disabled_roles()
 		self.validate_user_email_inbox()
 		ask_pass_update()
+		self.validate_roles()
 
 		if self.language == "Loading...":
 			self.language = None
@@ -74,6 +75,12 @@ class User(Document):
 		if (self.name not in ["Administrator", "Guest"]) and (not self.frappe_userid):
 			self.frappe_userid = frappe.generate_hash(length=39)
 
+	def validate_roles(self):
+		if self.role_profile_name:
+				role_profile = frappe.get_doc('Role Profile', self.role_profile_name)
+				self.set('roles', [])
+				self.append_roles(*[role.role for role in role_profile.roles])
+
 	def on_update(self):
 		# clear new password
 		self.validate_user_limit()
@@ -84,6 +91,7 @@ class User(Document):
 		if self.name not in ('Administrator', 'Guest') and not self.user_image:
 			frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name)
 
+
 	def has_website_permission(self, ptype, verbose=False):
 		"""Returns true if current user is the session user"""
 		return self.name == frappe.session.user
@@ -137,7 +145,7 @@ class User(Document):
 
 	def email_new_password(self, new_password=None):
 		if new_password and not self.flags.in_insert:
-			_update_password(self.name, new_password)
+			_update_password(user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions)
 
 			if self.send_password_update_notification:
 				self.password_update_mail(new_password)
@@ -183,7 +191,8 @@ class User(Document):
 				if self.name not in STANDARD_USERS:
 					if new_password:
 						# new password given, no email required
-						_update_password(self.name, new_password)
+						_update_password(user=self.name, pwd=new_password,
+							logout_all_sessions=self.logout_all_sessions)
 
 					if not self.flags.no_welcome_mail and self.send_welcome_email:
 						self.send_welcome_mail_to_user()
@@ -987,3 +996,17 @@ def throttle_user_creation():
 
 	if frappe.db.get_creation_count('User', 60) > frappe.local.conf.get("throttle_user_limit", 60):
 		frappe.throw(_('Throttled'))
+
+@frappe.whitelist()
+def get_role_profile(role_profile):
+	roles = frappe.get_doc('Role Profile', {'role_profile': role_profile})
+	return roles.roles
+
+def update_roles(role_profile):
+	users = frappe.get_all('User', filters={'role_profile_name': role_profile})
+	role_profile = frappe.get_doc('Role Profile', role_profile)
+	roles = [role.role for role in role_profile.roles]
+	for d in users:
+		user = frappe.get_doc('User', d)
+		user.set('roles', [])
+		user.add_roles(*roles)

frappe/core/doctype/user_permission_for_page_and_report/user_permission_for_page_and_report.js

@@ -11,16 +11,16 @@ frappe.ui.form.on('User Permission for Page and Report', {
 		if(!frm.roles_editor) {
 			frm.role_area = $('<div style="min-height: 300px">')
 				.appendTo(frm.fields_dict.roles_html.wrapper);
-			frm.roles_editor = new frappe.RoleEditor(frm.role_area);
+			frm.roles_editor = new frappe.RoleEditor(frm.role_area, frm);
 		}
 	},
 	
 	page: function(frm) {
-		frm.trigger("get_roles")
+		frm.trigger("get_roles");
 	},
 
 	report: function(frm){
-		frm.trigger("get_roles")
+		frm.trigger("get_roles");
 	},
 
 	get_roles: function(frm) {
@@ -30,26 +30,26 @@ frappe.ui.form.on('User Permission for Page and Report', {
 			method:"get_custom_roles",
 			doc: frm.doc,
 			callback: function(r) {
-				refresh_field('roles')
-				frm.roles_editor.show()
+				refresh_field('roles');
+				frm.roles_editor.show();
 			}
-		})
+		});
 	},
 
 	update: function(frm) {
 		if(frm.roles_editor) {
-			frm.roles_editor.set_roles_in_table()
+			frm.roles_editor.set_roles_in_table();
 		}
 
 		return frappe.call({
 			method:"set_custom_roles",
 			doc: frm.doc,
 			callback: function(r) {
-				refresh_field('roles')
-				frm.roles_editor.show()
-				frappe.msgprint(__("Successfully Updated"))
-				frm.reload_doc()
+				refresh_field('roles');
+				frm.roles_editor.show();
+				frappe.msgprint(__("Successfully Updated"));
+				frm.reload_doc();
 			}
-		})
+		});
 	}
 });

frappe/core/doctype/user_permission_for_page_and_report/user_permission_for_page_and_report.json

@@ -1,5 +1,6 @@
 {
  "allow_copy": 1, 
+ "allow_guest_to_view": 0, 
  "allow_import": 0, 
  "allow_rename": 0, 
  "beta": 0, 
@@ -12,6 +13,7 @@
  "engine": "InnoDB", 
  "fields": [
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -23,7 +25,7 @@
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
    "in_global_search": 0, 
-   "in_list_view": 0, 
+   "in_list_view": 1, 
    "in_standard_filter": 0, 
    "label": "Set Role For", 
    "length": 0, 
@@ -36,12 +38,13 @@
    "read_only": 0, 
    "remember_last_selected_value": 0, 
    "report_hide": 0, 
-   "reqd": 0, 
+   "reqd": 1, 
    "search_index": 0, 
    "set_only_once": 0, 
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -73,6 +76,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -104,6 +108,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -133,6 +138,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -163,6 +169,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -193,6 +200,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -221,6 +229,7 @@
    "unique": 0
   }, 
   {
+   "allow_bulk_edit": 0, 
    "allow_on_submit": 0, 
    "bold": 0, 
    "collapsible": 0, 
@@ -251,17 +260,17 @@
    "unique": 0
   }
  ], 
+ "has_web_view": 0, 
  "hide_heading": 0, 
  "hide_toolbar": 1, 
  "idx": 0, 
  "image_view": 0, 
  "in_create": 0, 
- "in_dialog": 0, 
  "is_submittable": 0, 
  "issingle": 1, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-02-22 18:07:29.954831", 
+ "modified": "2017-12-21 04:24:24.963988", 
  "modified_by": "Administrator", 
  "module": "Core", 
  "name": "User Permission for Page and Report", 

frappe/core/page/data_import_tool/__init__.py

@@ -1,4 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals

frappe/core/page/data_import_tool/data_import_main.html

@@ -1,118 +0,0 @@
-<div class="data-import-tool">
-    <div class="data-import-selector">
-        <h3>{%= __("Export Template") %}</h3>
-        <p class="text-muted">{%= __("To import or update records, you must first download the template for importing.") %}</p>
-        <h6>{%= __("Select Type of Document to Download") %}</h6>
-    	<div>
-            <select class="form-control doctype" style="width: 200px" placeholder="{%= __("Select Type") %}">
-                <option value=""></option>
-                {% for (var i=0, l= frappe.boot.user.can_import.length; i < l; i++) {
-                    var doctype = frappe.boot.user.can_import[i]; %}
-                <option value="{%= doctype %}">{%= __(doctype) %}</option>
-                {% } %}
-            </select>
-            <br>
-        </div>
-    </div>
-    <div class="export-import-section hide"  style="max-width: 700px;">
-		<h4>{{ __("1. Select Columns") }}</h4>
-        <p>
-			<a class="btn btn-default btn-xs btn-select-all" style="margin-right: 7px;">
-            {%= __("Select All") %}</a>
-			<a class="btn btn-default btn-xs btn-select-mandatory" style="margin-right: 7px;">
-            {%= __("Select Mandatory") %}</a>
-			<a class="btn btn-default btn-xs btn-unselect-all">
-            {%= __("Unselect All") %}</a>
-		</p>
-		<div class="select-columns">
-		</div>
-		<br>
-		<h4>{{ __("2. Download") }}</h4>
-		<div class="row">
-            <div class="col-sm-4">
-                <p><a class="btn btn-primary btn-xs btn-download-template">
-                    {%= __("Download Blank Template") %}</a></p>
-            </div>
-            <div class="col-sm-8">
-                <h6 class="text-muted">{%= __("Recommended for inserting new records.") %}</h6>
-            </div>
-        </div>
-        <div class="row">
-            <div class="col-sm-4">
-                <p><a class="btn btn-primary btn-xs btn-download-data">
-                    {%= __("Download with Data") %}</a></p>
-            </div>
-            <div class="col-sm-8">
-                <h6 class="text-muted">{%= __("Recommended bulk editing records via import, or understanding the import format.") %}</h6>
-            </div>
-        </div>
-        <div class="row">
-            <div class="col-sm-4">
-                <div class="checkbox" style="margin: 5px 0px;">
-                    <label>
-                        <input type="checkbox" class="excel-check" data-fieldname="excel_check" checked>
-                            <small>{%= __("Download in Excel File Format") %}</small>
-                    </label>
-                </div>
-            </div>
-        </div>
-    </div>
-    <div>
-        <hr style="margin-top: 50px;">
-        <h3>{%= __("Import") %}</h3>
-        <p class="text-muted">{%= __("Update the template and save in downloaded format before attaching.") %}</p>
-        <div class="row">
-            <div class="col-md-6">
-				<br>
-				<h4>{{ __("1. Select File") }}</h4>
-                <div class="upload-area"></div>
-				<br>
-
-				<h4>{{ __("2. Upload") }}</h4>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="always_insert">
-                            {%= __("Do not update, but insert new records.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="update_only">
-                            {%= __("Update only, do not insert new records.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="submit_after_import">
-                            {%= __("Submit after importing.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="ignore_encoding_errors">
-                            {%= __("Ignore encoding errors.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="skip_errors">
-                            {%= __("Skip rows with errors.") %}
-                    </label>
-                </div>
-                <div class="checkbox">
-                    <label>
-                        <input type="checkbox" name="no_email" checked>
-                            {%= __("Do not send emails.") %}
-                    </label>
-                </div>
-                <p>
-                    <button class="btn btn-sm btn-primary btn-import">Import</button>
-                </p>
-            </div>
-            <div class="import-log hide col-md-6">
-                <h3>Import Log</h3>
-                <div class="import-log-messages"></div>
-            </div>
-        </div>
-    </div>
-</div>

frappe/core/page/data_import_tool/data_import_tool.css

@@ -1,7 +0,0 @@
-.data-import-tool {
-	padding: 15px;
-}
-
-.data-import-tool hr {
-	margin: 10px -15px;
-}

frappe/core/page/data_import_tool/data_import_tool.js

@@ -1,233 +0,0 @@
-
-
-frappe.DataImportTool = Class.extend({
-	init: function(parent) {
-		this.page = frappe.ui.make_app_page({
-			parent: parent,
-			title: __("Data Import Tool"),
-			single_column: true
-		});
-		this.page.add_inner_button(__("Help"), function() {
-			frappe.help.show_video("6wiriRKPhmg");
-		});
-		this.make();
-		this.make_upload();
-	},
-	set_route_options: function() {
-		var doctype = null;
-		if(frappe.get_route()[1]) {
-			doctype = frappe.get_route()[1];
-		} else if(frappe.route_options && frappe.route_options.doctype) {
-			doctype = frappe.route_options.doctype;
-		}
-
-		if(in_list(frappe.boot.user.can_import, doctype)) {
-			this.select.val(doctype).change();
-		}
-
-		frappe.route_options = null;
-	},
-	make: function() {
-		var me = this;
-		frappe.boot.user.can_import = frappe.boot.user.can_import.sort();
-
-		$(frappe.render_template("data_import_main", this)).appendTo(this.page.main);
-
-		this.select = this.page.main.find("select.doctype");
-		this.select_columns = this.page.main.find('.select-columns');
-		this.select.on("change", function() {
-			me.doctype = $(this).val();
-			frappe.model.with_doctype(me.doctype, function() {
-				me.page.main.find(".export-import-section").toggleClass(!!me.doctype);
-				if(me.doctype) {
-
-					// render select columns
-					var parent_doctype = frappe.get_doc('DocType', me.doctype);
-					parent_doctype["reqd"] = true;
-					var doctype_list = [parent_doctype];
-
-					frappe.meta.get_table_fields(me.doctype).forEach(function(df) {
-						var d = frappe.get_doc('DocType', df.options);
-						d["reqd"]=df.reqd;
-						doctype_list.push(d);
-					});
-					$(frappe.render_template("data_import_tool_columns", {doctype_list: doctype_list}))
-						.appendTo(me.select_columns.empty());
-				}
-			});
-		});
-
-		this.page.main.find('.btn-select-all').on('click', function() {
-			me.select_columns.find('.select-column-check').prop('checked', true);
-		});
-
-		this.page.main.find('.btn-unselect-all').on('click', function() {
-			me.select_columns.find('.select-column-check').prop('checked', false);
-		});
-
-		this.page.main.find('.btn-select-mandatory').on('click', function() {
-			me.select_columns.find('.select-column-check').prop('checked', false);
-			me.select_columns.find('.select-column-check[data-reqd="1"]').prop('checked', true);
-		});
-
-		var get_template_url = '/api/method/frappe.core.page.data_import_tool.exporter.get_template';
-
-		this.page.main.find(".btn-download-template").on('click', function() {
-			open_url_post(get_template_url, me.get_export_params(false));
-		});
-
-		this.page.main.find(".btn-download-data").on('click', function() {
-			open_url_post(get_template_url, me.get_export_params(true));
-		});
-
-	},
-	get_export_params: function(with_data) {
-		var doctype = this.select.val();
-		var columns = {};
-
-		this.select_columns.find('.select-column-check:checked').each(function() {
-			var _doctype = $(this).attr('data-doctype');
-			var _fieldname = $(this).attr('data-fieldname');
-			if(!columns[_doctype]) {
-				columns[_doctype] = [];
-			}
-			columns[_doctype].push(_fieldname);
-		});
-
-		return {
-			doctype: doctype,
-			parent_doctype: doctype,
-			select_columns: JSON.stringify(columns),
-			with_data: with_data ? 'Yes' : 'No',
-			all_doctypes: 'Yes',
-			from_data_import: 'Yes',
-			excel_format: this.page.main.find(".excel-check").is(":checked") ? 'Yes' : 'No'
-		}
-	},
-	make_upload: function() {
-		var me = this;
-		frappe.upload.make({
-			no_socketio: true,
-			parent: this.page.main.find(".upload-area"),
-			btn: this.page.main.find(".btn-import"),
-			get_params: function() {
-				return {
-					submit_after_import: me.page.main.find('[name="submit_after_import"]').prop("checked"),
-					ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
-					skip_errors: me.page.main.find('[name="skip_errors"]').prop("checked"),
-					overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
-					update_only: me.page.main.find('[name="update_only"]').prop("checked"),
-					no_email: me.page.main.find('[name="no_email"]').prop("checked"),
-					from_data_import: 'Yes'
-				}
-			},
-			args: {
-				method: 'frappe.core.page.data_import_tool.importer.upload',
-			},
-			allow_multiple: 0,
-			onerror: function(r) {
-				me.onerror(r);
-			},
-			queued: function() {
-				// async, show queued
-				msg_dialog.clear();
-				frappe.msgprint(__("Import Request Queued. This may take a few moments, please be patient."));
-			},
-			running: function() {
-				// update async status as running
-				msg_dialog.clear();
-				frappe.msgprint(__("Importing..."));
-				me.write_messages([__("Importing")]);
-				me.has_progress = false;
-			},
-			progress: function(data) {
-				// show callback if async
-				if(data.progress) {
-					frappe.hide_msgprint(true);
-					me.has_progress = true;
-					frappe.show_progress(__("Importing"), data.progress[0],
-						data.progress[1]);
-				}
-			},
-			callback: function(attachment, r) {
-				if(r.message.error || r.message.messages.length==0) {
-					me.onerror(r);
-				} else {
-					if(me.has_progress) {
-						frappe.show_progress(__("Importing"), 1, 1);
-						setTimeout(frappe.hide_progress, 1000);
-					}
-
-					r.messages = ["<h5 style='color:green'>" + __("Import Successful!") + "</h5>"].
-						concat(r.message.messages)
-
-					me.write_messages(r.messages);
-				}
-			},
-			is_private: true
-		});
-
-		frappe.realtime.on("data_import_progress", function(data) {
-			if(data.progress) {
-				frappe.hide_msgprint(true);
-				me.has_progress = true;
-				frappe.show_progress(__("Importing"), data.progress[0],
-					data.progress[1]);
-			}
-		})
-
-	},
-	write_messages: function(data) {
-		this.page.main.find(".import-log").removeClass("hide");
-		var parent = this.page.main.find(".import-log-messages").empty();
-
-		// TODO render using template!
-		for (var i=0, l=data.length; i<l; i++) {
-			var v = data[i];
-			var $p = $('<p></p>').html(frappe.markdown(v)).appendTo(parent);
-			if(v.substr(0,5)=='Error') {
-				$p.css('color', 'red');
-			} else if(v.substr(0,8)=='Inserted') {
-				$p.css('color', 'green');
-			} else if(v.substr(0,7)=='Updated') {
-				$p.css('color', 'green');
-			} else if(v.substr(0,5)=='Valid') {
-				$p.css('color', '#777');
-			} else if(v.substr(0,7)=='Ignored') {
-				$p.css('color', '#777');
-			}
-		}
-	},
-	onerror: function(r) {
-		if(r.message) {
-			// bad design: moves r.messages to r.message.messages
-			r.messages = $.map(r.message.messages, function(v) {
-				var msg = v.replace("Inserted", "Valid")
-					.replace("Updated", "Valid").split("<");
-				if (msg.length > 1) {
-					v = msg[0] + (msg[1].split(">").slice(-1)[0]);
-				} else {
-					v = msg[0];
-				}
-				return v;
-			});
-
-			r.messages = ["<h4 style='color:red'>" + __("Import Failed") + "</h4>"]
-				.concat(r.messages);
-
-			r.messages.push("Please correct the format of the file and import again.");
-
-			frappe.show_progress(__("Importing"), 1, 1);
-
-			this.write_messages(r.messages);
-		}
-	}
-});
-
-frappe.pages['data-import-tool'].on_page_load = function(wrapper) {
-	frappe.data_import_tool = new frappe.DataImportTool(wrapper);
-}
-
-frappe.pages['data-import-tool'].on_page_show = function(wrapper) {
-	frappe.data_import_tool && frappe.data_import_tool.set_route_options();
-}

frappe/core/page/data_import_tool/data_import_tool.json

@@ -1,19 +0,0 @@
-{
- "content": null, 
- "creation": "2012-06-14 15:07:25", 
- "docstatus": 0, 
- "doctype": "Page", 
- "icon": "fa fa-upload", 
- "idx": 1, 
- "modified": "2016-05-11 03:37:53.385693", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "data-import-tool", 
- "owner": "Administrator", 
- "page_name": "data-import-tool", 
- "roles": [], 
- "script": null, 
- "standard": "Yes", 
- "style": null, 
- "title": "Data Import Tool"
-}

frappe/core/page/data_import_tool/data_import_tool_columns.html

@@ -1,22 +0,0 @@
-<div style="margin: 15px 0px;">
-{% for doctype in doctype_list %}
-	<h5 style="margin-top: 25px; margin-bottom: 5px;">{{ doctype.name }}</h5>
-	<div class="row">
-		{% for f in doctype.fields %}
-		{% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1 && !f.hidden) %}
-		{% doctype.reqd||(f.reqd=0);%}		
-		<div class="col-sm-4">
-            <div class="checkbox" style="margin: 5px 0px;">
-                <label>
-                    <input type="checkbox" class="select-column-check"
-						data-fieldname="{{ f.fieldname }}" data-reqd="{{ f.reqd }}"
-						data-doctype="{{ doctype.name }}" checked>
-						<small>{{ __(f.label) }}</small>
-                </label>
-            </div>
-		</div>
-		{% endif %}
-		{% endfor %}
-	</div>
-{% endfor %}
-</div>

frappe/core/page/desktop/desktop.js

@@ -138,9 +138,6 @@ $.extend(frappe.desktop, {
 	setup_wiggle: () => {
 		// Wiggle, Wiggle, Wiggle.
 		const DURATION_LONG_PRESS = 1000;
-		// lesser the antidode, more the wiggle (like your drunk uncle)
-		// 75 seems good to replicate the iOS feels.
-		const WIGGLE_ANTIDODE     = 75;
 
 		var   timer_id      = 0;
 		const $cases        = frappe.desktop.wrapper.find('.case-wrapper');
@@ -149,34 +146,29 @@ $.extend(frappe.desktop, {
 			// This hack is so bad, I should punch myself.
 			// Seriously, punch yourself.
 			const text      = $(object).find('.circle-text').html();
-			
+
 			return text;
 		}));
-		
+
 		const clearWiggle   = () => {
 			const $closes   = $cases.find('.module-remove');
 			$closes.hide();
 			$notis.show();
 
-			$icons.trigger('stopRumble');
+			$icons.removeClass('wiggle');
 
 			frappe.desktop.wiggling   = false;
 		};
 
-		// initiate wiggling.
-		$icons.jrumble({
-			speed: WIGGLE_ANTIDODE // seems neat enough to match the iOS way
-		});
-
 		frappe.desktop.wrapper.on('mousedown', '.app-icon', () => {
 			timer_id     = setTimeout(() => {
 				frappe.desktop.wiggling = true;
 				// hide all notifications.
 				$notis.hide();
-				
+
 				$cases.each((i) => {
 					const $case    = $($cases[i]);
-					const template = 
+					const template =
 					`
 						<div class="circle module-remove" style="background-color:#E0E0E0; color:#212121">
 							<div class="circle-text">
@@ -200,7 +192,7 @@ $.extend(frappe.desktop, {
 								method: 'frappe.desk.doctype.desktop_icon.desktop_icon.hide',
 								args: { name: name },
 								freeze: true,
-								callback: (response) => 
+								callback: (response) =>
 								{
 									if ( response.message ) {
 										location.reload();
@@ -209,7 +201,7 @@ $.extend(frappe.desktop, {
 							})
 
 							dialog.hide();
-							
+
 							clearWiggle();
 						});
 						// Hacks, Hacks and Hacks.
@@ -222,8 +214,9 @@ $.extend(frappe.desktop, {
 						dialog.show();
 					});
 				});
-			
-				$icons.trigger('startRumble');
+
+				$icons.addClass('wiggle');
+
 			}, DURATION_LONG_PRESS);
 		});
 		frappe.desktop.wrapper.on('mouseup mouseleave', '.app-icon', () => {

frappe/core/page/modules_setup/modules_setup.html

@@ -11,11 +11,11 @@
 		<div class="col-sm-3">
 			<select class="form-control" name="user">
 				{% for user in users %}
-				<option value="{{ user.name }}"
+				<option value="{{ user.name | e }}"
 					{% if user.name == frappe.user %}selected{% endif %}>
 					<!-- {{ variable | e }} "e" or "escape(s)" will escape the characters such "<, >, &, '"
 					in the HTML text (http://jinja.pocoo.org/docs/dev/templates/#escape) -->
-					{{ (user.first_name or "") | e }} {{ (user.last_name or "") | e }} ({{ user.name }})</option>
+					{{ (user.first_name or "") | e }} {{ (user.last_name or "") | e }} ({{ user.name | e }})</option>
 				{% endfor %}
 			</select>
 		</div>

frappe/core/page/permission_manager/permission_manager.js

@@ -155,7 +155,9 @@ frappe.PermissionEngine = Class.extend({
 				role: me.get_role()
 			},
 			callback: function(r) {
-				me.render(r.message);
+				frappe.model.with_doc('DocType', me.get_doctype(), () => {
+					me.render(r.message);
+				});
 			}
 		});
 	},
@@ -209,7 +211,10 @@ frappe.PermissionEngine = Class.extend({
 			var perm_cell = me.add_cell(row, d, "permissions").css("padding-top", 0);
 			var perm_container = $("<div class='row'></div>").appendTo(perm_cell);
 
-			$.each(me.rights, function(i, r) {
+			const { is_submittable } = frappe.model.get_doc('DocType', me.get_doctype());
+
+			me.rights.forEach(r => {
+				if (!is_submittable && ['submit', 'cancel', 'amend'].includes(r)) return;
 				me.add_check(perm_container, d, r);
 			});
 

frappe/core/utils.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import frappe
+from frappe import _
+
+def get_parent_doc(doc):
+	"""Returns document of `reference_doctype`, `reference_doctype`"""
+	if not hasattr(doc, "parent_doc"):
+		if doc.reference_doctype and doc.reference_name:
+			doc.parent_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
+		else:
+			doc.parent_doc = None
+	return doc.parent_doc
+
+def set_timeline_doc(doc):
+	"""Set timeline_doctype and timeline_name"""
+	parent_doc = get_parent_doc(doc)
+	if (doc.timeline_doctype and doc.timeline_name) or not parent_doc:
+		return
+
+	timeline_field = parent_doc.meta.timeline_field
+	if not timeline_field:
+		return
+
+	doctype = parent_doc.meta.get_link_doctype(timeline_field)
+	name = parent_doc.get(timeline_field)
+
+	if doctype and name:
+		doc.timeline_doctype = doctype
+		doc.timeline_name = name
+
+	else:	
+		return

frappe/custom/doctype/custom_field/custom_field.json

@@ -220,7 +220,7 @@
    "no_copy": 0, 
    "oldfieldname": "fieldtype", 
    "oldfieldtype": "Select", 
-   "options": "Attach\nAttach Image\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nText\nText Editor\nTime\nSignature", 
+   "options": "Attach\nAttach Image\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nGeolocation\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nText\nText Editor\nTime\nSignature", 
    "permlevel": 0, 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
@@ -1161,7 +1161,7 @@
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-07-06 17:23:43.835189", 
+ "modified": "2017-10-24 11:40:37.986457", 
  "modified_by": "Administrator", 
  "module": "Custom", 
  "name": "Custom Field", 

frappe/custom/doctype/custom_field/test_custom_field.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: Custom Field", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new Custom Field
+		() => frappe.tests.make('Custom Field', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/custom/doctype/customize_form/customize_form.js

@@ -176,6 +176,7 @@ frappe.customize_form.confirm = function(msg, frm) {
 						frappe.msgprint(r.exc);
 					} else {
 						d.hide();
+						frappe.show_alert({message:__('Customizations Reset'), indicator:'green'});
 						frappe.customize_form.clear_locals_and_refresh(frm);
 					}
 				}

frappe/custom/doctype/customize_form/customize_form.py

@@ -66,9 +66,9 @@ docfield_properties = {
 
 allowed_fieldtype_change = (('Currency', 'Float', 'Percent'), ('Small Text', 'Data'),
 	('Text', 'Data'), ('Text', 'Text Editor', 'Code', 'Signature'), ('Data', 'Select'),
-	('Text', 'Small Text'), ('Text', 'Data', 'Barcode'))
+	('Text', 'Small Text'), ('Text', 'Data', 'Barcode'), ('Code', 'Geolocation'))
 
-allowed_fieldtype_for_options_change = ('Read Only', 'HTML', 'Select',)
+allowed_fieldtype_for_options_change = ('Read Only', 'HTML', 'Select', 'Data')
 
 class CustomizeForm(Document):
 	def on_update(self):
@@ -108,7 +108,7 @@ class CustomizeForm(Document):
 		'''Create, update custom translation for this doctype'''
 		current = self.get_name_translation()
 		if current:
-			if self.label and current!=self.label:
+			if self.label and current.target_name != self.label:
 				frappe.db.set_value('Translation', current.name, 'target_name', self.label)
 				frappe.translate.clear_cache()
 			else:
@@ -163,16 +163,13 @@ class CustomizeForm(Document):
 					property_type=doctype_properties[property])
 
 		for df in self.get("fields"):
-			if df.get("__islocal"):
-				continue
-
 			meta_df = meta.get("fields", {"fieldname": df.fieldname})
 
 			if not meta_df or meta_df[0].get("is_custom_field"):
 				continue
 
 			for property in docfield_properties:
-				if property != "idx" and df.get(property) != meta_df[0].get(property):
+				if property != "idx" and (df.get(property) or '') != (meta_df[0].get(property) or ''):
 					if property == "fieldtype":
 						self.validate_fieldtype_change(df, meta_df[0].get(property), df.get(property))
 
@@ -329,6 +326,6 @@ class CustomizeForm(Document):
 			return
 
 		frappe.db.sql("""delete from `tabProperty Setter` where doc_type=%s
-			and ifnull(field_name, '')!='naming_series'""", self.doc_type)
+			and !(`field_name`='naming_series' and `property`='options')""", self.doc_type)
 		frappe.clear_cache(doctype=self.doc_type)
 		self.fetch_to_customize()

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -94,7 +94,7 @@
    "no_copy": 0, 
    "oldfieldname": "fieldtype", 
    "oldfieldtype": "Select", 
-   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nHeading\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nText\nText Editor\nTime", 
+   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nImage\nInt\nLink\nLong Text\nPassword\nPercent\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nText\nText Editor\nTime", 
    "permlevel": 0, 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
@@ -1202,7 +1202,7 @@
  "issingle": 0, 
  "istable": 1, 
  "max_attachments": 0, 
- "modified": "2017-10-11 06:45:20.172291", 
+ "modified": "2017-10-24 11:41:31.075929", 
  "modified_by": "Administrator", 
  "module": "Custom", 
  "name": "Customize Form Field", 

frappe/data_migration/doctype/data_migration_connector/connectors/postgres.py

@@ -1,43 +0,0 @@
-from __future__ import unicode_literals
-import frappe, psycopg2
-from .base import BaseConnection
-
-class PostGresConnection(BaseConnection):
-	def __init__(self, properties):
-		self.__dict__.update(properties)
-		self._connector = psycopg2.connect("host='{0}' dbname='{1}' user='{2}' password='{3}'".format(self.hostname,
-				self.database_name, self.username, self.password))
-		self.cursor = self._connector.cursor()
-
-	def get_objects(self, object_type, condition, selection):
-		if not condition:
-			condition = ''
-		else:
-			condition = ' WHERE ' + condition
-		self.cursor.execute('SELECT {0} FROM {1}{2}'.format(selection, object_type, condition))
-		raw_data = self.cursor.fetchall()
-		data = []
-		for r in raw_data:
-			row_dict = frappe._dict({})
-			for i, value in enumerate(r):
-				row_dict[self.cursor.description[i][0]] = value
-			data.append(row_dict)
-
-		return data
-
-	def get_join_objects(self, object_type, field, primary_key):
-		"""
-		field.formula 's first line will be list of tables that needs to be linked to fetch an item
-		The subsequent lines that follows will contain one to one mapping across tables keys
-		"""
-		condition = ""
-		key_mapping = field.formula.split('\n')
-		obj_type = key_mapping[0]
-		selection = field.source_fieldname
-
-		for d in key_mapping[1:]:
-			condition += d + ' AND '
-
-		condition += str(object_type) + ".id=" + str(primary_key)
-
-		return self.get_objects(obj_type, condition, selection)

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json

@@ -62,7 +62,7 @@
    "label": "Connector Type", 
    "length": 0, 
    "no_copy": 0, 
-   "options": "\nFrappe\nPostgres\nCustom", 
+   "options": "\nFrappe\nCustom", 
    "permlevel": 0, 
    "precision": "", 
    "print_hide": 0, 
@@ -268,7 +268,7 @@
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2017-10-26 12:03:40.646348", 
+ "modified": "2017-12-01 13:38:55.992499", 
  "modified_by": "Administrator", 
  "module": "Data Migration", 
  "name": "Data Migration Connector", 

frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py

@@ -8,7 +8,6 @@ from frappe.model.document import Document
 from frappe import _
 from frappe.modules.export_file import create_init_py
 from .connectors.base import BaseConnection
-from .connectors.postgres import PostGresConnection
 from .connectors.frappe_connection import FrappeConnection
 
 class DataMigrationConnector(Document):
@@ -27,10 +26,7 @@ class DataMigrationConnector(Document):
 			_class = get_connection_class(self.python_module)
 			return _class(self)
 		else:
-			if self.connector_type == 'Frappe':
-				self.connection = FrappeConnection(self)
-			elif self.connector_type == 'PostGres':
-				self.connection = PostGresConnection(self.as_dict())
+			self.connection = FrappeConnection(self)
 
 		return self.connection
 

frappe/database.py

@@ -5,9 +5,6 @@
 # --------------------
 
 from __future__ import unicode_literals
-import MySQLdb
-from MySQLdb.times import DateTimeDeltaType
-from markdown2 import UnicodeWithAttrs
 import warnings
 import datetime
 import frappe
@@ -17,11 +14,25 @@ import re
 import frappe.model.meta
 from frappe.utils import now, get_datetime, cstr
 from frappe import _
-from six import text_type, binary_type, string_types, integer_types
 from frappe.model.utils.link_count import flush_local_link_count
-from six import iteritems, text_type
 from frappe.utils.background_jobs import execute_job, get_queue
 
+# imports - compatibility imports
+from six import (
+	integer_types,
+	string_types,
+	binary_type,
+	text_type,
+	iteritems
+)
+
+# imports - third-party imports
+from markdown2 import UnicodeWithAttrs
+from pymysql.times import TimeDelta
+from pymysql.constants 	import ER, FIELD_TYPE
+from pymysql.converters import conversions
+import pymysql
+
 class Database:
 	"""
 	   Open a database connection with the given parmeters, if use_default is True, use the
@@ -50,7 +61,7 @@ class Database:
 
 	def connect(self):
 		"""Connects to a database as set in `site_config.json`."""
-		warnings.filterwarnings('ignore', category=MySQLdb.Warning)
+		warnings.filterwarnings('ignore', category=pymysql.Warning)
 		usessl = 0
 		if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key:
 			usessl = 1
@@ -59,19 +70,23 @@ class Database:
 				'cert':frappe.conf.db_ssl_cert,
 				'key':frappe.conf.db_ssl_key
 			}
+
+		conversions.update({
+			FIELD_TYPE.NEWDECIMAL: float,
+			FIELD_TYPE.DATETIME: get_datetime,
+			TimeDelta: conversions[binary_type],
+			UnicodeWithAttrs: conversions[text_type]
+		})
+
 		if usessl:
-			self._conn = MySQLdb.connect(self.host, self.user or '', self.password or '',
-				use_unicode=True, charset='utf8mb4', ssl=self.ssl)
+			self._conn = pymysql.connect(self.host, self.user or '', self.password or '',
+				charset='utf8mb4', use_unicode = True, ssl=self.ssl, conv = conversions)
 		else:
-			self._conn = MySQLdb.connect(self.host, self.user or '', self.password or '',
-				use_unicode=True, charset='utf8mb4')
-		self._conn.converter[246]=float
-		self._conn.converter[12]=get_datetime
-		self._conn.encoders[UnicodeWithAttrs] = self._conn.encoders[text_type]
-		self._conn.encoders[DateTimeDeltaType] = self._conn.encoders[binary_type]
+			self._conn = pymysql.connect(self.host, self.user or '', self.password or '',
+				charset='utf8mb4', use_unicode = True, conv = conversions)
 
-		MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1
-		self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF)
+		# MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1
+		# # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF)
 
 		self._cursor = self._conn.cursor()
 		if self.user != 'root':
@@ -142,7 +157,6 @@ class Database:
 						frappe.errprint(query % values)
 					except TypeError:
 						frappe.errprint([query, values])
-
 				if (frappe.conf.get("logging") or False)==2:
 					frappe.log("<<<< query")
 					frappe.log(query)
@@ -150,7 +164,6 @@ class Database:
 					frappe.log(values)
 					frappe.log(">>>>")
 				self._cursor.execute(query, values)
-
 			else:
 				if debug:
 					self.explain_query(query)
@@ -163,8 +176,8 @@ class Database:
 				self._cursor.execute(query)
 
 		except Exception as e:
-			# ignore data definition errors
-			if ignore_ddl and e.args[0] in (1146,1054,1091):
+			if ignore_ddl and e.args[0] in (ER.BAD_FIELD_ERROR, ER.NO_SUCH_TABLE,
+				ER.CANT_DROP_FIELD_OR_KEY):
 				pass
 
 			# NOTE: causes deadlock
@@ -175,7 +188,6 @@ class Database:
 			# 		as_dict=as_dict, as_list=as_list, formatted=formatted,
 			# 		debug=debug, ignore_ddl=ignore_ddl, as_utf8=as_utf8,
 			# 		auto_commit=auto_commit, update=update)
-
 			else:
 				raise
 
@@ -861,7 +873,7 @@ class Database:
 	def close(self):
 		"""Close database connection."""
 		if self._conn:
-			self._cursor.close()
+			# self._cursor.close()
 			self._conn.close()
 			self._cursor = None
 			self._conn = None
@@ -871,7 +883,7 @@ class Database:
 		if isinstance(s, text_type):
 			s = (s or "").encode("utf-8")
 
-		s = text_type(MySQLdb.escape_string(s), "utf-8").replace("`", "\\`")
+		s = text_type(pymysql.escape_string(s), "utf-8").replace("`", "\\`")
 
 		# NOTE separating % escape, because % escape should only be done when using LIKE operator
 		# or when you use python format string to generate query that already has a %s

frappe/desk/calendar.py

@@ -24,3 +24,23 @@ def get_event_conditions(doctype, filters=None):
 		frappe.throw(_("Not Permitted"), frappe.PermissionError)
 
 	return get_filters_cond(doctype, filters, [], with_match_conditions = True)
+
+@frappe.whitelist()
+def get_events(doctype, start, end, field_map, filters=None, fields=None):
+	field_map = frappe._dict(json.loads(field_map))
+
+	if filters:
+		filters = json.loads(filters or '')
+
+	if not fields:
+		fields = [field_map.start, field_map.end, field_map.title, 'name']
+
+	start_date = "ifnull(%s, '0000-00-00 00:00:00')" % field_map.start
+	end_date = "ifnull(%s, '2199-12-31 00:00:00')" % field_map.end
+
+	filters += [
+		[doctype, start_date, '<=', end],
+		[doctype, end_date, '>=', start],
+	]
+
+	return frappe.get_list(doctype, fields=fields, filters=filters)

frappe/desk/doctype/calendar_view/calendar_view.js

@@ -0,0 +1,35 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Calendar View', {
+	onload: function(frm) {
+		frm.trigger('reference_doctype');
+	},
+	refresh: function(frm) {
+		if (!frm.is_new()) {
+			frm.add_custom_button(__('Show Calendar'),
+				() => frappe.set_route('List', frm.doc.reference_doctype, 'Calendar', frm.doc.name));
+		}
+	},
+	reference_doctype: function(frm) {
+		const { reference_doctype } = frm.doc;
+		if (!reference_doctype) return;
+
+		frappe.model.with_doctype(reference_doctype, () => {
+			const meta = frappe.get_meta(reference_doctype);
+
+			const subject_options = meta.fields.filter(
+				df => !frappe.model.no_value_type.includes(df.fieldtype)
+			).map(df => df.fieldname);
+
+			const date_options = meta.fields.filter(
+				df => ['Date', 'Datetime'].includes(df.fieldtype)
+			).map(df => df.fieldname);
+
+			frm.set_df_property('subject_field', 'options', subject_options);
+			frm.set_df_property('start_date_field', 'options', date_options);
+			frm.set_df_property('end_date_field', 'options', date_options);
+			frm.refresh();
+		});
+	}
+});

frappe/desk/doctype/calendar_view/calendar_view.json

@@ -0,0 +1,204 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "autoname": "Prompt", 
+ "beta": 0, 
+ "creation": "2017-10-23 13:02:10.295824", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "reference_doctype", 
+   "fieldtype": "Link", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Reference DocType", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "DocType", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "subject_field", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Subject Field", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "start_date_field", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Start Date Field", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "end_date_field", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "End Date Field", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-11-14 14:14:11.544811", 
+ "modified_by": "Administrator", 
+ "module": "Desk", 
+ "name": "Calendar View", 
+ "name_case": "", 
+ "owner": "faris@erpnext.com", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 1, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 1, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }, 
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 0, 
+   "delete": 0, 
+   "email": 0, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 0, 
+   "read": 1, 
+   "report": 0, 
+   "role": "All", 
+   "set_user_permissions": 0, 
+   "share": 0, 
+   "submit": 0, 
+   "write": 0
+  }
+ ], 
+ "quick_entry": 0, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 0, 
+ "track_seen": 0
+}

frappe/desk/doctype/calendar_view/calendar_view.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+from frappe.model.document import Document
+
+class CalendarView(Document):
+	pass

frappe/desk/doctype/desktop_icon/desktop_icon.py

@@ -196,11 +196,13 @@ def set_desktop_icons(visible_list, ignore_duplicate=True):
 	if the desktop icon does not exist and the name is a DocType, then will create
 	an icon for the doctype'''
 
-	# clear all custom
-	frappe.db.sql('delete from `tabDesktop Icon` where standard=0')
+	# clear all custom only if setup is not complete
+	if not int(frappe.defaults.get_defaults().setup_complete or 0):
+		frappe.db.sql('delete from `tabDesktop Icon` where standard=0')
 
-	# set all as blocked
-	frappe.db.sql('update `tabDesktop Icon` set blocked=0, hidden=1')
+	# set standard as blocked and hidden if setting first active domain
+	if not frappe.flags.keep_desktop_icons:
+		frappe.db.sql('update `tabDesktop Icon` set blocked=0, hidden=1 where standard=1')
 
 	# set as visible if present, or add icon
 	for module_name in visible_list:

frappe/desk/doctype/todo/todo.js

@@ -18,7 +18,7 @@ frappe.ui.form.on("ToDo", {
 		}
 
 		if (!frm.doc.__islocal) {
-			if(frm.doc.status=="Open") {
+			if(frm.doc.status!=="Closed") {
 				frm.add_custom_button(__("Close"), function() {
 					frm.set_value("status", "Closed");
 					frm.save(null, function() {
@@ -27,7 +27,7 @@ frappe.ui.form.on("ToDo", {
 					});
 				}, "fa fa-check", "btn-success");
 			} else {
-				frm.add_custom_button(__("Re-open"), function() {
+				frm.add_custom_button(__("Reopen"), function() {
 					frm.set_value("status", "Open");
 					frm.save();
 				}, null, "btn-default");

frappe/desk/form/load.py

@@ -94,6 +94,7 @@ def get_docinfo(doc=None, doctype=None, name=None):
 	frappe.response["docinfo"] = {
 		"attachments": get_attachments(doc.doctype, doc.name),
 		"communications": _get_communications(doc.doctype, doc.name),
+		'total_comments': len(json.loads(doc.get('_comments') or '[]')),
 		'versions': get_versions(doc),
 		"assignments": get_assignments(doc.doctype, doc.name),
 		"permissions": get_doc_permissions(doc),

frappe/desk/form/utils.py

@@ -59,6 +59,17 @@ def add_comment(doc):
 
 	return doc.as_dict()
 
+@frappe.whitelist()
+def update_comment(name, content):
+	"""allow only owner to update comment"""
+	doc = frappe.get_doc('Communication', name)
+
+	if frappe.session.user not in ['Administrator', doc.owner]:
+		frappe.throw(_('Comment can only be edited by the owner'), frappe.PermissionError)
+
+	doc.content = content
+	doc.save(ignore_permissions=True)
+
 @frappe.whitelist()
 def get_next(doctype, value, prev, filters=None, order_by="modified desc"):
 

frappe/desk/page/activity/activity.js

@@ -78,12 +78,12 @@ frappe.pages['activity'].on_page_load = function(wrapper) {
 		}, 'fa fa-th')
 	}
 
-	this.page.add_menu_item(__('Authentication Log'), function() {
+	this.page.add_menu_item(__('Activity Log'), function() {
 		frappe.route_options = {
 			"user": frappe.session.user
 		}
 
-		frappe.set_route('Report', "Authentication Log");
+		frappe.set_route('Report', "Activity Log");
 	}, 'fa fa-th')
 
 	this.page.add_menu_item(__('Show Likes'), function() {
@@ -180,12 +180,14 @@ frappe.activity.render_heatmap = function(page) {
 		method: "frappe.desk.page.activity.activity.get_heatmap_data",
 		callback: function(r) {
 			if(r.message) {
-				var heatmap = new frappe.ui.HeatMap({
-					parent: $(".heatmap"),
+				var heatmap = new Chart({
+					parent: ".heatmap",
+					type: 'heatmap',
 					height: 100,
 					start: new Date(moment().subtract(1, 'year').toDate()),
 					count_label: "actions",
-					discrete_domains: 0
+					discrete_domains: 0,
+					data: {}
 				});
 
 				heatmap.update(r.message);

frappe/desk/page/activity/activity.py

@@ -4,25 +4,31 @@
 from __future__ import unicode_literals
 import frappe
 from frappe.utils import cint
-from frappe.core.doctype.communication.feed import get_feed_match_conditions
+from frappe.core.doctype.activity_log.feed import get_feed_match_conditions
 
 @frappe.whitelist()
 def get_feed(start, page_length, show_likes=False):
 	"""get feed"""
 	match_conditions = get_feed_match_conditions(frappe.session.user)
 
-	result = frappe.db.sql("""select name, owner, modified, creation, seen, comment_type,
+	result = frappe.db.sql("""select X.*
+		from (select name, owner, modified, creation, seen, comment_type,
 			reference_doctype, reference_name, link_doctype, link_name, subject,
 			communication_type, communication_medium, content
-		from `tabCommunication`
-		where
+			from `tabCommunication`
+			where
 			communication_type in ("Communication", "Comment")
 			and communication_medium != "Email"
 			and (comment_type is null or comment_type != "Like"
 				or (comment_type="Like" and (owner=%(user)s or reference_owner=%(user)s)))
 			{match_conditions}
 			{show_likes}
-		order by creation desc
+			union
+			select name, owner, modified, creation, '0', 'Updated',
+			reference_doctype, reference_name, link_doctype, link_name, subject,
+			'Comment', '', content
+			from `tabActivity Log`) X
+		order by X.creation DESC
 		limit %(start)s, %(page_length)s"""
 		.format(match_conditions="and {0}".format(match_conditions) if match_conditions else "",
 			show_likes="and comment_type='Like'" if show_likes else ""),
@@ -43,10 +49,8 @@ def get_feed(start, page_length, show_likes=False):
 @frappe.whitelist()
 def get_heatmap_data():
 	return dict(frappe.db.sql("""select unix_timestamp(date(creation)), count(name)
-		from `tabCommunication`
+		from `tabActivity Log`
 		where
-			communication_type in ("Communication", "Comment")
-			and communication_medium != "Email"
-			and date(creation) > subdate(curdate(), interval 1 year)
+			date(creation) > subdate(curdate(), interval 1 year)
 		group by date(creation)
 		order by creation asc"""))

frappe/desk/page/modules/modules.js

@@ -26,19 +26,40 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 		});
 	}
 
+	page.get_page_modules = () => {
+		return frappe.get_desktop_icons(true)
+			.filter(d => d.type==='module' && !d.blocked)
+			.sort((a, b) => { return (a._label > b._label) ? 1 : -1; });
+	};
+
+	let get_module_sidebar_item = (item) => `<li class="strong module-sidebar-item">
+		<a class="module-link" data-name="${item.module_name}" href="#modules/${item.module_name}">
+			<i class="fa fa-chevron-right pull-right" style="display: none;"></i>
+			<span>${item._label}</span>
+		</a>
+	</li>`;
+
+	let get_sidebar_html = () => {
+		let sidebar_items_html = page.get_page_modules()
+			.map(get_module_sidebar_item.bind(this)).join("");
+
+		return `<ul class="module-sidebar-nav overlay-sidebar nav nav-pills nav-stacked">
+			${sidebar_items_html}
+			<li class="divider"></li>
+		</ul>`;
+	};
+
 	// render sidebar
-	page.sidebar.html(frappe.render_template('modules_sidebar',
-		{modules: frappe.get_desktop_icons(true).sort(
-			function(a, b){ return (a._label > b._label) ? 1 : -1 })}));
+	page.sidebar.html(get_sidebar_html());
 
 	// help click
-	page.main.on("click", '.module-section-link[data-type="help"]', function(event) {
+	page.main.on("click", '.module-section-link[data-type="help"]', function() {
 		frappe.help.show_video($(this).attr("data-youtube-id"));
 		return false;
 	});
 
 	// notifications click
-	page.main.on("click", '.open-notification', function(event) {
+	page.main.on("click", '.open-notification', function() {
 		var doctype = $(this).attr('data-doctype');
 		if(doctype) {
 			frappe.ui.notifications.show_open_count_list(doctype);
@@ -50,9 +71,10 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 		page.wrapper.find('.module-sidebar-item.active, .module-link.active').removeClass('active');
 		$(link).addClass('active').parent().addClass("active");
 		show_section($(link).attr('data-name'));
-	}
+	};
 
 	var show_section = function(module_name) {
+		if (!module_name) return;
 		if(module_name in page.section_data) {
 			render_section(page.section_data[module_name]);
 		} else {
@@ -73,7 +95,7 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 			});
 		}
 
-	}
+	};
 
 	var render_section = function(m) {
 		page.set_title(__(m.label));
@@ -88,7 +110,7 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 
 		//setup_section_toggle();
 		frappe.app.update_notification_count_in_modules();
-	}
+	};
 
 	var process_data = function(module_name, data) {
 		frappe.module_links[module_name] = [];
@@ -103,7 +125,7 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 				}
 				if(!item.route) {
 					if(item.link) {
-						item.route=strip(item.link, "#")
+						item.route=strip(item.link, "#");
 					}
 					else if(item.type==="doctype") {
 						if(frappe.model.is_single(item.doctype)) {
@@ -112,16 +134,16 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 							if (item.filters) {
 								frappe.route_options=item.filters;
 							}
-							item.route="List/" + item.doctype
+							item.route="List/" + item.doctype;
 							//item.style = 'font-weight: 500;';
 						}
 						// item.style = 'font-weight: bold;';
 					}
 					else if(item.type==="report" && item.is_query_report) {
-						item.route="query-report/" + item.name
+						item.route="query-report/" + item.name;
 					}
 					else if(item.type==="report") {
-						item.route="Report/" + item.doctype + "/" + item.name
+						item.route="Report/" + item.doctype + "/" + item.name;
 					}
 					else if(item.type==="page") {
 						item.route=item.name;
@@ -130,7 +152,7 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 
 				if(item.route_options) {
 					item.route += "?" + $.map(item.route_options, function(value, key) {
-						return encodeURIComponent(key) + "=" + encodeURIComponent(value) }).join('&')
+						return encodeURIComponent(key) + "=" + encodeURIComponent(value); }).join('&');
 				}
 
 				if(item.type==="page" || item.type==="help" || item.type==="report" ||
@@ -139,22 +161,28 @@ frappe.pages['modules'].on_page_load = function(wrapper) {
 				}
 			});
 		});
-	}
-}
+	};
+};
 
 frappe.pages['modules'].on_page_show = function(wrapper) {
-	var route = frappe.get_route();
+	let route = frappe.get_route();
+	let modules = frappe.modules_page.get_page_modules().map(d => d.module_name);
 	$("body").attr("data-sidebar", 1);
 	if(route.length > 1) {
 		// activate section based on route
-		frappe.modules_page.activate_link(
-			frappe.modules_page.sidebar.find('.module-link[data-name="'+ route[1] +'"]'));
+		let module_name = route[1];
+		if(modules.includes(module_name)) {
+			frappe.modules_page.activate_link(
+				frappe.modules_page.sidebar.find('.module-link[data-name="'+ module_name +'"]'));
+		} else {
+			frappe.throw(__(`Module ${module_name} not found.`));
+		}
 	} else if(frappe.modules_page.last_link) {
 		// open last link
-		frappe.set_route('modules', frappe.modules_page.last_link.attr('data-name'))
+		frappe.set_route('modules', frappe.modules_page.last_link.attr('data-name'));
 	} else {
 		// first time, open the first page
 		frappe.modules_page.activate_link(frappe.modules_page.sidebar.find('.module-link:first'));
 	}
-}
+};
 

frappe/desk/page/modules/modules_sidebar.html

@@ -1,7 +0,0 @@
-<ul class="module-sidebar-nav overlay-sidebar nav nav-pills nav-stacked">
-    {% for (var i=0, l= modules.length; i < l; i++) { var item = modules[i];
-			if(item.type==="module" && !item.blocked) { %}
-			{{ frappe.render_template("modules_sidebar_item", {"item": item}) }}
-    {% }; } %}
-	<li class="divider"></li>
-</ul>

frappe/desk/page/modules/modules_sidebar_item.html

@@ -1,7 +0,0 @@
-    <li class="strong module-sidebar-item">
-        <a class="module-link" data-name="{{ item.module_name }}"
-					href="#modules/{{ item.module_name }}">
-          <i class="fa fa-chevron-right pull-right"
-						style="display: none;"></i>
-          <span>{{ item._label }}</span></a>
-    </li>

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -7,6 +7,7 @@ frappe.setup = {
 	events: {},
 	data: {},
 	utils: {},
+	domains: [],
 
 	on: function(event, fn) {
 		if(!frappe.setup.events[event]) {
@@ -26,7 +27,8 @@ frappe.setup = {
 }
 
 frappe.pages['setup-wizard'].on_page_load = function(wrapper) {
-	var requires = (frappe.boot.setup_wizard_requires || []);
+	let requires = (frappe.boot.setup_wizard_requires || []);
+
 
 	frappe.require(requires, function() {
 		frappe.call({
@@ -180,46 +182,79 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	}
 
 	action_on_complete() {
-		var me = this;
 		if (!this.current_slide.set_values()) return;
 		this.update_values();
 		this.show_working_state();
 		this.disable_keyboard_nav();
+		this.listen_for_setup_stages();
+
 		return frappe.call({
 			method: "frappe.desk.page.setup_wizard.setup_wizard.setup_complete",
 			args: {args: this.values},
-			callback: function() {
-				me.show_setup_complete_state();
-				if(frappe.setup.welcome_page) {
-					localStorage.setItem("session_last_route", frappe.setup.welcome_page);
+			callback: (r) => {
+				if(r.message.status === 'ok') {
+					this.post_setup_success();
+				} else if(r.message.fail !== undefined) {
+					this.abort_setup(r.message.fail);
 				}
-				setTimeout(function() {
-					// Reload
-					window.location.href = '';
-				}, 2000);
-				setTimeout(()=> {
-					$('body').removeClass('setup-state');
-				}, 20000);
 			},
-			error: function() {
-				var d = frappe.msgprint(__("There were errors."));
-				d.custom_onhide = function() {
-					$(me.parent).find('.page-card-container').remove();
-					$('body').removeClass('setup-state');
-					me.container.show();
-					frappe.set_route(me.page_name, me.slides.length - 1);
-				};
-			}
+			error: this.abort_setup.bind(this, "Error in setup", true)
 		});
 	}
 
+	post_setup_success() {
+		this.set_setup_complete_message(__("Setup Complete"), __("Refreshing..."));
+		if(frappe.setup.welcome_page) {
+			localStorage.setItem("session_last_route", frappe.setup.welcome_page);
+		}
+		setTimeout(function() {
+			// Reload
+			window.location.href = '';
+		}, 2000);
+	}
+
+	abort_setup(fail_msg, error=false) {
+		this.$working_state.find('.state-icon-container').html('');
+		fail_msg = fail_msg ? fail_msg : __("Failed to complete setup");
+
+		if(error && !frappe.boot.developer_mode) {
+			frappe.msgprint(`Don't worry. It's not you, it's us. We've
+				received the issue details and will get back to you on the solution.
+				Please feel free to contact us on support@erpnext.com in the meantime.`);
+		}
+
+		this.update_setup_message('Could not start up: ' + fail_msg);
+
+		this.$working_state.find('.title').html('Setup failed');
+
+		this.$abort_btn.show();
+	}
+
+	listen_for_setup_stages() {
+		frappe.realtime.on("setup_task", (data) => {
+			// console.log('data', data);
+			if(data.stage_status) {
+				// .html('Process '+ data.progress[0] + ' of ' + data.progress[1] + ': ' + data.stage_status);
+				this.update_setup_message(data.stage_status);
+				this.set_setup_load_percent((data.progress[0]+1)/data.progress[1] * 100);
+			}
+			if(data.fail_msg) {
+				this.abort_setup(data.fail_msg);
+			}
+		})
+	}
+
+	update_setup_message(message) {
+		this.$working_state.find('.setup-message').html(message);
+	}
+
 	get_setup_slides_filtered_by_domain() {
 		var filtered_slides = [];
 		frappe.setup.slides.forEach(function(slide) {
-			if(frappe.setup.domain) {
-				var domains = slide.domains;
-				if (domains.indexOf('all') !== -1 ||
-					domains.indexOf(frappe.setup.domain.toLowerCase()) !== -1) {
+			if(frappe.setup.domains) {
+				let active_domains = frappe.setup.domains;
+				if (!slide.domains ||
+					slide.domains.filter(d => active_domains.includes(d)).length > 0) {
 					filtered_slides.push(slide);
 				}
 			} else {
@@ -231,51 +266,56 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 
 	show_working_state() {
 		this.container.hide();
-		$('body').addClass('setup-state');
 		frappe.set_route(this.page_name);
 
-		this.working_state_message = this.get_message(
-			__("Setting Up"),
-			__("Sit tight while your system is being setup. This may take a few moments."),
-			true
-		).appendTo(this.parent);
+		this.$working_state = this.get_message(
+			__("Setting up your system"),
+			__("Starting Frappé ...")).appendTo(this.parent);
+
+		this.attach_abort_button();
 
 		this.current_id = this.slides.length;
 		this.current_slide = null;
-		this.completed_state_message = this.get_message(
-			__("Setup Complete"),
-			__("You're all set!")
-		);
 	}
 
-	show_setup_complete_state() {
-		this.working_state_message.hide();
-		this.completed_state_message.appendTo(this.parent);
+	attach_abort_button() {
+		this.$abort_btn = $(`<button class='btn btn-default btn-xs text-muted'
+			style="margin-bottom: 30px;">${__('Retry')}</button>`);
+		this.$working_state.find('.content').append(this.$abort_btn);
+
+		this.$abort_btn.on('click', () => {
+			$(this.parent).find('.setup-in-progress').remove();
+			this.container.show();
+			frappe.set_route(this.page_name, this.slides.length - 1);
+		});
+
+		this.$abort_btn.hide();
 	}
 
-	get_message(title, message="", loading=false) {
-		const loading_html = loading
-			? '<div style="width:100%;height:100%" class="lds-rolling state-icon"><div></div></div>'
-			: `<div style="width:100%;height:100%" class="state-icon">
-				<i class="fa fa-check-circle text-success"
-					style="font-size: 64px; margin-top: -8px;"></i>
-			</div>`;
-
-		return $(`<div class="page-card-container" data-state="setup">
-			<div class="page-card">
-				<div class="page-card-head">
-					${loading
-						? `<span class="indicator orange">${title}</span>`
-						: `<span class="indicator green">${title}</span>`
-					}
-				</div>
-				<p>${message}</p>
-				<div class="state-icon-container">
-				${loading_html}
-				</div>
+	get_message(title, message="") {
+		const loading_html = `<div class="progress-chart" style ="width: 150px;">
+			<div class="progress" style="margin-top: 70px; margin-bottom: 0px">
+				<div class="progress-bar" style="width: 2%; background-color: #5e64ff;"></div>
+			</div>
+		</div>`;
+
+		return $(`<div class="slides-wrapper setup-wizard-slide setup-in-progress">
+			<div class="content text-center">
+				<p class="title lead">${title}</p>
+				<div class="state-icon-container">${loading_html}</div>
+				<p class="setup-message text-muted" style="margin: 30px 0px;">${message}</p>
 			</div>
 		</div>`);
 	}
+
+	set_setup_complete_message(title, message) {
+		this.$working_state.find('.title').html(title);
+		this.$working_state.find('.setup-message').html(message);
+	}
+
+	set_setup_load_percent(percent) {
+		this.$working_state.find('.progress-bar').css({"width": percent + "%"});
+	}
 };
 
 frappe.setup.SetupWizardSlide = class SetupWizardSlide extends frappe.ui.Slide {
@@ -311,7 +351,6 @@ frappe.setup.slides_settings = [
 	{
 		// Welcome (language) slide
 		name: "welcome",
-		domains: ["all"],
 		title: __("Hello!"),
 		icon: "fa fa-world",
 		// help: __("Let's prepare the system for first use."),
@@ -344,7 +383,6 @@ frappe.setup.slides_settings = [
 	{
 		// Region slide
 		name: 'region',
-		domains: ["all"],
 		title: __("Select Your Region"),
 		icon: "fa fa-flag",
 		// help: __("Select your Country, Time Zone and Currency"),
@@ -376,7 +414,6 @@ frappe.setup.slides_settings = [
 	{
 		// Profile slide
 		name: 'user',
-		domains: ["all"],
 		title: __("The First User: You"),
 		icon: "fa fa-user",
 		fields: [

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -13,50 +13,117 @@ from werkzeug.useragents import UserAgent
 from . import install_fixtures
 from six import string_types
 
+def get_setup_stages(args):
+
+	# App setup stage functions should not include frappe.db.commit
+	# That is done by frappe after successful completion of all stages
+	stages = [
+		{
+			'status': 'Updating global settings',
+			'fail_msg': 'Failed to update global settings',
+			'tasks': [
+				{
+					'fn': update_global_settings,
+					'args': args,
+					'fail_msg': 'Failed to update global settings'
+				}
+			]
+		}
+	]
+
+	stages += get_stages_hooks(args) + get_setup_complete_hooks(args)
+
+	stages.append({
+		# post executing hooks
+		'status': 'Wrapping up',
+		'fail_msg': 'Failed to complete setup',
+		'tasks': [
+			{
+				'fn': run_post_setup_complete,
+				'args': args,
+				'fail_msg': 'Failed to complete setup'
+			}
+		]
+	})
+
+	return stages
+
 @frappe.whitelist()
 def setup_complete(args):
 	"""Calls hooks for `setup_wizard_complete`, sets home page as `desktop`
 	and clears cache. If wizard breaks, calls `setup_wizard_exception` hook"""
 
+	# Setup complete: do not throw an exception, let the user continue to desk
 	if cint(frappe.db.get_single_value('System Settings', 'setup_complete')):
-		# do not throw an exception if setup is already complete
-		# let the user continue to desk
 		return
-		#frappe.throw(_('Setup already complete'))
 
-	args = process_args(args)
+	args = parse_args(args)
 
-	try:
-		if args.language and args.language != "english":
-			set_default_language(get_language_code(args.lang))
+	stages = get_setup_stages(args)
 
-		frappe.clear_cache()
-
-		# update system settings
-		update_system_settings(args)
-		update_user_name(args)
-
-		for method in frappe.get_hooks("setup_wizard_complete"):
-			frappe.get_attr(method)(args)
+	try:
+		current_task = None
+		for idx, stage in enumerate(stages):
+			frappe.publish_realtime('setup_task', {"progress": [idx, len(stages)],
+				"stage_status": stage.get('status')}, user=frappe.session.user)
+
+			for task in stage.get('tasks'):
+				current_task = task
+				task.get('fn')(task.get('args'))
+
+	except Exception:
+		handle_setup_exception(args)
+		return {'status': 'fail', 'fail': current_task.get('fail_msg')}
+	else:
+		run_setup_success(args)
+		return {'status': 'ok'}
 
-		disable_future_access()
+def update_global_settings(args):
+	if args.language and args.language != "english":
+		set_default_language(get_language_code(args.lang))
+	frappe.clear_cache()
 
-		frappe.db.commit()
-		frappe.clear_cache()
-	except:
-		frappe.db.rollback()
-		if args:
-			traceback = frappe.get_traceback()
-			for hook in frappe.get_hooks("setup_wizard_exception"):
-				frappe.get_attr(hook)(traceback, args)
+	update_system_settings(args)
+	update_user_name(args)
 
-		raise
+def run_post_setup_complete(args):
+	disable_future_access()
+	frappe.db.commit()
+	frappe.clear_cache()
 
-	else:
-		for hook in frappe.get_hooks("setup_wizard_success"):
-			frappe.get_attr(hook)(args)
-		install_fixtures.install()
+def run_setup_success(args):
+	for hook in frappe.get_hooks("setup_wizard_success"):
+		frappe.get_attr(hook)(args)
+	install_fixtures.install()
+
+def get_stages_hooks(args):
+	stages = []
+	for method in frappe.get_hooks("setup_wizard_stages"):
+		stages += frappe.get_attr(method)(args)
+	return stages
+
+def get_setup_complete_hooks(args):
+	stages = []
+	for method in frappe.get_hooks("setup_wizard_complete"):
+		stages.append({
+			'status': 'Executing method',
+			'fail_msg': 'Failed to execute method',
+			'tasks': [
+				{
+					'fn': frappe.get_attr(method),
+					'args': args,
+					'fail_msg': 'Failed to execute method'
+				}
+			]
+		})
+	return stages
 
+def handle_setup_exception(args):
+	frappe.db.rollback()
+	if args:
+		traceback = frappe.get_traceback()
+		for hook in frappe.get_hooks("setup_wizard_exception"):
+			frappe.get_attr(hook)(traceback, args)
 
 def update_system_settings(args):
 	number_format = get_country_info(args.get("country")).get("number_format", "#,###.##")
@@ -126,7 +193,7 @@ def update_user_name(args):
 	if args.get('name'):
 		add_all_roles_to(args.get("name"))
 
-def process_args(args):
+def parse_args(args):
 	if not args:
 		args = frappe.local.form_dict
 	if isinstance(args, string_types):
@@ -234,14 +301,6 @@ def email_setup_wizard_exception(traceback, args):
 		user_agent = frappe._dict()
 
 	message = """
-#### Basic Information
-
-- **Site:** {site}
-- **User:** {user}
-- **Browser:** {user_agent.platform} {user_agent.browser} version: {user_agent.version} language: {user_agent.language}
-- **Browser Languages**: `{accept_languages}`
-
----
 
 #### Traceback
 
@@ -257,7 +316,16 @@ def email_setup_wizard_exception(traceback, args):
 
 #### Request Headers
 
-<pre>{headers}</pre>""".format(
+<pre>{headers}</pre>
+
+---
+
+#### Basic Information
+
+- **Site:** {site}
+- **User:** {user}
+- **Browser:** {user_agent.platform} {user_agent.browser} version: {user_agent.version} language: {user_agent.language}
+- **Browser Languages**: `{accept_languages}`""".format(
 		site=frappe.local.site,
 		traceback=traceback,
 		args="\n".join(pretty_args),
@@ -268,14 +336,13 @@ def email_setup_wizard_exception(traceback, args):
 
 	frappe.sendmail(recipients=frappe.local.conf.setup_wizard_exception_email,
 		sender=frappe.session.user,
-		subject="Exception in Setup Wizard - {}".format(frappe.local.site),
+		subject="Setup failed: {}".format(frappe.local.site),
 		message=message,
 		delayed=False)
 
 def get_language_code(lang):
 	return frappe.db.get_value('Language', {'language_name':lang})
 
-
 def enable_twofactor_all_roles():
 	all_role = frappe.get_doc('Role',{'role_name':'All'})
 	all_role.two_factor_auth = True

frappe/desk/query_builder.py

@@ -10,6 +10,9 @@ from frappe.utils import cint
 import frappe.defaults
 from six import text_type
 
+# imports - third-party imports
+import pymysql
+
 def get_sql_tables(q):
 	if q.find('WHERE') != -1:
 		tl = q.split('FROM')[1].split('WHERE')[0].split(',')
@@ -82,10 +85,9 @@ def guess_type(m):
 	"""
 		Returns fieldtype depending on the MySQLdb Description
 	"""
-	import MySQLdb
-	if m in MySQLdb.NUMBER:
+	if m in pymysql.NUMBER:
 		return 'Currency'
-	elif m in MySQLdb.DATE:
+	elif m in pymysql.DATE:
 		return 'Date'
 	else:
 		return 'Data'