Merge remote-tracking branch 'upstream/develop' into aaaa-website-enhancements-for-redesign

.github/frappe_linter/translation.py

@@ -7,22 +7,28 @@ start_pattern = re.compile(r"_{1,2}\([\"']{1,3}")
 
 # skip first argument
 files = sys.argv[1:]
-for _file in files:
-	if not _file.endswith(('.py', '.js')):
-		continue
+files_to_scan = [_file for _file in files if _file.endswith(('.py', '.js'))]
+
+for _file in files_to_scan:
 	with open(_file, 'r') as f:
 		print(f'Checking: {_file}')
-		for num, line in enumerate(f, 1):
-			all_matches = start_pattern.finditer(line)
-			if all_matches:
-				for match in all_matches:
-					verify = pattern.search(line)
-					if not verify:
-						errors_encounter += 1
-						print(f'A syntax error has been discovered at line number: {num}')
-						print(f'Syntax error occurred with: {line}')
+		file_lines = f.readlines()
+		for line_number, line in enumerate(file_lines, 1):
+			start_matches = start_pattern.search(line)
+			if start_matches:
+				match = pattern.search(line)
+				if not match and line.endswith(',\n'):
+					# concat remaining text to validate multiline pattern
+					line = "".join(file_lines[line_number - 1:])
+					line = line[start_matches.start() + 1:]
+					match = pattern.match(line)
+
+				if not match:
+					errors_encounter += 1
+					print(f'\nTranslation syntax error at line number: {line_number + 1}\n{line.strip()[:100]}')
+
 if errors_encounter > 0:
-	print('You can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.')
-	assert 1+1 == 3
+	print('\nYou can visit "https://frappeframework.com/docs/user/en/translations" to resolve this error.')
+	sys.exit(1)
 else:
-	print('Good To Go!')
+	print('\nGood To Go!')

.github/workflows/publish-assets-develop.yml

@@ -0,0 +1,43 @@
+name: Build and Publish Assets for Development
+
+on:
+  push:
+    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'frappe'
+      - uses: actions/setup-node@v1
+        with:
+          python-version: '12.x'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Set up bench for current push
+        run: |
+          npm install -g yarn
+          pip3 install -U frappe-bench
+          bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe
+          cd frappe-bench && bench build
+
+      - name: Package assets
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/build
+          tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+
+      - name: Publish assets to S3
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read
+        env:
+          AWS_S3_BUCKET: 'assets.frappeframework.com'
+          AWS_ACCESS_KEY_ID: ${{ secrets.S3_ASSETS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_ASSETS_SECRET_ACCESS_KEY }}
+          AWS_S3_ENDPOINT: 'http://s3.fr-par.scw.cloud'
+          AWS_REGION: 'fr-par'
+          SOURCE_DIR: '$GITHUB_WORKSPACE/build'

.github/workflows/publish-assets-releases.yml

@@ -0,0 +1,47 @@
+name: Build and Publish Assets built for Releases
+
+on:
+  release:
+    types: [ created ]
+
+env:
+  GITHUB_TOKEN: ${{ github.token }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'frappe'
+      - uses: actions/setup-node@v1
+        with:
+          python-version: '12.x'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Set up bench for current push
+        run: |
+          npm install -g yarn
+          pip3 install -U frappe-bench
+          bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe
+          cd frappe-bench && bench build
+
+      - name: Package assets
+        run: |
+          mkdir -p $GITHUB_WORKSPACE/build
+          tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css
+
+      - name: Get release
+        id: get_release
+        uses: bruceadams/get-release@v1.2.0
+
+      - name: Upload built Assets to Release
+        uses: actions/upload-release-asset@v1.0.2
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url }}
+          asset_path: build/assets.tar.gz
+          asset_name: assets.tar.gz
+          asset_content_type: application/octet-stream
+

.snyk

@@ -65,3 +65,37 @@ patch:
         patched: '2020-04-30T23:02:32.330Z'
     - quill-image-resize > lodash:
         patched: '2020-08-24T23:06:37.710Z'
+    - node-sass > lodash:
+        patched: '2020-09-15T23:06:41.931Z'
+    - node-sass > sass-graph > lodash:
+        patched: '2020-09-15T23:06:41.931Z'
+    - node-sass > gaze > globule > lodash:
+        patched: '2020-09-15T23:06:41.931Z'
+    - snyk > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-cpp-plugin > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-go-plugin > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-gradle-plugin > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-docker-plugin > snyk-nodejs-lockfile-parser > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-mvn-plugin > @snyk/java-call-graph-builder > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-php-plugin > @snyk/cli-interface > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-gradle-plugin > @snyk/cli-interface > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-mvn-plugin > @snyk/cli-interface > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > @snyk/dep-graph > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-nodejs-lockfile-parser > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'
+    - snyk > snyk-go-plugin > graphlib > lodash:
+        patched: '2020-09-16T23:06:38.881Z'

cypress/integration/recorder.js

@@ -61,10 +61,10 @@ context('Recorder', () => {
 
 		cy.visit('/desk#recorder');
 
-		cy.get('.list-row-container span').contains('frappe.desk.reportview.get').click();
+		cy.get('.list-row-container span').contains('/api/method/frappe').click();
 
 		cy.location('hash').should('contain', '#recorder/request/');
-		cy.get('form').should('contain', 'frappe.desk.reportview.get');
+		cy.get('form').should('contain', '/api/method/frappe');
 
 		cy.get('#page-recorder .primary-action').should('contain', 'Stop').click();
 		cy.get('#page-recorder .btn-secondary').should('contain', 'Clear').click();

frappe/__init__.py

@@ -182,6 +182,7 @@ def init(site, sites_path=None, new_site=False):
 	local.meta_cache = {}
 	local.form_dict = _dict()
 	local.session = _dict()
+	local.dev_server = os.environ.get('DEV_SERVER', False)
 
 	setup_module_map()
 
@@ -513,12 +514,15 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
 whitelisted = []
 guest_methods = []
 xss_safe_methods = []
-def whitelist(allow_guest=False, xss_safe=False):
+allowed_http_methods_for_whitelisted_func = {}
+
+def whitelist(allow_guest=False, xss_safe=False, methods=None):
 	"""
 	Decorator for whitelisting a function and making it accessible via HTTP.
 	Standard request will be `/api/method/[path.to.method]`
 
 	:param allow_guest: Allow non logged-in user to access this method.
+	:param methods: Allowed http method to access the method.
 
 	Use as:
 
@@ -526,10 +530,16 @@ def whitelist(allow_guest=False, xss_safe=False):
 		def myfunc(param1, param2):
 			pass
 	"""
+
+	if not methods:
+		methods = ['GET', 'POST', 'PUT', 'DELETE']
+
 	def innerfn(fn):
-		global whitelisted, guest_methods, xss_safe_methods
+		global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func
 		whitelisted.append(fn)
 
+		allowed_http_methods_for_whitelisted_func[fn] = methods
+
 		if allow_guest:
 			guest_methods.append(fn)
 
@@ -1109,8 +1119,8 @@ def get_newargs(fn, kwargs):
 		if (a in fnargs) or varkw:
 			newargs[a] = kwargs.get(a)
 
-	if "flags" in newargs:
-		del newargs["flags"]
+	newargs.pop("ignore_permissions", None)
+	newargs.pop("flags", None)
 
 	return newargs
 

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -403,6 +403,7 @@ def update_reference(docname, reference):
 
 @frappe.whitelist()
 def generate_message_preview(reference_dt, reference_doc, message=None, subject=None):
+	frappe.has_permission("Auto Repeat", "write", throw=True)
 	doc = frappe.get_doc(reference_dt, reference_doc)
 	subject_preview = _("Please add a subject to your email")
 	msg_preview = frappe.render_template(message, {'doc': doc})

frappe/build.py

@@ -11,24 +11,141 @@ import warnings
 import tempfile
 from distutils.spawn import find_executable
 
-from six import iteritems, text_type
-
 import frappe
 from frappe.utils.minify import JavascriptMinify
 
+import click
+from requests import get
+from six import iteritems, text_type
+from six.moves.urllib.parse import urlparse
+
 
 timestamps = {}
 app_paths = None
+sites_path = os.path.abspath(os.getcwd())
+
+
+def download_file(url, prefix):
+	filename = urlparse(url).path.split("/")[-1]
+	local_filename = os.path.join(prefix, filename)
+	with get(url, stream=True, allow_redirects=True) as r:
+		r.raise_for_status()
+		with open(local_filename, "wb") as f:
+			for chunk in r.iter_content(chunk_size=8192):
+				f.write(chunk)
+	return local_filename
+
+
+def build_missing_files():
+	# check which files dont exist yet from the build.json and tell build.js to build only those!
+	missing_assets = []
+	current_asset_files = []
+
+	for type in ["css", "js"]:
+		current_asset_files.extend(
+			[
+				"{0}/{1}".format(type, name)
+				for name in os.listdir(os.path.join(sites_path, "assets", type))
+			]
+		)
+
+	with open(os.path.join(sites_path, "assets", "frappe", "build.json")) as f:
+		all_asset_files = json.load(f).keys()
+
+	for asset in all_asset_files:
+		if asset.replace("concat:", "") not in current_asset_files:
+			missing_assets.append(asset)
+
+	if missing_assets:
+		from subprocess import check_call
+		from shlex import split
+
+		click.secho("\nBuilding missing assets...\n", fg="yellow")
+		command = split(
+			"node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets))
+		)
+		check_call(command, cwd=os.path.join("..", "apps", "frappe"))
+
+
+def get_assets_link(frappe_head):
+	from subprocess import getoutput
+	from requests import head
+
+	tag = getoutput(
+			"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+			" refs/tags/,,' -e 's/\^{}//'"
+			% frappe_head
+		)
+
+	if tag:
+		# if tag exists, download assets from github release
+		url = "https://github.com/frappe/frappe/releases/download/{0}/assets.tar.gz".format(tag)
+	else:
+		url = "http://assets.frappeframework.com/{0}.tar.gz".format(frappe_head)
+
+	if not head(url):
+		raise ValueError("URL {0} doesn't exist".format(url))
+
+	return url
+
+
+def download_frappe_assets(verbose=True):
+	"""Downloads and sets up Frappe assets if they exist based on the current
+	commit HEAD.
+	Returns True if correctly setup else returns False.
+	"""
+	from simple_chalk import green
+	from subprocess import getoutput
+	from tempfile import mkdtemp
+
+	assets_setup = False
+	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
+
+	if frappe_head:
+		try:
+			url = get_assets_link(frappe_head)
+			click.secho("Retreiving assets...", fg="yellow")
+			prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
+			assets_archive = download_file(url, prefix)
+			print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))
+
+			if assets_archive:
+				import tarfile
+
+				click.secho("\nExtracting assets...\n", fg="yellow")
+				with tarfile.open(assets_archive) as tar:
+					for file in tar:
+						if not file.isdir():
+							dest = "." + file.name.replace("./frappe-bench/sites", "")
+							show = dest.replace("./assets/", "")
+							tar.makefile(file, dest)
+							print("{0} Restored {1}".format(green('✔'), show))
+
+				build_missing_files()
+				return True
+			else:
+				raise
+		except Exception:
+			# TODO: log traceback in bench.log
+			click.secho("An Error occurred while downloading assets...", fg="red")
+			assets_setup = False
+		finally:
+			try:
+				shutil.rmtree(os.path.dirname(assets_archive))
+			except Exception:
+				pass
+
+	return assets_setup
 
 
 def symlink(target, link_name, overwrite=False):
-	'''
+	"""
 	Create a symbolic link named link_name pointing to target.
 	If link_name exists then FileExistsError is raised, unless overwrite=True.
 	When trying to overwrite a directory, IsADirectoryError is raised.
 
 	Source: https://stackoverflow.com/a/55742015/10309266
-	'''
+	"""
 
 	if not overwrite:
 		return os.symlink(target, link_name)
@@ -76,27 +193,28 @@ def setup():
 
 
 def get_node_pacman():
-	pacmans = ['yarn', 'npm']
-	for exec_ in pacmans:
-		exec_ = find_executable(exec_)
-		if exec_:
-			return exec_
-	raise ValueError('No Node.js Package Manager found.')
+	exec_ = find_executable("yarn")
+	if exec_:
+		return exec_
+	raise ValueError("Yarn not found")
 
 
-def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False):
+def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
 	"""concat / minify js files"""
 	setup()
 	make_asset_dirs(make_copy=make_copy, restore=restore)
 
 	pacman = get_node_pacman()
-	mode = 'build' if no_compress else 'production'
-	command = '{pacman} run {mode}'.format(pacman=pacman, mode=mode)
+	mode = "build" if no_compress else "production"
+	command = "{pacman} run {mode}".format(pacman=pacman, mode=mode)
 
 	if app:
-		command += ' --app {app}'.format(app=app)
+		command += " --app {app}".format(app=app)
+
+	if skip_frappe:
+		command += " --skip_frappe"
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..'))
+	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
 	check_yarn()
 	frappe.commands.popen(command, cwd=frappe_app_path)
 
@@ -107,22 +225,22 @@ def watch(no_compress):
 
 	pacman = get_node_pacman()
 
-	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], '..'))
+	frappe_app_path = os.path.abspath(os.path.join(app_paths[0], ".."))
 	check_yarn()
-	frappe_app_path = frappe.get_app_path('frappe', '..')
-	frappe.commands.popen('{pacman} run watch'.format(pacman=pacman), cwd=frappe_app_path)
+	frappe_app_path = frappe.get_app_path("frappe", "..")
+	frappe.commands.popen("{pacman} run watch".format(pacman=pacman), cwd=frappe_app_path)
 
 
 def check_yarn():
-	if not find_executable('yarn'):
-		print('Please install yarn using below command and try again.\nnpm install -g yarn')
+	if not find_executable("yarn"):
+		print("Please install yarn using below command and try again.\nnpm install -g yarn")
 
 
 def make_asset_dirs(make_copy=False, restore=False):
 	# don't even think of making assets_path absolute - rm -rf ahead.
 	assets_path = os.path.join(frappe.local.sites_path, "assets")
 
-	for dir_path in [os.path.join(assets_path, 'js'), os.path.join(assets_path, 'css')]:
+	for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
 		if not os.path.exists(dir_path):
 			os.makedirs(dir_path)
 
@@ -131,24 +249,27 @@ def make_asset_dirs(make_copy=False, restore=False):
 		app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
 
 		symlinks = []
-		app_public_path = os.path.join(app_base_path, 'public')
+		app_public_path = os.path.join(app_base_path, "public")
 		# app/public > assets/app
 		symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
 		# app/node_modules > assets/app/node_modules
 		if os.path.exists(os.path.abspath(app_public_path)):
-			symlinks.append([os.path.join(app_base_path, '..', 'node_modules'), os.path.join(
-				assets_path, app_name, 'node_modules')])
+			symlinks.append(
+				[
+					os.path.join(app_base_path, "..", "node_modules"),
+					os.path.join(assets_path, app_name, "node_modules"),
+				]
+			)
 
 		app_doc_path = None
-		if os.path.isdir(os.path.join(app_base_path, 'docs')):
-			app_doc_path = os.path.join(app_base_path, 'docs')
+		if os.path.isdir(os.path.join(app_base_path, "docs")):
+			app_doc_path = os.path.join(app_base_path, "docs")
 
-		elif os.path.isdir(os.path.join(app_base_path, 'www', 'docs')):
-			app_doc_path = os.path.join(app_base_path, 'www', 'docs')
+		elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
+			app_doc_path = os.path.join(app_base_path, "www", "docs")
 
 		if app_doc_path:
-			symlinks.append([app_doc_path, os.path.join(
-				assets_path, app_name + '_docs')])
+			symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
 
 		for source, target in symlinks:
 			source = os.path.abspath(source)
@@ -162,7 +283,7 @@ def make_asset_dirs(make_copy=False, restore=False):
 						shutil.copytree(source, target)
 				elif make_copy:
 					if os.path.exists(target):
-						warnings.warn('Target {target} already exists.'.format(target=target))
+						warnings.warn("Target {target} already exists.".format(target=target))
 					else:
 						shutil.copytree(source, target)
 				else:
@@ -174,7 +295,7 @@ def make_asset_dirs(make_copy=False, restore=False):
 					try:
 						symlink(source, target, overwrite=True)
 					except OSError:
-						print('Cannot link {} to {}'.format(source, target))
+						print("Cannot link {} to {}".format(source, target))
 			else:
 				# warnings.warn('Source {source} does not exist.'.format(source = source))
 				pass
@@ -193,7 +314,7 @@ def get_build_maps():
 
 	build_maps = {}
 	for app_path in app_paths:
-		path = os.path.join(app_path, 'public', 'build.json')
+		path = os.path.join(app_path, "public", "build.json")
 		if os.path.exists(path):
 			with open(path) as f:
 				try:
@@ -202,8 +323,7 @@ def get_build_maps():
 						source_paths = []
 						for source in sources:
 							if isinstance(source, list):
-								s = frappe.get_pymodule_path(
-									source[0], *source[1].split("/"))
+								s = frappe.get_pymodule_path(source[0], *source[1].split("/"))
 							else:
 								s = os.path.join(app_path, source)
 							source_paths.append(s)
@@ -211,36 +331,42 @@ def get_build_maps():
 						build_maps[target] = source_paths
 				except ValueError as e:
 					print(path)
-					print('JSON syntax error {0}'.format(str(e)))
+					print("JSON syntax error {0}".format(str(e)))
 	return build_maps
 
 
 def pack(target, sources, no_compress, verbose):
 	from six import StringIO
 
-	outtype, outtxt = target.split(".")[-1], ''
+	outtype, outtxt = target.split(".")[-1], ""
 	jsm = JavascriptMinify()
 
 	for f in sources:
 		suffix = None
-		if ':' in f:
-			f, suffix = f.split(':')
+		if ":" in f:
+			f, suffix = f.split(":")
 		if not os.path.exists(f) or os.path.isdir(f):
 			print("did not find " + f)
 			continue
 		timestamps[f] = os.path.getmtime(f)
 		try:
-			with open(f, 'r') as sourcefile:
-				data = text_type(sourcefile.read(), 'utf-8', errors='ignore')
+			with open(f, "r") as sourcefile:
+				data = text_type(sourcefile.read(), "utf-8", errors="ignore")
 
 			extn = f.rsplit(".", 1)[1]
 
-			if outtype == "js" and extn == "js" and (not no_compress) and suffix != "concat" and (".min." not in f):
-				tmpin, tmpout = StringIO(data.encode('utf-8')), StringIO()
+			if (
+				outtype == "js"
+				and extn == "js"
+				and (not no_compress)
+				and suffix != "concat"
+				and (".min." not in f)
+			):
+				tmpin, tmpout = StringIO(data.encode("utf-8")), StringIO()
 				jsm.minify(tmpin, tmpout)
 				minified = tmpout.getvalue()
 				if minified:
-					outtxt += text_type(minified or '', 'utf-8').strip('\n') + ';'
+					outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
 
 				if verbose:
 					print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@@ -248,27 +374,27 @@ def pack(target, sources, no_compress, verbose):
 				# add to frappe.templates
 				outtxt += html_to_js_template(f, data)
 			else:
-				outtxt += ('\n/*\n *\t%s\n */' % f)
-				outtxt += '\n' + data + '\n'
+				outtxt += "\n/*\n *\t%s\n */" % f
+				outtxt += "\n" + data + "\n"
 
 		except Exception:
 			print("--Error in:" + f + "--")
 			print(frappe.get_traceback())
 
-	with open(target, 'w') as f:
+	with open(target, "w") as f:
 		f.write(outtxt.encode("utf-8"))
 
-	print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target)/1024))))
+	print("Wrote %s - %sk" % (target, str(int(os.path.getsize(target) / 1024))))
 
 
 def html_to_js_template(path, content):
-	'''returns HTML template content as Javascript code, adding it to `frappe.templates`'''
+	"""returns HTML template content as Javascript code, adding it to `frappe.templates`"""
 	return """frappe.templates["{key}"] = '{content}';\n""".format(
 		key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content))
 
 
 def scrub_html_template(content):
-	'''Returns HTML content with removed whitespace and comments'''
+	"""Returns HTML content with removed whitespace and comments"""
 	# remove whitespace to a single space
 	content = re.sub("\s+", " ", content)
 
@@ -281,12 +407,12 @@ def scrub_html_template(content):
 def files_dirty():
 	for target, sources in iteritems(get_build_maps()):
 		for f in sources:
-			if ':' in f:
-				f, suffix = f.split(':')
+			if ":" in f:
+				f, suffix = f.split(":")
 			if not os.path.exists(f) or os.path.isdir(f):
 				continue
 			if os.path.getmtime(f) != timestamps.get(f):
-				print(f + ' dirty')
+				print(f + " dirty")
 				return True
 	else:
 		return False

frappe/client.py

@@ -107,7 +107,7 @@ def get_single_value(doctype, field):
 	value = frappe.db.get_single_value(doctype, field)
 	return value
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def set_value(doctype, name, fieldname, value=None):
 	'''Set a value using get_doc, group of values
 
@@ -142,7 +142,7 @@ def set_value(doctype, name, fieldname, value=None):
 
 	return doc.as_dict()
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def insert(doc=None):
 	'''Insert a document
 
@@ -160,7 +160,7 @@ def insert(doc=None):
 		doc = frappe.get_doc(doc).insert()
 		return doc.as_dict()
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def insert_many(docs=None):
 	'''Insert multiple documents
 
@@ -186,7 +186,7 @@ def insert_many(docs=None):
 
 	return out
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def save(doc):
 	'''Update (save) an existing document
 
@@ -199,7 +199,7 @@ def save(doc):
 
 	return doc.as_dict()
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def rename_doc(doctype, old_name, new_name, merge=False):
 	'''Rename document
 
@@ -209,7 +209,7 @@ def rename_doc(doctype, old_name, new_name, merge=False):
 	new_name = frappe.rename_doc(doctype, old_name, new_name, merge=merge)
 	return new_name
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def submit(doc):
 	'''Submit a document
 
@@ -222,7 +222,7 @@ def submit(doc):
 
 	return doc.as_dict()
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def cancel(doctype, name):
 	'''Cancel a document
 
@@ -233,7 +233,7 @@ def cancel(doctype, name):
 
 	return wrapper.as_dict()
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['DELETE', 'POST'])
 def delete(doctype, name):
 	'''Delete a remote document
 
@@ -241,13 +241,13 @@ def delete(doctype, name):
 	:param name: name of the document to be deleted'''
 	frappe.delete_doc(doctype, name, ignore_missing=False)
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def set_default(key, value, parent=None):
 	"""set a user default value"""
 	frappe.db.set_default(key, value, parent or frappe.session.user)
 	frappe.clear_cache(user=frappe.session.user)
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def make_width_property_setter(doc):
 	'''Set width Property Setter
 
@@ -257,7 +257,7 @@ def make_width_property_setter(doc):
 	if doc["doctype"]=="Property Setter" and doc["property"]=="width":
 		frappe.get_doc(doc).insert(ignore_permissions = True)
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def bulk_update(docs):
 	'''Bulk update documents
 
@@ -333,7 +333,7 @@ def get_time_zone():
 	'''Returns default time zone'''
 	return {"time_zone": frappe.defaults.get_defaults().get("time_zone")}
 
-@frappe.whitelist()
+@frappe.whitelist(methods=['POST', 'PUT'])
 def attach_file(filename=None, filedata=None, doctype=None, docname=None, folder=None, decode_base64=False, is_private=None, docfield=None):
 	'''Attach a file to Document (POST)
 

frappe/commands/utils.py

@@ -1,17 +1,17 @@
 # -*- coding: utf-8 -*-
 
-from __future__ import unicode_literals, absolute_import, print_function
-import click
-import json, os, sys, subprocess
+import json
+import os
+import subprocess
+import sys
 from distutils.spawn import find_executable
+
+import click
+
 import frappe
-from frappe.commands import pass_context, get_site
+from frappe.commands import get_site, pass_context
 from frappe.exceptions import SiteNotSpecifiedError
-from frappe.utils import update_progress_bar, get_bench_path
-from frappe.utils.response import json_handler
-from coverage import Coverage
-import cProfile, pstats
-from six import StringIO
+from frappe.utils import get_bench_path, update_progress_bar
 
 
 @click.command('build')
@@ -19,14 +19,22 @@ from six import StringIO
 @click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
 @click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
 @click.option('--verbose', is_flag=True, default=False, help='Verbose')
-def build(app=None, make_copy=False, restore = False, verbose=False):
+@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
+def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
 	"Minify + concatenate JS and CSS files, build translations"
 	import frappe.build
-	import frappe
 	frappe.init('')
 	# don't minify in developer_mode for faster builds
 	no_compress = frappe.local.conf.developer_mode or False
-	frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore = restore, verbose=verbose)
+
+	# dont try downloading assets if force used, app specified or running via CI
+	if not (force or app or os.environ.get('CI')):
+		# skip building frappe if assets exist remotely
+		skip_frappe = frappe.build.download_frappe_assets(verbose=verbose)
+	else:
+		skip_frappe = False
+
+	frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
 
 
 @click.command('watch')
@@ -133,6 +141,7 @@ def reset_perms(context):
 def execute(context, method, args=None, kwargs=None, profile=False):
 	"Execute a function"
 	for site in context.sites:
+		ret = ""
 		try:
 			frappe.init(site=site)
 			frappe.connect()
@@ -151,12 +160,19 @@ def execute(context, method, args=None, kwargs=None, profile=False):
 				kwargs = {}
 
 			if profile:
+				import cProfile
 				pr = cProfile.Profile()
 				pr.enable()
 
-			ret = frappe.get_attr(method)(*args, **kwargs)
+			try:
+				ret = frappe.get_attr(method)(*args, **kwargs)
+			except Exception:
+				ret = frappe.safe_eval(method + "(*args, **kwargs)", eval_globals=globals(), eval_locals=locals())
 
 			if profile:
+				import pstats
+				from six import StringIO
+
 				pr.disable()
 				s = StringIO()
 				pstats.Stats(pr, stream=s).sort_stats('cumulative').print_stats(.5)
@@ -167,6 +183,7 @@ def execute(context, method, args=None, kwargs=None, profile=False):
 		finally:
 			frappe.destroy()
 		if ret:
+			from frappe.utils.response import json_handler
 			print(json.dumps(ret, default=json_handler))
 
 	if not context.sites:
@@ -492,6 +509,8 @@ def run_tests(context, app=None, module=None, doctype=None, test=(),
 	frappe.flags.skip_test_records = skip_test_records
 
 	if coverage:
+		from coverage import Coverage
+
 		# Generate coverage report only for app that is being tested
 		source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
 		cov = Coverage(source=[source_path], omit=[

frappe/core/doctype/activity_log/activity_log.json

@@ -1,731 +1,184 @@
- {
- "allow_copy": 0,
- "allow_guest_to_view": 0,
+{
+ "actions": [],
  "allow_import": 1,
- "allow_rename": 0,
- "autoname": "",
- "beta": 0,
  "creation": "2017-10-05 11:10:38.780133",
- "custom": 0,
  "description": "Keep track of all update feeds",
- "docstatus": 0,
  "doctype": "DocType",
  "document_type": "Setup",
- "editable_grid": 0,
  "engine": "InnoDB",
+ "field_order": [
+  "subject",
+  "section_break_8",
+  "content",
+  "column_break_5",
+  "additional_info",
+  "communication_date",
+  "column_break_7",
+  "operation",
+  "status",
+  "reference_section",
+  "reference_doctype",
+  "reference_name",
+  "reference_owner",
+  "column_break_14",
+  "timeline_doctype",
+  "timeline_name",
+  "link_doctype",
+  "link_name",
+  "user",
+  "full_name"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "subject",
    "fieldtype": "Small Text",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_global_search": 1,
    "in_list_view": 1,
-   "in_standard_filter": 0,
    "label": "Subject",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "section_break_8",
-   "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "fieldtype": "Section Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "content",
    "fieldtype": "Text Editor",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Message",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0,
    "width": "400"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "column_break_5",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
    "collapsible": 1,
-   "columns": 0,
    "fieldname": "additional_info",
    "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "More Information",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "More Information"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "Now",
    "fieldname": "communication_date",
    "fieldtype": "Datetime",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Date",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Date"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "column_break_7",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "operation",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Operation",
-   "length": 0,
-   "no_copy": 0,
-   "options": "\nLogin\nLogout",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "\nLogin\nLogout"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "status",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
    "label": "Status",
-   "length": 0,
-   "no_copy": 0,
-   "options": "\nSuccess\nFailed\nLinked\nClosed",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "\nSuccess\nFailed\nLinked\nClosed"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
    "collapsible": 1,
-   "columns": 0,
    "fieldname": "reference_section",
    "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Reference",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Reference"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "reference_doctype",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Reference Document Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "DocType"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "reference_name",
    "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Reference Name",
-   "length": 0,
-   "no_copy": 0,
-   "options": "reference_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "reference_doctype"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fetch_from": "reference_name.owner",
    "fieldname": "reference_owner",
    "fieldtype": "Read Only",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Reference Owner",
-   "length": 0,
-   "no_copy": 0,
-   "options": "",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 1,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "search_index": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "column_break_14",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "timeline_doctype",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Timeline DocType",
-   "length": 0,
-   "no_copy": 0,
-   "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "DocType"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "timeline_name",
    "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Timeline Name",
-   "length": 0,
-   "no_copy": 0,
-   "options": "timeline_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "timeline_doctype"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "link_doctype",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Link DocType",
-   "length": 0,
-   "no_copy": 0,
    "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "link_name",
    "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Link Name",
-   "length": 0,
-   "no_copy": 0,
    "options": "link_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "__user",
    "fieldname": "user",
    "fieldtype": "Link",
-   "hidden": 0,
    "ignore_user_permissions": 1,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "User",
-   "length": 0,
-   "no_copy": 0,
    "options": "User",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "full_name",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
-   "label": "Full Name",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Full Name"
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
  "icon": "fa fa-comment",
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2019-09-05 14:22:27.664645",
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2020-08-28 11:43:57.504565",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Activity Log",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "cancel": 0,
-   "create": 1,
-   "delete": 1,
    "email": 1,
-   "export": 0,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
-   "print": 0,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
-   "share": 1,
-   "submit": 0,
-   "write": 0
+   "share": 1
   },
   {
-   "amend": 0,
-   "cancel": 0,
-   "create": 0,
-   "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 1,
+   "if_owner": 1,
    "print": 1,
    "read": 1,
    "report": 1,
-   "role": "System Manager",
-   "set_user_permissions": 0,
-   "share": 1,
-   "submit": 0,
-   "write": 0
-  },
-  {
-   "amend": 0,
-   "cancel": 0,
-   "create": 0,
-   "delete": 1,
-   "email": 1,
-   "export": 0,
-   "if_owner": 1,
-   "import": 0,
-   "permlevel": 0,
-   "print": 0,
-   "read": 1,
-   "report": 0,
    "role": "All",
-   "set_user_permissions": 0,
-   "share": 0,
-   "submit": 0,
-   "write": 0
+   "share": 1
   }
  ],
- "quick_entry": 0,
- "read_only": 0,
- "read_only_onload": 0,
  "search_fields": "subject",
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
  "title_field": "subject",

frappe/core/doctype/activity_log/activity_log.py

@@ -25,9 +25,6 @@ class ActivityLog(Document):
 		if self.reference_doctype and self.reference_name:
 			self.status = "Linked"
 
-	def on_trash(self): # pylint: disable=no-self-use
-		frappe.throw(_("Sorry! You cannot delete auto-generated comments"))
-
 def on_doctype_update():
 	"""Add indexes in `tabActivity Log`"""
 	frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"])

frappe/core/doctype/communication/communication.py

@@ -455,18 +455,18 @@ def update_parent_document_on_communication(doc):
 			# update the modified date for document
 			parent.update_modified()
 
-	update_mins_to_first_communication(parent, doc)
+	update_first_response_time(parent, doc)
 	set_avg_response_time(parent, doc)
 	parent.run_method("notify_communication", doc)
 	parent.notify_update()
 
-def update_mins_to_first_communication(parent, communication):
-	if parent.meta.has_field("mins_to_first_response") and not parent.get("mins_to_first_response"):
+def update_first_response_time(parent, communication):
+	if parent.meta.has_field("first_response_time") and not parent.get("first_response_time"):
 		if is_system_user(communication.sender):
 			first_responded_on = communication.creation
 			if parent.meta.has_field("first_responded_on") and communication.sent_or_received == "Sent":
 				parent.db_set("first_responded_on", first_responded_on)
-			parent.db_set("mins_to_first_response", round(time_diff_in_seconds(first_responded_on, parent.creation) / 60), 2)
+			parent.db_set("first_response_time", round(time_diff_in_seconds(first_responded_on, parent.creation), 2))
 
 def set_avg_response_time(parent, communication):
 	if parent.meta.has_field("avg_response_time") and communication.sent_or_received == "Sent":

frappe/core/doctype/doctype/doctype.py

@@ -234,6 +234,8 @@ class DocType(Document):
 
 		if not autoname and self.get("fields", {"fieldname":"naming_series"}):
 			self.autoname = "naming_series:"
+		elif self.autoname == "naming_series:" and not self.get("fields", {"fieldname":"naming_series"}):
+			frappe.throw(_("Invalid fieldname '{0}' in autoname").format(self.autoname))
 
 		# validate field name if autoname field:fieldname is used
 		# Create unique index on autoname field automatically.

frappe/core/doctype/file/file.py

@@ -278,25 +278,26 @@ class File(Document):
 		base_url = os.path.dirname(self.file_url)
 
 		files = []
-		with zipfile.ZipFile(zip_path) as zf:
-			zf.extractall(os.path.dirname(zip_path))
-			for info in zf.infolist():
-				if not info.filename.startswith('__MACOSX'):
-					file_url = file_url = base_url + '/' + info.filename
-					file_name = frappe.db.get_value('File', dict(file_url=file_url))
-					if file_name:
-						file_doc = frappe.get_doc('File', file_name)
-					else:
-						file_doc = frappe.new_doc("File")
-					file_doc.file_name = info.filename
-					file_doc.file_size = info.file_size
-					file_doc.folder = self.folder
-					file_doc.is_private = self.is_private
-					file_doc.file_url = file_url
-					file_doc.attached_to_doctype = self.attached_to_doctype
-					file_doc.attached_to_name = self.attached_to_name
-					file_doc.save()
-					files.append(file_doc)
+		with zipfile.ZipFile(zip_path) as z:
+			for file in z.filelist:
+				if file.is_dir() or file.filename.startswith('__MACOSX/'):
+					# skip directories and macos hidden directory
+					continue
+
+				filename = os.path.basename(file.filename)
+				if filename.startswith('.'):
+					# skip hidden files
+					continue
+
+				file_doc = frappe.new_doc('File')
+				file_doc.content = z.read(file.filename)
+				file_doc.file_name = filename
+				file_doc.folder = self.folder
+				file_doc.is_private = self.is_private
+				file_doc.attached_to_doctype = self.attached_to_doctype
+				file_doc.attached_to_name = self.attached_to_name
+				file_doc.save()
+				files.append(file_doc)
 
 		frappe.delete_doc('File', self.name)
 		return files
@@ -359,6 +360,9 @@ class File(Document):
 		"""write file to disk with a random name (to compare)"""
 		file_path = get_files_path(is_private=self.is_private)
 
+		if os.path.sep in self.file_name:
+			frappe.throw(_('File name cannot have {0}').format(os.path.sep))
+
 		# create directory (if not exists)
 		frappe.create_folder(file_path)
 		# write the file
@@ -938,7 +942,7 @@ def attach_files_to_document(doc, event):
 		# we dont want the update to fail if file cannot be attached for some reason
 		try:
 			value = doc.get(df.fieldname)
-			if not value.startswith(("/files", "/private/files")):
+			if not (value or '').startswith(("/files", "/private/files")):
 				return
 
 			if frappe.db.exists("File", {

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -109,12 +109,14 @@ class ScheduledJobType(Document):
 	def on_trash(self):
 		frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name)
 
+
 @frappe.whitelist()
 def execute_event(doc):
 	frappe.only_for('System Manager')
 	doc = json.loads(doc)
 	frappe.get_doc('Scheduled Job Type', doc.get('name')).enqueue()
 
+
 def run_scheduled_job(job_type):
 	'''This is a wrapper function that runs a hooks.scheduler_events method'''
 	try:
@@ -122,44 +124,62 @@ def run_scheduled_job(job_type):
 	except Exception:
 		print(frappe.get_traceback())
 
-def sync_jobs():
-	frappe.reload_doc('core', 'doctype', 'scheduled_job_type')
-	all_events = []
-	scheduler_events = frappe.get_hooks("scheduler_events")
-	insert_events(all_events, scheduler_events)
-	clear_events(all_events, scheduler_events)
 
-def insert_events(all_events, scheduler_events):
+def sync_jobs(hooks=None):
+	frappe.reload_doc("core", "doctype", "scheduled_job_type")
+	scheduler_events = hooks or frappe.get_hooks("scheduler_events")
+	all_events = insert_events(scheduler_events)
+	clear_events(all_events)
+
+
+def insert_events(scheduler_events):
+	cron_jobs, event_jobs = [], []
 	for event_type in scheduler_events:
 		events = scheduler_events.get(event_type)
 		if isinstance(events, dict):
-			insert_cron_event(events, all_events)
+			cron_jobs += insert_cron_jobs(events)
 		else:
 			# hourly, daily etc
-			insert_event_list(events, event_type, all_events)
+			event_jobs += insert_event_jobs(events, event_type)
+	return cron_jobs + event_jobs
+
 
-def insert_cron_event(events, all_events):
+def insert_cron_jobs(events):
+	cron_jobs = []
 	for cron_format in events:
 		for event in events.get(cron_format):
-			all_events.append(event)
-			insert_single_event('Cron', event, cron_format)
+			cron_jobs.append(event)
+			insert_single_event("Cron", event, cron_format)
+	return cron_jobs
+
 
-def insert_event_list(events, event_type, all_events):
+def insert_event_jobs(events, event_type):
+	event_jobs = []
 	for event in events:
-		all_events.append(event)
+		event_jobs.append(event)
 		frequency = event_type.replace('_', ' ').title()
 		insert_single_event(frequency, event)
+	return event_jobs
+
+
+def insert_single_event(frequency, event, cron_format=None):
+	cron_expr = {"cron_format": cron_format} if cron_format else {}
+	doc = frappe.get_doc({
+		"doctype": "Scheduled Job Type",
+		"method": event,
+		"cron_format": cron_format,
+		"frequency": frequency
+	})
+
+	if not frappe.db.exists("Scheduled Job Type", {"method": event, "frequency": frequency, **cron_expr }):
+		try:
+			doc.insert()
+		except frappe.DuplicateEntryError:
+			doc.delete()
+			doc.insert()
+
 
-def insert_single_event(frequency, event, cron_format = None):
-	if not frappe.db.exists('Scheduled Job Type', dict(method=event)):
-		frappe.get_doc(dict(
-			doctype = 'Scheduled Job Type',
-			method = event,
-			cron_format = cron_format,
-			frequency = frequency
-		)).insert()
-
-def clear_events(all_events, scheduler_events):
-	for event in frappe.get_all('Scheduled Job Type', ('name', 'method')):
+def clear_events(all_events):
+	for event in frappe.get_all("Scheduled Job Type", ("name", "method")):
 		if event.method not in all_events:
-			frappe.delete_doc('Scheduled Job Type', event.name)
+			frappe.delete_doc("Scheduled Job Type", event.name)

frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py

@@ -11,11 +11,10 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 
 class TestScheduledJobType(unittest.TestCase):
 	def setUp(self):
-		if not frappe.get_all('Scheduled Job Type', limit=1):
-			frappe.db.rollback()
-			frappe.db.sql('truncate `tabScheduled Job Type`')
-			sync_jobs()
-			frappe.db.commit()
+		frappe.db.rollback()
+		frappe.db.sql('truncate `tabScheduled Job Type`')
+		sync_jobs()
+		frappe.db.commit()
 
 	def test_sync_jobs(self):
 		all_job = frappe.get_doc('Scheduled Job Type',
@@ -32,6 +31,12 @@ class TestScheduledJobType(unittest.TestCase):
 		self.assertEqual(cron_job.frequency, 'Cron')
 		self.assertEqual(cron_job.cron_format, '0/15 * * * *')
 
+		# check if jobs are synced after change in hooks
+		updated_scheduler_events = { "hourly": ["frappe.email.queue.flush"] }
+		sync_jobs(updated_scheduler_events)
+		updated_scheduled_job = frappe.get_doc("Scheduled Job Type", {"method": "frappe.email.queue.flush"})
+		self.assertEqual(updated_scheduled_job.frequency, "Hourly")
+
 	def test_daily_job(self):
 		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.clear_outbox'))
 		job.db_set('last_execution', '2019-01-01 00:00:00')

frappe/core/doctype/system_settings/system_settings.json

@@ -40,6 +40,7 @@
   "password_settings",
   "logout_on_password_reset",
   "force_user_to_reset_password",
+  "password_reset_limit",
   "column_break_31",
   "enable_password_policy",
   "minimum_password_score",
@@ -415,6 +416,13 @@
    "fieldtype": "Int",
    "label": "Run Jobs only Daily if Inactive For (Days)"
   },
+  {
+   "default": "3",
+   "description": "Hourly rate limit for generating password reset links",
+   "fieldname": "password_reset_limit",
+   "fieldtype": "Int",
+   "label": "Password Reset Link Generation Limit"
+  },
   {
    "default": "1",
    "fieldname": "logout_on_password_reset",

frappe/core/doctype/user/test_user.py

@@ -19,6 +19,7 @@ class TestUser(unittest.TestCase):
 		# disable password strength test
 		frappe.db.set_value("System Settings", "System Settings", "enable_password_policy", 0)
 		frappe.db.set_value("System Settings", "System Settings", "minimum_password_score", "")
+		frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 3)
 
 	def test_user_type(self):
 		new_user = frappe.get_doc(dict(doctype='User', email='test-for-type@example.com',
@@ -222,6 +223,19 @@ class TestUser(unittest.TestCase):
 		self.assertEqual(extract_mentions(comment)[0], "test_user@example.com")
 		self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com")
 
+	def test_rate_limiting_for_reset_password(self):
+		from frappe.utils.password import delete_password_reset_cache
+		delete_password_reset_cache()
+
+		frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 1)
+
+		user = frappe.get_doc("User", "testperm@example.com")
+		link = user.reset_password()
+		self.assertRegex(link, "\/update-password\?key=[A-Za-z0-9]*")
+
+		self.assertRaises(frappe.ValidationError, user.reset_password, False)
+
+
 def delete_contact(user):
 	frappe.db.sql("DELETE FROM `tabContact` WHERE `email_id`= %s", user)
 	frappe.db.sql("DELETE FROM `tabContact Email` WHERE `email_id`= %s", user)

frappe/core/doctype/user/user.py

@@ -13,15 +13,16 @@ from frappe.utils.user import get_system_managers
 from bs4 import BeautifulSoup
 import frappe.permissions
 import frappe.share
-import re
-import json
 
 from frappe.website.utils import is_signup_enabled
 from frappe.utils.background_jobs import enqueue
 
 STANDARD_USERS = ("Guest", "Administrator")
 
-class MaxUsersReachedError(frappe.ValidationError): pass
+
+class MaxUsersReachedError(frappe.ValidationError):
+	pass
+
 
 class User(Document):
 	__new_password = None
@@ -225,6 +226,11 @@ class User(Document):
 	def reset_password(self, send_email=False, password_expired=False):
 		from frappe.utils import random_string, get_url
 
+		rate_limit = frappe.db.get_single_value("System Settings", "password_reset_limit")
+
+		if rate_limit:
+			check_password_reset_limit(self.name, rate_limit)
+
 		key = random_string(32)
 		self.db_set("reset_password_key", key)
 
@@ -236,6 +242,7 @@ class User(Document):
 		if send_email:
 			self.password_reset_mail(link)
 
+		update_password_reset_limit(self.name)
 		return link
 
 	def get_other_system_managers(self):
@@ -1110,3 +1117,16 @@ def generate_keys(user):
 
 		return {"api_secret": api_secret}
 	frappe.throw(frappe._("Not Permitted"), frappe.PermissionError)
+
+def update_password_reset_limit(user):
+	generated_link_count = get_generated_link_count(user)
+	generated_link_count += 1
+	frappe.cache().hset("password_reset_link_count", user, generated_link_count)
+
+def check_password_reset_limit(user, rate_limit):
+	generated_link_count = get_generated_link_count(user)
+	if generated_link_count >= rate_limit:
+		frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later."))
+
+def get_generated_link_count(user):
+	return cint(frappe.cache().hget("password_reset_link_count", user)) or 0

frappe/core/doctype/user_permission/test_user_permission.py

@@ -26,8 +26,7 @@ class TestUserPermission(unittest.TestCase):
 		user = create_user('test_user_perm1@example.com', 'Website Manager')
 		for category in ['general', 'public']:
 			if not frappe.db.exists('Blog Category', category):
-				frappe.get_doc({'doctype': 'Blog Category',
-					'category_name': category, 'title': category}).insert()
+				frappe.get_doc({'doctype': 'Blog Category', 'title': category}).insert()
 
 		param = get_params(user, 'Blog Category', 'general', is_default=1)
 		add_user_permissions(param)

frappe/custom/doctype/package_publish_tool/package_publish_tool.py

@@ -100,6 +100,7 @@ def export_package():
 @frappe.whitelist()
 def import_package(package=None):
 	"""Import package from JSON."""
+	frappe.only_for("System Manager")
 	if isinstance(package, string_types):
 		package = json.loads(package)
 

frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py

@@ -5,11 +5,12 @@
 from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
+from frappe.utils.safe_exec import get_safe_globals
 
 class DataMigrationMapping(Document):
 	def get_filters(self):
 		if self.condition:
-			return frappe.safe_eval(self.condition, dict(frappe=frappe))
+			return frappe.safe_eval(self.condition, get_safe_globals())
 
 	def get_fields(self):
 		fields = []
@@ -63,7 +64,7 @@ def get_value_from_fieldname(field_map, fieldname_field, doc):
 	field_name = get_source_value(field_map, fieldname_field)
 
 	if field_name.startswith('eval:'):
-		value = frappe.safe_eval(field_name[5:], dict(frappe=frappe))
+		value = frappe.safe_eval(field_name[5:], get_safe_globals())
 	elif field_name[0] in ('"', "'"):
 		value = field_name[1:-1]
 	else:

frappe/desk/desktop.py

@@ -40,7 +40,7 @@ class Workspace:
 
 		self.doc = self.get_page_for_user()
 
-		if self.doc.module not in self.allowed_modules:
+		if self.doc.module and self.doc.module not in self.allowed_modules:
 			raise frappe.PermissionError
 
 		self.can_read = self.get_cached('user_perm_can_read', self.get_can_read_items)

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -60,11 +60,11 @@ def has_permission(doc, ptype, user):
 
 
 	if doc.chart_type == 'Report':
-		allowed_reports = tuple([key.encode('UTF8') for key in get_allowed_reports()])
+		allowed_reports = [key if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()]
 		if doc.report_name in allowed_reports:
 			return True
 	else:
-		allowed_doctypes = tuple(frappe.permissions.get_doctypes_with_read())
+		allowed_doctypes = [frappe.permissions.get_doctypes_with_read()]
 		if doc.document_type in allowed_doctypes:
 			return True
 

frappe/desk/doctype/desk_page/desk_page.py

@@ -38,7 +38,7 @@ class DeskPage(Document):
 
 		pages = frappe.get_all("Desk Page", fields=["name", "module"], filters=filters, as_list=1)
 
-		return { page[1]: page[0]  for page in pages }
+		return { page[1]: page[0] for page in pages if page[1] }
 
 def disable_saving_as_standard():
 	return frappe.flags.in_install or \

frappe/desk/query_builder.py

@@ -1,263 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-from __future__ import unicode_literals
-import frappe
-
-out = frappe.response
-
-from frappe.utils import cint
-import frappe.defaults
-from six import text_type
-
-def get_sql_tables(q):
-	if q.find('WHERE') != -1:
-		tl = q.split('FROM')[1].split('WHERE')[0].split(',')
-	elif q.find('GROUP BY') != -1:
-		tl = q.split('FROM')[1].split('GROUP BY')[0].split(',')
-	else:
-		tl = q.split('FROM')[1].split('ORDER BY')[0].split(',')
-	return [t.strip().strip('`')[3:] for t in tl]
-
-def get_parent_dt(dt):
-	pdt = ''
-	if frappe.db.sql('select name from `tabDocType` where istable=1 and name=%s', dt):
-		import frappe.model.meta
-		return frappe.model.meta.get_parent_dt(dt)
-	return pdt
-
-def get_sql_meta(tl):
-	std_columns = {
-		'owner':('Owner', '', '', '100'),
-		'creation':('Created on', 'Date', '', '100'),
-		'modified':('Last modified on', 'Date', '', '100'),
-		'modified_by':('Modified By', '', '', '100')
-	}
-
-	meta = {}
-
-	for dt in tl:
-		meta[dt] = std_columns.copy()
-
-		# for table doctype, the ID is the parent id
-		pdt = get_parent_dt(dt)
-		if pdt:
-			meta[dt]['parent'] = ('ID', 'Link', pdt, '200')
-
-		# get the field properties from DocField
-		res = frappe.db.sql("select fieldname, label, fieldtype, options, width \
-			from tabDocField where parent=%s", dt)
-		for r in res:
-			if r[0]:
-				meta[dt][r[0]] = (r[1], r[2], r[3], r[4]);
-
-		# name
-		meta[dt]['name'] = ('ID', 'Link', dt, '200')
-
-	return meta
-
-def add_match_conditions(q, tl):
-	from frappe.desk.reportview import build_match_conditions
-	sl = []
-	for dt in tl:
-		s = build_match_conditions(dt)
-		if s:
-			sl.append(s)
-
-	# insert the conditions
-	if sl:
-		condition_st  = q.find('WHERE')!=-1 and ' AND ' or ' WHERE '
-		condition_end = q.find('ORDER BY')!=-1 and 'ORDER BY' or 'LIMIT'
-		condition_end = q.find('GROUP BY')!=-1 and 'GROUP BY' or condition_end
-
-		if q.find('ORDER BY')!=-1 or q.find('LIMIT')!=-1 or q.find('GROUP BY')!=-1: # if query continues beyond conditions
-			q = q.split(condition_end)
-			q = q[0] + condition_st + '(' + ' OR '.join(sl) + ') ' + condition_end + q[1]
-		else:
-			q = q + condition_st + '(' + ' OR '.join(sl) + ')'
-
-	return q
-
-def guess_type(m):
-	"""
-		Returns fieldtype depending on the MySQLdb Description
-	"""
-	if frappe.db.is_type_number(m):
-		return 'Currency'
-	elif m in frappe.is_type_datetime(m):
-		return 'Date'
-	else:
-		return 'Data'
-
-def build_description_simple():
-	colnames, coltypes, coloptions, colwidths = [], [], [], []
-
-	for m in frappe.db.get_description():
-		colnames.append(m[0])
-		coltypes.append(guess_type[m[1]])
-		coloptions.append('')
-		colwidths.append('100')
-
-	return colnames, coltypes, coloptions, colwidths
-
-def build_description_standard(meta, tl):
-
-	desc = frappe.db.get_description()
-
-	colnames, coltypes, coloptions, colwidths = [], [], [], []
-
-	# merged metadata - used if we are unable to
-	# get both the table name and field name from
-	# the description - in case of joins
-	merged_meta = {}
-	for d in meta:
-		merged_meta.update(meta[d])
-
-	for f in desc:
-		fn, dt = f[0], ''
-		if '.' in fn:
-			dt, fn = fn.split('.')
-
-		if (not dt) and merged_meta.get(fn):
-			# no "AS" given, find type from merged description
-
-			desc = merged_meta[fn]
-			colnames.append(desc[0] or fn)
-			coltypes.append(desc[1] or '')
-			coloptions.append(desc[2] or '')
-			colwidths.append(desc[3] or '100')
-
-		elif fn in meta.get(dt,{}):
-			# type specified for a multi-table join
-			# usually from Report Builder
-
-			desc = meta[dt][fn]
-			colnames.append(desc[0] or fn)
-			coltypes.append(desc[1] or '')
-			coloptions.append(desc[2] or '')
-			colwidths.append(desc[3] or '100')
-
-		else:
-			# nothing found
-			# guess
-			colnames.append(fn)
-			coltypes.append(guess_type(f[1]))
-			coloptions.append('')
-			colwidths.append('100')
-
-	return colnames, coltypes, coloptions, colwidths
-
-@frappe.whitelist()
-def runquery(q='', ret=0, from_export=0):
-	import frappe.utils
-
-	formatted = cint(frappe.form_dict.get('formatted'))
-
-	# CASE A: Simple Query
-	# --------------------
-	if frappe.form_dict.get('simple_query') or frappe.form_dict.get('is_simple'):
-		if not q: q = frappe.form_dict.get('simple_query') or frappe.form_dict.get('query')
-		if q.split()[0].lower() != 'select':
-			raise Exception('Query must be a SELECT')
-
-		as_dict = cint(frappe.form_dict.get('as_dict'))
-		res = frappe.db.sql(q, as_dict = as_dict, as_list = not as_dict, formatted=formatted)
-
-		# build colnames etc from metadata
-		colnames, coltypes, coloptions, colwidths = [], [], [], []
-
-	# CASE B: Standard Query
-	# -----------------------
-	else:
-		if not q: q = frappe.form_dict.get('query')
-
-		tl = get_sql_tables(q)
-		meta = get_sql_meta(tl)
-
-		q = add_match_conditions(q, tl)
-
-		# replace special variables
-		q = q.replace('__user', frappe.session.user)
-		q = q.replace('__today', frappe.utils.nowdate())
-
-		res = frappe.db.sql(q, as_list=1, formatted=formatted)
-
-		colnames, coltypes, coloptions, colwidths = build_description_standard(meta, tl)
-
-	# run server script
-	# -----------------
-	style, header_html, footer_html, page_template = '', '', '', ''
-
-	out['colnames'] = colnames
-	out['coltypes'] = coltypes
-	out['coloptions'] = coloptions
-	out['colwidths'] = colwidths
-	out['header_html'] = header_html
-	out['footer_html'] = footer_html
-	out['page_template'] = page_template
-
-	if style:
-		out['style'] = style
-
-	# just the data - return
-	if ret==1:
-		return res
-
-	out['values'] = res
-
-	# return num of entries
-	qm = frappe.form_dict.get('query_max') or ''
-	if qm and qm.strip():
-		if qm.split()[0].lower() != 'select':
-			raise Exception('Query (Max) must be a SELECT')
-		if not frappe.form_dict.get('simple_query'):
-			qm = add_match_conditions(qm, tl)
-
-		out['n_values'] = frappe.utils.cint(frappe.db.sql(qm)[0][0])
-
-
-@frappe.whitelist()
-def runquery_csv():
-	global out
-
-	q = frappe.form_dict.get('query')
-
-	rep_name = frappe.form_dict.get('report_name')
-	if not frappe.form_dict.get('simple_query'):
-
-		# Report Name
-		if not rep_name:
-			rep_name = get_sql_tables(q)[0]
-
-	if not rep_name: rep_name = 'DataExport'
-
-	rows = [[rep_name], out['colnames']] + out['values']
-
-	from six import StringIO
-	import csv
-
-	f = StringIO()
-	writer = csv.writer(f)
-	for r in rows:
-		# encode only unicode type strings and not int, floats etc.
-		writer.writerow(map(lambda v: isinstance(v, text_type) and v.encode('utf-8') or v, r))
-
-	f.seek(0)
-	out['result'] = text_type(f.read(), 'utf-8')
-	out['type'] = 'csv'
-	out['doctype'] = rep_name
-
-def add_limit_to_query(query, args):
-	"""
-		Add limit condition to query
-		can be used by methods called in listing to add limit condition
-	"""
-	if args.get('limit_page_length'):
-		query += """
-			limit %(limit_start)s, %(limit_page_length)s"""
-
-		import frappe.utils
-		args['limit_start'] = frappe.utils.cint(args.get('limit_start'))
-		args['limit_page_length'] = frappe.utils.cint(args.get('limit_page_length'))
-
-	return query, args

frappe/desk/query_report.py

@@ -462,6 +462,9 @@ def add_total_row(result, columns, meta = None):
 @frappe.whitelist()
 def get_data_for_custom_field(doctype, field):
 
+	if not frappe.has_permission(doctype, "read"):
+		frappe.throw(_("Not Permitted"), frappe.PermissionError)
+
 	value_map = frappe._dict(frappe.get_all(doctype,
 		fields=["name", field],
 		as_list=1))

frappe/desk/reportview.py

@@ -36,6 +36,7 @@ def get_form_params():
 	data.pop('data', None)
 	data.pop('ignore_permissions', None)
 	data.pop('view', None)
+	data.pop('user', None)
 
 	if "csrf_token" in data:
 		del data["csrf_token"]

frappe/desk/treeview.py

@@ -6,12 +6,11 @@ import frappe
 from frappe import _
 
 @frappe.whitelist()
-def get_all_nodes(doctype, parent, tree_method, **filters):
+def get_all_nodes(doctype, label, parent, tree_method, **filters):
 	'''Recursively gets all data from tree nodes'''
 
 	if 'cmd' in filters:
 		del filters['cmd']
-
 	filters.pop('data', None)
 
 	tree_method = frappe.get_attr(tree_method)
@@ -20,7 +19,7 @@ def get_all_nodes(doctype, parent, tree_method, **filters):
 		frappe.throw(_("Not Permitted"), frappe.PermissionError)
 
 	data = tree_method(doctype, parent, **filters)
-	out = [dict(parent=parent, data=data)]
+	out = [dict(parent=label, data=data)]
 
 	if 'is_root' in filters:
 		del filters['is_root']

frappe/email/doctype/newsletter/newsletter.py

@@ -85,12 +85,12 @@ class Newsletter(WebsiteGenerator):
 			self.db_set("scheduled_to_send", len(self.recipients))
 
 	def get_message(self):
-		
+
 		return {
 			'Rich Text': self.message,
 			'Markdown': markdown(self.message_md),
 			'HTML': self.message_html
-		}[self.content_type]
+		}[self.content_type or 'Rich Text']
 
 	def get_recipients(self):
 		"""Get recipients from Email Group"""

frappe/email/doctype/notification/notification.js

@@ -19,9 +19,12 @@ frappe.notification = {
 		}
 
 		frappe.model.with_doctype(frm.doc.document_type, function() {
-			let get_select_options = function(df) {
+			let get_select_options = function(df, parent_field) {
+				// Append parent_field name along with fieldname for child table fields
+				let select_value = parent_field ? df.fieldname + ',' + parent_field : df.fieldname;
+
 				return {
-					value: df.fieldname,
+					value: select_value,
 					label: df.fieldname + ' (' + __(df.label) + ')'
 				};
 			};
@@ -59,9 +62,21 @@ frappe.notification = {
 			let receiver_fields = [];
 			if (frm.doc.channel === 'Email') {
 				receiver_fields = $.map(fields, function(d) {
-					return d.options == 'Email' ||
-						(d.options == 'User' && d.fieldtype == 'Link')
-						? get_select_options(d) : null;
+
+					// Add User and Email fields from child into select dropdown
+					if (d.fieldtype == 'Table') {
+						let child_fields = frappe.get_doc('DocType', d.options).fields;
+						return $.map(child_fields, function(df) {
+							return df.options == 'Email' ||
+								(df.options == 'User' && df.fieldtype == 'Link')
+								? get_select_options(df, d.fieldname) : null;
+						});
+					// Add User and Email fields from parent into select dropdown
+					} else {
+						return d.options == 'Email' ||
+							(d.options == 'User' && d.fieldtype == 'Link')
+							? get_select_options(d) : null;
+					}
 				});
 			} else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) {
 				receiver_fields = $.map(fields, function(d) {
@@ -87,7 +102,7 @@ frappe.notification = {
 <h5>Message Example</h5>
 
 <pre>
-Your {{ doc.name }} order of {{ doc.total }} has shipped and should be delivered on {{ doc.date }}. Details : {{doc.customer}}
+Your appointment is coming up on {{ doc.date }} at {{ doc.time }}
 </pre>`;
 		} else if (frm.doc.channel === 'Email') {
 			template = `<h5>Message Example</h5>
@@ -151,6 +166,7 @@ frappe.ui.form.on('Notification', {
 	},
 	refresh: function(frm) {
 		frappe.notification.setup_fieldname_select(frm);
+		frappe.notification.setup_example_message(frm);
 		frm.get_field('is_standard').toggle(frappe.boot.developer_mode);
 		frm.trigger('event');
 	},

frappe/email/doctype/notification/notification.json

@@ -34,6 +34,7 @@
   "set_property_after_alert",
   "property_value",
   "column_break_5",
+  "send_to_all_assignees",
   "recipients",
   "message_sb",
   "message",
@@ -66,7 +67,7 @@
   },
   {
    "depends_on": "eval:doc.channel=='Slack'",
-   "description": "To use Slack Channel, add a <a href=\"\\#Form/Slack Webhook URL\">Slack Webhook URL</a>.",
+   "description": "To use Slack Channel, add a <a href=\"#List/Slack%20Webhook%20URL/List\">Slack Webhook URL</a>.",
    "fieldname": "slack_webhook_url",
    "fieldtype": "Link",
    "label": "Slack Channel",
@@ -216,7 +217,7 @@
    "fieldname": "recipients",
    "fieldtype": "Table",
    "label": "Recipients",
-   "mandatory_depends_on": "eval:doc.channel!=='Slack'",
+   "mandatory_depends_on": "eval:doc.channel!=='Slack' && !doc.send_to_all_assignees",
    "options": "Notification Recipient"
   },
   {
@@ -268,6 +269,7 @@
    "fieldname": "twilio_number",
    "fieldtype": "Link",
    "label": "Twilio Number",
+   "mandatory_depends_on": "eval: doc.channel==='WhatsApp'",
    "options": "Twilio Number Group"
   },
   {
@@ -277,11 +279,19 @@
    "fieldname": "send_system_notification",
    "fieldtype": "Check",
    "label": "Send System Notification"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.channel == 'Email'",
+   "fieldname": "send_to_all_assignees",
+   "fieldtype": "Check",
+   "label": "Send To All Assignees"
   }
  ],
  "icon": "fa fa-envelope",
+ "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-08-11 19:24:35.479373",
+ "modified": "2020-09-03 10:33:23.084590",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification",

frappe/email/doctype/notification/notification.py

@@ -10,6 +10,7 @@ from frappe.model.document import Document
 from frappe.core.doctype.role.role import get_info_based_on_role, get_user_info
 from frappe.utils import validate_email_address, nowdate, parse_val, is_html, add_to_date
 from frappe.utils.jinja import validate_template
+from frappe.utils.safe_exec import get_safe_globals
 from frappe.modules.utils import export_module_json, get_doc_module
 from six import string_types
 from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
@@ -42,6 +43,7 @@ class Notification(Document):
 		self.validate_forbidden_types()
 		self.validate_condition()
 		self.validate_standard()
+		self.validate_twilio_settings()
 		frappe.cache().hdel('notifications', self.document_type)
 
 	def on_update(self):
@@ -68,6 +70,11 @@ def get_context(context):
 		if self.is_standard and not frappe.conf.developer_mode:
 			frappe.throw(_('Cannot edit Standard Notification. To edit, please disable this and duplicate it'))
 
+	def validate_twilio_settings(self):
+		if self.enabled and self.channel == "WhatsApp" \
+			and not frappe.db.get_single_value("Twilio Settings", "enabled"):
+			frappe.throw(_("Please enable Twilio settings to send WhatsApp messages"))
+
 	def validate_condition(self):
 		temp_doc = frappe.new_doc(self.document_type)
 		if self.condition:
@@ -166,8 +173,13 @@ def get_context(context):
 			subject = frappe.render_template(self.subject, context)
 
 		attachments = self.get_attachment(doc)
+
 		recipients, cc, bcc = self.get_list_of_recipients(doc, context)
+
 		users = recipients + cc + bcc
+		
+		if not users:
+			return
 
 		notification_doc = {
 			'type': 'Alert',
@@ -189,6 +201,7 @@ def get_context(context):
 		recipients, cc, bcc = self.get_list_of_recipients(doc, context)
 		if not (recipients or cc or bcc):
 			return
+
 		sender = None
 		if self.sender and self.sender_email:
 			sender = formataddr((self.sender, self.sender_email))
@@ -234,13 +247,20 @@ def get_context(context):
 				if not frappe.safe_eval(recipient.condition, None, context):
 					continue
 			if recipient.receiver_by_document_field:
-				email_ids_value = doc.get(recipient.receiver_by_document_field)
-				if validate_email_address(email_ids_value):
-					email_ids = email_ids_value.replace(",", "\n")
-					recipients = recipients + email_ids.split("\n")
+				fields = recipient.receiver_by_document_field.split(',')
+				# fields from child table
+				if len(fields) > 1:
+					for d in doc.get(fields[1]):
+						email_id = d.get(fields[0])
+						if validate_email_address(email_id):
+							recipients.append(email_id)
+				# field from parent doc
+				else:
+					email_ids_value = doc.get(fields[0])
+					if validate_email_address(email_ids_value):
+						email_ids = email_ids_value.replace(",", "\n")
+						recipients = recipients + email_ids.split("\n")
 
-				# else:
-				# 	print "invalid email"
 			if recipient.cc and "{" in recipient.cc:
 				recipient.cc = frappe.render_template(recipient.cc, context)
 
@@ -262,8 +282,9 @@ def get_context(context):
 				for email in emails:
 					recipients = recipients + email.split("\n")
 
-		if not recipients and not cc and not bcc:
-			return None, None, None
+		if self.send_to_all_assignees:
+			recipients = recipients + get_assignees(doc)
+
 		return list(set(recipients)), list(set(cc)), list(set(bcc))
 
 	def get_receiver_list(self, doc, context):
@@ -404,4 +425,13 @@ def evaluate_alert(doc, alert, event):
 			frappe.utils.get_link_to_form('Error Log', error_log.name)))
 
 def get_context(doc):
-	return {"doc": doc, "nowdate": nowdate, "frappe": frappe._dict(utils=frappe.utils)}
+	return {"doc": doc, "nowdate": nowdate, "frappe": frappe._dict(utils=get_safe_globals().get("frappe").get("utils"))}
+
+def get_assignees(doc):
+	assignees = []
+	assignees = frappe.get_all('ToDo', filters={'status': 'Open', 'reference_name': doc.name,
+		'reference_type': doc.doctype}, fields=['owner'])
+
+	recipients = [d.owner for d in assignees]
+
+	return recipients

frappe/email/doctype/notification/test_notification.py

@@ -4,6 +4,7 @@
 from __future__ import unicode_literals
 
 import frappe, frappe.utils, frappe.utils.scheduler
+from frappe.desk.form import assign_to
 import unittest
 
 test_records = frappe.get_test_records('Notification')
@@ -13,7 +14,31 @@ test_dependencies = ["User"]
 class TestNotification(unittest.TestCase):
 	def setUp(self):
 		frappe.db.sql("""delete from `tabEmail Queue`""")
-		frappe.set_user("test1@example.com")
+		frappe.set_user("test@example.com")
+
+		if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'):
+			notification = frappe.new_doc('Notification')
+			notification.name = 'ToDo Status Update'
+			notification.subject = 'ToDo Status Update'
+			notification.document_type = 'ToDo'
+			notification.event = 'Value Change'
+			notification.value_changed = 'status'
+			notification.send_to_all_assignees = 1
+			notification.save()
+
+		if not frappe.db.exists('Notification', {'name': 'Contact Status Update'}, 'name'):
+			notification = frappe.new_doc('Notification')
+			notification.name = 'Contact Status Update'
+			notification.subject = 'Contact Status Update'
+			notification.document_type = 'Contact'
+			notification.event = 'Value Change'
+			notification.value_changed = 'status'
+			notification.message = 'Test Contact Update'
+			notification.append('recipients', {
+				'receiver_by_document_field': 'email_id,email_ids'
+			})
+			notification.save()
+
 
 	def tearDown(self):
 		frappe.set_user("Administrator")
@@ -177,3 +202,65 @@ class TestNotification(unittest.TestCase):
 		frappe.db.sql("""delete from `tabUser` where email='test_jinja@example.com'""")
 		frappe.db.sql("""delete from `tabEmail Queue`""")
 		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
+
+	def test_notification_to_assignee(self):
+		todo = frappe.new_doc('ToDo')
+		todo.description = 'Test Notification'
+		todo.save()
+
+		assign_to.add({
+			"assign_to": ["test2@example.com"],
+			"doctype": todo.doctype,
+			"name": todo.name,
+			"description": "Close this Todo"
+		})
+
+		assign_to.add({
+			"assign_to": ["test1@example.com"],
+			"doctype": todo.doctype,
+			"name": todo.name,
+			"description": "Close this Todo"
+		})
+
+		#change status of todo
+		todo.status = 'Closed'
+		todo.save()
+
+		email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'ToDo',
+			'reference_name': todo.name})
+
+		self.assertTrue(email_queue)
+
+		recipients = [d.recipient for d in email_queue.recipients]
+		self.assertTrue('test2@example.com' in recipients)
+		self.assertTrue('test1@example.com' in recipients)
+
+	def test_notification_by_child_table_field(self):
+		contact = frappe.new_doc('Contact')
+		contact.first_name = 'John Doe'
+		contact.status = 'Open'
+		contact.append('email_ids', {
+			'email_id': 'test2@example.com',
+			'is_primary': 1
+		})
+
+		contact.append('email_ids', {
+			'email_id': 'test1@example.com'
+		})
+
+		contact.save()
+
+		#change status of contact
+		contact.status = 'Replied'
+		contact.save()
+
+		email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'Contact',
+			'reference_name': contact.name})
+
+		self.assertTrue(email_queue)
+
+		recipients = [d.recipient for d in email_queue.recipients]
+		self.assertTrue('test2@example.com' in recipients)
+		self.assertTrue('test1@example.com' in recipients)
+
+

frappe/email/doctype/notification_recipient/notification_recipient.json

@@ -46,9 +46,10 @@
    "options": "Role"
   }
  ],
+ "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-02-21 11:18:40.125233",
+ "modified": "2020-09-01 17:40:27.289105",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification Recipient",

frappe/email/email_body.py

@@ -207,7 +207,7 @@ class EMail:
 
 	def set_in_reply_to(self, in_reply_to):
 		"""Used to send the Message-Id of a received email back as In-Reply-To"""
-		self.msg_root["In-Reply-To"] = in_reply_to
+		self.set_header('In-Reply-To', in_reply_to)
 
 	def make(self):
 		"""build into msg_root"""
@@ -234,7 +234,10 @@ class EMail:
 		if key in self.msg_root:
 			del self.msg_root[key]
 
-		self.msg_root[key] = value
+		try:
+			self.msg_root[key] = value
+		except ValueError:
+			self.msg_root[key] = sanitize_email_header(value)
 
 	def as_string(self):
 		"""validate, build message and convert to string"""
@@ -458,3 +461,6 @@ def get_header(header=None):
 	})
 
 	return email_header
+
+def sanitize_email_header(str):
+	return str.replace('\r', '').replace('\n', '')

frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py

@@ -102,7 +102,7 @@ class DocumentTypeMapping(Document):
 		filters = json.loads(mapping.remote_value_filters)
 		for key, value in iteritems(filters):
 			if value.startswith('eval:'):
-				val = frappe.safe_eval(value[5:], dict(frappe=frappe))
+				val = frappe.safe_eval(value[5:], None, dict(doc=doc))
 				filters[key] = val
 			if doc.get(value):
 				filters[key] = doc.get(value)

frappe/event_streaming/doctype/event_consumer/event_consumer.json

@@ -13,8 +13,7 @@
   "api_secret",
   "column_break_6",
   "user",
-  "incoming_change",
-  "in_test"
+  "incoming_change"
  ],
  "fields": [
   {
@@ -22,6 +21,7 @@
    "fieldtype": "Data",
    "in_list_view": 1,
    "label": "Callback URL",
+   "read_only": 1,
    "reqd": 1,
    "unique": 1
   },
@@ -29,19 +29,20 @@
    "fieldname": "api_key",
    "fieldtype": "Data",
    "label": "API Key",
-   "read_only": 1
+   "reqd": 1
   },
   {
    "fieldname": "api_secret",
    "fieldtype": "Password",
    "label": "API Secret",
-   "read_only": 1
+   "reqd": 1
   },
   {
    "fieldname": "user",
    "fieldtype": "Link",
    "label": "Event Subscriber",
    "options": "User",
+   "read_only": 1,
    "reqd": 1
   },
   {
@@ -60,14 +61,6 @@
    "label": "Incoming Change",
    "read_only": 1
   },
-  {
-   "default": "0",
-   "fieldname": "in_test",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "label": "In Test",
-   "read_only": 1
-  },
   {
    "fieldname": "consumer_doctypes",
    "fieldtype": "Table",
@@ -78,7 +71,7 @@
  ],
  "in_create": 1,
  "links": [],
- "modified": "2019-12-30 11:52:16.276047",
+ "modified": "2020-09-08 16:42:39.828085",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Consumer",

frappe/event_streaming/doctype/event_consumer/event_consumer.py

@@ -6,6 +6,8 @@ from __future__ import unicode_literals
 import frappe
 import json
 import requests
+import os
+from frappe import _
 from frappe.model.document import Document
 from frappe.frappeclient import FrappeClient
 from frappe.utils.data import get_url
@@ -14,13 +16,18 @@ from frappe.utils.background_jobs import get_jobs
 
 class EventConsumer(Document):
 	def validate(self):
-		if self.in_test:
+		# approve subscribed doctypes for tests
+		# frappe.flags.in_test won't work here as tests are running on the consumer site
+		if os.environ.get('CI'):
 			for entry in self.consumer_doctypes:
 				entry.status = 'Approved'
-			self.in_test = False
 
 	def on_update(self):
 		if not self.incoming_change:
+			doc_before_save = self.get_doc_before_save()
+			if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret:
+				return
+
 			self.update_consumer_status()
 		else:
 			frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
@@ -56,17 +63,26 @@ class EventConsumer(Document):
 			return 'offline'
 		return 'online'
 
-
-@frappe.whitelist(allow_guest=True)
+@frappe.whitelist()
 def register_consumer(data):
 	"""create an event consumer document for registering a consumer"""
 	data = json.loads(data)
 	# to ensure that consumer is created only once
 	if frappe.db.exists('Event Consumer', data['event_consumer']):
 		return None
+
+	user = data['user']
+	if not frappe.db.exists('User', user):
+		frappe.throw(_('User {0} not found on the producer site').format(user))
+
+	if "System Manager" not in frappe.get_roles(user):
+		frappe.throw(_("Event Subscriber has to be a System Manager."))
+
 	consumer = frappe.new_doc('Event Consumer')
 	consumer.callback_url = data['event_consumer']
 	consumer.user = data['user']
+	consumer.api_key = data['api_key']
+	consumer.api_secret = data['api_secret']
 	consumer.incoming_change = True
 	consumer_doctypes = json.loads(data['consumer_doctypes'])
 
@@ -76,19 +92,13 @@ def register_consumer(data):
 			'status': 'Pending'
 		})
 
-	api_key = frappe.generate_hash(length=10)
-	api_secret = frappe.generate_hash(length=10)
-	consumer.api_key = api_key
-	consumer.api_secret = api_secret
-	consumer.in_test = data['in_test']
-	consumer.insert(ignore_permissions=True)
-	frappe.db.commit()
+	consumer.insert()
 
 	# consumer's 'last_update' field should point to the latest update
 	# in producer's update log when subscribing
 	# so that, updates after subscribing are consumed and not the old ones.
 	last_update = str(get_last_update())
-	return json.dumps({'api_key': api_key, 'api_secret': api_secret, 'last_update': last_update})
+	return json.dumps({'last_update': last_update})
 
 
 def get_consumer_site(consumer_url):
@@ -97,8 +107,7 @@ def get_consumer_site(consumer_url):
 	consumer_site = FrappeClient(
 		url=consumer_url,
 		api_key=consumer_doc.api_key,
-		api_secret=consumer_doc.get_password('api_secret'),
-		frappe_authorization_source='Event Producer'
+		api_secret=consumer_doc.get_password('api_secret')
 	)
 	return consumer_site
 

frappe/event_streaming/doctype/event_producer/event_producer.json

@@ -32,23 +32,26 @@
    "read_only": 1
   },
   {
+   "description": "API Key of the user(Event Subscriber) on the producer site",
    "fieldname": "api_key",
    "fieldtype": "Data",
    "label": "API Key",
-   "read_only": 1
+   "reqd": 1
   },
   {
+   "description": "API Secret of the user(Event Subscriber) on the producer site",
    "fieldname": "api_secret",
    "fieldtype": "Password",
    "label": "API Secret",
-   "read_only": 1
+   "reqd": 1
   },
   {
    "fieldname": "user",
    "fieldtype": "Link",
    "label": "Event Subscriber",
    "options": "User",
-   "reqd": 1
+   "reqd": 1,
+   "set_only_once": 1
   },
   {
    "fieldname": "column_break_6",
@@ -74,7 +77,7 @@
   }
  ],
  "links": [],
- "modified": "2019-12-26 13:04:11.438349",
+ "modified": "2020-09-08 18:50:57.687979",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Producer",

frappe/event_streaming/doctype/event_producer/event_producer.py

@@ -12,7 +12,8 @@ from frappe import _
 from frappe.model.document import Document
 from frappe.frappeclient import FrappeClient
 from frappe.utils.background_jobs import get_jobs
-from frappe.utils.data import get_url
+from frappe.utils.data import get_url, get_link_to_form
+from frappe.utils.password import get_decrypted_password
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.integrations.oauth2 import validate_url
 
@@ -20,19 +21,35 @@ from frappe.integrations.oauth2 import validate_url
 class EventProducer(Document):
 	def before_insert(self):
 		self.check_url()
+		self.validate_event_subscriber()
 		self.incoming_change = True
 		self.create_event_consumer()
 		self.create_custom_fields()
 
 	def validate(self):
+		self.validate_event_subscriber()
 		if frappe.flags.in_test:
 			for entry in self.producer_doctypes:
 				entry.status = 'Approved'
 
+	def validate_event_subscriber(self):
+		if not frappe.db.get_value('User', self.user, 'api_key'):
+			frappe.throw(_('Please generate keys for the Event Subscriber User {0} first.').format(
+				frappe.bold(get_link_to_form('User', self.user))
+			))
+
 	def on_update(self):
 		if not self.incoming_change:
-			self.update_event_consumer()
-			self.create_custom_fields()
+			if frappe.db.exists('Event Producer', self.name):
+				if not self.api_key or not self.api_secret:
+					frappe.throw(_('Please set API Key and Secret on the producer and consumer sites first.'))
+				else:
+					doc_before_save = self.get_doc_before_save()
+					if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret:
+						return
+
+					self.update_event_consumer()
+					self.create_custom_fields()
 		else:
 			# when producer doc is updated it updates the consumer doc, set flag to avoid deadlock
 			self.db_set('incoming_change', 0)
@@ -50,15 +67,18 @@ class EventProducer(Document):
 	def create_event_consumer(self):
 		"""register event consumer on the producer site"""
 		if self.is_producer_online():
-			producer_site = FrappeClient(self.producer_url, verify=False)
+			producer_site = FrappeClient(
+				url=self.producer_url,
+				api_key=self.api_key,
+				api_secret=self.get_password('api_secret')
+			)
+
 			response = producer_site.post_api(
 				'frappe.event_streaming.doctype.event_consumer.event_consumer.register_consumer',
 				params={'data': json.dumps(self.get_request_data())}
 			)
 			if response:
 				response = json.loads(response)
-				self.api_key = response['api_key']
-				self.api_secret = response['api_secret']
 				self.last_update = response['last_update']
 			else:
 				frappe.throw(_('Failed to create an Event Consumer or an Event Consumer for the current site is already registered.'))
@@ -72,11 +92,14 @@ class EventProducer(Document):
 			else:
 				consumer_doctypes.append(entry.ref_doctype)
 
+		user_key = frappe.db.get_value('User', self.user, 'api_key')
+		user_secret = get_decrypted_password('User', self.user, 'api_secret')
 		return {
 			'event_consumer': get_url(),
 			'consumer_doctypes': json.dumps(consumer_doctypes),
 			'user': self.user,
-			'in_test': frappe.flags.in_test
+			'api_key': user_key,
+			'api_secret': user_secret
 		}
 
 	def create_custom_fields(self):
@@ -110,8 +133,6 @@ class EventProducer(Document):
 						'status': get_approval_status(config, ref_doctype),
 						'unsubscribed': entry.unsubscribe
 					})
-				if frappe.flags.in_test:
-					event_consumer.in_test = True
 				event_consumer.user = self.user
 				event_consumer.incoming_change = True
 				producer_site.update(event_consumer)
@@ -134,8 +155,7 @@ def get_producer_site(producer_url):
 	producer_site = FrappeClient(
 		url=producer_url,
 		api_key=producer_doc.api_key,
-		api_secret=producer_doc.get_password('api_secret'),
-		frappe_authorization_source='Event Consumer'
+		api_secret=producer_doc.get_password('api_secret')
 	)
 	return producer_site
 

frappe/event_streaming/doctype/event_producer/test_event_producer.py

@@ -8,6 +8,7 @@ import unittest
 import json
 from frappe.frappeclient import FrappeClient
 from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node
+from frappe.core.doctype.user.user import generate_keys
 
 producer_url = 'http://test_site_producer:8000'
 
@@ -166,16 +167,6 @@ class TestEventProducer(unittest.TestCase):
 	def pull_producer_data(self):
 		pull_from_node(producer_url)
 
-	def get_remote_site(self):
-		producer_doc = frappe.get_doc('Event Producer', producer_url)
-		producer_site = FrappeClient(
-			url=producer_doc.producer_url,
-			api_key=producer_doc.api_key,
-			api_secret=producer_doc.get_password('api_secret'),
-			frappe_authorization_source='Event Consumer'
-		)
-		return producer_site
-
 	def test_mapping(self):
 		producer = get_remote_site()
 		event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
@@ -298,6 +289,20 @@ def create_event_producer(producer_url):
 		event_producer.save()
 		return
 
+	generate_keys('Administrator')
+
+	producer_site = connect()
+
+	response = producer_site.post_api(
+		'frappe.core.doctype.user.user.generate_keys',
+		params={'user': 'Administrator'}
+	)
+
+	api_secret = response.get('api_secret')
+
+	response = producer_site.get_value('User', 'api_key', {'name': 'Administrator'})
+	api_key = response.get('api_key')
+
 	event_producer = frappe.new_doc('Event Producer')
 	event_producer.producer_doctypes = []
 	event_producer.producer_url = producer_url
@@ -310,6 +315,8 @@ def create_event_producer(producer_url):
 		'use_same_name': 1
 	})
 	event_producer.user = 'Administrator'
+	event_producer.api_key = api_key
+	event_producer.api_secret = api_secret
 	event_producer.save()
 
 def reset_configuration(producer_url):
@@ -331,9 +338,9 @@ def get_remote_site():
 	producer_doc = frappe.get_doc('Event Producer', producer_url)
 	producer_site = FrappeClient(
 		url=producer_doc.producer_url,
-		api_key=producer_doc.api_key,
-		api_secret=producer_doc.get_password('api_secret'),
-		frappe_authorization_source='Event Consumer'
+		username='Administrator',
+		password='admin',
+		verify=False
 	)
 	return producer_site
 
@@ -341,4 +348,17 @@ def unsubscribe_doctypes(producer_url):
 	event_producer = frappe.get_doc('Event Producer', producer_url)
 	for entry in event_producer.producer_doctypes:
 		entry.unsubscribe = 1
-	event_producer.save()
+	event_producer.save()
+
+def connect():
+	def _connect():
+		return FrappeClient(
+			url=producer_url,
+			username='Administrator',
+			password='admin',
+			verify=False
+		)
+	try:
+		return _connect()
+	except Exception:
+		return _connect()

frappe/handler.py

@@ -65,16 +65,21 @@ def execute_cmd(cmd, from_async=False):
 		method = method.queue
 
 	is_whitelisted(method)
+	is_valid_http_method(method)
 
 	return frappe.call(method, **frappe.form_dict)
 
+def is_valid_http_method(method):
+	http_method = frappe.local.request.method
+
+	if http_method not in frappe.allowed_http_methods_for_whitelisted_func[method]:
+		frappe.throw(_("Not permitted"), frappe.PermissionError)
 
 def is_whitelisted(method):
 	# check if whitelisted
 	if frappe.session['user'] == 'Guest':
 		if (method not in frappe.guest_methods):
-			frappe.msgprint(_("Not permitted"))
-			raise frappe.PermissionError('Not Allowed, {0}'.format(method))
+			frappe.throw(_("Not permitted"), frappe.PermissionError)
 
 		if method not in frappe.xss_safe_methods:
 			# strictly sanitize form_dict
@@ -85,8 +90,7 @@ def is_whitelisted(method):
 
 	else:
 		if not method in frappe.whitelisted:
-			frappe.msgprint(_("Not permitted"))
-			raise frappe.PermissionError('Not Allowed, {0}'.format(method))
+			frappe.throw(_("Not permitted"), frappe.PermissionError)
 
 @frappe.whitelist(allow_guest=True)
 def version():

frappe/hooks.py

@@ -201,7 +201,8 @@ scheduler_events = {
 		"frappe.deferred_insert.save_to_db",
 		"frappe.desk.form.document_follow.send_hourly_updates",
 		"frappe.integrations.doctype.google_calendar.google_calendar.sync",
-		"frappe.email.doctype.newsletter.newsletter.send_scheduled_email"
+		"frappe.email.doctype.newsletter.newsletter.send_scheduled_email",
+		"frappe.utils.password.delete_password_reset_cache"
 	],
 	"daily": [
 		"frappe.email.queue.clear_outbox",

frappe/integrations/doctype/dropbox_settings/dropbox_settings.py

@@ -97,7 +97,7 @@ def backup_to_dropbox(upload_db_backup=True):
 		if frappe.flags.create_new_backup:
 			backup = new_backup(ignore_files=True)
 			filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
-			site_config = os.path.join(get_backups_path(), os.path.basename(backup.site_config_backup_path))
+			site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf))
 		else:
 			filename, site_config = get_latest_backup_file()
 

frappe/integrations/doctype/google_drive/google_drive.py

@@ -191,7 +191,7 @@ def upload_system_backup_to_google_drive():
 		backup = new_backup()
 		file_urls = []
 		file_urls.append(backup.backup_path_db)
-		file_urls.append(backup.site_config_backup_path)
+		file_urls.append(backup.backup_path_conf)
 
 		if account.file_backup:
 			file_urls.append(backup.backup_path_files)

frappe/integrations/doctype/razorpay_settings/razorpay_settings.py

@@ -65,6 +65,7 @@ import frappe
 from frappe import _
 import json
 import hmac
+import razorpay
 import hashlib
 from six.moves.urllib.parse import urlencode
 from frappe.model.document import Document
@@ -75,6 +76,11 @@ from frappe.integrations.utils import (make_get_request, make_post_request, crea
 class RazorpaySettings(Document):
 	supported_currencies = ["INR"]
 
+	def init_client(self):
+		if self.api_key:
+			secret = self.get_password(fieldname="api_secret", raise_exception=False)
+			self.client = razorpay.Client(auth=(self.api_key, secret))
+
 	def validate(self):
 		create_payment_gateway('Razorpay')
 		call_hook_method('payment_gateway_enabled', gateway='Razorpay')

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py

@@ -118,7 +118,7 @@ def backup_to_s3():
 		backup = new_backup(ignore_files=False, backup_path_db=None,
 						backup_path_files=None, backup_path_private_files=None, force=True)
 		db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
-		site_config = os.path.join(get_backups_path(), os.path.basename(backup.site_config_backup_path))
+		site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf))
 		if backup_files:
 			files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
 			private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))

frappe/integrations/doctype/twilio_settings/twilio_settings.json

@@ -5,6 +5,7 @@
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
+  "enabled",
   "account_sid",
   "auth_token",
   "column_break_2",
@@ -14,12 +15,14 @@
   {
    "fieldname": "account_sid",
    "fieldtype": "Data",
-   "label": "Account SID"
+   "label": "Account SID",
+   "mandatory_depends_on": "eval: doc.enabled"
   },
   {
    "fieldname": "auth_token",
    "fieldtype": "Password",
-   "label": "Auth Token"
+   "label": "Auth Token",
+   "mandatory_depends_on": "eval: doc.enabled"
   },
   {
    "fieldname": "column_break_2",
@@ -30,11 +33,18 @@
    "fieldtype": "Table",
    "label": "Twilio Number",
    "options": "Twilio Number Group"
+  },
+  {
+   "default": "0",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "label": "Enabled"
   }
  ],
+ "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-08-11 15:28:57.860554",
+ "modified": "2020-09-03 10:17:21.318743",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Twilio Settings",

frappe/integrations/doctype/twilio_settings/twilio_settings.py

@@ -5,14 +5,16 @@
 from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
-from twilio.rest import Client
 from frappe import _
 from frappe.utils.password import get_decrypted_password
+from twilio.rest import Client
 from six import string_types
+from json import loads
 
 class TwilioSettings(Document):
-	def validate(self):
-		self.validate_twilio_credentials()
+	def on_update(self):
+		if self.enabled:
+			self.validate_twilio_credentials()
 
 	def validate_twilio_credentials(self):
 		try:
@@ -23,14 +25,15 @@ class TwilioSettings(Document):
 			frappe.throw(_("Invalid Account SID or Auth Token."))
 
 def send_whatsapp_message(sender, receiver_list, message):
-	import json
+	twilio_settings = frappe.get_doc("Twilio Settings")
+	if not twilio_settings.enabled:
+		frappe.throw(_("Please enable twilio settings before sending WhatsApp messages"))
+
 	if isinstance(receiver_list, string_types):
-		receiver_list = json.loads(receiver_list)
+		receiver_list = loads(receiver_list)
 		if not isinstance(receiver_list, list):
 			receiver_list = [receiver_list]
 
-
-	twilio_settings = frappe.get_doc("Twilio Settings")
 	auth_token = get_decrypted_password("Twilio Settings", "Twilio Settings", 'auth_token')
 	client = Client(twilio_settings.account_sid, auth_token)
 	args = {

frappe/integrations/doctype/webhook/webhook.py

@@ -18,6 +18,7 @@ import frappe
 from frappe import _
 from frappe.model.document import Document
 from frappe.utils.jinja import validate_template
+from frappe.utils.safe_exec import get_safe_globals
 
 WEBHOOK_SECRET_HEADER = "X-Frappe-Webhook-Signature"
 
@@ -75,8 +76,7 @@ class Webhook(Document):
 
 
 def get_context(doc):
-	return {"doc": doc, "utils": frappe.utils}
-
+	return {'doc': doc, 'utils': get_safe_globals().get('frappe').get('utils')}
 
 def enqueue_webhook(doc, webhook):
 	webhook = frappe.get_doc("Webhook", webhook.get("name"))

frappe/model/db_query.py

@@ -391,7 +391,10 @@ class DatabaseQuery(object):
 				ref_doctype = frappe.get_meta(f.doctype).get_field(f.fieldname).options
 
 			result=[]
-			lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"])
+
+			lft, rgt = '', ''
+			if f.value:
+				lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"])
 
 			# Get descendants elements of a DocType with a tree structure
 			if f.operator.lower() in ('descendants of', 'not descendants of') :
@@ -769,6 +772,7 @@ def get_list(doctype, *args, **kwargs):
 	kwargs.pop('ignore_permissions', None)
 	kwargs.pop('data', None)
 	kwargs.pop('strict', None)
+	kwargs.pop('user', None)
 
 	# If doctype is child table
 	if frappe.is_table(doctype):

frappe/model/document.py

@@ -905,9 +905,9 @@ class Document(BaseDocument):
 		"""Cancel the document. Sets `docstatus` = 2, then saves."""
 		self._cancel()
 
-	def delete(self):
+	def delete(self, ignore_permissions=False):
 		"""Delete document."""
-		frappe.delete_doc(self.doctype, self.name, flags=self.flags)
+		frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags)
 
 	def run_before_save_methods(self):
 		"""Run standard methods before  `INSERT` or `UPDATE`. Standard Methods are:

frappe/model/rename_doc.py

@@ -25,7 +25,6 @@ def update_document_title(doctype, docname, title_field=None, old_title=None, ne
 
 	return docname
 
-@frappe.whitelist()
 def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=False, ignore_if_exists=False, show_alert=True):
 	"""
 		Renames a doc(dt, old) to doc(dt, new) and

frappe/patches.txt

@@ -305,6 +305,9 @@ frappe.patches.v12_0.fix_email_id_formatting
 frappe.patches.v13_0.add_toggle_width_in_navbar_settings
 frappe.patches.v13_0.rename_notification_fields
 frappe.patches.v13_0.remove_duplicate_navbar_items
+frappe.patches.v12_0.set_default_password_reset_limit
+frappe.patches.v13_0.set_route_for_blog_category
 frappe.patches.v13_0.enable_custom_script
 frappe.patches.v13_0.update_newsletter_content_type
 execute:frappe.db.set_value('Website Settings', 'Website Settings', {'navbar_template': 'Standard Navbar', 'footer_template': 'Standard Footer'})
+frappe.patches.v13_0.delete_event_producer_and_consumer_keys

frappe/patches/v12_0/set_default_password_reset_limit.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import frappe
+
+
+def execute():
+	frappe.reload_doc("core", "doctype", "system_settings", force=1)
+	frappe.db.set_value('System Settings', None, "password_reset_limit", 3)

frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py

@@ -0,0 +1,11 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+
+def execute():
+	if frappe.db.exists("DocType", "Event Producer"):
+		frappe.db.sql("""UPDATE `tabEvent Producer` SET api_key='', api_secret=''""")
+	if frappe.db.exists("DocType", "Event Consumer"):
+		frappe.db.sql("""UPDATE `tabEvent Consumer` SET api_key='', api_secret=''""")

frappe/patches/v13_0/set_route_for_blog_category.py

@@ -0,0 +1,8 @@
+import frappe
+
+def execute():
+	categories = frappe.get_list("Blog Category")
+	for category in categories:
+		doc = frappe.get_doc("Blog Category", category["name"])
+		doc.set_route()
+		doc.save()

frappe/public/build.json

@@ -128,7 +128,7 @@
 		"public/js/lib/Sortable.min.js",
 		"public/js/lib/jquery/jquery.hotkeys.js",
 		"public/js/lib/bootstrap.min.js",
-		"node_modules/vue/dist/vue.js",
+		"node_modules/vue/dist/vue.min.js",
 		"node_modules/moment/min/moment-with-locales.min.js",
 		"node_modules/moment-timezone/builds/moment-timezone-with-data.min.js",
 		"public/js/lib/socket.io.min.js",

frappe/public/js/frappe/form/controls/data.js

@@ -61,7 +61,7 @@ frappe.ui.form.ControlData = frappe.ui.form.ControlInput.extend({
 						// check if name exists
 						frappe.db.get_value(this.doctype, this.$input.val(),
 							'name', (val) => {
-								if (val) {
+								if (val && val.name) {
 									this.set_description(__('{0} already exists. Select another name', [val.name]));
 								}
 							},

frappe/public/js/frappe/form/grid_row.js

@@ -393,11 +393,16 @@ export default class GridRow {
 		// sync get_query
 		field.get_query = this.grid.get_field(df.fieldname).get_query;
 
-		var field_on_change_function = field.df.onchange;
-		field.df.onchange = function(e) {
-			field_on_change_function && field_on_change_function(e);
-			me.grid.grid_rows[this.doc.idx - 1].refresh_field(field.df.fieldname);
-		};
+		if (!field.df.onchange_modified) {
+			var field_on_change_function = field.df.onchange;
+			field.df.onchange = function(e) {
+				field_on_change_function && field_on_change_function(e);
+				me.grid.grid_rows[this.doc.idx - 1].refresh_field(this.df.fieldname);
+			};
+
+			field.df.onchange_modified = true;
+		}
+
 		field.refresh();
 		if(field.$input) {
 			field.$input

frappe/public/js/frappe/ui/tree.js

@@ -5,17 +5,20 @@ frappe.provide('frappe.ui');
 
 frappe.ui.Tree = class {
 	constructor({
-		parent, label, icon_set, toolbar, expandable, with_skeleton=1, 	// eslint-disable-line
+		parent, label, root_value, icon_set, toolbar, expandable, with_skeleton=1, 	// eslint-disable-line
 
 		args, method, get_label, on_render, on_click 		// eslint-disable-line
 	}) {
 		$.extend(this, arguments[0]);
+		if (root_value == null) {
+			this.root_value = label;
+		}
 		this.setup_treenode_class();
 		this.nodes = {};
 		this.wrapper = $('<div class="tree">').appendTo(this.parent);
-		if(with_skeleton) this.wrapper.addClass('with-skeleton');
+		if (with_skeleton) this.wrapper.addClass('with-skeleton');
 
-		if(!icon_set) {
+		if (!icon_set) {
 			this.icon_set = {
 				open: 'fa fa-fw fa-folder-open',
 				closed: 'fa fa-fw fa-folder',
@@ -42,8 +45,9 @@ frappe.ui.Tree = class {
 		});
 	}
 
-	get_all_nodes(value, is_root) {
+	get_all_nodes(value, is_root, label) {
 		var args = Object.assign({}, this.args);
+		args.label = label || value;
 		args.parent = value;
 		args.is_root = is_root;
 
@@ -88,7 +92,7 @@ frappe.ui.Tree = class {
 			expandable: true,
 			is_root: true,
 			data: {
-				value: this.label
+				value: this.root_value
 			}
 		});
 		this.expand_node(this.root_node, false);
@@ -144,25 +148,25 @@ frappe.ui.Tree = class {
 	}
 
 	load_children(node, deep=false) {
-		let value = node.data.value, is_root = node.is_root;
+		let lab = node.label, value = node.data.value, is_root = node.is_root;
 
 		if(!deep) {
 			frappe.run_serially([
-				() => {return this.get_nodes(value, is_root);},
-				(data_set) => { this.render_node_children(node, data_set); },
-				() => { this.set_selected_node(node); }
+				() => this.get_nodes(value, is_root),
+				(data_set) => this.render_node_children(node, data_set),
+				() => this.set_selected_node(node)
 			]);
 		} else {
 			frappe.run_serially([
-				() => {return this.get_all_nodes(value, is_root);},
-				(data_list) => { this.render_children_of_all_nodes(data_list); },
-				() => { this.set_selected_node(node); }
+				() => this.get_all_nodes(value, is_root, lab),
+				(data_list) => this.render_children_of_all_nodes(data_list),
+				() => this.set_selected_node(node)
 			]);
 		}
 	}
 
 	render_children_of_all_nodes(data_list) {
-		data_list.map(d => { this.render_node_children(this.nodes[d.parent], d.data); });
+		data_list.map(d => this.render_node_children(this.nodes[d.parent], d.data));
 	}
 
 	render_node_children(node, data_set) {

frappe/public/js/frappe/views/reports/print_grid.html

@@ -37,16 +37,20 @@
 
 					<td {% if row.bold == 1 %} style="font-weight: bold" {% endif %}>
 						<span {% if col._index == 0 %} style="padding-left: {%= cint(row.indent) * 2 %}em" {% endif %}>
-							{% format_data = row.is_total_row ? data[0] : row %}
-							{{
-								col.formatter
-									? col.formatter(row._index, col._index, value, col, format_data, true)
-									: col.format
-										? col.format(value, row, col, format_data)
-										: col.docfield
-											? frappe.format(value, col.docfield)
-											: value
-							}}
+							{% format_data = row.is_total_row && ["Currency", "Float"].includes(col.fieldtype) ? data[0] : row %}
+							{% if (row.is_total_row && col._index == 0) { %}
+								{{ __("Total") }}
+							{% } else { %}
+								{{
+									col.formatter
+										? col.formatter(row._index, col._index, value, col, format_data, true)
+										: col.format
+											? col.format(value, row, col, format_data)
+											: col.docfield
+												? frappe.format(value, col.docfield)
+												: value
+								}}
+							{% } %}
 						</span>
 					</td>
 				{% endif %}
@@ -55,4 +59,3 @@
 		{% endfor %}
 	</tbody>
 </table>
-

frappe/public/js/frappe/views/reports/query_report.js

@@ -1259,7 +1259,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			return;
 		}
 
-		this.export_dialog = frappe.prompt([
+		let export_dialog_fields = [
 			{
 				label: __('Select File Format'),
 				fieldname: 'file_format',
@@ -1267,13 +1267,18 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				options: ['Excel', 'CSV'],
 				default: 'Excel',
 				reqd: 1
-			},
-			{
+			}
+		];
+
+		if (this.tree_report) {
+			export_dialog_fields.push({
 				label: __("Include indentation"),
 				fieldname: "include_indentation",
 				fieldtype: "Check",
-			}
-		], ({ file_format, include_indentation }) => {
+			});
+		}
+
+		this.export_dialog = frappe.prompt(export_dialog_fields, ({ file_format, include_indentation }) => {
 			this.make_access_log('Export', file_format);
 			if (file_format === 'CSV') {
 				const column_row = this.columns.reduce((acc, col) => {

frappe/public/js/frappe/views/reports/report_view.js

@@ -50,6 +50,12 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 		super.setup_new_doc_event();
 	}
 
+	toggle_side_bar() {
+		super.toggle_side_bar();
+		// refresh datatable when sidebar is toggled to accomodate extra space
+		this.render(true);
+	}
+
 	setup_result_area() {
 		super.setup_result_area();
 		this.setup_charts_area();

frappe/public/js/frappe/views/treeview.js

@@ -39,6 +39,7 @@ frappe.views.TreeView = Class.extend({
 		this.get_permissions();
 		this.make_page();
 		this.make_filters();
+		this.root_value = null;
 
 		if (me.opts.get_tree_root) {
 			this.get_root();
@@ -129,7 +130,13 @@ frappe.views.TreeView = Class.extend({
 			args: me.args,
 			callback: function(r) {
 				if (r.message) {
-					me.root_label = r.message[0]["value"];
+					if (r.message.length > 1) {
+						me.root_label = me.doctype;
+						me.root_value = "";
+					} else {
+						me.root_label = r.message[0]["value"];
+						me.root_value = me.root_label;
+					}
 					me.make_tree();
 				}
 			}
@@ -138,9 +145,15 @@ frappe.views.TreeView = Class.extend({
 	make_tree: function() {
 		$(this.parent).find(".tree").remove();
 
+		var use_label = this.args[this.opts.root_label] || this.root_label || this.opts.root_label;
+		var use_value = this.root_value;
+		if (use_value == null) {
+			use_value = use_label;
+		}
 		this.tree = new frappe.ui.Tree({
 			parent: this.body,
-			label: this.args[this.opts.root_label] || this.root_label || this.opts.root_label,
+			label: use_label,
+			root_value: use_value,
 			expandable: true,
 
 			args: this.args,

frappe/public/js/frappe/widgets/shortcut_widget.js

@@ -89,11 +89,10 @@ export default class ShortcutWidget extends Widget {
 		const label = get_label();
 		const buttons = $(`<div class="small pill">${label}</div>`);
 		if (this.color) {
-			buttons.css("background-color", this.color);
-			buttons.css(
-				"color",
-				frappe.ui.color.get_contrast_color(this.color)
-			);
+			let bg_color = count ? this.color: '#EEEEEE';
+			let text_color = count ? frappe.ui.color.get_contrast_color(bg_color): '#8D99A6';
+			buttons.css("background-color", bg_color);
+			buttons.css("color", text_color);
 		}
 
 		buttons.appendTo(this.action_area);

frappe/public/less/report.less

@@ -72,7 +72,7 @@
 	margin-bottom: 10px;
 }
 
-.report-wrapper {
+.report-wrapper, .datatable-wrapper {
 	overflow: auto;
 }
 

frappe/public/scss/login.scss

@@ -0,0 +1,203 @@
+/* login-css */
+
+#page-login {
+	.hero-and-content {
+		/*background-color: #f5f7fa;*/
+		background-color: #fafbfc;
+	}
+
+	.page-sidebar,
+	#wrap-footer,
+	.page-header {
+		display: none;
+	}
+
+	.page-content {
+		right: 0%;
+		width: 100%;
+	}
+
+	.icon-twitter,
+	.icon-twitter-sign {
+		color: #00a0d1;
+	}
+
+	.icon-linkedin,
+	.icon-linkedin-sign {
+		color: #4875b4;
+	}
+
+	#wrap {
+		background-color: #7575ff;
+	}
+
+	.for-login {
+		display: none;
+	}
+
+	.for-forgot {
+		display: none;
+	}
+
+	.for-signup {
+		display: none;
+	}
+
+	.form-signin {
+		.form-signin-heading,
+		.checkbox {
+			margin-bottom: 10px;
+		}
+		.checkbox {
+			font-weight: normal;
+		}
+		.form-control {
+			position: relative;
+			height: auto;
+			-webkit-box-sizing: border-box;
+			-moz-box-sizing: border-box;
+			box-sizing: border-box;
+			padding: 6px;
+			font-size: 14px;
+			margin-bottom: 10px;
+		}
+		.form-control:focus {
+			z-index: 2;
+		}
+	}
+
+	.btn-social {
+		margin: 10px;
+	}
+
+	.social-logins .fa {
+		margin-right: 5px;
+		color: #8d99a6;
+	}
+
+	.form-footer {
+		margin-top: -45px;
+		text-align: center;
+		font-size: 12px;
+		color: #8d99a6;
+		font-weight: bold;
+
+		a {
+			font-size: 12px;
+			color: #8d99a6;
+			font-weight: bold;
+		}
+
+		h6 {
+			font-size: 12px;
+			color: #8d99a6;
+			font-weight: bold;
+		}
+
+		.btn-default {
+			color: #36414c;
+		}
+	}
+
+	h5 {
+		position: relative;
+		text-align: center;
+		margin-top: 20px;
+		margin-bottom: 20px;
+	}
+
+	p {
+		margin-bottom: 20px;
+	}
+
+	.login-content .btn {
+		font-size: 14px;
+		margin-top: 45px;
+	}
+
+	.page-card {
+		max-width: 360px;
+		padding: 15px;
+		margin: 70px auto;
+		border: 1px solid #d1d8dd;
+		border-radius: 4px;
+		background-color: #fff;
+		box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.1);
+
+		.page-card-head {
+			padding: 10px 15px;
+			margin: -15px;
+			margin-bottom: 15px;
+			border-bottom: 1px solid #d1d8dd;
+		}
+
+		.page-card-head .indicator {
+			color: #36414c;
+			font-size: 14px;
+		}
+
+		.page-card-head .indicator::before {
+			margin: 0 6px 0.5px 0px;
+		}
+
+		.btn {
+			margin-top: 30px;
+		}
+	}
+
+	.bordered {
+		border: 1px solid #d1d8dd;
+		padding: 10px;
+		border-radius: 4px;
+	}
+
+	.toggle-password {
+		right: 9px;
+		top: 9px;
+		position: absolute;
+		z-index: 2;
+		cursor: pointer;
+		font-size: 12px;
+	}
+
+	.invalid-login {
+		-webkit-animation: wiggle 0.5s linear;
+		animation: wiggle 0.5s linear;
+	}
+
+	@-webkit-keyframes wiggle {
+		8%, 41% {
+			-webkit-transform: translateX(-10px);
+		}
+		25%, 58% {
+			-webkit-transform: translateX(10px);
+		}
+		75% {
+			-webkit-transform: translateX(-5px);
+		}
+		92% {
+			-webkit-transform: translateX(5px);
+		}
+		0%, 100% {
+			-webkit-transform: translateX(0);
+		}
+	}
+
+	@keyframes wiggle {
+		8%, 41% {
+			transform: translate(-10px);
+		}
+		25%, 58% {
+			transform: translate(10px);
+		}
+		75% {
+			transform: translate(-5px);
+		}
+		92% {
+			transform: translate(5px);
+		}
+		0%, 100% {
+			transform: translate(0);
+		} 
+	}
+}

frappe/public/scss/website.scss

@@ -15,6 +15,7 @@
 @import 'doc';
 @import 'navbar';
 @import 'footer';
+@import 'login';
 
 .ql-editor.read-mode {
 	padding: 0;

frappe/search/full_text_search.py

@@ -46,7 +46,8 @@ class FullTextSearch:
 			doc_name (str): name of the document to be updated
 		"""
 		document = self.get_document_to_index(doc_name)
-		self.update_index(document)
+		if document:
+			self.update_index(document)
 
 	def remove_document_from_index(self, doc_name):
 		"""Remove document from search index

frappe/templates/includes/login/login.css

@@ -1,167 +0,0 @@
-/* login-css */
-
-.hero-and-content {
-	/*background-color: #f5f7fa;*/
-	background-color: #fafbfc;
-}
-
-.page-sidebar, #wrap-footer, .page-header {
-	display: none;
-}
-
-.page-content {
-	right: 0%;
-	width: 100%;
-}
-
-.icon-twitter, .icon-twitter-sign{
-    color: #00a0d1;
-}
-
-.icon-linkedin, .icon-linkedin-sign{
-    color: #4875B4;
-}
-
-#wrap {
-	background-color: #7575ff;
-}
-
-.for-login {
-	display: none;
-}
-
-.for-forgot {
-	display: none;
-}
-
-.for-signup {
-	display: none;
-}
-
-.form-signin .form-signin-heading,
-.form-signin .checkbox {
-  margin-bottom: 10px;
-}
-.form-signin .checkbox {
-  font-weight: normal;
-}
-.form-signin .form-control {
-  position: relative;
-  height: auto;
-  -webkit-box-sizing: border-box;
-     -moz-box-sizing: border-box;
-          box-sizing: border-box;
-  padding: 6px;
-  font-size: 14px;
-  margin-bottom: 10px;
-}
-.form-signin .form-control:focus {
-  z-index: 2;
-}
-
-.btn-social {
-	margin: 10px;
-}
-
-.social-logins .fa {
-	margin-right: 5px;
-	color: #8D99A6;
-}
-
-.form-footer {
-	margin-top: -45px;
-	text-align: center;
-}
-
-.form-footer, .form-footer a, .form-footer h6 {
-	font-size: 12px;
-	color: #8D99A6;
-	font-weight: bold;
-}
-
-.form-footer .btn-default {
-	color: #36414C;
-}
-
-h5 {
-  position: relative;
-  text-align: center;
-  margin-top:20px;
-  margin-bottom:20px;
-}
-
-p {
-	margin-bottom:20px;
-}
-
-.login-content .btn {
-	font-size: 14px;
-	margin-top: 45px;
-}
-
-.page-card {
-  max-width: 360px;
-  padding: 15px;
-  margin: 70px auto;
-  border: 1px solid #d1d8dd;
-  border-radius: 4px;
-  background-color: #fff;
-  box-shadow: 0px 3px 6px rgba(0, 0, 0, 0.1);
-}
-.page-card .page-card-head {
-  padding: 10px 15px;
-  margin: -15px;
-  margin-bottom: 15px;
-  border-bottom: 1px solid #d1d8dd;
-}
-.page-card .page-card-head .indicator {
-  color: #36414C;
-  font-size: 14px;
-}
-.page-card .page-card-head .indicator::before {
-  margin: 0 6px 0.5px 0px;
-}
-.page-card .btn {
-  margin-top: 30px;
-}
-
-.bordered {
-	border: 1px solid  #d1d8dd;
-	padding: 10px;
-	border-radius: 4px;
-}
-
-.toggle-password {
-  right: 9px;
-  top: 9px;
-  position: absolute;
-  z-index: 2;
-  cursor: pointer;
-  font-size: 12px;
-}
-
-.invalid-login {
-   -webkit-animation: wiggle 0.5s linear;
-}
-
-@-webkit-keyframes wiggle {
-  8%,
-  41% {
-    -webkit-transform: translateX(-10px);
-  }
-  25%,
-  58% {
-    -webkit-transform: translateX(10px);
-  }
-  75% {
-    -webkit-transform: translateX(-5px);
-  }
-  92% {
-    -webkit-transform: translateX(5px);
-  }
-  0%,
-  100% {
-    -webkit-transform: translateX(0);
-  }
-}
-

frappe/tests/test_client.py

@@ -21,3 +21,37 @@ class TestClient(unittest.TestCase):
 
 		self.assertFalse(frappe.db.exists("ToDo", todo.name))
 		self.assertRaises(frappe.DoesNotExistError, delete, "ToDo", todo.name)
+
+	def test_http_valid_method_access(self):
+		from frappe.client import delete
+		from frappe.handler import execute_cmd
+
+		frappe.set_user("Administrator")
+
+		frappe.local.request = frappe._dict()
+		frappe.local.request.method = 'POST'
+
+		frappe.local.form_dict = frappe._dict({
+			'doc': dict(doctype='ToDo', description='Valid http method'),
+			'cmd': 'frappe.client.save'
+		})
+		todo = execute_cmd('frappe.client.save')
+
+		self.assertEqual(todo.get('description'), 'Valid http method')
+
+		delete("ToDo", todo.name)
+
+	def test_http_invalid_method_access(self):
+		from frappe.handler import execute_cmd
+
+		frappe.set_user("Administrator")
+
+		frappe.local.request = frappe._dict()
+		frappe.local.request.method = 'GET'
+
+		frappe.local.form_dict = frappe._dict({
+			'doc': dict(doctype='ToDo', description='Invalid http method'),
+			'cmd': 'frappe.client.save'
+		})
+
+		self.assertRaises(frappe.PermissionError, execute_cmd, 'frappe.client.save')

frappe/tests/test_commands.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+
+# imports - standard imports
+import shlex
+import subprocess
+import unittest
+
+# imports - module imports
+import frappe
+
+
+def clean(value):
+	if isinstance(value, (bytes, str)):
+		value = value.decode().strip()
+	return value
+
+
+class BaseTestCommands:
+	def execute(self, command):
+		command = command.format(**{"site": frappe.local.site})
+		command = shlex.split(command)
+		self._proc = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+		self.stdout = clean(self._proc.stdout)
+		self.stderr = clean(self._proc.stderr)
+		self.returncode = clean(self._proc.returncode)
+
+
+class TestCommands(BaseTestCommands, unittest.TestCase):
+	def test_execute(self):
+		# test 1: execute a command expecting a numeric output
+		self.execute("bench --site {site} execute frappe.db.get_database_size")
+		self.assertEquals(self.returncode, 0)
+		self.assertIsInstance(float(self.stdout), float)
+
+		# test 2: execute a command expecting an errored output as local won't exist
+		self.execute("bench --site {site} execute frappe.local.site")
+		self.assertEquals(self.returncode, 1)
+		self.assertIsNotNone(self.stderr)
+
+		# test 3: execute a command with kwargs
+		# Note:
+		# terminal command has been escaped to avoid .format string replacement
+		# The returned value has quotes which have been trimmed for the test
+		self.execute("""bench --site {site} execute frappe.bold --kwargs '{{"text": "DocType"}}'""")
+		self.assertEquals(self.returncode, 0)
+		self.assertEquals(self.stdout[1:-1], frappe.bold(text='DocType'))

frappe/tests/test_form_load.py

@@ -24,7 +24,7 @@ class TestFormLoad(unittest.TestCase):
 	def test_fieldlevel_permissions_in_load(self):
 		blog = frappe.get_doc({
 			"doctype": "Blog Post",
-			"blog_category": "_Test Blog Category 1",
+			"blog_category": "-test-blog-category-1",
 			"blog_intro": "Test Blog Intro",
 			"blogger": "_Test Blogger 1",
 			"content": "Test Blog Content",
@@ -40,7 +40,7 @@ class TestFormLoad(unittest.TestCase):
 		user.remove_roles(*user_roles)
 		user.add_roles('Blogger')
 
-		make_property_setter('Blog Post', 'published', 'permlevel', 1, 'Int')
+		blog_post_property_setter = make_property_setter('Blog Post', 'published', 'permlevel', 1, 'Int')
 		reset('Blog Post')
 		add('Blog Post', 'Website Manager', 1)
 		update('Blog Post', 'Website Manager', 1, 'write', 1)
@@ -80,6 +80,7 @@ class TestFormLoad(unittest.TestCase):
 		user.add_roles(*user_roles)
 
 		blog_doc.delete()
+		frappe.delete_doc(blog_post_property_setter.doctype, blog_post_property_setter.name)
 
 	def test_fieldlevel_permissions_in_load_for_child_table(self):
 		contact = frappe.new_doc('Contact')

frappe/tests/test_permissions.py

@@ -59,7 +59,7 @@ class TestPermissions(unittest.TestCase):
 		self.assertTrue(post.has_permission("read"))
 
 	def test_user_permissions_in_doc(self):
-		add_user_permission("Blog Category", "_Test Blog Category 1",
+		add_user_permission("Blog Category", "-test-blog-category-1",
 			"test2@example.com")
 
 		frappe.set_user("test2@example.com")
@@ -73,7 +73,7 @@ class TestPermissions(unittest.TestCase):
 		self.assertTrue(get_doc_permissions(post1).get("read"))
 
 	def test_user_permissions_in_report(self):
-		add_user_permission("Blog Category", "_Test Blog Category 1", "test2@example.com")
+		add_user_permission("Blog Category", "-test-blog-category-1", "test2@example.com")
 
 		frappe.set_user("test2@example.com")
 		names = [d.name for d in frappe.get_list("Blog Post", fields=["name", "blog_category"])]
@@ -86,23 +86,23 @@ class TestPermissions(unittest.TestCase):
 		self.assertFalse(doc.get("blog_category"))
 
 		# Fetch default based on single user permission
-		add_user_permission("Blog Category", "_Test Blog Category 1", "test2@example.com")
+		add_user_permission("Blog Category", "-test-blog-category-1", "test2@example.com")
 
 		frappe.set_user("test2@example.com")
 		doc = frappe.new_doc("Blog Post")
-		self.assertEqual(doc.get("blog_category"), "_Test Blog Category 1")
+		self.assertEqual(doc.get("blog_category"), "-test-blog-category-1")
 
 		# Don't fetch default if user permissions is more than 1
-		add_user_permission("Blog Category", "_Test Blog Category", "test2@example.com", ignore_permissions=True)
+		add_user_permission("Blog Category", "-test-blog-category", "test2@example.com", ignore_permissions=True)
 		frappe.clear_cache()
 		doc = frappe.new_doc("Blog Post")
 		self.assertFalse(doc.get("blog_category"))
 
 		# Fetch user permission set as default from multiple user permission
-		add_user_permission("Blog Category", "_Test Blog Category 2", "test2@example.com", ignore_permissions=True, is_default=1)
+		add_user_permission("Blog Category", "-test-blog-category-2", "test2@example.com", ignore_permissions=True, is_default=1)
 		frappe.clear_cache()
 		doc = frappe.new_doc("Blog Post")
-		self.assertEqual(doc.get("blog_category"), "_Test Blog Category 2")
+		self.assertEqual(doc.get("blog_category"), "-test-blog-category-2")
 
 	def test_user_link_match_doc(self):
 		blogger = frappe.get_doc("Blogger", "_Test Blogger 1")
@@ -215,7 +215,7 @@ class TestPermissions(unittest.TestCase):
 		frappe.clear_cache(doctype='DocType')
 
 	def test_user_permission_doctypes(self):
-		add_user_permission("Blog Category", "_Test Blog Category 1",
+		add_user_permission("Blog Category", "-test-blog-category-1",
 			"test2@example.com")
 		add_user_permission("Blogger", "_Test Blogger 1",
 			"test2@example.com")
@@ -235,7 +235,7 @@ class TestPermissions(unittest.TestCase):
 	def if_owner_setup(self):
 		update('Blog Post', 'Blogger', 0, 'if_owner', 1)
 
-		add_user_permission("Blog Category", "_Test Blog Category 1",
+		add_user_permission("Blog Category", "-test-blog-category-1",
 			"test2@example.com")
 		add_user_permission("Blogger", "_Test Blogger 1",
 			"test2@example.com")
@@ -254,7 +254,7 @@ class TestPermissions(unittest.TestCase):
 
 		doc = frappe.get_doc({
 			"doctype": "Blog Post",
-			"blog_category": "_Test Blog Category",
+			"blog_category": "-test-blog-category",
 			"blogger": "_Test Blogger 1",
 			"title": "_Test Blog Post Title",
 			"content": "_Test Blog Post Content"
@@ -263,14 +263,14 @@ class TestPermissions(unittest.TestCase):
 		self.assertRaises(frappe.PermissionError, doc.insert)
 
 		frappe.set_user('test1@example.com')
-		add_user_permission("Blog Category", "_Test Blog Category",
+		add_user_permission("Blog Category", "-test-blog-category",
 			"test2@example.com")
 
 		frappe.set_user("test2@example.com")
 		doc.insert()
 
 		frappe.set_user("Administrator")
-		remove_user_permission("Blog Category", "_Test Blog Category",
+		remove_user_permission("Blog Category", "-test-blog-category",
 			"test2@example.com")
 
 		frappe.set_user("test2@example.com")
@@ -286,13 +286,13 @@ class TestPermissions(unittest.TestCase):
 	def test_ignore_user_permissions_if_missing(self):
 		"""If there are no user permissions, then allow as per role"""
 
-		add_user_permission("Blog Category", "_Test Blog Category",
+		add_user_permission("Blog Category", "-test-blog-category",
 			"test2@example.com")
 		frappe.set_user("test2@example.com")
 
 		doc = frappe.get_doc({
 			"doctype": "Blog Post",
-			"blog_category": "_Test Blog Category 2",
+			"blog_category": "-test-blog-category-2",
 			"blogger": "_Test Blogger 1",
 			"title": "_Test Blog Post Title",
 			"content": "_Test Blog Post Content"
@@ -301,7 +301,7 @@ class TestPermissions(unittest.TestCase):
 		self.assertFalse(doc.has_permission("write"))
 
 		frappe.set_user("Administrator")
-		remove_user_permission("Blog Category", "_Test Blog Category",
+		remove_user_permission("Blog Category", "-test-blog-category",
 			"test2@example.com")
 
 		frappe.set_user("test2@example.com")
@@ -420,7 +420,7 @@ class TestPermissions(unittest.TestCase):
 
 		doc = frappe.get_doc({
 			"doctype": "Blog Post",
-			"blog_category": "_Test Blog Category",
+			"blog_category": "-test-blog-category",
 			"blogger": "_Test Blogger 1",
 			"title": "_Test Blog Post Title",
 			"content": "_Test Blog Post Content"
@@ -454,7 +454,7 @@ class TestPermissions(unittest.TestCase):
 
 		add_user_permission('Blog Post', '-test-blog-post-1', 'test2@example.com')
 		add_user_permission('Blog Post', '-test-blog-post-2', 'test2@example.com')
-		add_user_permission("Blog Category", '_Test Blog Category 1', 'test2@example.com')
+		add_user_permission("Blog Category", '-test-blog-category-1', 'test2@example.com')
 
 		deleted_user_permission_count = clear_user_permissions('test2@example.com', 'Blog Post')
 

frappe/tests/test_safe_exec.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 import unittest, frappe
-from frappe.utils.safe_exec import safe_exec
+from frappe.utils.safe_exec import safe_exec, get_safe_globals
 
 class TestSafeExec(unittest.TestCase):
 	def test_import_fails(self):
@@ -9,6 +9,15 @@ class TestSafeExec(unittest.TestCase):
 	def test_internal_attributes(self):
 		self.assertRaises(SyntaxError, safe_exec, '().__class__.__call__')
 
+	def test_utils(self):
+		_locals = dict(out=None)
+		safe_exec('''out = frappe.utils.cint("1")''', None, _locals)
+		self.assertEqual(_locals['out'], 1)
+
+	def test_safe_eval(self):
+		self.assertEqual(frappe.safe_eval('1+1'), 2)
+		self.assertRaises(AttributeError, frappe.safe_eval, 'frappe.utils.os.path', get_safe_globals())
+
 	def test_sql(self):
 		_locals = dict(out=None)
 		safe_exec('''out = frappe.db.sql("select name from tabDocType where name='DocType'")''', None, _locals)

frappe/tests/test_website.py

@@ -9,6 +9,7 @@ from frappe.utils import set_request
 
 
 class TestWebsite(unittest.TestCase):
+
 	def test_home_page_for_role(self):
 		frappe.delete_doc_if_exists('User', 'test-user-for-home-page@example.com')
 		frappe.delete_doc_if_exists('Role', 'home-page-test')
@@ -42,8 +43,6 @@ class TestWebsite(unittest.TestCase):
 		frappe.cache().hdel('home_page', frappe.session.user)
 		self.assertEqual(get_home_page(), 'test-portal-home')
 
-
-
 	def test_page_load(self):
 		frappe.set_user('Guest')
 		set_request(method='POST', path='login')
@@ -53,7 +52,6 @@ class TestWebsite(unittest.TestCase):
 
 		html = frappe.safe_decode(response.get_data())
 
-		self.assertTrue('/* login-css */' in html)
 		self.assertTrue('// login.js' in html)
 		self.assertTrue('<!-- login.html -->' in html)
 		frappe.set_user('Administrator')

frappe/tests/ui_test_helpers.py

@@ -9,6 +9,9 @@ def create_if_not_exists(doc):
 	:param doc: dict of field value pairs. can be a list of dict for multiple records.
 	'''
 
+	if not frappe.local.dev_server:
+		frappe.throw('This method can only be accessed in development', frappe.PermissionError)
+
 	doc = frappe.parse_json(doc)
 
 	if not isinstance(doc, list):

frappe/utils/__init__.py

@@ -135,7 +135,8 @@ def validate_email_address(email_str, throw=False):
 
 		if not _valid:
 			if throw:
-				frappe.throw(frappe._("{0} is not a valid Email Address").format(e),
+				invalid_email = frappe.utils.escape_html(e)
+				frappe.throw(frappe._("{0} is not a valid Email Address").format(invalid_email),
 					frappe.InvalidEmailAddressError)
 			return None
 		else:

frappe/utils/backups.py

@@ -68,6 +68,12 @@ class BackupGenerator:
 				dir = os.path.dirname(file_path)
 				os.makedirs(dir, exist_ok=True)
 
+	@property
+	def site_config_backup_path(self):
+		# For backwards compatibility
+		import click
+		click.secho("BackupGenerator.site_config_backup_path has been deprecated in favour of BackupGenerator.backup_path_conf", fg="yellow")
+		return getattr(self, "backup_path_conf", None)
 
 	def get_backup(self, older_than=24, ignore_files=False, force=False):
 		"""
@@ -96,7 +102,7 @@ class BackupGenerator:
 			self.backup_path_files = last_file
 			self.backup_path_db = last_db
 			self.backup_path_private_files = last_private_file
-			self.site_config_backup_path = site_config_backup_path
+			self.backup_path_conf = site_config_backup_path
 
 	def set_backup_file_name(self):
 		#Generate a random name using today's date and a 8 digit random number

frappe/utils/change_log.py

@@ -9,7 +9,6 @@ import frappe
 import requests
 import subprocess # nosec
 from frappe.utils import cstr
-from frappe.utils.gitutils import get_app_branch
 from frappe import _, safe_decode
 
 

frappe/utils/data.py

@@ -3,10 +3,8 @@
 
 from __future__ import unicode_literals
 
-# IMPORTANT: only import safe functions as this module will be included in jinja environment
 import frappe
 from dateutil.parser._parser import ParserError
-import subprocess
 import operator
 import json
 import re, datetime, math, time
@@ -427,19 +425,6 @@ def flt(s, precision=None):
 
 	return num
 
-def get_wkhtmltopdf_version():
-	wkhtmltopdf_version = frappe.cache().hget("wkhtmltopdf_version", None)
-
-	if not wkhtmltopdf_version:
-		try:
-			res = subprocess.check_output(["wkhtmltopdf", "--version"])
-			wkhtmltopdf_version = res.decode('utf-8').split(" ")[1]
-			frappe.cache().hset("wkhtmltopdf_version", None, wkhtmltopdf_version)
-		except Exception:
-			pass
-
-	return (wkhtmltopdf_version or '0')
-
 def cint(s):
 	"""Convert to integer"""
 	try: num = int(float(s))
@@ -754,7 +739,7 @@ def get_thumbnail_base64_for_image(src):
 	if not src:
 		frappe.throw('Invalid source for image: {0}'.format(src))
 
-	if not src.startswith('/files'):
+	if not src.startswith('/files') or '..' in src:
 		return
 
 	if src.endswith('.svg'):

frappe/utils/gitutils.py

@@ -1,24 +0,0 @@
-from __future__ import unicode_literals
-
-import subprocess
-
-def get_app_branch(app):
-	'''Returns branch of an app'''
-	try:
-		branch = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app),
-			shell=True)
-		branch = branch.decode('utf-8')
-		branch = branch.strip()
-		return branch
-	except Exception:
-		return ''
-
-def get_app_last_commit_ref(app):
-	try:
-		commit_id = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD'.format(app),
-			shell=True)
-		commit_id = commit_id.decode('utf-8')
-		commit_id = commit_id.strip()[:7]
-		return commit_id
-	except Exception:
-		return ''

frappe/utils/password.py

@@ -50,6 +50,7 @@ def get_decrypted_password(doctype, name, fieldname='password', raise_exception=
 	elif raise_exception:
 		frappe.throw(_('Password not found'), frappe.AuthenticationError)
 
+
 def set_encrypted_password(doctype, name, pwd, fieldname='password'):
 	try:
 		frappe.db.sql("""insert into `__Auth` (doctype, name, fieldname, `password`, encrypted)
@@ -63,6 +64,7 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'):
 			frappe.throw("Most probably your password is too long.", exc=e)
 		raise e
 
+
 def check_password(user, pwd, doctype='User', fieldname='password'):
 	'''Checks if user and password are correct, else raises frappe.AuthenticationError'''
 
@@ -82,11 +84,20 @@ def check_password(user, pwd, doctype='User', fieldname='password'):
 
 	return user
 
+
 def delete_login_failed_cache(user):
 	frappe.cache().hdel('last_login_tried', user)
 	frappe.cache().hdel('login_failed_count', user)
 	frappe.cache().hdel('locked_account_time', user)
 
+
+def delete_password_reset_cache(user=None):
+	if user:
+		frappe.cache().hdel('password_reset_link_count', user)
+	else:
+		frappe.cache().delete_key('password_reset_link_count')
+
+
 def update_password(user, pwd, doctype='User', fieldname='password', logout_all_sessions=False):
 	'''
 		Update the password for the User
@@ -115,6 +126,7 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_
 		from frappe.sessions import clear_sessions
 		clear_sessions(user=user, keep_current=True, force=True)
 
+
 def delete_all_passwords_for(doctype, name):
 	try:
 		frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""",
@@ -123,26 +135,31 @@ def delete_all_passwords_for(doctype, name):
 		if not frappe.db.is_missing_column(e):
 			raise
 
+
 def rename_password(doctype, old_name, new_name):
 	# NOTE: fieldname is not considered, since the document is renamed
 	frappe.db.sql("""update `__Auth` set name=%(new_name)s
 		where doctype=%(doctype)s and name=%(old_name)s""",
 		{ 'doctype': doctype, 'new_name': new_name, 'old_name': old_name })
 
+
 def rename_password_field(doctype, old_fieldname, new_fieldname):
 	frappe.db.sql('''update `__Auth` set fieldname=%(new_fieldname)s
 		where doctype=%(doctype)s and fieldname=%(old_fieldname)s''',
 		{ 'doctype': doctype, 'old_fieldname': old_fieldname, 'new_fieldname': new_fieldname })
 
+
 def create_auth_table():
 	# same as Framework.sql
 	frappe.db.create_auth_table()
 
+
 def encrypt(pwd):
 	cipher_suite = Fernet(encode(get_encryption_key()))
 	cipher_text = cstr(cipher_suite.encrypt(encode(pwd)))
 	return cipher_text
 
+
 def decrypt(pwd):
 	try:
 		cipher_suite = Fernet(encode(get_encryption_key()))
@@ -152,6 +169,7 @@ def decrypt(pwd):
 		# encryption_key in site_config is changed and not valid
 		frappe.throw(_('Encryption key is invalid, Please check site_config.json'))
 
+
 def get_encryption_key():
 	from frappe.installer import update_site_config
 

frappe/utils/pdf.py

@@ -6,6 +6,7 @@ import io
 import os
 import re
 from distutils.version import LooseVersion
+import subprocess
 
 import pdfkit
 import six
@@ -14,7 +15,7 @@ from PyPDF2 import PdfFileReader, PdfFileWriter
 
 import frappe
 from frappe import _
-from frappe.utils import get_wkhtmltopdf_version, scrub_urls
+from frappe.utils import scrub_urls
 
 
 PDF_CONTENT_ERRORS = ["ContentNotFoundError", "ContentOperationNotPermittedError",
@@ -51,6 +52,8 @@ def get_pdf(html, options=None, output=None):
 				output.appendPagesFromReader(reader)
 		else:
 			raise
+	finally:
+		cleanup(options)
 
 	if "password" in options:
 		password = options["password"]
@@ -109,8 +112,7 @@ def prepare_options(html, options):
 	options.update(html_options or {})
 
 	# cookies
-	if frappe.session and frappe.session.sid:
-		options['cookie'] = [('sid', '{0}'.format(frappe.session.sid))]
+	options.update(get_cookie_options())
 
 	# page size
 	if not options.get("page-size"):
@@ -119,6 +121,22 @@ def prepare_options(html, options):
 	return html, options
 
 
+def get_cookie_options():
+	options = {}
+	if frappe.session and frappe.session.sid:
+		# Use wkhtmltopdf's cookie-jar feature to set cookies and restrict them to host domain
+		cookiejar = "/tmp/{}.jar".format(frappe.generate_hash())
+
+		# Remove port from request.host
+		# https://werkzeug.palletsprojects.com/en/0.16.x/wrappers/#werkzeug.wrappers.BaseRequest.host
+		domain = frappe.local.request.host.split(":", 1)[0]
+		with open(cookiejar, "w") as f:
+			f.write("sid={}; Domain={};\n".format(frappe.session.sid, domain))
+
+		options['cookie-jar'] = cookiejar
+
+	return options
+
 def read_options_from_html(html):
 	options = {}
 	soup = BeautifulSoup(html, "html5lib")
@@ -183,15 +201,11 @@ def prepare_header_footer(soup):
 	return options
 
 
-def cleanup(fname, options):
-	if os.path.exists(fname):
-		os.remove(fname)
-
-	for key in ("header-html", "footer-html"):
+def cleanup(options):
+	for key in ("header-html", "footer-html", "cookie-jar"):
 		if options.get(key) and os.path.exists(options[key]):
 			os.remove(options[key])
 
-
 def toggle_visible_pdf(soup):
 	for tag in soup.find_all(attrs={"class": "visible-pdf"}):
 		# remove visible-pdf class to unhide
@@ -200,3 +214,16 @@ def toggle_visible_pdf(soup):
 	for tag in soup.find_all(attrs={"class": "hidden-pdf"}):
 		# remove tag from html
 		tag.extract()
+
+def get_wkhtmltopdf_version():
+	wkhtmltopdf_version = frappe.cache().hget("wkhtmltopdf_version", None)
+
+	if not wkhtmltopdf_version:
+		try:
+			res = subprocess.check_output(["wkhtmltopdf", "--version"])
+			wkhtmltopdf_version = res.decode('utf-8').split(" ")[1]
+			frappe.cache().hset("wkhtmltopdf_version", None, wkhtmltopdf_version)
+		except Exception:
+			pass
+
+	return (wkhtmltopdf_version or '0')

frappe/utils/safe_exec.py

@@ -39,7 +39,7 @@ def get_safe_globals():
 		date_format = "yyyy-mm-dd"
 		time_format = "HH:mm:ss"
 
-	add_module_properties(frappe.utils.data, datautils, lambda obj: hasattr(obj, "__call__"))
+	add_data_utils(datautils)
 
 	if "_" in getattr(frappe.local, 'form_dict', {}):
 		del frappe.local.form_dict["_"]
@@ -162,6 +162,11 @@ def _write(obj):
 	# allow writing to any object
 	return obj
 
+def add_data_utils(data):
+	for key, obj in frappe.utils.data.__dict__.items():
+		if key in VALID_UTILS:
+			data[key] = obj
+
 def add_module_properties(module, data, filter_method):
 	for key, obj in module.__dict__.items():
 		if key.startswith("_"):
@@ -171,3 +176,106 @@ def add_module_properties(module, data, filter_method):
 		if filter_method(obj):
 			# only allow functions
 			data[key] = obj
+
+VALID_UTILS = (
+"DATE_FORMAT",
+"TIME_FORMAT",
+"DATETIME_FORMAT",
+"is_invalid_date_string",
+"getdate",
+"get_datetime",
+"to_timedelta",
+"add_to_date",
+"add_days",
+"add_months",
+"add_years",
+"date_diff",
+"month_diff",
+"time_diff",
+"time_diff_in_seconds",
+"time_diff_in_hours",
+"now_datetime",
+"get_timestamp",
+"get_eta",
+"get_time_zone",
+"convert_utc_to_user_timezone",
+"now",
+"nowdate",
+"today",
+"nowtime",
+"get_first_day",
+"get_quarter_start",
+"get_first_day_of_week",
+"get_year_start",
+"get_last_day_of_week",
+"get_last_day",
+"get_time",
+"get_datetime_str",
+"get_date_str",
+"get_time_str",
+"get_user_date_format",
+"get_user_time_format",
+"format_date",
+"format_time",
+"format_datetime",
+"format_duration",
+"get_weekdays",
+"get_weekday",
+"get_timespan_date_range",
+"global_date_format",
+"has_common",
+"flt",
+"cint",
+"floor",
+"ceil",
+"cstr",
+"rounded",
+"remainder",
+"safe_div",
+"round_based_on_smallest_currency_fraction",
+"encode",
+"parse_val",
+"fmt_money",
+"get_number_format_info",
+"money_in_words",
+"in_words",
+"is_html",
+"is_image",
+"get_thumbnail_base64_for_image",
+"image_to_base64",
+"strip_html",
+"escape_html",
+"pretty_date",
+"comma_or",
+"comma_and",
+"comma_sep",
+"new_line_sep",
+"filter_strip_join",
+"get_url",
+"get_host_name_from_request",
+"url_contains_port",
+"get_host_name",
+"get_link_to_form",
+"get_link_to_report",
+"get_absolute_url",
+"get_url_to_form",
+"get_url_to_list",
+"get_url_to_report",
+"get_url_to_report_with_filters",
+"evaluate_filters",
+"compare",
+"get_filter",
+"make_filter_tuple",
+"make_filter_dict",
+"sanitize_column",
+"scrub_urls",
+"expand_relative_urls",
+"quoted",
+"quote_urls",
+"unique",
+"strip",
+"to_markdown",
+"md_to_html",
+"is_subset",
+"generate_hash"
+)

frappe/website/doctype/blog_category/blog_category.json

@@ -2,26 +2,17 @@
  "actions": [],
  "allow_guest_to_view": 1,
  "allow_import": 1,
- "autoname": "field:category_name",
+ "allow_rename": 1,
  "creation": "2013-03-08 09:41:11",
  "doctype": "DocType",
  "document_type": "Setup",
  "engine": "InnoDB",
  "field_order": [
-  "category_name",
   "title",
   "published",
   "route"
  ],
  "fields": [
-  {
-   "fieldname": "category_name",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Category Name",
-   "reqd": 1,
-   "unique": 1
-  },
   {
    "fieldname": "title",
    "fieldtype": "Data",
@@ -31,7 +22,7 @@
    "reqd": 1
   },
   {
-   "default": "0",
+   "default": "1",
    "fieldname": "published",
    "fieldtype": "Check",
    "in_list_view": 1,
@@ -42,15 +33,17 @@
    "fieldname": "route",
    "fieldtype": "Data",
    "label": "Route",
+   "read_only": 1,
    "unique": 1
   }
  ],
  "has_web_view": 1,
  "icon": "fa fa-tag",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2020-07-29 21:14:47.210446",
+ "modified": "2020-08-21 11:40:36.919321",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Category",
@@ -78,5 +71,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "title_field": "title",
  "track_changes": 1
 }

frappe/website/doctype/blog_category/blog_category.py

@@ -8,12 +8,11 @@ from frappe.website.render import clear_cache
 class BlogCategory(WebsiteGenerator):
 	def autoname(self):
 		# to override autoname of WebsiteGenerator
-		self.name = self.category_name
+		self.name = self.scrub(self.title)
 
 	def on_update(self):
 		clear_cache()
 
-	def validate(self):
-		if not self.route:
-			self.route = 'blog/' + self.scrub(self.name)
-		super(BlogCategory, self).validate()
+	def set_route(self):
+		# Override blog route since it has to been templated
+		self.route = 'blog/' + self.name

frappe/website/doctype/blog_category/test_blog_category.py

@@ -3,5 +3,7 @@
 from __future__ import unicode_literals
 
 import frappe
+import unittest
 
-test_records = frappe.get_test_records('Blog Category')
+class TestBlogCategory(unittest.TestCase):
+	pass

frappe/website/doctype/blog_category/test_records.json

@@ -1,18 +1,15 @@
 [
 	{
-		"category_name": "_Test Blog Category",
 		"doctype": "Blog Category",
 		"parent_website_route": "blog",
 		"title": "_Test Blog Category"
 	},
 	{
-		"category_name": "_Test Blog Category 1",
 		"doctype": "Blog Category",
 		"parent_website_route": "blog",
 		"title": "_Test Blog Category 1"
 	},
 	{
-		"category_name": "_Test Blog Category 2",
 		"doctype": "Blog Category",
 		"parent_website_route": "blog",
 		"title": "_Test Blog Category 2"

frappe/website/doctype/blog_post/blog_post.js

@@ -11,18 +11,31 @@ frappe.ui.form.on('Blog Post', {
 	},
 	title: function(frm) {
 		generate_google_search_preview(frm);
+		frm.trigger('set_route');
 	},
 	meta_description: function(frm) {
 		generate_google_search_preview(frm);
 	},
 	blog_intro: function(frm) {
 		generate_google_search_preview(frm);
+	},
+	blog_category(frm) {
+		frm.trigger('set_route');
+	},
+	set_route(frm) {
+		if (frm.doc.route) return;
+		if (frm.doc.title && frm.doc.blog_category) {
+			frm.call('make_route').then(r => {
+				frm.set_value('route', r.message);
+			});
+		}
 	}
 });
 
 function generate_google_search_preview(frm) {
+	if (!(frm.doc.meta_title || frm.doc.title)) return;
 	let google_preview = frm.get_field("google_preview");
-	let seo_title = (frm.doc.title).slice(0, 60);
+	let seo_title = (frm.doc.meta_title || frm.doc.title).slice(0, 60);
 	let seo_description =  (frm.doc.meta_description || frm.doc.blog_intro || "").slice(0, 160);
 	let date = frm.doc.published_on ? new frappe.datetime.datetime(frm.doc.published_on).moment.format('ll') + ' - ' : '';
 	let route_array = frm.doc.route ? frm.doc.route.split('/') : [];

frappe/website/doctype/blog_post/blog_post.json

@@ -26,6 +26,7 @@
   "content_html",
   "email_sent",
   "meta_tags",
+  "meta_title",
   "meta_description",
   "column_break_18",
   "meta_image",
@@ -110,7 +111,6 @@
    "depends_on": "eval:doc.content_type === 'Markdown'",
    "fieldname": "content_md",
    "fieldtype": "Markdown Editor",
-   "ignore_xss_filter": 1,
    "label": "Content (Markdown)"
   },
   {
@@ -185,6 +185,12 @@
    "fieldtype": "Check",
    "hidden": 1,
    "label": "Hide CTA"
+  },
+  {
+   "fieldname": "meta_title",
+   "fieldtype": "Data",
+   "label": "Meta Title",
+   "length": 60
   }
  ],
  "has_web_view": 1,
@@ -194,7 +200,7 @@
  "is_published_field": "published",
  "links": [],
  "max_attachments": 5,
- "modified": "2020-08-31 16:55:03.687862",
+ "modified": "2020-08-31 21:01:51.100349",
  "modified_by": "Administrator",
  "module": "Website",
  "name": "Blog Post",

frappe/website/doctype/blog_post/blog_post.py

@@ -36,6 +36,11 @@ class BlogPost(WebsiteGenerator):
 		if self.blog_intro:
 			self.blog_intro = self.blog_intro[:200]
 
+		if not self.meta_title:
+			self.meta_title = self.title[:60]
+		else:
+			self.meta_title = self.meta_title[:60]
+
 		if not self.meta_description:
 			self.meta_description = self.blog_intro[:140]
 		else:
@@ -88,7 +93,7 @@ class BlogPost(WebsiteGenerator):
 		context.description = self.meta_description or self.blog_intro or strip_html_tags(context.content[:140])
 
 		context.metatags = {
-			"name": self.title,
+			"name": self.meta_title,
 			"description": context.description,
 		}
 
@@ -242,7 +247,7 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len
 		and t1.blogger = t2.name
 		%(condition)s
 		order by featured desc, published_on desc, name asc
-		limit %(start)s, %(page_len)s""" % {
+		limit %(page_len)s OFFSET %(start)s""" % {
 			"start": limit_start, "page_len": limit_page_length,
 				"condition": (" and " + " and ".join(conditions)) if conditions else ""
 		}

frappe/website/doctype/blog_post/templates/blog_post.html

@@ -12,7 +12,7 @@
 			<div>
 				<a class="mr-2" href="/blog">{{ _('Blog') }}</a>
 				<span class="text-muted">/</span>
-				<a class="ml-2" href="/blog/{{ category.title }}">{{ category.title }}</a>
+				<a class="ml-2" href="/{{ category.route }}">{{ category.title }}</a>
 			</div>
 			<h1 itemprop="headline" class="blog-title">{{ title }}</h1>
 			<p class="blog-intro">

frappe/website/doctype/blog_post/test_blog_post.py

@@ -3,10 +3,14 @@
 from __future__ import unicode_literals
 import frappe
 import unittest
+from bs4 import BeautifulSoup
+import re
 
 from frappe.utils import set_request
 from frappe.website.render import render
 from frappe.utils import random_string
+from frappe.website.doctype.blog_post.blog_post import get_blog_list
+from frappe.website.website_generator import WebsiteGenerator
 
 class TestBlogPost(unittest.TestCase):
 	def test_generator_view(self):
@@ -32,12 +36,62 @@ class TestBlogPost(unittest.TestCase):
 
 		self.assertTrue(response.status_code, 404)
 
-def make_test_blog():
-	if not frappe.db.exists('Blog Category', 'Test Blog Category'):
+	def test_category_link(self):
+		# Make a temporary Blog Post (and a Blog Category)
+		blog = make_test_blog()
+
+		# Visit the blog post page
+		set_request(path=blog.route)
+		blog_page_response = render()
+		blog_page_html = frappe.safe_decode(blog_page_response.get_data())
+
+		# On blog post page find link to the category page
+		soup = BeautifulSoup(blog_page_html, "lxml")
+		category_page_link = list(soup.find_all('a', href=re.compile(blog.blog_category)))[0]
+		category_page_url = category_page_link["href"]
+
+		# Visit the category page (by following the link found in above stage)
+		set_request(path=category_page_url)
+		category_page_response = render()
+		category_page_html = frappe.safe_decode(category_page_response.get_data())
+
+		# Category page should contain the blog post title
+		self.assertIn(blog.title, category_page_html)
+
+		# Cleanup afterwords
+		frappe.delete_doc("Blog Post", blog.name)
+		frappe.delete_doc("Blog Category", blog.blog_category)
+
+	def test_blog_pagination(self):
+		# Create some Blog Posts for a Blog Category
+		category_title, blogs, BLOG_COUNT = "List Category", [], 4
+
+		for index in range(BLOG_COUNT):
+			blog = make_test_blog(category_title)
+			blogs.append(blog)
+
+		filters = frappe._dict({"blog_category": scrub(category_title)})
+		# Assert that get_blog_list returns results as expected
+
+		self.assertEqual(len(get_blog_list(None, None, filters, 0, 3)), 3)
+		self.assertEqual(len(get_blog_list(None, None, filters, 0, BLOG_COUNT)), BLOG_COUNT)
+		self.assertEqual(len(get_blog_list(None, None, filters, 0, 2)), 2)
+		self.assertEqual(len(get_blog_list(None, None, filters, 2, BLOG_COUNT)), 2)
+
+		# Cleanup Blog Post and linked Blog Category
+		for blog in blogs:
+			frappe.delete_doc(blog.doctype, blog.name)
+		frappe.delete_doc("Blog Category", blogs[0].blog_category)
+
+def scrub(text):
+	return WebsiteGenerator.scrub(None, text)
+
+def make_test_blog(category_title="Test Blog Category"):
+	category_name = scrub(category_title)
+	if not frappe.db.exists('Blog Category', category_name):
 		frappe.get_doc(dict(
 			doctype = 'Blog Category',
-			category_name = 'Test Blog Category',
-			title='Test Blog Category')).insert()
+			title=category_title)).insert()
 	if not frappe.db.exists('Blogger', 'test-blogger'):
 		frappe.get_doc(dict(
 			doctype = 'Blogger',
@@ -45,7 +99,7 @@ def make_test_blog():
 			full_name='Test Blogger')).insert()
 	test_blog = frappe.get_doc(dict(
 		doctype = 'Blog Post',
-		blog_category = 'Test Blog Category',
+		blog_category = category_name,
 		blogger = 'test-blogger',
 		title = random_string(20),
 		route = random_string(20),

frappe/website/doctype/blog_post/test_records.json

@@ -1,6 +1,6 @@
 [
 	{
-		"blog_category": "_Test Blog Category",
+		"blog_category": "-test-blog-category",
 		"blog_intro": "Test Blog Intro",
 		"blogger": "_Test Blogger",
 		"content": "Test Blog Content",
@@ -9,7 +9,7 @@
 		"published": 1
 	},
 	{
-		"blog_category": "_Test Blog Category 1",
+		"blog_category": "-test-blog-category-1",
 		"blog_intro": "Test Blog Intro",
 		"blogger": "_Test Blogger",
 		"content": "Test Blog Content",
@@ -18,7 +18,7 @@
 		"published": 1
 	},
 	{
-		"blog_category": "_Test Blog Category 1",
+		"blog_category": "-test-blog-category-1",
 		"blog_intro": "Test Blog Intro",
 		"blogger": "_Test Blogger 1",
 		"content": "Test Blog Content",
@@ -27,7 +27,7 @@
 		"published": 0
 	},
 	{
-		"blog_category": "_Test Blog Category 1",
+		"blog_category": "-test-blog-category-1",
 		"blog_intro": "Test Blog Intro",
 		"blogger": "_Test Blogger 2",
 		"content": "Test Blog Content",
@@ -35,4 +35,4 @@
 		"title": "_Test Blog Post 3",
 		"published": 0
 	}
-]
+]

frappe/website/web_template/section_with_tabs/section_with_tabs.html

@@ -1,5 +1,8 @@
 <h2 class="section-title">{{ title }}</h2>
+
+{%- if subtitle -%}
 <p class="section-description">{{ subtitle }}</p>
+{%- endif -%}
 
 <div class="mt-12">
 	{% set ns = namespace(tabs=[]) %}

frappe/www/login.html

@@ -1,11 +1,5 @@
 {% extends "templates/web.html" %}
 
-{% block style %}
-<style>
-	{% include "templates/includes/login/login.css" %}
-</style>
-{% endblock %}
-
 {% block page_content %}
 <!-- {{ for_test }} -->
 <div style='min-height: 360px'>