Merge branch 'version-13-beta-pre-release' into version-13-beta

.editorconfig

@@ -0,0 +1,14 @@
+# Root editor config file
+root = true
+
+# Common settings
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+
+# python, js indentation settings
+[{*.py,*.js}]
+indent_style = tab
+indent_size = 4

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Community Forum
+    url: https://discuss.erpnext.com/
+    about: For general QnA, discussions and community help.

.github/helper/documentation.py

@@ -21,8 +21,8 @@ def docs_link_exists(body):
 			if word.startswith('http') and uri_validator(word):
 				parsed_url = urlparse(word)
 				if parsed_url.netloc == "github.com":
-					_, org, repo, _type, ref = parsed_url.path.split('/')
-					if org == "frappe" and repo in docs_repos:
+					parts = parsed_url.path.split('/')
+					if len(parts) == 5 and parts[1] == "frappe" and parts[2] in docs_repos:
 						return True
 
 

.github/workflows/docker-release.yml

@@ -1,9 +1,10 @@
-name: Trigger Docker build on release
+name: 'Trigger Docker build on release'
 on:
   release:
     types: [released]
 jobs:
   curl:
+    name: 'Trigger Docker build on release'
     runs-on: ubuntu-latest
     container:
       image: alpine:latest

.github/workflows/docs-checker.yml

@@ -1,10 +1,11 @@
-name: 'Documentation Required'
+name: 'Documentation Check'
 on:
   pull_request:
     types: [ opened, synchronize, reopened, edited ]
 
 jobs:
-  build:
+  docs-required:
+    name: 'Documentation Required'
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/publish-assets-develop.yml

@@ -1,11 +1,12 @@
-name: Build and Publish Assets for Development
+name: 'Frappe Assets'
 
 on:
   push:
     branches: [ develop ]
 
 jobs:
-  build:
+  build-dev-and-publish:
+    name: 'Build and Publish Assets for Development'
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/publish-assets-releases.yml

@@ -1,4 +1,4 @@
-name: Build and Publish Assets built for Releases
+name: 'Frappe Assets'
 
 on:
   release:
@@ -8,7 +8,8 @@ env:
   GITHUB_TOKEN: ${{ github.token }}
 
 jobs:
-  build:
+  build-release-and-publish:
+    name: 'Build and Publish Assets built for Releases'
     runs-on: ubuntu-latest
 
     steps:
@@ -44,4 +45,3 @@ jobs:
           asset_path: build/assets.tar.gz
           asset_name: assets.tar.gz
           asset_content_type: application/octet-stream
-

.mergify.yml

@@ -5,7 +5,7 @@ pull_request_rules:
       - status-success=Semantic Pull Request
       - status-success=Travis CI - Pull Request
       - status-success=security/snyk (frappe)
-      - label!=don't-merge
+      - label!=dont-merge
       - label!=squash
       - "#approved-reviews-by>=1"
     actions:
@@ -17,7 +17,7 @@ pull_request_rules:
       - status-success=Semantic Pull Request
       - status-success=Travis CI - Pull Request
       - status-success=security/snyk (frappe)
-      - label!=don't-merge
+      - label!=dont-merge
       - label=squash
       - "#approved-reviews-by>=1"
     actions:

frappe/__init__.py

@@ -23,7 +23,7 @@ if PY2:
 	reload(sys)
 	sys.setdefaultencoding("utf-8")
 
-__version__ = '13.0.0-beta.8'
+__version__ = '13.0.0-beta.9'
 
 __title__ = "Frappe Framework"
 
@@ -349,7 +349,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False,
 
 	if as_table and type(msg) in (list, tuple):
 		out.as_table = 1
-	
+
 	if as_list and type(msg) in (list, tuple) and len(msg) > 1:
 		out.as_list = 1
 
@@ -797,11 +797,17 @@ def get_doc(*args, **kwargs):
 
 	return doc
 
-def get_last_doc(doctype):
+def get_last_doc(doctype, filters=None, order_by="creation desc"):
 	"""Get last created document of this type."""
-	d = get_all(doctype, ["name"], order_by="creation desc", limit_page_length=1)
+	d = get_all(
+		doctype,
+		filters=filters,
+		limit_page_length=1,
+		order_by=order_by,
+		pluck="name"
+	)
 	if d:
-		return get_doc(doctype, d[0].name)
+		return get_doc(doctype, d[0])
 	else:
 		raise DoesNotExistError
 
@@ -1155,6 +1161,7 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp
 			'doctype_or_field': args.doctype_or_field,
 			'doc_type': doctype,
 			'field_name': args.fieldname,
+			'row_name': args.row_name,
 			'property': args.property,
 			'value': args.value,
 			'property_type': args.property_type or "Data",

frappe/app.py

@@ -160,6 +160,10 @@ def handle_exception(e):
 	http_status_code = getattr(e, "http_status_code", 500)
 	return_as_message = False
 
+	if frappe.conf.get('developer_mode'):
+		# don't fail silently
+		print(frappe.get_traceback())
+
 	if frappe.get_request_header('Accept') and (frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept')):
 		# handle ajax responses first
 		# if the request is ajax, send back the trace or error message

frappe/automation/doctype/assignment_rule/assignment_rule.js

@@ -57,7 +57,8 @@ frappe.ui.form.on('Assignment Rule', {
 		frm.set_fields_as_options(
 			'field',
 			doctype,
-			(df) => df.fieldtype == 'Link' && df.options == 'User',
+			(df) => ['Dynamic Link', 'Data'].includes(df.fieldtype)
+				|| (df.fieldtype == 'Link' && df.options == 'User'),
 			[{ label: 'Owner', value: 'owner' }]
 		);
 		if (doctype) {

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -82,7 +82,7 @@ class AssignmentRule(Document):
 		elif self.rule == 'Load Balancing':
 			return self.get_user_load_balancing()
 		elif self.rule == 'Based on Field':
-			return doc.get(self.field)
+			return self.get_user_based_on_field(doc)
 
 	def get_user_round_robin(self):
 		'''
@@ -119,6 +119,11 @@ class AssignmentRule(Document):
 		# pick the first user
 		return sorted_counts[0].get('user')
 
+	def get_user_based_on_field(self, doc):
+		val = doc.get(self.field)
+		if frappe.db.exists('User', val):
+			return val
+
 	def safe_eval(self, fieldname, doc):
 		try:
 			if self.get(fieldname):

frappe/automation/doctype/auto_repeat/auto_repeat.js

@@ -44,6 +44,20 @@ frappe.ui.form.on('Auto Repeat', {
 
 		// auto repeat schedule
 		frappe.auto_repeat.render_schedule(frm);
+
+		frm.trigger('toggle_submit_on_creation');
+	},
+
+	reference_doctype: function(frm) {
+		frm.trigger('toggle_submit_on_creation');
+	},
+
+	toggle_submit_on_creation: function(frm) {
+		// submit on creation checkbox
+		frappe.model.with_doctype(frm.doc.reference_doctype, () => {
+			let meta = frappe.get_meta(frm.doc.reference_doctype);
+			frm.toggle_display('submit_on_creation', meta.is_submittable);
+		});
 	},
 
 	template: function(frm) {

frappe/automation/doctype/auto_repeat/auto_repeat.json

@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "allow_import": 1,
  "allow_rename": 1,
  "autoname": "format:AUT-AR-{#####}",
@@ -12,6 +13,7 @@
   "section_break_3",
   "reference_doctype",
   "reference_document",
+  "submit_on_creation",
   "column_break_5",
   "start_date",
   "end_date",
@@ -186,9 +188,16 @@
    "fieldname": "repeat_on_last_day",
    "fieldtype": "Check",
    "label": "Repeat on Last Day of the Month"
+  },
+  {
+   "default": "0",
+   "fieldname": "submit_on_creation",
+   "fieldtype": "Check",
+   "label": "Submit on Creation"
   }
  ],
- "modified": "2019-07-17 11:30:51.412317",
+ "links": [],
+ "modified": "2020-12-10 10:43:13.449172",
  "modified_by": "Administrator",
  "module": "Automation",
  "name": "Auto Repeat",

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -21,6 +21,7 @@ class AutoRepeat(Document):
 	def validate(self):
 		self.update_status()
 		self.validate_reference_doctype()
+		self.validate_submit_on_creation()
 		self.validate_dates()
 		self.validate_email_id()
 		self.set_dates()
@@ -60,6 +61,11 @@ class AutoRepeat(Document):
 		if not frappe.get_meta(self.reference_doctype).allow_auto_repeat:
 			frappe.throw(_("Enable Allow Auto Repeat for the doctype {0} in Customize Form").format(self.reference_doctype))
 
+	def validate_submit_on_creation(self):
+		if self.submit_on_creation and not frappe.get_meta(self.reference_doctype).is_submittable:
+			frappe.throw(_('Cannot enable {0} for a non-submittable doctype').format(
+				frappe.bold('Submit on Creation')))
+
 	def validate_dates(self):
 		if frappe.flags.in_patch:
 			return
@@ -150,6 +156,9 @@ class AutoRepeat(Document):
 		self.update_doc(new_doc, reference_doc)
 		new_doc.insert(ignore_permissions = True)
 
+		if self.submit_on_creation:
+			new_doc.submit()
+
 		return new_doc
 
 	def update_doc(self, new_doc, reference_doc):
@@ -160,7 +169,7 @@ class AutoRepeat(Document):
 		if new_doc.meta.get_field('auto_repeat'):
 			new_doc.set('auto_repeat', self.name)
 
-		for fieldname in ['naming_series', 'ignore_pricing_rule', 'posting_time', 'select_print_heading', 'remarks', 'owner']:
+		for fieldname in ['naming_series', 'ignore_pricing_rule', 'posting_time', 'select_print_heading', 'user_remark', 'remarks', 'owner']:
 			if new_doc.meta.get_field(fieldname):
 				new_doc.set(fieldname, reference_doc.get(fieldname))
 

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -111,6 +111,25 @@ class TestAutoRepeat(unittest.TestCase):
 		doc = make_auto_repeat(frequency='Daily', reference_document=todo.name, start_date=add_days(today(), -2))
 		self.assertEqual(getdate(doc.next_schedule_date), current_date)
 
+	def test_submit_on_creation(self):
+		doctype = 'Test Submittable DocType'
+		create_submittable_doctype(doctype)
+
+		current_date = getdate()
+		submittable_doc = frappe.get_doc(dict(doctype=doctype, test='test submit on creation')).insert()
+		submittable_doc.submit()
+		doc = make_auto_repeat(frequency='Daily', reference_doctype=doctype, reference_document=submittable_doc.name,
+			start_date=add_days(current_date, -1), submit_on_creation=1)
+
+		data = get_auto_repeat_entries(current_date)
+		create_repeated_entries(data)
+		docnames = frappe.db.get_all(doc.reference_doctype,
+			filters={'auto_repeat': doc.name},
+			fields=['docstatus'],
+			limit=1
+		)
+		self.assertEquals(docnames[0].docstatus, 1)
+
 
 def make_auto_repeat(**args):
 	args = frappe._dict(args)
@@ -118,6 +137,7 @@ def make_auto_repeat(**args):
 		'doctype': 'Auto Repeat',
 		'reference_doctype': args.reference_doctype or 'ToDo',
 		'reference_document': args.reference_document or frappe.db.get_value('ToDo', 'name'),
+		'submit_on_creation': args.submit_on_creation or 0,
 		'frequency': args.frequency or 'Daily',
 		'start_date': args.start_date or add_days(today(), -1),
 		'end_date': args.end_date or "",
@@ -128,3 +148,34 @@ def make_auto_repeat(**args):
 	}).insert(ignore_permissions=True)
 
 	return doc
+
+
+def create_submittable_doctype(doctype):
+	if frappe.db.exists('DocType', doctype):
+		return
+	else:
+		doc = frappe.get_doc({
+			'doctype': 'DocType',
+			'__newname': doctype,
+			'module': 'Custom',
+			'custom': 1,
+			'is_submittable': 1,
+			'fields': [{
+				'fieldname': 'test',
+				'label': 'Test',
+				'fieldtype': 'Data'
+			}],
+			'permissions': [{
+				'role': 'System Manager',
+				'read': 1,
+				'write': 1,
+				'create': 1,
+				'delete': 1,
+				'submit': 1,
+				'cancel': 1,
+				'amend': 1
+			}]
+		}).insert()
+
+		doc.allow_auto_repeat = 1
+		doc.save()

frappe/build.py

@@ -105,7 +105,7 @@ def download_frappe_assets(verbose=True):
 	if frappe_head:
 		try:
 			url = get_assets_link(frappe_head)
-			click.secho("Retreiving assets...", fg="yellow")
+			click.secho("Retrieving assets...", fg="yellow")
 			prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
 			assets_archive = download_file(url, prefix)
 			print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))

frappe/change_log/v13/v13_0_0-beta_9.md

@@ -0,0 +1,121 @@
+### Version 13.0.0 Beta 9 Release Notes
+
+#### Features and Enhancements
+
+- Permission Query via Server Script ([#12034](https://github.com/frappe/frappe/pull/12034))
+- Option to strip EXIF data from image files before uploading ([#12014](https://github.com/frappe/frappe/pull/12014))
+- Provision to open child table in customize form ([#12030](https://github.com/frappe/frappe/pull/12030))
+- Added date format support DD-Mon-YY ([#12056](https://github.com/frappe/frappe/pull/12056))
+- Partial Backups & Restores ([#11104](https://github.com/frappe/frappe/pull/11104))
+- Non negative check for Int, Float and Currency fields ([#11818](https://github.com/frappe/frappe/pull/11818))
+- Allow html in email templates ([#12018](https://github.com/frappe/frappe/pull/12018))
+- Web form fields support for property depends on fields (read-on… ([#11927](https://github.com/frappe/frappe/pull/11927))
+- Conditional Events in Event Streaming ([#10868](https://github.com/frappe/frappe/pull/10868))
+- Added filters and order_by fields in frappe.get_last_doc API ([#11870](https://github.com/frappe/frappe/pull/11870))
+- Check if auto_repeat field is already present ([#11970](https://github.com/frappe/frappe/pull/11970))
+- as_raw update to display content inline not always download as attachment ([#11569](https://github.com/frappe/frappe/pull/11569))
+- Reset scroll position on list paging ([#11673](https://github.com/frappe/frappe/pull/11673))
+- Validate sql file before restoring site ([#11878](https://github.com/frappe/frappe/pull/11878))
+- Allow custom freeze message in Open Mapped Doc ([#11976](https://github.com/frappe/frappe/pull/11976))
+- Show absolute value in print format ([#12019](https://github.com/frappe/frappe/pull/12019))
+- Add links and actions to customize form and cleanup code ([#11565](https://github.com/frappe/frappe/pull/11565))
+- Add module field in get_desk_sidebar_items ([#11781](https://github.com/frappe/frappe/pull/11781))
+
+#### Fixes
+
+- Maximum attachment limit validation ([#11940](https://github.com/frappe/frappe/pull/11940))
+- Define chunk size based on backup file size to avoid timeout issues ([#11526](https://github.com/frappe/frappe/pull/11526))
+- Fetch doc from db in get_transitions ([#11883](https://github.com/frappe/frappe/pull/11883))
+- Remove Package Publish Tool ([#11863](https://github.com/frappe/frappe/pull/11863))
+- Make strings translatable ([#11825](https://github.com/frappe/frappe/pull/11825))
+- Clear cache after updating defaults manually ([#11830](https://github.com/frappe/frappe/pull/11830))
+- Doctype query in create new shortcut ([#11864](https://github.com/frappe/frappe/pull/11864))
+- Ace editor fixes ([#11920](https://github.com/frappe/frappe/pull/11920))
+- Shorten number card percentage stat ([#11846](https://github.com/frappe/frappe/pull/11846))
+- Error page rendering shouldn't fail when recorder is active ([#11806](https://github.com/frappe/frappe/pull/11806))
+- Rename Doc considers invalid table name on renaming record ([#11743](https://github.com/frappe/frappe/pull/11743))
+- Use modified by or owner to send notification from ([#11981](https://github.com/frappe/frappe/pull/11981))
+- Bust cache by passing build_version to link and script src ([#11903](https://github.com/frappe/frappe/pull/11903))
+- Customize options in desk page rtl layout ([#11772](https://github.com/frappe/frappe/pull/11772))
+- Use sql_ddl to avoid exception triggered while running drop sql ([#11831](https://github.com/frappe/frappe/pull/11831))
+- Render template for HTML content type ([#12035](https://github.com/frappe/frappe/pull/12035))
+- Display style removed from emails ([#11963](https://github.com/frappe/frappe/pull/11963))
+- Handle FileAlreadyAttachedException while pulling email ([#11713](https://github.com/frappe/frappe/pull/11713))
+- Skip Email Account and Email Domain in Document Follow ([#11973](https://github.com/frappe/frappe/pull/11973))
+- URI encode in case white spaces exist in docname ([#11783](https://github.com/frappe/frappe/pull/11783))
+- Don't run a query when table is missing ([#11801](https://github.com/frappe/frappe/pull/11801))
+- Submit on Creation configuration in Auto Repeat ([#12069](https://github.com/frappe/frappe/pull/12069))
+- Add missing space in description ([#11787](https://github.com/frappe/frappe/pull/11787))
+- Query to handle user value having special characters ([#11694](https://github.com/frappe/frappe/pull/11694))
+- TypeError on saving report with child table ([#11925](https://github.com/frappe/frappe/pull/11925))
+- Translate Kanban board title ([#11985](https://github.com/frappe/frappe/pull/11985))
+- Set label as State/Province in Address instead of just State ([#11748](https://github.com/frappe/frappe/pull/11748))
+- Allow "Default Print Language" for custom Print Format ([#11800](https://github.com/frappe/frappe/pull/11800))
+- Validate SQL files better ([#11930](https://github.com/frappe/frappe/pull/11930))
+- Validate links table data ([#11884](https://github.com/frappe/frappe/pull/11884))
+- Use html.unescape for Python 3.9 compatibility. ([#12005](https://github.com/frappe/frappe/pull/12005))
+- Use set_header to set Message-Id in header ([#11778](https://github.com/frappe/frappe/pull/11778))
+- Typo when creating an SMTP server without port ([#11996](https://github.com/frappe/frappe/pull/11996))
+- Add rename_doc utils for external API usages ([#12011](https://github.com/frappe/frappe/pull/12011))
+- Treats the scrollbar as an overlay ([#11790](https://github.com/frappe/frappe/pull/11790))
+- Dashboard not visible bug ([#11918](https://github.com/frappe/frappe/pull/11918))
+- Total Row in Checkbox Column Reports ([#11872](https://github.com/frappe/frappe/pull/11872))
+- Remove scrolling on focusout event for touchscreen devices ([#11888](https://github.com/frappe/frappe/pull/11888))
+- Add default to Web Template Field ([#11780](https://github.com/frappe/frappe/pull/11780))
+- Order of HTML closing tags ([#11923](https://github.com/frappe/frappe/pull/11923))
+- Show total text with value if first column is numeric ([#11813](https://github.com/frappe/frappe/pull/11813))
+- Milestone not created for fields updated after submission ([#11793](https://github.com/frappe/frappe/pull/11793))
+- Cint seconds before operations ([#12067](https://github.com/frappe/frappe/pull/12067))
+- Validate email id before passing to formataddr ([#11720](https://github.com/frappe/frappe/pull/11720))
+- Use frappe.utils.shorten_number ([#12039](https://github.com/frappe/frappe/pull/12039))
+- Mandatory depends on in grid form ([#11834](https://github.com/frappe/frappe/pull/11834))
+- Calculate chart data from beginning of period ([#11794](https://github.com/frappe/frappe/pull/11794))
+- Move unnecessary compileall outside migrate ([#11833](https://github.com/frappe/frappe/pull/11833))
+- PDF generation shouldn't fail in background jobs and tests ([#11792](https://github.com/frappe/frappe/pull/11792))
+- Throw exception if template not found ([#11843](https://github.com/frappe/frappe/pull/11843))
+- Security upgrade snyk from 1.398.1 to 1.425.4 ([#11990](https://github.com/frappe/frappe/pull/11990))
+- Show custom message for invalid login credentials ([#11853](https://github.com/frappe/frappe/pull/11853))
+- Remove twilio integration ([#11841](https://github.com/frappe/frappe/pull/11841))
+- Navbar logo height and width ([#11822](https://github.com/frappe/frappe/pull/11822))
+- Allow all Data, Link, Dynamic Link fields to be set in based on field ([#11922](https://github.com/frappe/frappe/pull/11922))
+- Error on trying to check semantic version ([#11916](https://github.com/frappe/frappe/pull/11916))
+- Notification settings ([#11862](https://github.com/frappe/frappe/pull/11862))
+- Manage private images via get_local_image ([#11935](https://github.com/frappe/frappe/pull/11935))
+- Filter dashboards, dashboard charts, number cards by modules ([#12057](https://github.com/frappe/frappe/pull/12057))
+- Label for _assign field ([#12038](https://github.com/frappe/frappe/pull/12038))
+- Bypass validation if force is passed ([#11915](https://github.com/frappe/frappe/pull/11915))
+- Max slices for aggregate charts ([#11808](https://github.com/frappe/frappe/pull/11808))
+- Delete prepared reports in batches ([#11869](https://github.com/frappe/frappe/pull/11869))
+- Allow empty type for Web Templates that are not sections ([#11628](https://github.com/frappe/frappe/pull/11628))
+- Allow other github links in same PR ([#11982](https://github.com/frappe/frappe/pull/11982))
+- Don't set context.no_cache ([#11799](https://github.com/frappe/frappe/pull/11799))
+- Remove telephony related code ([#12017](https://github.com/frappe/frappe/pull/12017))
+- Don't throw if filter is invalid ([#11866](https://github.com/frappe/frappe/pull/11866))
+- Show 0 instead of undefined when value not set in dashboard ([#11816](https://github.com/frappe/frappe/pull/11816))
+- Disable chart form condition ([#11844](https://github.com/frappe/frappe/pull/11844))
+- Enable disable save when navigating between docs ([#11867](https://github.com/frappe/frappe/pull/11867))
+- Don't enqueue a job if it is being executed ([#11655](https://github.com/frappe/frappe/pull/11655))
+- Clear localstorage if quota exceeds ([#12002](https://github.com/frappe/frappe/pull/12002))
+- Uninstall app enhancements ([#11911](https://github.com/frappe/frappe/pull/11911))
+- Make route and action readonly for standard navbar items ([#11842](https://github.com/frappe/frappe/pull/11842))
+- Show scrollbar for datatable ([#11910](https://github.com/frappe/frappe/pull/11910))
+- Open console even if Frappe imports failed ([#11832](https://github.com/frappe/frappe/pull/11832))
+- Set user selected timezone in user defaults ([#11902](https://github.com/frappe/frappe/pull/11902))
+- Email password prompt missing field name for submit button ([#11840](https://github.com/frappe/frappe/pull/11840))
+- Add semicolons to end unicode  ([#11993](https://github.com/frappe/frappe/pull/11993))
+- Replace target field in Top Bar Item table with a checkbox ([#11763](https://github.com/frappe/frappe/pull/11763))
+- shorten_number function ([#12050](https://github.com/frappe/frappe/pull/12050))
+- Not able to save Domain Settings ([#11984](https://github.com/frappe/frappe/pull/11984))
+- uninstall-app enhancements ([#11969](https://github.com/frappe/frappe/pull/11969))
+- Remove @ from relevance query ([#11837](https://github.com/frappe/frappe/pull/11837))
+- Login Code Size too Small on Mobile ([#11742](https://github.com/frappe/frappe/pull/11742))
+- Add namespaces to build_summary_item, generate_route, short… ([#11868](https://github.com/frappe/frappe/pull/11868))
+- allow "only image" in comments ([#11914](https://github.com/frappe/frappe/pull/11914))
+- oauth2 ([#11966](https://github.com/frappe/frappe/pull/11966))
+- Server scripts enhancements ([#12008](https://github.com/frappe/frappe/pull/12008))
+- Multiple y rows in charts ([#12031](https://github.com/frappe/frappe/pull/12031))
+- Update child values for existing rows ([#11737](https://github.com/frappe/frappe/pull/11737))
+- Auto contact creation in email account ([#11732](https://github.com/frappe/frappe/pull/11732))
+- Display web template after save ([#11809](https://github.com/frappe/frappe/pull/11809))
+- Create auto_repeat field only if docfield/custom field does not exist ([#11827](https://github.com/frappe/frappe/pull/11827))
+- frappe.utils.formatdate not working in the jinja template ([#11871](https://github.com/frappe/frappe/pull/11871))

frappe/chat/util/util.py

@@ -1,27 +1,21 @@
 from __future__ import unicode_literals
 
-# imports - third-party imports
-import requests
-
-# imports - compatibility imports
-import six
-
 # imports - standard imports
-from   collections import Sequence, MutableSequence, Mapping, MutableMapping
-if six.PY2:
-	from urlparse import urlparse 	  # PY2
-else:
-	from urllib.parse import urlparse # PY3
 import json
+from collections.abc import MutableMapping, MutableSequence, Sequence
+
+# imports - third-party imports
+import requests
+from urllib.parse import urlparse
 
 # imports - module imports
-from   frappe.model.document import Document
-from   frappe.exceptions import DuplicateEntryError
-from   frappe import _dict
 import frappe
+from frappe.exceptions import DuplicateEntryError
+from frappe.model.document import Document
 
 session = frappe.session
 
+
 def get_user_doc(user = None):
 	if isinstance(user, Document):
 		return user
@@ -38,12 +32,12 @@ def squashify(what):
 	return what
 
 def safe_json_loads(*args):
-	results = [ ]
+	results = []
 
 	for arg in args:
 		try:
 			arg = json.loads(arg)
-		except Exception as e:
+		except Exception:
 			pass
 
 		results.append(arg)
@@ -81,7 +75,7 @@ def dictify(arg):
 		for i, a in enumerate(arg):
 			arg[i] = dictify(a)
 	elif isinstance(arg, MutableMapping):
-		arg = _dict(arg)
+		arg = frappe._dict(arg)
 
 	return arg
 
@@ -113,4 +107,4 @@ def get_emojis():
 			emojis = resp.json()
 			redis.hset('frappe_emojis', 'emojis', emojis)
 
-	return dictify(emojis)
+	return dictify(emojis)

frappe/commands/site.py

@@ -9,7 +9,7 @@ import click
 import frappe
 from frappe.commands import get_site, pass_context
 from frappe.exceptions import SiteNotSpecifiedError
-from frappe.utils import get_site_path, touch_file
+from frappe.installer import _new_site
 
 
 @click.command('new-site')
@@ -42,57 +42,6 @@ def new_site(site, mariadb_root_username=None, mariadb_root_password=None, admin
 	if len(frappe.utils.get_sites()) == 1:
 		use(site)
 
-def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=None,
-			  admin_password=None, verbose=False, install_apps=None, source_sql=None, force=False,
-			  no_mariadb_socket=False, reinstall=False,  db_password=None, db_type=None, db_host=None,
-			  db_port=None, new_site=False):
-	"""Install a new Frappe site"""
-
-	if not force and os.path.exists(site):
-		print('Site {0} already exists'.format(site))
-		sys.exit(1)
-
-	if no_mariadb_socket and not db_type == "mariadb":
-		print('--no-mariadb-socket requires db_type to be set to mariadb.')
-		sys.exit(1)
-
-	if not db_name:
-		import hashlib
-		db_name = '_' + hashlib.sha1(site.encode()).hexdigest()[:16]
-
-	from frappe.commands.scheduler import _is_scheduler_enabled
-	from frappe.installer import install_db, make_site_dirs
-	from frappe.installer import install_app as _install_app
-	import frappe.utils.scheduler
-
-	frappe.init(site=site)
-
-	try:
-
-		# enable scheduler post install?
-		enable_scheduler = _is_scheduler_enabled()
-	except Exception:
-		enable_scheduler = False
-
-	make_site_dirs()
-
-	installing = touch_file(get_site_path('locks', 'installing.lock'))
-
-	install_db(root_login=mariadb_root_username, root_password=mariadb_root_password, db_name=db_name,
-		admin_password=admin_password, verbose=verbose, source_sql=source_sql, force=force, reinstall=reinstall,
-		db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port, no_mariadb_socket=no_mariadb_socket)
-	apps_to_install = ['frappe'] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
-	for app in apps_to_install:
-		_install_app(app, verbose=verbose, set_as_patched=not source_sql)
-
-	os.remove(installing)
-
-	frappe.utils.scheduler.toggle_scheduler(enable_scheduler)
-	frappe.db.commit()
-
-	scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
-	print("*** Scheduler is", scheduler_status, "***")
-
 
 @click.command('restore')
 @click.argument('sql-file-path')
@@ -103,36 +52,45 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N
 @click.option('--install-app', multiple=True, help='Install app after installation')
 @click.option('--with-public-files', help='Restores the public files of the site, given path to its tar file')
 @click.option('--with-private-files', help='Restores the private files of the site, given path to its tar file')
-@click.option('--force', is_flag=True, default=False, help='Ignore the site downgrade warning, if applicable')
+@click.option('--force', is_flag=True, default=False, help='Ignore the validations and downgrade warnings. This action is not recommended')
 @pass_context
 def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None):
 	"Restore site database from an sql file"
-	from frappe.installer import extract_sql_gzip, extract_files, is_downgrade
-	force = context.force or force
+	from frappe.installer import (
+		extract_sql_from_archive,
+		extract_files,
+		is_downgrade,
+		is_partial,
+		validate_database_sql
+	)
 
-	# Extract the gzip file if user has passed *.sql.gz file instead of *.sql file
-	if not os.path.exists(sql_file_path):
-		base_path = '..'
-		sql_file_path = os.path.join(base_path, sql_file_path)
-		if not os.path.exists(sql_file_path):
-			print('Invalid path {0}'.format(sql_file_path[3:]))
-			sys.exit(1)
-	elif sql_file_path.startswith(os.sep):
-		base_path = os.sep
-	else:
-		base_path = '.'
+	force = context.force or force
+	decompressed_file_name = extract_sql_from_archive(sql_file_path)
+
+	# check if partial backup
+	if is_partial(decompressed_file_name):
+		click.secho(
+			"Partial Backup file detected. You cannot use a partial file to restore a Frappe Site.",
+			fg="red"
+		)
+		click.secho(
+			"Use `bench partial-restore` to restore a partial backup to an existing site.",
+			fg="yellow"
+		)
+		sys.exit(1)
 
-	if sql_file_path.endswith('sql.gz'):
-		decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path))
-	else:
-		decompressed_file_name = sql_file_path
+	# check if valid SQL file
+	validate_database_sql(decompressed_file_name, _raise=not force)
 
 	site = get_site(context)
 	frappe.init(site=site)
 
 	# dont allow downgrading to older versions of frappe without force
 	if not force and is_downgrade(decompressed_file_name, verbose=True):
-		warn_message = "This is not recommended and may lead to unexpected behaviour. Do you want to continue anyway?"
+		warn_message = (
+			"This is not recommended and may lead to unexpected behaviour. "
+			"Do you want to continue anyway?"
+		)
 		click.confirm(warn_message, abort=True)
 
 	_new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username,
@@ -142,22 +100,39 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
 
 	# Extract public and/or private files to the restored site, if user has given the path
 	if with_public_files:
-		with_public_files = os.path.join(base_path, with_public_files)
-		public = extract_files(site, with_public_files, 'public')
+		public = extract_files(site, with_public_files)
 		os.remove(public)
 
 	if with_private_files:
-		with_private_files = os.path.join(base_path, with_private_files)
-		private = extract_files(site, with_private_files, 'private')
+		private = extract_files(site, with_private_files)
 		os.remove(private)
 
 	# Removing temporarily created file
 	if decompressed_file_name != sql_file_path:
 		os.remove(decompressed_file_name)
 
-	success_message = "Site {0} has been restored{1}".format(site, " with files" if (with_public_files or with_private_files) else "")
+	success_message = "Site {0} has been restored{1}".format(
+		site,
+		" with files" if (with_public_files or with_private_files) else ""
+	)
 	click.secho(success_message, fg="green")
 
+
+@click.command('partial-restore')
+@click.argument('sql-file-path')
+@click.option("--verbose", "-v", is_flag=True)
+@pass_context
+def partial_restore(context, sql_file_path, verbose):
+	from frappe.installer import partial_restore
+	verbose = context.verbose or verbose
+
+	site = get_site(context)
+	frappe.init(site=site)
+	frappe.connect(site=site)
+	partial_restore(sql_file_path, verbose)
+	frappe.destroy()
+
+
 @click.command('reinstall')
 @click.option('--admin-password', help='Administrator Password for reinstalled site')
 @click.option('--mariadb-root-username', help='Root username for MariaDB')
@@ -222,15 +197,51 @@ def install_app(context, apps):
 	sys.exit(exit_code)
 
 
-@click.command('list-apps')
+@click.command("list-apps")
 @pass_context
 def list_apps(context):
 	"List apps in site"
-	site = get_site(context)
-	frappe.init(site=site)
-	frappe.connect()
-	print("\n".join(frappe.get_installed_apps()))
-	frappe.destroy()
+
+	def fix_whitespaces(text):
+		if site == context.sites[-1]:
+			text = text.rstrip()
+		if len(context.sites) == 1:
+			text = text.lstrip()
+		return text
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+		site_title = (
+			click.style(f"{site}", fg="green") if len(context.sites) > 1 else ""
+		)
+		apps = frappe.get_single("Installed Applications").installed_applications
+
+		if apps:
+			name_len, ver_len = [
+				max([len(x.get(y)) for x in apps])
+				for y in ["app_name", "app_version"]
+			]
+			template = "{{0:{0}}} {{1:{1}}} {{2}}".format(name_len, ver_len)
+
+			installed_applications = [
+				template.format(app.app_name, app.app_version, app.git_branch)
+				for app in apps
+			]
+			applications_summary = "\n".join(installed_applications)
+			summary = f"{site_title}\n{applications_summary}\n"
+
+		else:
+			applications_summary = "\n".join(frappe.get_installed_apps())
+			summary = f"{site_title}\n{applications_summary}\n"
+
+		summary = fix_whitespaces(summary)
+
+		if applications_summary and summary:
+			print(summary)
+
+		frappe.destroy()
+
 
 @click.command('add-system-manager')
 @click.argument('email')
@@ -265,14 +276,12 @@ def disable_user(context, email):
 		user.save(ignore_permissions=True)
 		frappe.db.commit()
 
-
 @click.command('migrate')
 @click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run")
 @click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents")
 @pass_context
 def migrate(context, skip_failing=False, skip_search_index=False):
 	"Run patches, sync schema and rebuild files/translations"
-	import compileall
 	import re
 	from frappe.migrate import migrate
 
@@ -291,9 +300,6 @@ def migrate(context, skip_failing=False, skip_search_index=False):
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
-	print("Compiling Python files...")
-	compileall.compile_dir('../apps', quiet=1, rx=re.compile('.*node_modules.*'))
-
 @click.command('migrate-to')
 @click.argument('frappe_provider')
 @pass_context
@@ -310,15 +316,16 @@ def migrate_to(context, frappe_provider):
 
 @click.command('run-patch')
 @click.argument('module')
+@click.option('--force', is_flag=True)
 @pass_context
-def run_patch(context, module):
+def run_patch(context, module, force):
 	"Run a particular patch"
 	import frappe.modules.patch_handler
 	for site in context.sites:
 		frappe.init(site=site)
 		try:
 			frappe.connect()
-			frappe.modules.patch_handler.run_single(module, force=context.force)
+			frappe.modules.patch_handler.run_single(module, force=force or context.force)
 		finally:
 			frappe.destroy()
 	if not context.sites:
@@ -383,16 +390,20 @@ def use(site, sites_path='.'):
 
 @click.command('backup')
 @click.option('--with-files', default=False, is_flag=True, help="Take backup with files")
+@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas")
+@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas")
 @click.option('--backup-path', default=None, help="Set path for saving all the files in this operation")
 @click.option('--backup-path-db', default=None, help="Set path for saving database file")
 @click.option('--backup-path-files', default=None, help="Set path for saving public file")
 @click.option('--backup-path-private-files', default=None, help="Set path for saving private file")
 @click.option('--backup-path-conf', default=None, help="Set path for saving config file")
+@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config")
 @click.option('--verbose', default=False, is_flag=True, help="Add verbosity")
 @click.option('--compress', default=False, is_flag=True, help="Compress private and public files")
 @pass_context
 def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None,
-	backup_path_private_files=None, backup_path_conf=None, verbose=False, compress=False):
+	backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False,
+	compress=False, include="", exclude=""):
 	"Backup"
 	from frappe.utils.backups import scheduled_backup
 	verbose = verbose or context.verbose
@@ -402,11 +413,27 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac
 		try:
 			frappe.init(site=site)
 			frappe.connect()
-			odb = scheduled_backup(ignore_files=not with_files, backup_path=backup_path, backup_path_db=backup_path_db, backup_path_files=backup_path_files, backup_path_private_files=backup_path_private_files, backup_path_conf=backup_path_conf, force=True, verbose=verbose, compress=compress)
+			odb = scheduled_backup(
+				ignore_files=not with_files,
+				backup_path=backup_path,
+				backup_path_db=backup_path_db,
+				backup_path_files=backup_path_files,
+				backup_path_private_files=backup_path_private_files,
+				backup_path_conf=backup_path_conf,
+				ignore_conf=ignore_backup_conf,
+				include_doctypes=include,
+				exclude_doctypes=exclude,
+				compress=compress,
+				verbose=verbose,
+				force=True
+			)
 		except Exception:
 			click.secho("Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), fg="red")
+			if verbose:
+				print(frappe.get_traceback())
 			exit_code = 1
 			continue
+
 		odb.print_summary()
 		click.secho("Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), fg="green")
 		frappe.destroy()
@@ -479,13 +506,14 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path=
 		if force:
 			pass
 		else:
-			click.echo("="*80)
-			click.echo("Error: The operation has stopped because backup of {s}'s database failed.".format(s=site))
-			click.echo("Reason: {reason}{sep}".format(reason=str(err), sep="\n"))
-			click.echo("Fix the issue and try again.")
-			click.echo(
-				"Hint: Use 'bench drop-site {s} --force' to force the removal of {s}".format(sep="\n", tab="\t", s=site)
-			)
+			messages = [
+				"=" * 80,
+				"Error: The operation has stopped because backup of {0}'s database failed.".format(site),
+				"Reason: {0}\n".format(str(err)),
+				"Fix the issue and try again.",
+				"Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site)
+			]
+			click.echo("\n".join(messages))
 			sys.exit(1)
 
 	drop_user_and_database(frappe.conf.db_name, root_login, root_password)
@@ -615,8 +643,10 @@ def browse(context, site):
 @click.command('start-recording')
 @pass_context
 def start_recording(context):
+	import frappe.recorder
 	for site in context.sites:
 		frappe.init(site=site)
+		frappe.set_user("Administrator")
 		frappe.recorder.start()
 	if not context.sites:
 		raise SiteNotSpecifiedError
@@ -625,8 +655,10 @@ def start_recording(context):
 @click.command('stop-recording')
 @pass_context
 def stop_recording(context):
+	import frappe.recorder
 	for site in context.sites:
 		frappe.init(site=site)
+		frappe.set_user("Administrator")
 		frappe.recorder.stop()
 	if not context.sites:
 		raise SiteNotSpecifiedError
@@ -697,5 +729,6 @@ commands = [
 	stop_recording,
 	add_to_hosts,
 	start_ngrok,
-	build_search_index
+	build_search_index,
+	partial_restore
 ]

frappe/commands/utils.py

@@ -460,11 +460,21 @@ def console(context):
 	frappe.init(site=site)
 	frappe.connect()
 	frappe.local.lang = frappe.db.get_default("lang")
+
 	import IPython
 	all_apps = frappe.get_installed_apps()
+	failed_to_import = []
+
 	for app in all_apps:
-		locals()[app] = __import__(app)
+		try:
+			locals()[app] = __import__(app)
+		except ModuleNotFoundError:
+			failed_to_import.append(app)
+
 	print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
+	if failed_to_import:
+		print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
+
 	IPython.embed(display_banner="", header="", colors="neutral")
 
 

frappe/config/customization.py

@@ -54,12 +54,6 @@ def get_data():
 					"label": _("Custom Translations"),
 					"name": "Translation",
 					"description": _("Add your own translations")
-				},
-				{
-					"type": "doctype",
-					"label": _("Package"),
-					"name": "Package",
-					"description": _("Import and Export Packages.")
 				}
 			]
 		}

frappe/contacts/doctype/address/address.json

@@ -75,7 +75,7 @@
   {
    "fieldname": "state",
    "fieldtype": "Data",
-   "label": "State"
+   "label": "State/Province"
   },
   {
    "fieldname": "country",
@@ -148,7 +148,7 @@
  "icon": "fa fa-map-marker",
  "idx": 5,
  "links": [],
- "modified": "2020-10-14 17:38:08.971776",
+ "modified": "2020-10-21 16:14:37.284830",
  "modified_by": "Administrator",
  "module": "Contacts",
  "name": "Address",

frappe/core/doctype/communication/communication.py

@@ -260,10 +260,8 @@ class Communication(Document):
 	# Timeline Links
 	def set_timeline_links(self):
 		contacts = []
-		if (self.email_account and frappe.db.get_value("Email Account", self.email_account, "create_contact")) or \
-			frappe.flags.in_test:
-
-			contacts = get_contacts([self.sender, self.recipients, self.cc, self.bcc])
+		create_contact_enabled = self.email_account and frappe.db.get_value("Email Account", self.email_account, "create_contact")
+		contacts = get_contacts([self.sender, self.recipients, self.cc, self.bcc], auto_create_contact=create_contact_enabled)
 
 		for contact_name in contacts:
 			self.add_link('Contact', contact_name)
@@ -342,7 +340,7 @@ def get_permission_query_conditions_for_communication(user):
 		return """`tabCommunication`.email_account in ({email_accounts})"""\
 			.format(email_accounts=','.join(email_accounts))
 
-def get_contacts(email_strings):
+def get_contacts(email_strings, auto_create_contact=False):
 	email_addrs = []
 
 	for email_string in email_strings:
@@ -357,7 +355,7 @@ def get_contacts(email_strings):
 		email = get_email_without_link(email)
 		contact_name = get_contact_name(email)
 
-		if not contact_name and email:
+		if not contact_name and email and auto_create_contact:
 			email_parts = email.split("@")
 			first_name = frappe.unscrub(email_parts[0])
 

frappe/core/doctype/docfield/docfield.json

@@ -13,6 +13,7 @@
   "fieldname",
   "precision",
   "length",
+  "non_negative",
   "hide_days",
   "hide_seconds",
   "reqd",
@@ -473,13 +474,20 @@
    "fieldname": "hide_border",
    "fieldtype": "Check",
    "label": "Hide Border"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:in_list([\"Int\", \"Float\", \"Currency\"], doc.fieldtype)",
+   "fieldname": "non_negative",
+   "fieldtype": "Check",
+   "label": "Non Negative"
   }
  ],
  "idx": 1,
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-08-28 11:28:21.252853",
+ "modified": "2020-10-29 06:09:26.454990",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocField",

frappe/core/doctype/doctype/doctype.py

@@ -56,7 +56,8 @@ class DocType(Document):
 		- Check fieldnames (duplication etc)
 		- Clear permission table for child tables
 		- Add `amended_from` and `amended_by` if Amendable
-		- Add custom field `auto_repeat` if Repeatable"""
+		- Add custom field `auto_repeat` if Repeatable
+		- Check if links point to valid fieldnames"""
 
 		self.check_developer_mode()
 
@@ -88,6 +89,7 @@ class DocType(Document):
 		self.make_repeatable()
 		self.validate_nestedset()
 		self.validate_website()
+		self.validate_links_table_fieldnames()
 
 		if not self.is_new():
 			self.before_update = frappe.get_doc('DocType', self.name)
@@ -392,7 +394,10 @@ class DocType(Document):
 			frappe.db.sql("""update tabSingles set value=%s
 				where doctype=%s and field='name' and value = %s""", (new, new, old))
 		else:
-			frappe.db.sql("rename table `tab%s` to `tab%s`" % (old, new))
+			frappe.db.multisql({
+				"mariadb": f"RENAME TABLE `tab{old}` TO `tab{new}`",
+				"postgres": f"ALTER TABLE `tab{old}` RENAME TO `tab{new}`"
+			})
 
 	def rename_files_and_folders(self, old, new):
 		# move files
@@ -657,6 +662,19 @@ class DocType(Document):
 		if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
 			frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
 
+	def validate_links_table_fieldnames(self):
+		"""Validate fieldnames in Links table"""
+		if frappe.flags.in_patch: return
+		if frappe.flags.in_fixtures: return
+		if not self.links: return
+
+		for index, link in enumerate(self.links):
+			meta = frappe.get_meta(link.link_doctype)
+			if not meta.get_field(link.link_fieldname):
+				message = _("Row #{0}: Could not find field {1} in {2} DocType").format(index+1, frappe.bold(link.link_fieldname), frappe.bold(link.link_doctype))
+				frappe.throw(message, InvalidFieldNameError, _("Invalid Fieldname"))
+
+
 
 def validate_fields_for_doctype(doctype):
 	doc = frappe.get_doc("DocType", doctype)
@@ -753,8 +771,8 @@ def validate_fields(meta):
 	def check_illegal_default(d):
 		if d.fieldtype == "Check" and not d.default:
 			d.default = '0'
-		if d.fieldtype == "Check" and d.default not in ('0', '1'):
-			frappe.throw(_("Default for 'Check' type of field must be either '0' or '1'"))
+		if d.fieldtype == "Check" and cint(d.default) not in (0, 1):
+			frappe.throw(_("Default for 'Check' type of field {0} must be either '0' or '1'").format(frappe.bold(d.fieldname)))
 		if d.fieldtype == "Select" and d.default:
 			if not d.options:
 				frappe.throw(_("Options for {0} must be set before setting the default value.").format(frappe.bold(d.fieldname)))

frappe/core/doctype/doctype/test_doctype.py

@@ -12,41 +12,22 @@ from frappe.core.doctype.doctype.doctype import UniqueFieldnameError, IllegalMan
 
 
 class TestDocType(unittest.TestCase):
-	def new_doctype(self, name, unique=0, depends_on=''):
-		return frappe.get_doc({
-			"doctype": "DocType",
-			"module": "Core",
-			"custom": 1,
-			"fields": [{
-				"label": "Some Field",
-				"fieldname": "some_fieldname",
-				"fieldtype": "Data",
-				"unique": unique,
-				"depends_on": depends_on,
-			}],
-			"permissions": [{
-				"role": "System Manager",
-				"read": 1,
-			}],
-			"name": name
-		})
-
 	def test_validate_name(self):
-		self.assertRaises(frappe.NameError, self.new_doctype("_Some DocType").insert)
-		self.assertRaises(frappe.NameError, self.new_doctype("8Some DocType").insert)
-		self.assertRaises(frappe.NameError, self.new_doctype("Some (DocType)").insert)
+		self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert)
+		self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert)
+		self.assertRaises(frappe.NameError, new_doctype("Some (DocType)").insert)
 		for name in ("Some DocType", "Some_DocType"):
 			if frappe.db.exists("DocType", name):
 				frappe.delete_doc("DocType", name)
 
-			doc = self.new_doctype(name).insert()
+			doc = new_doctype(name).insert()
 			doc.delete()
 
 	def test_doctype_unique_constraint_dropped(self):
 		if frappe.db.exists("DocType", "With_Unique"):
 			frappe.delete_doc("DocType", "With_Unique")
 
-		dt = self.new_doctype("With_Unique", unique=1)
+		dt = new_doctype("With_Unique", unique=1)
 		dt.insert()
 
 		doc1 = frappe.new_doc("With_Unique")
@@ -67,7 +48,7 @@ class TestDocType(unittest.TestCase):
 		doc2.delete()
 
 	def test_validate_search_fields(self):
-		doc = self.new_doctype("Test Search Fields")
+		doc = new_doctype("Test Search Fields")
 		doc.search_fields = "some_fieldname"
 		doc.insert()
 		self.assertEqual(doc.name, "Test Search Fields")
@@ -85,7 +66,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(frappe.ValidationError, doc.save)
 
 	def test_depends_on_fields(self):
-		doc = self.new_doctype("Test Depends On", depends_on="eval:doc.__islocal == 0")
+		doc = new_doctype("Test Depends On", depends_on="eval:doc.__islocal == 0")
 		doc.insert()
 
 		# check if the assignment operation is allowed in depends_on
@@ -261,7 +242,7 @@ class TestDocType(unittest.TestCase):
 			frappe.flags.allow_doctype_export = 0
 
 	def test_unique_field_name_for_two_fields(self):
-		doc = self.new_doctype('Test Unique Field')
+		doc = new_doctype('Test Unique Field')
 		field_1 = doc.append('fields', {})
 		field_1.fieldname  = 'some_fieldname_1'
 		field_1.fieldtype = 'Data'
@@ -273,7 +254,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(UniqueFieldnameError, doc.insert)
 
 	def test_fieldname_is_not_name(self):
-		doc = self.new_doctype('Test Name Field')
+		doc = new_doctype('Test Name Field')
 		field_1 = doc.append('fields', {})
 		field_1.label  = 'Name'
 		field_1.fieldtype = 'Data'
@@ -283,7 +264,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(InvalidFieldNameError, doc.save)
 
 	def test_illegal_mandatory_validation(self):
-		doc = self.new_doctype('Test Illegal mandatory')
+		doc = new_doctype('Test Illegal mandatory')
 		field_1 = doc.append('fields', {})
 		field_1.fieldname  = 'some_fieldname_1'
 		field_1.fieldtype = 'Section Break'
@@ -292,7 +273,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(IllegalMandatoryError, doc.insert)
 
 	def test_link_with_wrong_and_no_options(self):
-		doc = self.new_doctype('Test link')
+		doc = new_doctype('Test link')
 		field_1 = doc.append('fields', {})
 		field_1.fieldname  = 'some_fieldname_1'
 		field_1.fieldtype = 'Link'
@@ -304,7 +285,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(WrongOptionsDoctypeLinkError, doc.insert)
 
 	def test_hidden_and_mandatory_without_default(self):
-		doc = self.new_doctype('Test hidden and mandatory')
+		doc = new_doctype('Test hidden and mandatory')
 		field_1 = doc.append('fields', {})
 		field_1.fieldname  = 'some_fieldname_1'
 		field_1.fieldtype = 'Data'
@@ -314,7 +295,7 @@ class TestDocType(unittest.TestCase):
 		self.assertRaises(HiddenAndMandatoryWithoutDefaultError, doc.insert)
 
 	def test_field_can_not_be_indexed_validation(self):
-		doc = self.new_doctype('Test index')
+		doc = new_doctype('Test index')
 		field_1 = doc.append('fields', {})
 		field_1.fieldname  = 'some_fieldname_1'
 		field_1.fieldtype = 'Long Text'
@@ -327,14 +308,14 @@ class TestDocType(unittest.TestCase):
 		from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs
 
 		#create doctype
-		link_doc = self.new_doctype('Test Linked Doctype')
+		link_doc = new_doctype('Test Linked Doctype')
 		link_doc.is_submittable = 1
 		for data in link_doc.get('permissions'):
 			data.submit = 1
 			data.cancel = 1
 		link_doc.insert()
 
-		doc = self.new_doctype('Test Doctype')
+		doc = new_doctype('Test Doctype')
 		doc.is_submittable = 1
 		field_2 = doc.append('fields', {})
 		field_2.label = 'Test Linked Doctype'
@@ -377,12 +358,12 @@ class TestDocType(unittest.TestCase):
 		doc.delete()
 		frappe.db.commit()
 
-	def test_ignore_cancelation_of_linked_doctype_during_cancell(self):
+	def test_ignore_cancelation_of_linked_doctype_during_cancel(self):
 		import json
 		from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs
 
 		#create linked doctype
-		link_doc = self.new_doctype('Test Linked Doctype 1')
+		link_doc = new_doctype('Test Linked Doctype 1')
 		link_doc.is_submittable = 1
 		for data in link_doc.get('permissions'):
 			data.submit = 1
@@ -390,7 +371,7 @@ class TestDocType(unittest.TestCase):
 		link_doc.insert()
 
 		#create first parent doctype
-		test_doc_1 = self.new_doctype('Test Doctype 1')
+		test_doc_1 = new_doctype('Test Doctype 1')
 		test_doc_1.is_submittable = 1
 
 		field_2 = test_doc_1.append('fields', {})
@@ -405,7 +386,7 @@ class TestDocType(unittest.TestCase):
 		test_doc_1.insert()
 
 		#crete second parent doctype
-		doc = self.new_doctype('Test Doctype 2')
+		doc = new_doctype('Test Doctype 2')
 		doc.is_submittable = 1
 
 		field_2 = doc.append('fields', {})
@@ -469,3 +450,55 @@ class TestDocType(unittest.TestCase):
 		doc.delete()
 		test_doc_1.delete()
 		frappe.db.commit()
+
+	def test_links_table_fieldname_validation(self):
+		doc = new_doctype("Test Links Table Validation")
+
+		# check valid data
+		doc.append("links", {
+			'link_doctype': "User",
+			'link_fieldname': "first_name"
+		})
+		doc.validate_links_table_fieldnames() # no error
+		doc.links = [] # reset links table
+
+		# check invalid doctype
+		doc.append("links", {
+			'link_doctype': "User2",
+			'link_fieldname': "first_name"
+		})
+		self.assertRaises(frappe.DoesNotExistError, doc.validate_links_table_fieldnames)
+		doc.links = [] # reset links table
+
+		# check invalid fieldname
+		doc.append("links", {
+			'link_doctype': "User",
+			'link_fieldname': "a_field_that_does_not_exists"
+		})
+		self.assertRaises(InvalidFieldNameError, doc.validate_links_table_fieldnames)
+
+
+def new_doctype(name, unique=0, depends_on='', fields=None):
+	doc = frappe.get_doc({
+		"doctype": "DocType",
+		"module": "Core",
+		"custom": 1,
+		"fields": [{
+			"label": "Some Field",
+			"fieldname": "some_fieldname",
+			"fieldtype": "Data",
+			"unique": unique,
+			"depends_on": depends_on,
+		}],
+		"permissions": [{
+			"role": "System Manager",
+			"read": 1,
+		}],
+		"name": name
+	})
+
+	if fields:
+		for f in fields:
+			doc.append('fields', f)
+
+	return doc

frappe/core/doctype/doctype_action/doctype_action.json

@@ -9,7 +9,8 @@
   "action_type",
   "action",
   "group",
-  "hidden"
+  "hidden",
+  "custom"
  ],
  "fields": [
   {
@@ -48,12 +49,19 @@
    "fieldname": "hidden",
    "fieldtype": "Check",
    "label": "Hidden"
+  },
+  {
+   "default": "0",
+   "fieldname": "custom",
+   "fieldtype": "Check",
+   "hidden": 1,
+   "label": "Custom"
   }
  ],
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-08-21 14:44:03.845315",
+ "modified": "2020-09-24 14:19:05.549835",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType Action",

frappe/core/doctype/doctype_link/doctype_link.json

@@ -7,7 +7,9 @@
  "field_order": [
   "link_doctype",
   "link_fieldname",
-  "group"
+  "group",
+  "hidden",
+  "custom"
  ],
  "fields": [
   {
@@ -30,10 +32,25 @@
    "fieldtype": "Data",
    "in_list_view": 1,
    "label": "Group"
+  },
+  {
+   "default": "0",
+   "fieldname": "hidden",
+   "fieldtype": "Check",
+   "label": "Hidden"
+  },
+  {
+   "default": "0",
+   "fieldname": "custom",
+   "fieldtype": "Check",
+   "hidden": 1,
+   "label": "Custom"
   }
  ],
+ "index_web_pages_for_search": 1,
  "istable": 1,
- "modified": "2019-09-24 11:41:25.291377",
+ "links": [],
+ "modified": "2020-09-24 14:19:25.189511",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType Link",

frappe/core/doctype/domain_settings/domain_settings.js

@@ -18,6 +18,9 @@ frappe.ui.form.on('Domain Settings', {
 								checked: active_domains.includes(domain)
 							};
 						});
+					},
+					on_change: () => {
+						frm.dirty();
 					}
 				},
 				render_input: true

frappe/core/doctype/file/file.py

@@ -30,7 +30,7 @@ import frappe
 from frappe import _, conf
 from frappe.model.document import Document
 from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, get_hook_method, random_string, strip
-
+from frappe.utils.image import strip_exif_data
 
 class MaxFileSizeReachedError(frappe.ValidationError):
 	pass
@@ -93,6 +93,7 @@ class File(Document):
 			self.set_is_private()
 			self.set_file_name()
 			self.validate_duplicate_entry()
+			self.validate_attachment_limit()
 		self.validate_folder()
 
 		if not self.file_url and not self.flags.ignore_file_validate:
@@ -140,6 +141,26 @@ class File(Document):
 		if self.file_url and (self.is_private != self.file_url.startswith('/private')):
 			frappe.throw(_('Invalid file URL. Please contact System Administrator.'))
 
+	def validate_attachment_limit(self):
+		attachment_limit = 0
+		if self.attached_to_doctype and self.attached_to_name:
+			attachment_limit = cint(frappe.get_meta(self.attached_to_doctype).max_attachments)
+
+		if attachment_limit:
+			current_attachment_count = len(frappe.get_all('File', filters={
+				'attached_to_doctype': self.attached_to_doctype,
+				'attached_to_name': self.attached_to_name,
+			}, limit=attachment_limit + 1))
+
+			if current_attachment_count >= attachment_limit:
+				frappe.throw(
+					_("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format(
+						frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name
+					),
+					exc=frappe.exceptions.AttachmentLimitReached,
+					title=_('Attachment Limit Reached')
+				)
+
 	def set_folder_name(self):
 		"""Make parent folders if not exists based on reference doctype and name"""
 		if self.attached_to_doctype and not self.folder:
@@ -435,6 +456,7 @@ class File(Document):
 	def save_file(self, content=None, decode=False, ignore_existing_file_check=False):
 		file_exists = False
 		self.content = content
+		
 		if decode:
 			if isinstance(content, text_type):
 				self.content = content.encode("utf-8")
@@ -445,10 +467,19 @@ class File(Document):
 
 		if not self.is_private:
 			self.is_private = 0
-		self.file_size = self.check_max_file_size()
-		self.content_hash = get_content_hash(self.content)
+		
 		self.content_type = mimetypes.guess_type(self.file_name)[0]
+		
+		self.file_size = self.check_max_file_size()
+		
+		if (
+			self.content_type and "image" in self.content_type
+			and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
+		):
+			self.content = strip_exif_data(self.content, self.content_type)			
 
+		self.content_hash = get_content_hash(self.content)
+		
 		duplicate_file = None
 
 		# check if a file exists with the same content hash and is also in the same folder (public or private)
@@ -612,7 +643,12 @@ def get_extension(filename, extn, content):
 	return extn
 
 def get_local_image(file_url):
-	file_path = frappe.get_site_path("public", file_url.lstrip("/"))
+	if file_url.startswith("/private"):
+		file_url_path = (file_url.lstrip("/"), )
+	else:
+		file_url_path = ("public", file_url.lstrip("/"))
+
+	file_path = frappe.get_site_path(*file_url_path)
 
 	try:
 		image = Image.open(file_path)

frappe/core/doctype/file/test_file.py

@@ -160,6 +160,31 @@ class TestSameContent(unittest.TestCase):
 	def test_saved_content(self):
 		self.assertFalse(os.path.exists(get_files_path(self.dup_filename)))
 
+	def test_attachment_limit(self):
+		doctype, docname = make_test_doc()
+		from frappe.custom.doctype.property_setter.property_setter import make_property_setter
+		limit_property = make_property_setter('ToDo', None, 'max_attachments', 1, 'int', for_doctype=True)
+		file1 = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'test-attachment',
+			"attached_to_doctype": doctype,
+			"attached_to_name": docname,
+			"content": 'test'
+		})
+
+		file1.insert()
+
+		file2 = frappe.get_doc({
+			"doctype": "File",
+			"file_name": 'test-attachment',
+			"attached_to_doctype": doctype,
+			"attached_to_name": docname,
+			"content": 'test2'
+		})
+
+		self.assertRaises(frappe.exceptions.AttachmentLimitReached, file2.insert)
+		limit_property.delete()
+		frappe.clear_cache(doctype='ToDo')
 
 	def tearDown(self):
 		# File gets deleted on rollback, so blank

frappe/core/doctype/navbar_item/navbar_item.json

@@ -2,7 +2,6 @@
  "actions": [],
  "creation": "2020-08-01 23:38:41.783206",
  "doctype": "DocType",
- "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
   "item_label",
@@ -30,6 +29,7 @@
    "in_list_view": 1,
    "label": "Item Type",
    "options": "Route\nAction\nSeparator",
+   "read_only_depends_on": "eval:doc.is_standard",
    "show_days": 1,
    "show_seconds": 1
   },
@@ -59,6 +59,7 @@
    "in_list_view": 1,
    "label": "Route",
    "mandatory_depends_on": "eval:doc.item_type == 'Route'",
+   "read_only_depends_on": "eval:doc.is_standard",
    "show_days": 1,
    "show_seconds": 1
   },
@@ -68,13 +69,14 @@
    "fieldtype": "Data",
    "label": "Action",
    "mandatory_depends_on": "eval:doc.item_type == 'Action'",
+   "read_only_depends_on": "eval:doc.is_standard",
    "show_days": 1,
    "show_seconds": 1
   }
  ],
  "istable": 1,
  "links": [],
- "modified": "2020-08-06 16:32:49.597060",
+ "modified": "2020-11-02 10:57:37.709262",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Navbar Item",

frappe/core/doctype/prepared_report/prepared_report.py

@@ -89,20 +89,18 @@ def delete_expired_prepared_reports():
 				'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)]
 			})
 
-		args = {
-			'reports': prepared_reports_to_delete,
-			'limit': 50
-		}
-
-		enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
+		batches = frappe.utils.create_batch(prepared_reports_to_delete, 100)
+		for batch in batches:
+			args = {
+				'reports': batch,
+			}
+			enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args)
 
 @frappe.whitelist()
-def delete_prepared_reports(reports, limit=None):
+def delete_prepared_reports(reports):
 	reports = frappe.parse_json(reports)
-	for index, doc in enumerate(reports):
-		if limit and index == limit:
-			return
-		frappe.delete_doc('Prepared Report', doc['name'], ignore_permissions=True)
+	for report in reports:
+		frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True)
 
 def create_json_gz_file(data, dt, dn):
 	# Storing data in CSV file causes information loss

frappe/core/doctype/report/report.py

@@ -49,8 +49,8 @@ class Report(Document):
 		self.export_doc()
 
 	def on_trash(self):
-		if (self.is_standard == 'Yes' 
-			and not cint(getattr(frappe.local.conf, 'developer_mode', 0)) 
+		if (self.is_standard == 'Yes'
+			and not cint(getattr(frappe.local.conf, 'developer_mode', 0))
 			and not frappe.flags.in_patch):
 			frappe.throw(_("You are not allowed to delete Standard Report"))
 		delete_custom_role('report', self.name)
@@ -61,8 +61,9 @@ class Report(Document):
 	def set_doctype_roles(self):
 		if not self.get('roles') and self.is_standard == 'No':
 			meta = frappe.get_meta(self.ref_doctype)
-			roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
-			self.set('roles', roles)
+			if not meta.istable:
+				roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0]
+				self.set('roles', roles)
 
 	def is_permitted(self):
 		"""Returns true if Has Role is not set or the user is allowed."""

frappe/core/doctype/role/role.py

@@ -37,7 +37,7 @@ class Role(Document):
 def get_info_based_on_role(role, field='email'):
 	''' Get information of all users that have been assigned this role '''
 	users = frappe.get_list("Has Role", filters={"role": role, "parenttype": "User"},
-		fields=["parent"])
+		fields=["parent as user_name"])
 
 	return get_user_info(users, field)
 
@@ -45,7 +45,7 @@ def get_user_info(users, field='email'):
 	''' Fetch details about users for the specified field '''
 	info_list = []
 	for user in users:
-		user_info, enabled = frappe.db.get_value("User", user.parent, [field, "enabled"])
+		user_info, enabled = frappe.db.get_value("User", user.get("user_name"), [field, "enabled"])
 		if enabled and user_info not in ["admin@example.com", "guest@example.com"]:
 			info_list.append(user_info)
 	return info_list

frappe/core/doctype/scheduled_job_type/scheduled_job_type.json

@@ -36,7 +36,7 @@
   },
   {
    "default": "0",
-   "depends_on": "eval:doc.queue==='All'",
+   "depends_on": "eval:doc.frequency==='All'",
    "fieldname": "create_log",
    "fieldtype": "Check",
    "label": "Create Log"
@@ -49,7 +49,7 @@
   },
   {
    "allow_in_quick_entry": 1,
-   "depends_on": "eval:doc.queue==='Cron'",
+   "depends_on": "eval:doc.frequency==='Cron'",
    "fieldname": "cron_format",
    "fieldtype": "Data",
    "label": "Cron Format",
@@ -81,7 +81,7 @@
    "link_fieldname": "scheduled_job_type"
   }
  ],
- "modified": "2020-04-05 17:27:33.480562",
+ "modified": "2020-10-07 10:39:24.519460",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Scheduled Job Type",

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -20,9 +20,9 @@ class ScheduledJobType(Document):
 			# force logging for all events other than continuous ones (ALL)
 			self.create_log = 1
 
-	def enqueue(self):
+	def enqueue(self, force=False):
 		# enqueue event if last execution is done
-		if self.is_event_due():
+		if self.is_event_due() or force:
 			if frappe.flags.enqueued_jobs:
 				frappe.flags.enqueued_jobs.append(self.method)
 
@@ -114,7 +114,7 @@ class ScheduledJobType(Document):
 def execute_event(doc):
 	frappe.only_for('System Manager')
 	doc = json.loads(doc)
-	frappe.get_doc('Scheduled Job Type', doc.get('name')).enqueue()
+	frappe.get_doc('Scheduled Job Type', doc.get('name')).enqueue(force=True)
 
 
 def run_scheduled_job(job_type):

frappe/core/doctype/server_script/server_script.js

@@ -48,29 +48,33 @@ frappe.ui.form.on('Server Script', {
 
 	setup_help(frm) {
 		frm.get_field('help_html').html(`
-<h3>Examples</h3>
 <h4>DocType Event</h4>
-<pre><code>
+<p>Add logic for standard doctype events like Before Insert, After Submit, etc.</p>
+<pre>
+	<code>
 # set property
 if "test" in doc.description:
-    doc.status = 'Closed'
+	doc.status = 'Closed'
 
 
 # validate
 if "validate" in doc.description:
-    raise frappe.ValidationError
+	raise frappe.ValidationError
 
 # auto create another document
-if doc.allocted_to:
-    frappe.get_doc(dict(
-        doctype = 'ToDo'
-        owner = doc.allocated_to,
-        description = doc.subject
-    )).insert()
-</code></pre>
+if doc.allocated_to:
+	frappe.get_doc(dict(
+		doctype = 'ToDo'
+		owner = doc.allocated_to,
+		description = doc.subject
+	)).insert()
+</code>
+</pre>
+
 <hr>
 
 <h4>API Call</h4>
+<p>Respond to <code>/api/method/&lt;method-name&gt;</code> calls, just like whitelisted methods</p>
 <pre><code>
 # respond to API
 
@@ -79,6 +83,21 @@ if frappe.form_dict.message == "ping":
 else:
 	frappe.response['message'] = "ok"
 </code></pre>
+
+<hr>
+
+<h4>Permission Query</h4>
+<p>Add conditions to the where clause of list queries.</p>
+<pre><code>
+# generate dynamic conditions and set it in the conditions variable
+tenant_id = frappe.db.get_value(...)
+conditions = 'tenant_id = {}'.format(tenant_id)
+
+# resulting select query
+select name from \`tabPerson\`
+where tenant_id = 2
+order by creation desc
+</code></pre>
 `);
 	}
 

frappe/core/doctype/server_script/server_script.json

@@ -24,17 +24,18 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Script Type",
-   "options": "DocType Event\nScheduler Event\nAPI",
+   "options": "DocType Event\nScheduler Event\nPermission Query\nAPI",
    "reqd": 1
   },
   {
    "fieldname": "script",
    "fieldtype": "Code",
    "label": "Script",
+   "options": "Python",
    "reqd": 1
   },
   {
-   "depends_on": "eval:doc.script_type==='DocType Event'",
+   "depends_on": "eval:['DocType Event', 'Permission Query'].includes(doc.script_type)",
    "fieldname": "reference_doctype",
    "fieldtype": "Link",
    "in_list_view": 1,
@@ -87,7 +88,7 @@
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-08-24 16:44:41.060350",
+ "modified": "2020-12-03 22:42:02.708148",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Server Script",

frappe/core/doctype/server_script/server_script.py

@@ -4,6 +4,8 @@
 
 from __future__ import unicode_literals
 
+import ast
+
 import frappe
 from frappe.model.document import Document
 from frappe.utils.safe_exec import safe_exec
@@ -11,9 +13,9 @@ from frappe import _
 
 
 class ServerScript(Document):
-	@staticmethod
-	def validate():
+	def validate(self):
 		frappe.only_for('Script Manager', True)
+		ast.parse(self.script)
 
 	@staticmethod
 	def on_update():
@@ -41,6 +43,12 @@ class ServerScript(Document):
 			# wrong report type!
 			raise frappe.DoesNotExistError
 
+	def get_permission_query_conditions(self, user):
+		locals = {"user": user, "conditions": ""}
+		safe_exec(self.script, None, locals)
+		if locals["conditions"]:
+			return locals["conditions"]
+
 @frappe.whitelist()
 def setup_scheduler_events(script_name, frequency):
 	method = frappe.scrub('{0}-{1}'.format(script_name, frequency))

frappe/core/doctype/server_script/server_script_utils.py

@@ -50,6 +50,9 @@ def get_server_script_map():
 	# 	},
 	# 	'_api': {
 	# 		'[path]': '[server script]'
+	# 	},
+	# 	'permission_query': {
+	# 		'DocType': '[server script]'
 	# 	}
 	# }
 	if frappe.flags.in_patch and not frappe.db.table_exists('Server Script'):
@@ -57,16 +60,20 @@ def get_server_script_map():
 
 	script_map = frappe.cache().get_value('server_script_map')
 	if script_map is None:
-		script_map = {}
+		script_map = {
+			'permission_query': {}
+		}
 		enabled_server_scripts = frappe.get_all('Server Script',
 			fields=('name', 'reference_doctype', 'doctype_event','api_method', 'script_type'),
 			filters={'disabled': 0})
 		for script in enabled_server_scripts:
 			if script.script_type == 'DocType Event':
 				script_map.setdefault(script.reference_doctype, {}).setdefault(script.doctype_event, []).append(script.name)
+			elif script.script_type == 'Permission Query':
+				script_map['permission_query'][script.reference_doctype] = script.name
 			else:
 				script_map.setdefault('_api', {})[script.api_method] = script.name
 
 		frappe.cache().set_value('server_script_map', script_map)
 
-	return script_map
+	return script_map

frappe/core/doctype/server_script/test_server_script.py

@@ -45,6 +45,22 @@ frappe.response['message'] = 'hello'
 		allow_guest = 1,
 		script = '''
 frappe.flags = 'hello'
+'''
+	),
+	dict(
+		name='test_permission_query',
+		script_type = 'Permission Query',
+		reference_doctype = 'ToDo',
+		script = '''
+conditions = '1 = 1'
+'''),
+  dict(
+		name='test_invalid_namespace_method',
+		script_type = 'DocType Event',
+		doctype_event = 'Before Insert',
+		reference_doctype = 'Note',
+		script = '''
+frappe.method_that_doesnt_exist("do some magic")
 '''
 	)
 ]
@@ -85,3 +101,12 @@ class TestServerScript(unittest.TestCase):
 
 	def test_api_return(self):
 		self.assertEqual(frappe.get_doc('Server Script', 'test_return_value').execute_method(), 'hello')
+
+	def test_permission_query(self):
+		self.assertTrue('where (1 = 1)' in frappe.db.get_list('ToDo', return_query=1))
+		self.assertTrue(isinstance(frappe.db.get_list('ToDo'), list))
+
+	def test_attribute_error(self):
+		"""Raise AttributeError if method not found in Namespace"""
+		note = frappe.get_doc({"doctype": "Note", "title": "Test Note: Server Script"})
+		self.assertRaises(AttributeError, note.insert)

frappe/core/doctype/system_settings/system_settings.js

@@ -1,37 +1,36 @@
-frappe.ui.form.on("System Settings", "refresh", function(frm) {
-	frappe.call({
-		method: "frappe.core.doctype.system_settings.system_settings.load",
-		callback: function(data) {
-			frappe.all_timezones = data.message.timezones;
-			frm.set_df_property("time_zone", "options", frappe.all_timezones);
+frappe.ui.form.on("System Settings", {
+	refresh: function(frm) {
+		frappe.call({
+			method: "frappe.core.doctype.system_settings.system_settings.load",
+			callback: function(data) {
+				frappe.all_timezones = data.message.timezones;
+				frm.set_df_property("time_zone", "options", frappe.all_timezones);
 
-			$.each(data.message.defaults, function(key, val) {
-				frm.set_value(key, val);
-				frappe.sys_defaults[key] = val;
-			})
+				$.each(data.message.defaults, function(key, val) {
+					frm.set_value(key, val);
+					frappe.sys_defaults[key] = val;
+				});
+			}
+		});
+	},
+	enable_password_policy: function(frm) {
+		if (frm.doc.enable_password_policy == 0) {
+			frm.set_value("minimum_password_score", "");
+		} else {
+			frm.set_value("minimum_password_score", "2");
 		}
-	});
-});
-
-frappe.ui.form.on("System Settings", "enable_password_policy", function(frm) {
-	if(frm.doc.enable_password_policy == 0){
-		frm.set_value("minimum_password_score", "");
-	} else {
-		frm.set_value("minimum_password_score", "2");
-	}
-});
-
-frappe.ui.form.on("System Settings", "enable_two_factor_auth", function(frm) {
-	if(frm.doc.enable_two_factor_auth == 0){
-		frm.set_value("bypass_2fa_for_retricted_ip_users", 0);
-		frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0);
-	}
-});
-
-frappe.ui.form.on("System Settings", "enable_prepared_report_auto_deletion", function(frm) {
-	if (frm.doc.enable_prepared_report_auto_deletion) {
-		if (!frm.doc.prepared_report_expiry_period) {
-			frm.set_value('prepared_report_expiry_period', 7);
+	},
+	enable_two_factor_auth: function(frm) {
+		if (frm.doc.enable_two_factor_auth == 0) {
+			frm.set_value("bypass_2fa_for_retricted_ip_users", 0);
+			frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0);
+		}
+	},
+	enable_prepared_report_auto_deletion: function(frm) {
+		if (frm.doc.enable_prepared_report_auto_deletion) {
+			if (!frm.doc.prepared_report_expiry_period) {
+				frm.set_value('prepared_report_expiry_period', 7);
+			}
 		}
 	}
 });

frappe/core/doctype/system_settings/system_settings.json

@@ -37,6 +37,7 @@
   "allow_login_using_mobile_number",
   "allow_login_using_user_name",
   "allow_error_traceback",
+  "strip_exif_metadata_from_uploaded_images",
   "password_settings",
   "logout_on_password_reset",
   "force_user_to_reset_password",
@@ -460,12 +461,18 @@
    "fieldname": "prepared_report_section",
    "fieldtype": "Section Break",
    "label": "Prepared Report"
+  },
+  {
+   "default": "1",
+   "fieldname": "strip_exif_metadata_from_uploaded_images",
+   "fieldtype": "Check",
+   "label": "Strip EXIF tags from uploaded images"
   }
  ],
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2020-08-12 14:35:45.214327",
+ "modified": "2020-11-30 18:52:22.161391",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

frappe/core/doctype/user/user.py

@@ -13,7 +13,7 @@ from frappe.utils.user import get_system_managers
 from bs4 import BeautifulSoup
 import frappe.permissions
 import frappe.share
-
+import frappe.defaults
 from frappe.website.utils import is_signup_enabled
 from frappe.utils.background_jobs import enqueue
 
@@ -107,6 +107,10 @@ class User(Document):
 		)
 		if self.name not in ('Administrator', 'Guest') and not self.user_image:
 			frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name)
+		
+		# Set user selected timezone
+		if self.time_zone:
+			frappe.defaults.set_default("time_zone", self.time_zone, self.name)
 
 	def has_website_permission(self, ptype, user, verbose=False):
 		"""Returns true if current user is the session user"""
@@ -1129,4 +1133,4 @@ def check_password_reset_limit(user, rate_limit):
 		frappe.throw(_("You have reached the hourly limit for generating password reset links. Please try again later."))
 
 def get_generated_link_count(user):
-	return cint(frappe.cache().hget("password_reset_link_count", user)) or 0
+	return cint(frappe.cache().hget("password_reset_link_count", user)) or 0

frappe/custom/doctype/custom_field/custom_field.json

@@ -30,6 +30,7 @@
   "mandatory_depends_on",
   "read_only_depends_on",
   "properties",
+  "non_negative",
   "reqd",
   "unique",
   "read_only",
@@ -403,13 +404,20 @@
    "fieldname": "hide_border",
    "fieldtype": "Check",
    "label": "Hide Border"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:in_list([\"Int\", \"Float\", \"Currency\"], doc.fieldtype)",
+   "fieldname": "non_negative",
+   "fieldtype": "Check",
+   "label": "Non Negative"
   }
  ],
  "icon": "fa fa-glass",
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-08-28 11:28:44.377753",
+ "modified": "2020-10-29 06:14:43.073329",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Custom Field",

frappe/custom/doctype/custom_link/custom_link.js

@@ -1,20 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Custom Link', {
-	refresh: function(frm) {
-		frm.set_query("document_type", function () {
-			return {
-				filters: {
-					custom: 0,
-					istable: 0,
-					module: ['not in', ["Email", "Core", "Custom", "Event Streaming", "Social", "Data Migration", "Geo", "Desk"]]
-				}
-			};
-		});
-
-		frm.add_custom_button(__('Go to {0} List', [frm.doc.document_type]), function() {
-			frappe.set_route('List', frm.doc.document_type);
-		});
-	}
-});

frappe/custom/doctype/custom_link/custom_link.py

@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and contributors
-# For license information, please see license.txt
-
-from __future__ import unicode_literals
-# import frappe
-from frappe.model.document import Document
-
-class CustomLink(Document):
-	pass

frappe/custom/doctype/customize_form/customize_form.js

@@ -5,6 +5,7 @@ frappe.provide("frappe.customize_form");
 
 frappe.ui.form.on("Customize Form", {
 	onload: function(frm) {
+		frm.disable_save();
 		frm.set_query("doc_type", function() {
 			return {
 				translate_values: false,
@@ -27,7 +28,7 @@ frappe.ui.form.on("Customize Form", {
 		});
 
 		$(frm.wrapper).on("grid-row-render", function(e, grid_row) {
-			if(grid_row.doc && grid_row.doc.fieldtype=="Section Break") {
+			if (grid_row.doc && grid_row.doc.fieldtype=="Section Break") {
 				$(grid_row.row).css({"font-weight": "bold"});
 			}
 		});
@@ -40,19 +41,25 @@ frappe.ui.form.on("Customize Form", {
 			frm.trigger("setup_sortable");
 		});
 
+		if (localStorage['customize_doctype']) {
+			// set default value from customize form
+			frm.set_value('doc_type', localStorage['customize_doctype']);
+		}
+
 	},
 
 	doc_type: function(frm) {
-		if(frm.doc.doc_type) {
+		if (frm.doc.doc_type) {
 			return frm.call({
 				method: "fetch_to_customize",
 				doc: frm.doc,
 				freeze: true,
 				callback: function(r) {
-					if(r) {
-						if(r._server_messages && r._server_messages.length) {
+					if (r) {
+						if (r._server_messages && r._server_messages.length) {
 							frm.set_value("doc_type", "");
 						} else {
+							localStorage['customize_doctype'] = frm.doc.doc_type;
 							frm.refresh();
 							frm.trigger("setup_sortable");
 						}
@@ -69,11 +76,16 @@ frappe.ui.form.on("Customize Form", {
 		frm.doc.fields.forEach(function(f, i) {
 			var data_row = frm.page.body.find('[data-fieldname="fields"] [data-idx="'+ f.idx +'"] .data-row');
 
-			if(f.is_custom_field) {
+			if (f.is_custom_field) {
 				data_row.addClass("highlight");
 			} else {
 				f._sortable = false;
 			}
+			if (f.fieldtype == "Table") {
+				frm.add_custom_button(f.options, function() {
+					frm.set_value('doc_type', f.options);
+				}, __('Customize Child Table'));
+			}
 		});
 		frm.fields_dict.fields.grid.refresh();
 	},
@@ -82,26 +94,26 @@ frappe.ui.form.on("Customize Form", {
 		frm.disable_save();
 		frm.page.clear_icons();
 
-		if(frm.doc.doc_type) {
+		if (frm.doc.doc_type) {
 			frappe.customize_form.set_primary_action(frm);
 
 			frm.add_custom_button(__('Go to {0} List', [frm.doc.doc_type]), function() {
 				frappe.set_route('List', frm.doc.doc_type);
-			});
+			}, __('Actions'));
 
-			frm.add_custom_button(__('Refresh Form'), function() {
+			frm.add_custom_button(__('Reload'), function() {
 				frm.script_manager.trigger("doc_type");
-			}, "fa fa-refresh", "btn-default");
+			}, __('Actions'));
 
 			frm.add_custom_button(__('Reset to defaults'), function() {
 				frappe.customize_form.confirm(__('Remove all customizations?'), frm);
-			}, "fa fa-eraser", "btn-default");
+			}, __('Actions'));
 
 			frm.add_custom_button(__('Set Permissions'), function() {
 				frappe.set_route('permission-manager', frm.doc.doc_type);
-			}, "fa fa-lock", "btn-default");
+			}, __('Actions'));
 
-			if(frappe.boot.developer_mode) {
+			if (frappe.boot.developer_mode) {
 				frm.add_custom_button(__('Export Customizations'), function() {
 					frappe.prompt(
 						[
@@ -124,34 +136,36 @@ frappe.ui.form.on("Customize Form", {
 							});
 						},
 						__("Select Module"));
-				});
+				}, __('Actions'));
 			}
 		}
 
 		// sort order select
-		if(frm.doc.doc_type) {
+		if (frm.doc.doc_type) {
 			var fields = $.map(frm.doc.fields,
-					function(df) { return frappe.model.is_value_type(df.fieldtype) ? df.fieldname : null; });
+				function(df) {
+					return frappe.model.is_value_type(df.fieldtype) ? df.fieldname : null;
+				});
 			fields = ["", "name", "modified"].concat(fields);
 			frm.set_df_property("sort_field", "options", fields);
 		}
 
-		if(frappe.route_options && frappe.route_options.doc_type) {
+		if (frappe.route_options && frappe.route_options.doc_type) {
 			setTimeout(function() {
 				frm.set_value("doc_type", frappe.route_options.doc_type);
 				frappe.route_options = null;
 			}, 1000);
 		}
-
 	}
 });
 
+// can't delete standard fields
 frappe.ui.form.on("Customize Form Field", {
 	before_fields_remove: function(frm, doctype, name) {
 		var row = frappe.get_doc(doctype, name);
-		if(!(row.is_custom_field || row.__islocal)) {
+		if (!(row.is_custom_field || row.__islocal)) {
 			frappe.msgprint(__("Cannot delete standard field. You can hide it if you want"));
-			throw "cannot delete custom field";
+			throw "cannot delete standard field";
 		}
 	},
 	fields_add: function(frm, cdt, cdn) {
@@ -160,16 +174,46 @@ frappe.ui.form.on("Customize Form Field", {
 	}
 });
 
+// can't delete standard links
+frappe.ui.form.on("DocType Link", {
+	before_links_remove: function(frm, doctype, name) {
+		let row = frappe.get_doc(doctype, name);
+		if (!(row.custom || row.__islocal)) {
+			frappe.msgprint(__("Cannot delete standard link. You can hide it if you want"));
+			throw "cannot delete standard link";
+		}
+	},
+	links_add: function(frm, cdt, cdn) {
+		let f = frappe.model.get_doc(cdt, cdn);
+		f.custom = 1;
+	}
+});
+
+// can't delete standard actions
+frappe.ui.form.on("DocType Action", {
+	before_actions_remove: function(frm, doctype, name) {
+		let row = frappe.get_doc(doctype, name);
+		if (!(row.custom || row.__islocal)) {
+			frappe.msgprint(__("Cannot delete standard action. You can hide it if you want"));
+			throw "cannot delete standard action";
+		}
+	},
+	actions_add: function(frm, cdt, cdn) {
+		let f = frappe.model.get_doc(cdt, cdn);
+		f.custom = 1;
+	}
+});
+
 frappe.customize_form.set_primary_action = function(frm) {
 	frm.page.set_primary_action(__("Update"), function() {
-		if(frm.doc.doc_type) {
+		if (frm.doc.doc_type) {
 			return frm.call({
 				doc: frm.doc,
 				freeze: true,
 				btn: frm.page.btn_primary,
 				method: "save_customization",
 				callback: function(r) {
-					if(!r.exc) {
+					if (!r.exc) {
 						frappe.customize_form.clear_locals_and_refresh(frm);
 						frm.script_manager.trigger("doc_type");
 					}
@@ -180,7 +224,7 @@ frappe.customize_form.set_primary_action = function(frm) {
 };
 
 frappe.customize_form.confirm = function(msg, frm) {
-	if(!frm.doc.doc_type) return;
+	if (!frm.doc.doc_type) return;
 
 	var d = new frappe.ui.Dialog({
 		title: 'Reset To Defaults',
@@ -192,7 +236,7 @@ frappe.customize_form.confirm = function(msg, frm) {
 				doc: frm.doc,
 				method: "reset_to_defaults",
 				callback: function(r) {
-					if(r.exc) {
+					if (r.exc) {
 						frappe.msgprint(r.exc);
 					} else {
 						d.hide();

frappe/custom/doctype/customize_form/customize_form.json

@@ -10,8 +10,9 @@
   "doc_type",
   "properties",
   "label",
-  "default_print_format",
   "max_attachments",
+  "search_fields",
+  "column_break_5",
   "allow_copy",
   "istable",
   "editable_grid",
@@ -20,22 +21,27 @@
   "track_views",
   "allow_auto_repeat",
   "allow_import",
-  "show_preview_popup",
-  "image_view",
-  "column_break_5",
+  "fields_section_break",
+  "fields",
+  "view_settings_section",
   "title_field",
   "image_field",
-  "search_fields",
-  "section_break_8",
-  "sort_field",
-  "column_break_10",
-  "sort_order",
-  "section_break_23",
+  "default_print_format",
+  "column_break_29",
+  "show_preview_popup",
+  "image_view",
+  "email_settings_section",
   "email_append_to",
   "sender_field",
   "subject_field",
-  "fields_section_break",
-  "fields"
+  "document_actions_section",
+  "actions",
+  "document_links_section",
+  "links",
+  "section_break_8",
+  "sort_field",
+  "column_break_10",
+  "sort_order"
  ],
  "fields": [
   {
@@ -130,9 +136,11 @@
    "label": "Search Fields"
   },
   {
+   "collapsible": 1,
    "depends_on": "doc_type",
    "fieldname": "section_break_8",
-   "fieldtype": "Section Break"
+   "fieldtype": "Section Break",
+   "label": "List Settings"
   },
   {
    "fieldname": "sort_field",
@@ -161,7 +169,8 @@
    "fieldname": "fields",
    "fieldtype": "Table",
    "label": "Fields",
-   "options": "Customize Form Field"
+   "options": "Customize Form Field",
+   "reqd": 1
   },
   {
    "default": "0",
@@ -200,24 +209,67 @@
    "fieldtype": "Check",
    "label": "Allow document creation via Email"
   },
-  {
-   "depends_on": "doc_type",
-   "fieldname": "section_break_23",
-   "fieldtype": "Section Break"
-  },
   {
    "default": "0",
    "fieldname": "show_preview_popup",
    "fieldtype": "Check",
    "label": "Show Preview Popup"
+  },
+  {
+   "collapsible": 1,
+   "depends_on": "doc_type",
+   "fieldname": "view_settings_section",
+   "fieldtype": "Section Break",
+   "label": "View Settings"
+  },
+  {
+   "fieldname": "column_break_29",
+   "fieldtype": "Column Break"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "email_append_to",
+   "depends_on": "doc_type",
+   "fieldname": "email_settings_section",
+   "fieldtype": "Section Break",
+   "label": "Email Settings"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "links",
+   "depends_on": "doc_type",
+   "fieldname": "document_links_section",
+   "fieldtype": "Section Break",
+   "label": "Document Links"
+  },
+  {
+   "fieldname": "links",
+   "fieldtype": "Table",
+   "label": "Links",
+   "options": "DocType Link"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "actions",
+   "depends_on": "doc_type",
+   "fieldname": "document_actions_section",
+   "fieldtype": "Section Break",
+   "label": "Document Actions"
+  },
+  {
+   "fieldname": "actions",
+   "fieldtype": "Table",
+   "label": "Actions",
+   "options": "DocType Action"
   }
  ],
  "hide_toolbar": 1,
  "icon": "fa fa-glass",
  "idx": 1,
+ "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-04-10 12:16:01.320411",
+ "modified": "2020-09-24 14:16:49.594012",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",

frappe/custom/doctype/customize_form/customize_form.py

@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 	Customize Form is a Single DocType used to mask the Property Setter
 	Thus providing a better UI from user perspective
 """
+import json
 import frappe
 import frappe.translate
 from frappe import _
@@ -14,80 +15,9 @@ from frappe.model.document import Document
 from frappe.model import no_value_fields, core_doctypes_list
 from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype, check_email_append_to
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
+from frappe.custom.doctype.property_setter.property_setter import delete_property_setter
 from frappe.model.docfield import supports_translation
 
-doctype_properties = {
-	'search_fields': 'Data',
-	'title_field': 'Data',
-	'image_field': 'Data',
-	'sort_field': 'Data',
-	'sort_order': 'Data',
-	'default_print_format': 'Data',
-	'allow_copy': 'Check',
-	'istable': 'Check',
-	'quick_entry': 'Check',
-	'editable_grid': 'Check',
-	'max_attachments': 'Int',
-	'track_changes': 'Check',
-	'track_views': 'Check',
-	'allow_auto_repeat': 'Check',
-	'allow_import': 'Check',
-	'show_preview_popup': 'Check',
-	'email_append_to': 'Check',
-	'subject_field': 'Data',
-	'sender_field': 'Data'
-}
-
-docfield_properties = {
-	'idx': 'Int',
-	'label': 'Data',
-	'fieldtype': 'Select',
-	'options': 'Text',
-	'fetch_from': 'Small Text',
-	'fetch_if_empty': 'Check',
-	'permlevel': 'Int',
-	'width': 'Data',
-	'print_width': 'Data',
-	'reqd': 'Check',
-	'unique': 'Check',
-	'ignore_user_permissions': 'Check',
-	'in_list_view': 'Check',
-	'in_standard_filter': 'Check',
-	'in_global_search': 'Check',
-	'in_preview': 'Check',
-	'bold': 'Check',
-	'hidden': 'Check',
-	'collapsible': 'Check',
-	'collapsible_depends_on': 'Data',
-	'print_hide': 'Check',
-	'print_hide_if_no_value': 'Check',
-	'report_hide': 'Check',
-	'allow_on_submit': 'Check',
-	'translatable': 'Check',
-	'mandatory_depends_on': 'Data',
-	'read_only_depends_on': 'Data',
-	'depends_on': 'Data',
-	'description': 'Text',
-	'default': 'Text',
-	'precision': 'Select',
-	'read_only': 'Check',
-	'length': 'Int',
-	'columns': 'Int',
-	'remember_last_selected_value': 'Check',
-	'allow_bulk_edit': 'Check',
-	'auto_repeat': 'Link',
-	'allow_in_quick_entry': 'Check',
-	'hide_border': 'Check',
-	'hide_days': 'Check',
-	'hide_seconds': 'Check'
-}
-
-allowed_fieldtype_change = (('Currency', 'Float', 'Percent'), ('Small Text', 'Data'),
-	('Text', 'Data'), ('Text', 'Text Editor', 'Code', 'Signature', 'HTML Editor'), ('Data', 'Select'),
-	('Text', 'Small Text'), ('Text', 'Data', 'Barcode'), ('Code', 'Geolocation'), ('Table', 'Table MultiSelect'))
-
-allowed_fieldtype_for_options_change = ('Read Only', 'HTML', 'Select', 'Data')
-
 class CustomizeForm(Document):
 	def on_update(self):
 		frappe.db.sql("delete from tabSingles where doctype='Customize Form'")
@@ -100,30 +30,54 @@ class CustomizeForm(Document):
 
 		meta = frappe.get_meta(self.doc_type)
 
+		self.validate_doctype(meta)
+
+		# load the meta properties on the customize (self) object
+		self.load_properties(meta)
+
+		# load custom translation
+		translation = self.get_name_translation()
+		self.label = translation.translated_text if translation else ''
+
+		self.create_auto_repeat_custom_field_if_required(meta)
+
+		# NOTE doc (self) is sent to clientside by run_method
+
+	def validate_doctype(self, meta):
+		'''
+		Check if the doctype is allowed to be customized.
+		'''
 		if self.doc_type in core_doctypes_list:
-			return frappe.msgprint(_("Core DocTypes cannot be customized."))
+			frappe.throw(_("Core DocTypes cannot be customized."))
 
 		if meta.issingle:
-			return frappe.msgprint(_("Single DocTypes cannot be customized."))
+			frappe.throw(_("Single DocTypes cannot be customized."))
 
 		if meta.custom:
-			return frappe.msgprint(_("Only standard DocTypes are allowed to be customized from Customize Form."))
+			frappe.throw(_("Only standard DocTypes are allowed to be customized from Customize Form."))
 
+	def load_properties(self, meta):
+		'''
+		Load the customize object (this) with the metadata properties
+		'''
 		# doctype properties
-		for property in doctype_properties:
-			self.set(property, meta.get(property))
+		for prop in doctype_properties:
+			self.set(prop, meta.get(prop))
 
 		for d in meta.get("fields"):
 			new_d = {"fieldname": d.fieldname, "is_custom_field": d.get("is_custom_field"), "name": d.name}
-			for property in docfield_properties:
-				new_d[property] = d.get(property)
+			for prop in docfield_properties:
+				new_d[prop] = d.get(prop)
 			self.append("fields", new_d)
 
-		# load custom translation
-		translation = self.get_name_translation()
-		self.label = translation.translated_text if translation else ''
+		for fieldname in ('links', 'actions'):
+			for d in meta.get(fieldname):
+				self.append(fieldname, d)
 
-		#If allow_auto_repeat is set, add auto_repeat custom field.
+	def create_auto_repeat_custom_field_if_required(self, meta):
+		'''
+		Create auto repeat custom field if it's not already present
+		'''
 		if self.allow_auto_repeat:
 			all_fields = [df.fieldname for df in meta.fields]
 
@@ -140,7 +94,6 @@ class CustomizeForm(Document):
 				read_only=1, no_copy=1, print_hide=1
 			))
 
-		# NOTE doc is sent to clientside by run_method
 
 	def get_name_translation(self):
 		'''Get translation object if exists of current doctype name in the default language'''
@@ -205,72 +158,142 @@ class CustomizeForm(Document):
 
 	def set_property_setters(self):
 		meta = frappe.get_meta(self.doc_type)
-		# doctype property setters
 
-		for property in doctype_properties:
-			if self.get(property) != meta.get(property):
-				self.make_property_setter(property=property, value=self.get(property),
-					property_type=doctype_properties[property])
+		# doctype
+		self.set_property_setters_for_doctype(meta)
 
+		# docfield
 		for df in self.get("fields"):
 			meta_df = meta.get("fields", {"fieldname": df.fieldname})
-
 			if not meta_df or meta_df[0].get("is_custom_field"):
 				continue
+			self.set_property_setters_for_docfield(meta, df, meta_df)
+
+		# action and links
+		self.set_property_setters_for_actions_and_links(meta)
+
+	def set_property_setters_for_doctype(self, meta):
+		for prop, prop_type in doctype_properties.items():
+			if self.get(prop) != meta.get(prop):
+				self.make_property_setter(prop, self.get(prop), prop_type)
+
+	def set_property_setters_for_docfield(self, meta, df, meta_df):
+		for prop, prop_type in docfield_properties.items():
+			if prop != "idx" and (df.get(prop) or '') != (meta_df[0].get(prop) or ''):
+				if not self.allow_property_change(prop, meta_df, df):
+					continue
+
+				self.make_property_setter(prop, df.get(prop), prop_type,
+					fieldname=df.fieldname)
+
+	def allow_property_change(self, prop, meta_df, df):
+		if prop == "fieldtype":
+			self.validate_fieldtype_change(df, meta_df[0].get(prop), df.get(prop))
+
+		elif prop == "allow_on_submit" and df.get(prop):
+			if not frappe.db.get_value("DocField",
+				{"parent": self.doc_type, "fieldname": df.fieldname}, "allow_on_submit"):
+				frappe.msgprint(_("Row {0}: Not allowed to enable Allow on Submit for standard fields")\
+					.format(df.idx))
+				return False
+
+		elif prop == "reqd" and \
+			((frappe.db.get_value("DocField",
+				{"parent":self.doc_type,"fieldname":df.fieldname}, "reqd") == 1) \
+				and (df.get(prop) == 0)):
+			frappe.msgprint(_("Row {0}: Not allowed to disable Mandatory for standard fields")\
+					.format(df.idx))
+			return False
+
+		elif prop == "in_list_view" and df.get(prop) \
+			and df.fieldtype!="Attach Image" and df.fieldtype in no_value_fields:
+					frappe.msgprint(_("'In List View' not allowed for type {0} in row {1}")
+						.format(df.fieldtype, df.idx))
+					return False
+
+		elif prop == "precision" and cint(df.get("precision")) > 6 \
+				and cint(df.get("precision")) > cint(meta_df[0].get("precision")):
+			self.flags.update_db = True
+
+		elif prop == "unique":
+			self.flags.update_db = True
+
+		elif (prop == "read_only" and cint(df.get("read_only"))==0
+				and frappe.db.get_value("DocField", {"parent": self.doc_type,
+				"fieldname": df.fieldname}, "read_only")==1):
+			# if docfield has read_only checked and user is trying to make it editable, don't allow it
+			frappe.msgprint(_("You cannot unset 'Read Only' for field {0}").format(df.label))
+			return False
+
+		elif prop == "options" and df.get("fieldtype") not in ALLOWED_OPTIONS_CHANGE:
+			frappe.msgprint(_("You can't set 'Options' for field {0}").format(df.label))
+			return False
+
+		elif prop == 'translatable' and not supports_translation(df.get('fieldtype')):
+			frappe.msgprint(_("You can't set 'Translatable' for field {0}").format(df.label))
+			return False
+
+		elif (prop == 'in_global_search' and
+			df.in_global_search != meta_df[0].get("in_global_search")):
+			self.flags.rebuild_doctype_for_global_search = True
+
+		return True
+
+	def set_property_setters_for_actions_and_links(self, meta):
+		'''
+		Apply property setters or create custom records for DocType Action and DocType Link
+		'''
+		for doctype, fieldname, field_map in (
+				('DocType Link', 'links', doctype_link_properties),
+				('DocType Action', 'actions', doctype_action_properties)
+			):
+			has_custom = False
+			items = []
+			for i, d in enumerate(self.get(fieldname) or []):
+				d.idx = i
+				if frappe.db.exists(doctype, d.name) and not d.custom:
+					# check property and apply property setter
+					original = frappe.get_doc(doctype, d.name)
+					for prop, prop_type in field_map.items():
+						if d.get(prop) != original.get(prop):
+							self.make_property_setter(prop, d.get(prop), prop_type,
+								apply_on=doctype, row_name=d.name)
+					items.append(d.name)
+				else:
+					# custom - just insert/update
+					d.parent = self.doc_type
+					d.custom = 1
+					d.save(ignore_permissions=True)
+					has_custom = True
+					items.append(d.name)
+
+			self.update_order_property_setter(has_custom, fieldname)
+			self.clear_removed_items(doctype, items)
+
+	def update_order_property_setter(self, has_custom, fieldname):
+		'''
+		We need to maintain the order of the link/actions if the user has shuffled them.
+		So we create a new property (ex `links_order`) to keep a list of items.
+		'''
+		property_name = '{}_order'.format(fieldname)
+		if has_custom:
+			# save the order of the actions and links
+			self.make_property_setter(property_name,
+				json.dumps([d.name for d in self.get(fieldname)]), 'Small Text')
+		else:
+			frappe.db.delete('Property Setter', dict(property=property_name,
+				doc_type=self.doc_type))
 
-			for property in docfield_properties:
-				if property != "idx" and (df.get(property) or '') != (meta_df[0].get(property) or ''):
-					if property == "fieldtype":
-						self.validate_fieldtype_change(df, meta_df[0].get(property), df.get(property))
-
-					elif property == "allow_on_submit" and df.get(property):
-						if not frappe.db.get_value("DocField",
-							{"parent": self.doc_type, "fieldname": df.fieldname}, "allow_on_submit"):
-							frappe.msgprint(_("Row {0}: Not allowed to enable Allow on Submit for standard fields")\
-								.format(df.idx))
-							continue
-
-					elif property == "reqd" and \
-						((frappe.db.get_value("DocField",
-							{"parent":self.doc_type,"fieldname":df.fieldname}, "reqd") == 1) \
-							and (df.get(property) == 0)):
-						frappe.msgprint(_("Row {0}: Not allowed to disable Mandatory for standard fields")\
-								.format(df.idx))
-						continue
-
-					elif property == "in_list_view" and df.get(property) \
-						and df.fieldtype!="Attach Image" and df.fieldtype in no_value_fields:
-								frappe.msgprint(_("'In List View' not allowed for type {0} in row {1}")
-									.format(df.fieldtype, df.idx))
-								continue
-
-					elif property == "precision" and cint(df.get("precision")) > 6 \
-							and cint(df.get("precision")) > cint(meta_df[0].get("precision")):
-						self.flags.update_db = True
-
-					elif property == "unique":
-						self.flags.update_db = True
-
-					elif (property == "read_only" and cint(df.get("read_only"))==0
-						and frappe.db.get_value("DocField", {"parent": self.doc_type, "fieldname": df.fieldname}, "read_only")==1):
-						# if docfield has read_only checked and user is trying to make it editable, don't allow it
-						frappe.msgprint(_("You cannot unset 'Read Only' for field {0}").format(df.label))
-						continue
-
-					elif property == "options" and df.get("fieldtype") not in allowed_fieldtype_for_options_change:
-						frappe.msgprint(_("You can't set 'Options' for field {0}").format(df.label))
-						continue
-
-					elif property == 'translatable' and not supports_translation(df.get('fieldtype')):
-						frappe.msgprint(_("You can't set 'Translatable' for field {0}").format(df.label))
-						continue
-
-					elif (property == 'in_global_search' and
-						df.in_global_search != meta_df[0].get("in_global_search")):
-						self.flags.rebuild_doctype_for_global_search = True
-
-					self.make_property_setter(property=property, value=df.get(property),
-						property_type=docfield_properties[property], fieldname=df.fieldname)
+
+	def clear_removed_items(self, doctype, items):
+		'''
+		Clear rows that do not appear in `items`. These have been removed by the user.
+		'''
+		if items:
+			frappe.db.delete(doctype, dict(parent=self.doc_type, custom=1,
+				name=('not in', items)))
+		else:
+			frappe.db.delete(doctype, dict(parent=self.doc_type, custom=1))
 
 	def update_custom_fields(self):
 		for i, df in enumerate(self.get("fields")):
@@ -288,8 +311,8 @@ class CustomizeForm(Document):
 
 		d.dt = self.doc_type
 
-		for property in docfield_properties:
-			d.set(property, df.get(property))
+		for prop in docfield_properties:
+			d.set(prop, df.get(prop))
 
 		if i!=0:
 			d.insert_after = self.fields[i-1].fieldname
@@ -307,12 +330,12 @@ class CustomizeForm(Document):
 
 		custom_field = frappe.get_doc("Custom Field", meta_df[0].name)
 		changed = False
-		for property in docfield_properties:
-			if df.get(property) != custom_field.get(property):
-				if property == "fieldtype":
-					self.validate_fieldtype_change(df, meta_df[0].get(property), df.get(property))
+		for prop in docfield_properties:
+			if df.get(prop) != custom_field.get(prop):
+				if prop == "fieldtype":
+					self.validate_fieldtype_change(df, meta_df[0].get(prop), df.get(prop))
 
-				custom_field.set(property, df.get(property))
+				custom_field.set(prop, df.get(prop))
 				changed = True
 
 		# check and update `insert_after` property
@@ -338,32 +361,28 @@ class CustomizeForm(Document):
 			if df.get("is_custom_field"):
 				frappe.delete_doc("Custom Field", df.name)
 
-	def make_property_setter(self, property, value, property_type, fieldname=None):
-		self.delete_existing_property_setter(property, fieldname)
+	def make_property_setter(self, prop, value, property_type, fieldname=None,
+		apply_on=None, row_name = None):
+		delete_property_setter(self.doc_type, prop, fieldname)
 
-		property_value = self.get_existing_property_value(property, fieldname)
+		property_value = self.get_existing_property_value(prop, fieldname)
 
 		if property_value==value:
 			return
 
+		if not apply_on:
+			apply_on = "DocField" if fieldname else "DocType"
+
 		# create a new property setter
-		# ignore validation becuase it will be done at end
 		frappe.make_property_setter({
 			"doctype": self.doc_type,
-			"doctype_or_field": "DocField" if fieldname else "DocType",
+			"doctype_or_field": apply_on,
 			"fieldname": fieldname,
-			"property": property,
+			"row_name": row_name,
+			"property": prop,
 			"value": value,
 			"property_type": property_type
-		}, ignore_validate=True)
-
-	def delete_existing_property_setter(self, property, fieldname=None):
-		# first delete existing property setter
-		existing_property_setter = frappe.db.get_value("Property Setter", {"doc_type": self.doc_type,
-			"property": property, "field_name['']": fieldname or ''})
-
-		if existing_property_setter:
-			frappe.db.sql("delete from `tabProperty Setter` where name=%s", existing_property_setter)
+		})
 
 	def get_existing_property_value(self, property_name, fieldname=None):
 		# check if there is any need to make property setter!
@@ -371,20 +390,17 @@ class CustomizeForm(Document):
 			property_value = frappe.db.get_value("DocField", {"parent": self.doc_type,
 				"fieldname": fieldname}, property_name)
 		else:
-			try:
+			if frappe.db.has_column("DocType", property_name):
 				property_value = frappe.db.get_value("DocType", self.doc_type, property_name)
-			except Exception as e:
-				if frappe.db.is_column_missing(e):
-					property_value = None
-				else:
-					raise
+			else:
+				property_value = None
 
 		return property_value
 
 	def validate_fieldtype_change(self, df, old_value, new_value):
 		allowed = False
 		self.check_length_for_fieldtypes = []
-		for allowed_changes in allowed_fieldtype_change:
+		for allowed_changes in ALLOWED_FIELDTYPE_CHANGE:
 			if (old_value in allowed_changes and new_value in allowed_changes):
 				allowed = True
 				old_value_length = cint(frappe.db.type_map.get(old_value)[1])
@@ -444,4 +460,100 @@ def reset_customization(doctype):
 			and `field_name`!='naming_series'
 			and `property`!='options'
 		""", doctype)
-	frappe.clear_cache(doctype=doctype)
+	frappe.clear_cache(doctype=doctype)
+
+doctype_properties = {
+	'search_fields': 'Data',
+	'title_field': 'Data',
+	'image_field': 'Data',
+	'sort_field': 'Data',
+	'sort_order': 'Data',
+	'default_print_format': 'Data',
+	'allow_copy': 'Check',
+	'istable': 'Check',
+	'quick_entry': 'Check',
+	'editable_grid': 'Check',
+	'max_attachments': 'Int',
+	'track_changes': 'Check',
+	'track_views': 'Check',
+	'allow_auto_repeat': 'Check',
+	'allow_import': 'Check',
+	'show_preview_popup': 'Check',
+	'email_append_to': 'Check',
+	'subject_field': 'Data',
+	'sender_field': 'Data'
+}
+
+docfield_properties = {
+	'idx': 'Int',
+	'label': 'Data',
+	'fieldtype': 'Select',
+	'options': 'Text',
+	'fetch_from': 'Small Text',
+	'fetch_if_empty': 'Check',
+	'permlevel': 'Int',
+	'width': 'Data',
+	'print_width': 'Data',
+	'non_negative': 'Check',
+	'reqd': 'Check',
+	'unique': 'Check',
+	'ignore_user_permissions': 'Check',
+	'in_list_view': 'Check',
+	'in_standard_filter': 'Check',
+	'in_global_search': 'Check',
+	'in_preview': 'Check',
+	'bold': 'Check',
+	'hidden': 'Check',
+	'collapsible': 'Check',
+	'collapsible_depends_on': 'Data',
+	'print_hide': 'Check',
+	'print_hide_if_no_value': 'Check',
+	'report_hide': 'Check',
+	'allow_on_submit': 'Check',
+	'translatable': 'Check',
+	'mandatory_depends_on': 'Data',
+	'read_only_depends_on': 'Data',
+	'depends_on': 'Data',
+	'description': 'Text',
+	'default': 'Text',
+	'precision': 'Select',
+	'read_only': 'Check',
+	'length': 'Int',
+	'columns': 'Int',
+	'remember_last_selected_value': 'Check',
+	'allow_bulk_edit': 'Check',
+	'auto_repeat': 'Link',
+	'allow_in_quick_entry': 'Check',
+	'hide_border': 'Check',
+	'hide_days': 'Check',
+	'hide_seconds': 'Check'
+}
+
+doctype_link_properties = {
+	'link_doctype': 'Link',
+	'link_fieldname': 'Data',
+	'group': 'Data',
+	'hidden': 'Check'
+}
+
+doctype_action_properties = {
+	'label': 'Link',
+	'action_type': 'Select',
+	'action': 'Small Text',
+	'group': 'Data',
+	'hidden': 'Check'
+}
+
+
+ALLOWED_FIELDTYPE_CHANGE = (
+	('Currency', 'Float', 'Percent'),
+	('Small Text', 'Data'),
+	('Text', 'Data'),
+	('Text', 'Text Editor', 'Code', 'Signature', 'HTML Editor'),
+	('Data', 'Select'),
+	('Text', 'Small Text'),
+	('Text', 'Data', 'Barcode'),
+	('Code', 'Geolocation'),
+	('Table', 'Table MultiSelect'))
+
+ALLOWED_OPTIONS_CHANGE = ('Read Only', 'HTML', 'Select', 'Data')

frappe/custom/doctype/customize_form/test_customize_form.py

@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 import frappe, unittest, json
 from frappe.test_runner import make_test_records_for_doctype
 from frappe.core.doctype.doctype.doctype import InvalidFieldNameError
+from frappe.core.doctype.doctype.test_doctype import new_doctype
 
 test_dependencies = ["Custom Field", "Property Setter"]
 class TestCustomizeForm(unittest.TestCase):
@@ -24,6 +25,7 @@ class TestCustomizeForm(unittest.TestCase):
 
 	def setUp(self):
 		self.insert_custom_field()
+		frappe.db.delete('Property Setter', dict(doc_type='Event'))
 		frappe.db.commit()
 		frappe.clear_cache(doctype="Event")
 
@@ -185,9 +187,75 @@ class TestCustomizeForm(unittest.TestCase):
 		d.run_method("save_customization")
 
 	def test_core_doctype_customization(self):
-		d = self.get_customize_form('User')
-		e = self.get_customize_form('Custom Field')
+		self.assertRaises(frappe.ValidationError, self.get_customize_form, 'User')
 
-		# core doctype is invalid, hence no attributes are set
-		self.assertEquals(d.get("fields"), [])
-		self.assertEquals(e.get("fields"), [])
+	def test_custom_link(self):
+		try:
+			# create a dummy doctype linked to Event
+			testdt_name = 'Test Link for Event'
+			testdt = new_doctype(testdt_name, fields=[
+				dict(fieldtype='Link', fieldname='event', options='Event')
+			]).insert()
+
+			testdt_name1 = 'Test Link for Event 1'
+			testdt1 = new_doctype(testdt_name1, fields=[
+				dict(fieldtype='Link', fieldname='event', options='Event')
+			]).insert()
+
+			# add a custom link
+			d = self.get_customize_form("Event")
+
+			d.append('links', dict(link_doctype=testdt_name, link_fieldname='event', group='Tests'))
+			d.append('links', dict(link_doctype=testdt_name1, link_fieldname='event', group='Tests'))
+
+			d.run_method("save_customization")
+
+			frappe.clear_cache()
+			event = frappe.get_meta('Event')
+
+			# check links exist
+			self.assertTrue([d.name for d in event.links if d.link_doctype == testdt_name])
+			self.assertTrue([d.name for d in event.links if d.link_doctype == testdt_name1])
+
+			# check order
+			order = json.loads(event.links_order)
+			self.assertListEqual(order, [d.name for d in event.links])
+
+			# remove the link
+			d = self.get_customize_form("Event")
+			d.links = []
+			d.run_method("save_customization")
+
+			frappe.clear_cache()
+			event = frappe.get_meta('Event')
+			self.assertFalse([d.name for d in (event.links or []) if d.link_doctype == testdt_name])
+		finally:
+			testdt.delete()
+			testdt1.delete()
+
+	def test_custom_action(self):
+		test_route = '#List/DocType'
+
+		# create a dummy action (route)
+		d = self.get_customize_form("Event")
+		d.append('actions', dict(label='Test Action', action_type='Route', action=test_route))
+		d.run_method("save_customization")
+
+		frappe.clear_cache()
+		event = frappe.get_meta('Event')
+
+		# check if added to meta
+		action = [d for d in event.actions if d.label=='Test Action']
+		self.assertEqual(len(action), 1)
+		self.assertEqual(action[0].action, test_route)
+
+		# clear the action
+		d = self.get_customize_form("Event")
+		d.actions = []
+		d.run_method("save_customization")
+
+		frappe.clear_cache()
+		event = frappe.get_meta('Event')
+
+		action = [d for d in event.actions if d.label=='Test Action']
+		self.assertEqual(len(action), 0)

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -11,8 +11,7 @@
   "label",
   "fieldtype",
   "fieldname",
-  "hide_seconds",
-  "hide_days",
+  "non_negative",
   "reqd",
   "unique",
   "in_list_view",
@@ -23,6 +22,7 @@
   "allow_in_quick_entry",
   "translatable",
   "column_break_7",
+  "default",
   "precision",
   "length",
   "options",
@@ -47,8 +47,9 @@
   "column_break_33",
   "read_only_depends_on",
   "display",
-  "default",
   "in_filter",
+  "hide_seconds",
+  "hide_days",
   "column_break_21",
   "description",
   "print_hide",
@@ -100,6 +101,7 @@
    "depends_on": "eval:!in_list([\"Section Break\", \"Column Break\", \"Button\", \"HTML\"], doc.fieldtype)",
    "fieldname": "reqd",
    "fieldtype": "Check",
+   "in_list_view": 1,
    "label": "Mandatory",
    "oldfieldname": "reqd",
    "oldfieldtype": "Check",
@@ -283,7 +285,7 @@
   },
   {
    "fieldname": "default",
-   "fieldtype": "Text",
+   "fieldtype": "Small Text",
    "label": "Default",
    "oldfieldname": "default",
    "oldfieldtype": "Text"
@@ -413,13 +415,20 @@
    "fieldname": "hide_border",
    "fieldtype": "Check",
    "label": "Hide Border"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:in_list([\"Int\", \"Float\", \"Currency\"], doc.fieldtype)",
+   "fieldname": "non_negative",
+   "fieldtype": "Check",
+   "label": "Non Negative"
   }
  ],
  "idx": 1,
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-08-28 11:28:59.084060",
+ "modified": "2020-10-29 06:11:57.661039",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",

frappe/custom/doctype/package_document_type/package_document_type.json

@@ -1,65 +0,0 @@
-{
- "actions": [],
- "creation": "2020-05-14 16:45:47.196395",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "document_type",
-  "column_break_2",
-  "attachments",
-  "overwrite",
-  "section_break_4",
-  "filters_json"
- ],
- "fields": [
-  {
-   "fieldname": "document_type",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "reqd": 1
-  },
-  {
-   "fieldname": "column_break_2",
-   "fieldtype": "Column Break"
-  },
-  {
-   "default": "0",
-   "fieldname": "attachments",
-   "fieldtype": "Check",
-   "in_list_view": 1,
-   "label": "Include Attachments"
-  },
-  {
-   "default": "0",
-   "fieldname": "overwrite",
-   "fieldtype": "Check",
-   "in_list_view": 1,
-   "label": "Overwrite"
-  },
-  {
-   "fieldname": "section_break_4",
-   "fieldtype": "Section Break"
-  },
-  {
-   "fieldname": "filters_json",
-   "fieldtype": "Code",
-   "label": "Filters",
-   "options": "JSON"
-  }
- ],
- "istable": 1,
- "links": [],
- "modified": "2020-05-14 16:45:47.196395",
- "modified_by": "Administrator",
- "module": "Custom",
- "name": "Package Document Type",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}

frappe/custom/doctype/package_publish_target/package_publish_target.json

@@ -1,47 +0,0 @@
-{
- "actions": [],
- "creation": "2020-05-13 16:04:32.724663",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "instance_url",
-  "username",
-  "password"
- ],
- "fields": [
-  {
-   "fieldname": "instance_url",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Site URL",
-   "reqd": 1
-  },
-  {
-   "fieldname": "username",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Username",
-   "reqd": 1
-  },
-  {
-   "fieldname": "password",
-   "fieldtype": "Password",
-   "in_list_view": 1,
-   "label": "Password",
-   "reqd": 1
-  }
- ],
- "istable": 1,
- "links": [],
- "modified": "2020-05-15 17:35:16.282235",
- "modified_by": "Administrator",
- "module": "Custom",
- "name": "Package Publish Target",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}

frappe/custom/doctype/package_publish_target/package_publish_target.py

@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and contributors
-# For license information, please see license.txt
-
-from __future__ import unicode_literals
-# import frappe
-from frappe.model.document import Document
-
-class PackagePublishTarget(Document):
-	pass

frappe/custom/doctype/package_publish_tool/package_publish_tool.js

@@ -1,159 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Package Publish Tool', {
-	refresh: function(frm) {
-		frm.set_query("document_type", "package_details", function () {
-			return {
-				filters: {
-					"istable": 0,
-				}
-			};
-		});
-
-		frappe.realtime.on("package", (data) => {
-			frm.dashboard.show_progress(data.prefix, data.progress / data.total * 100, __("{0}", [data.message]));
-			if ((data.progress+1) != data.total) {
-				frm.dashboard.show_progress(data.prefix, data.progress / data.total * 100, __("{0}", [data.message]));
-			} else {
-				frm.dashboard.hide_progress();
-			}
-		});
-
-		frm.trigger("show_instructions");
-		frm.trigger("last_deployed_on");
-		frm.trigger("set_dirty_trigger");
-		frm.trigger("set_deploy_primary_action");
-	},
-	last_deployed_on: function(frm) {
-		if (frm.doc.last_deployed_on) {
-			frm.trigger("show_indicator");
-		}
-	},
-	show_indicator: function(frm) {
-		let pretty_date = frappe.datetime.prettyDate(frm.doc.last_deployed_on);
-		frm.page.set_indicator(__("Last published {0}", [pretty_date]), "blue");
-	},
-	set_dirty_trigger: function(frm) {
-		$(frm.wrapper).on("dirty", function() {
-			frm.page.set_primary_action(__('Save'), () => frm.save());
-		});
-	},
-	set_deploy_primary_action: function(frm) {
-		if (frm.doc.package_details.length && frm.doc.instances.length) {
-			frm.page.set_primary_action(__("Publish"), function () {
-				frappe.show_alert({
-					message: __("Publishing documents..."),
-					indicator: "green"
-				});
-
-				frappe.call({
-					method: "frappe.custom.doctype.package_publish_tool.package_publish_tool.deploy_package",
-					callback: function() {
-						frm.reload_doc();
-						frappe.msgprint(__("Documents have been published."));
-					}
-				});
-			});
-		}
-	},
-	show_instructions: function(frm) {
-		let field = frm.get_field("html_info");
-		field.html(`
-			<p class="text-muted text-medium">
-				Package Publish Tool let's you copy documents from your site to any other remote site.
-				Follow the steps below to publish.
-			</p>
-			<ol class="text-muted small">
-				<li>Add Document Types that you want to copy from the table below. You can also add filters by expanding the row.</li>
-				<li>Add the Sites URL where you want to copy these documents, and enter the Username and Password.</li>
-				<li>Click on Save. Now, you can click on Publish and the documents will be copied.</li>
-			</ol>
-		`);
-	}
-});
-
-frappe.ui.form.on('Package Document Type', {
-	form_render: function (frm, cdt, cdn) {
-		function _show_filters(filters, table) {
-			table.find('tbody').empty();
-
-			if (filters.length > 0) {
-				filters.forEach(filter => {
-					const filter_row =
-						$(`<tr>
-							<td>${filter[1]}</td>
-							<td>${filter[2] || ""}</td>
-							<td>${filter[3]}</td>
-						</tr>`);
-
-					table.find('tbody').append(filter_row);
-				});
-			} else {
-				const filter_row = $(`<tr><td colspan="3" class="text-muted text-center">
-					${__("Click to Set Filters")}</td></tr>`);
-				table.find('tbody').append(filter_row);
-			}
-		}
-
-		let row = frappe.get_doc(cdt, cdn);
-
-		let wrapper = $(`[data-fieldname="filters_json"]`).empty();
-		let table = $(`<table class="table table-bordered" style="cursor:pointer; margin:0px;">
-				<thead>
-					<tr>
-						<th style="width: 33%">${__('Filter')}</th>
-						<th style="width: 33%">${__('Condition')}</th>
-						<th>${__('Value')}</th>
-					</tr>
-				</thead>
-				<tbody>
-				</tbody>
-			</table>`).appendTo(wrapper);
-		$(`<p class="text-muted small">${__("Click table to edit")}</p>`).appendTo(wrapper);
-
-		let filters = JSON.parse(row.filters_json || '[]');
-		_show_filters(filters, table);
-
-		table.on('click', () => {
-			if (!row.document_type) {
-				frappe.msgprint(__("Select Document Type."));
-				return;
-			}
-
-			frappe.model.with_doctype(row.document_type, function() {
-				let dialog = new frappe.ui.Dialog({
-					title: __('Set Filters'),
-					fields: [
-						{
-							fieldtype: 'HTML',
-							label: 'Filters',
-							fieldname: 'filter_area',
-						}
-					],
-					primary_action: function() {
-						let values = filter_group.get_filters();
-						let flt = [];
-						if (values) {
-							values.forEach(function(value) {
-								flt.push([value[0], value[1], value[2], value[3]]);
-							});
-						}
-						row.filters_json = JSON.stringify(flt);
-						_show_filters(flt, table);
-						dialog.hide();
-					},
-					primary_action_label: "Set"
-				});
-
-				let filter_group = new frappe.ui.FilterGroup({
-					parent: dialog.get_field('filter_area').$wrapper,
-					doctype: row.document_type,
-					on_change: () => {},
-				});
-				filter_group.add_filters_to_filter_group(filters);
-				dialog.show();
-			});
-		});
-	},
-});

frappe/custom/doctype/package_publish_tool/package_publish_tool.json

@@ -1,84 +0,0 @@
-{
- "actions": [],
- "creation": "2020-05-13 15:54:38.082657",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "html_info",
-  "sb_00",
-  "package_details",
-  "sb_01",
-  "instances",
-  "last_deployed_on"
- ],
- "fields": [
-  {
-   "description": "Click on the row for accessing filters.",
-   "fieldname": "package_details",
-   "fieldtype": "Table",
-   "label": "Document Types",
-   "options": "Package Document Type",
-   "reqd": 1
-  },
-  {
-   "fieldname": "instances",
-   "fieldtype": "Table",
-   "label": "Sites",
-   "options": "Package Publish Target",
-   "reqd": 1
-  },
-  {
-   "fieldname": "html_info",
-   "fieldtype": "HTML"
-  },
-  {
-   "fieldname": "last_deployed_on",
-   "fieldtype": "Datetime",
-   "hidden": 1,
-   "label": "Last Deployed On",
-   "read_only": 1
-  },
-  {
-   "fieldname": "sb_00",
-   "fieldtype": "Section Break"
-  },
-  {
-   "fieldname": "sb_01",
-   "fieldtype": "Section Break"
-  }
- ],
- "issingle": 1,
- "links": [],
- "modified": "2020-05-15 17:31:37.060199",
- "modified_by": "Administrator",
- "module": "Custom",
- "name": "Package Publish Tool",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  },
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "role": "All",
-   "share": 1,
-   "write": 1
-  }
- ],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}

frappe/custom/doctype/package_publish_tool/package_publish_tool.py

@@ -1,178 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and contributors
-# For license information, please see license.txt
-
-from __future__ import unicode_literals
-import frappe
-import json
-import datetime
-import base64
-from frappe.model.document import Document
-from frappe.utils.file_manager import save_file, get_file
-from frappe import _
-from six import string_types
-from frappe.frappeclient import FrappeClient
-from frappe.utils import get_datetime_str, get_datetime
-from frappe.utils.password import get_decrypted_password
-
-class PackagePublishTool(Document):
-	pass
-
-@frappe.whitelist()
-def deploy_package():
-	package, doc = export_package()
-
-	file_name = "Package-" + get_datetime_str(get_datetime())
-
-	length = len(doc.instances)
-	for idx, instance in enumerate(doc.instances):
-		frappe.publish_realtime("package",  {"progress": idx, "total": length, "message": instance.instance_url, "prefix": _("Deploying")},
-			user=frappe.session.user)
-
-		install_package_to_remote(package, instance)
-
-	frappe.db.set_value("Package Publish Tool", "Package Publish Tool", "last_deployed_on", frappe.utils.now_datetime())
-
-def install_package_to_remote(package, instance):
-	try:
-		connection = FrappeClient(instance.instance_url, instance.username, get_decrypted_password(instance.doctype, instance.name))
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-		frappe.throw(_("Couldn't connect to site {0}. Please check Error Logs.").format(instance.instance_url))
-
-	try:
-		connection.post_request({
-			"cmd": "frappe.custom.doctype.package_publish_tool.package_publish_tool.import_package",
-			"package": json.dumps(package)
-		})
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-		frappe.throw(_("Error while installing package to site {0}. Please check Error Logs.").format(instance.instance_url))
-
-@frappe.whitelist()
-def export_package():
-	"""Export package as JSON."""
-	package_doc = frappe.get_single("Package Publish Tool")
-	package = []
-
-	for doctype in package_doc.package_details:
-		filters = []
-
-		if doctype.get("filters_json"):
-			filters = json.loads(doctype.get("filters_json"))
-
-		docs = frappe.get_all(doctype.get("document_type"), filters=filters)
-		length = len(docs)
-
-		for idx, doc in enumerate(docs):
-			frappe.publish_realtime("package", {
-					"progress":idx, "total":length,
-					"message":doctype.get("document_type"),
-					"prefix": _("Exporting")
-				},
-				user=frappe.session.user)
-
-			document = frappe.get_doc(doctype.get("document_type"), doc.name).as_dict()
-			attachments = []
-
-			if doctype.attachments:
-				filters = {
-					"attached_to_doctype": document.get("doctype"),
-					"attached_to_name": document.get("name")
-				}
-
-				for f in frappe.get_list("File", filters=filters):
-					fname, fcontents = get_file(f.name)
-					attachments.append({
-						"fname": fname,
-						"content": base64.b64encode(fcontents).decode('ascii')
-					})
-
-			document.update({
-				"__attachments": attachments,
-				"__overwrite": True if doctype.overwrite else False
-			})
-
-			package.append(document)
-
-	return post_process(package), package_doc
-
-@frappe.whitelist()
-def import_package(package=None):
-	"""Import package from JSON."""
-	frappe.only_for("System Manager")
-	if isinstance(package, string_types):
-		package = json.loads(package)
-
-	for doc in package:
-		modified = doc.pop("modified")
-		overwrite = doc.pop("__overwrite")
-		attachments = doc.pop("__attachments")
-		exists = frappe.db.exists(doc.get("doctype"), doc.get("name"))
-
-		if not exists:
-			d = frappe.get_doc(doc).insert(ignore_permissions=True, ignore_if_duplicate=True)
-			if attachments:
-				add_attachment(attachments, d)
-		else:
-			docname = doc.pop("name")
-			document = frappe.get_doc(doc.get("doctype"), docname)
-
-			if overwrite:
-				update_document(document, doc, attachments)
-
-			else:
-				if frappe.utils.get_datetime(document.modified) < frappe.utils.get_datetime(modified):
-					update_document(document, doc, attachments)
-
-def update_document(document, doc, attachments):
-	document.update(doc)
-	document.save()
-	if attachments:
-		add_attachment(attachments, document)
-
-def add_attachment(attachments, doc):
-	for attachment in attachments:
-		save_file(attachment.get("fname"), base64.b64decode(attachment.get("content")), doc.get("doctype"), doc.get("name"))
-
-def post_process(package):
-	"""Remove the keys from Document and Child Document. Convert datetime, date, time to str."""
-	del_keys = ('modified_by', 'creation', 'owner', 'idx', 'docstatus')
-	child_del_keys = ('modified_by', 'creation', 'owner', 'idx', 'docstatus', 'name')
-
-	for doc in package:
-		for key in del_keys:
-			if key in doc:
-				del doc[key]
-
-		for key, value in doc.items():
-			stringified_value = get_stringified_value(value)
-			if stringified_value:
-				doc[key] = stringified_value
-
-			if not isinstance(value, list):
-				continue
-
-			for child in value:
-				for child_key in child_del_keys:
-					if child_key in child:
-						del child[child_key]
-
-				for child_key, child_value in child.items():
-					stringified_value = get_stringified_value(child_value)
-					if stringified_value:
-						child[child_key] = stringified_value
-
-	return package
-
-def get_stringified_value(value):
-	if isinstance(value, datetime.datetime):
-		return frappe.utils.get_datetime_str(value)
-
-	if isinstance(value, datetime.date):
-		return frappe.utils.get_date_str(value)
-
-	if isinstance(value, datetime.timedelta):
-		return frappe.utils.get_time_str(value)
-
-	return None

frappe/custom/doctype/package_publish_tool/test_package_publish_tool.py

@@ -1,10 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# See license.txt
-from __future__ import unicode_literals
-
-# import frappe
-import unittest
-
-class TestPackagePublishTool(unittest.TestCase):
-	pass

frappe/custom/doctype/property_setter/property_setter.json

@@ -1,358 +1,133 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2013-01-10 16:34:04", 
- "custom": 0, 
- "description": "Property Setter overrides a standard DocType or Field property", 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 0, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2013-01-10 16:34:04",
+ "description": "Property Setter overrides a standard DocType or Field property",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "engine": "InnoDB",
+ "field_order": [
+  "help",
+  "sb0",
+  "doctype_or_field",
+  "doc_type",
+  "field_name",
+  "row_name",
+  "column_break0",
+  "property",
+  "property_type",
+  "value",
+  "default_value"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "help", 
-   "fieldtype": "HTML", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Help", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "<div class=\"alert\">Please don't update it as it can mess up your form. Use the Customize Form View and Custom Fields to set properties!</div>", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "help",
+   "fieldtype": "HTML",
+   "label": "Help",
+   "options": "<div class=\"alert\">Please don't update it as it can mess up your form. Use the Customize Form View and Custom Fields to set properties!</div>"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "sb0", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "sb0",
+   "fieldtype": "Section Break"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:doc.__islocal", 
-   "fieldname": "doctype_or_field", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 1, 
-   "label": "DocType or Field", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "\nDocField\nDocType", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "doctype_or_field",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Applied On",
+   "options": "\nDocField\nDocType\nDocType Link\nDocType Action",
+   "read_only_depends_on": "eval:!doc.__islocal",
+   "reqd": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "New value to be set", 
-   "fieldname": "value", 
-   "fieldtype": "Text", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Set Value", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "description": "New value to be set",
+   "fieldname": "value",
+   "fieldtype": "Small Text",
+   "in_list_view": 1,
+   "label": "Set Value"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break0", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "column_break0",
+   "fieldtype": "Column Break"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "doc_type", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 1, 
-   "label": "DocType", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "DocType", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 1, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "doc_type",
+   "fieldtype": "Link",
+   "in_standard_filter": 1,
+   "label": "DocType",
+   "options": "DocType",
+   "reqd": 1,
+   "search_index": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "depends_on": "eval:doc.doctype_or_field=='DocField'", 
-   "description": "ID (name) of the entity whose property is to be set", 
-   "fieldname": "field_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 1, 
-   "label": "Field Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 1, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "depends_on": "eval:doc.doctype_or_field=='DocField'",
+   "description": "ID (name) of the entity whose property is to be set",
+   "fieldname": "field_name",
+   "fieldtype": "Data",
+   "in_standard_filter": 1,
+   "label": "Field Name",
+   "search_index": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "property", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 1, 
-   "label": "Property", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 1, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "property",
+   "fieldtype": "Data",
+   "in_standard_filter": 1,
+   "label": "Property",
+   "reqd": 1,
+   "search_index": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "property_type", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Property Type", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "property_type",
+   "fieldtype": "Data",
+   "label": "Property Type"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "default_value", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Default Value", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "default_value",
+   "fieldtype": "Data",
+   "label": "Default Value"
+  },
+  {
+   "description": "For DocType Link / DocType Action",
+   "fieldname": "row_name",
+   "fieldtype": "Data",
+   "label": "Row Name"
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-glass", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:39:50.172883", 
- "modified_by": "Administrator", 
- "module": "Custom", 
- "name": "Property Setter", 
- "owner": "Administrator", 
+ ],
+ "icon": "fa fa-glass",
+ "idx": 1,
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2020-09-24 14:42:38.599684",
+ "modified_by": "Administrator",
+ "module": "Custom",
+ "name": "Property Setter",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Administrator", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Administrator",
+   "share": 1,
    "write": 1
-  }, 
+  },
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "search_fields": "doc_type,property", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "search_fields": "doc_type,property",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
 }

frappe/custom/doctype/property_setter/property_setter.py

@@ -11,13 +11,16 @@ not_allowed_fieldtype_change = ['naming_series']
 
 class PropertySetter(Document):
 	def autoname(self):
-		self.name = self.doc_type + "-" \
-			+ (self.field_name and (self.field_name + "-")  or "") \
-			+ self.property
+		self.name = '{doctype}-{field}-{property}'.format(
+			doctype = self.doc_type,
+			field = self.field_name or self.row_name or 'main',
+			property = self.property
+		)
 
 	def validate(self):
 		self.validate_fieldtype_change()
-		self.delete_property_setter()
+		if self.is_new():
+			delete_property_setter(self.doc_type, self.property, self.field_name)
 
 		# clear cache
 		frappe.clear_cache(doctype = self.doc_type)
@@ -27,15 +30,6 @@ class PropertySetter(Document):
 			self.property == 'fieldtype':
 			frappe.throw(_("Field type cannot be changed for {0}").format(self.field_name))
 
-	def delete_property_setter(self):
-		"""delete other property setters on this, if this is new"""
-		if self.get('__islocal'):
-			frappe.db.sql("""delete from `tabProperty Setter` where
-				doctype_or_field = %(doctype_or_field)s
-				and doc_type = %(doc_type)s
-				and coalesce(field_name,'') = coalesce(%(field_name)s, '')
-				and property = %(property)s""", self.get_valid_dict())
-
 	def get_property_list(self, dt):
 		return frappe.db.get_all('DocField',
 			fields=['fieldname', 'label', 'fieldtype'],
@@ -89,3 +83,12 @@ def make_property_setter(doctype, fieldname, property, value, property_type, for
 	property_setter.flags.validate_fields_for_doctype = validate_fields_for_doctype
 	property_setter.insert()
 	return property_setter
+
+def delete_property_setter(doc_type, property, field_name=None):
+	"""delete other property setters on this, if this is new"""
+	filters = dict(doc_type = doc_type, property=property)
+	if field_name:
+		filters['field_name'] = field_name
+
+	frappe.db.delete('Property Setter', filters)
+

frappe/database/database.py

@@ -319,8 +319,7 @@ class Database(object):
 			nres.append(nr)
 		return nres
 
-	@staticmethod
-	def build_conditions(filters):
+	def build_conditions(self, filters):
 		"""Convert filters sent as dict, lists to SQL conditions. filter's key
 		is passed by map function, build conditions like:
 
@@ -341,18 +340,12 @@ class Database(object):
 			value = filters.get(key)
 			values[key] = value
 			if isinstance(value, (list, tuple)):
-				# value is a tuble like ("!=", 0)
+				# value is a tuple like ("!=", 0)
 				_operator = value[0]
 				values[key] = value[1]
 				if isinstance(value[1], (tuple, list)):
 					# value is a list in tuple ("in", ("A", "B"))
-					inner_list = []
-					for i, v in enumerate(value[1]):
-						inner_key = "{0}_{1}".format(key, i)
-						values[inner_key] = v
-						inner_list.append("%({0})s".format(inner_key))
-
-					_rhs = " ({0})".format(", ".join(inner_list))
+					_rhs = " ({0})".format(", ".join([self.escape(v) for v in value[1]]))
 					del values[key]
 
 			if _operator not in ["=", "!=", ">", ">=", "<", "<=", "like", "in", "not in", "not like"]:
@@ -787,6 +780,9 @@ class Database(object):
 		"""Returns True if table for given doctype exists."""
 		return ("tab" + doctype) in self.get_tables()
 
+	def has_table(self, doctype):
+		return self.table_exists(doctype)
+
 	def get_tables(self):
 		tables = frappe.cache().get_value('db_tables')
 		if not tables:
@@ -959,13 +955,13 @@ class Database(object):
 		query = sql_dict.get(current_dialect)
 		return self.sql(query, values, **kwargs)
 
-	def delete(self, doctype, conditions):
+	def delete(self, doctype, conditions, debug=False):
 		if conditions:
 			conditions, values = self.build_conditions(conditions)
 			return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format(
 				doctype=doctype,
 				conditions=conditions
-			), values)
+			), values, debug=debug)
 		else:
 			frappe.throw(_('No conditions provided'))
 

frappe/database/db_manager.py

@@ -3,7 +3,6 @@ import frappe
 
 
 class DbManager:
-
 	def __init__(self, db):
 		"""
 		Pass root_conn here for access to all databases.
@@ -66,10 +65,10 @@ class DbManager:
 		esc = make_esc('$ ')
 
 		from distutils.spawn import find_executable
-		pipe = find_executable('pv')
-		if pipe:
-			pipe = '{pipe} {source} |'.format(
-				pipe=pipe,
+		pv = find_executable('pv')
+		if pv:
+			pipe = '{pv} {source} |'.format(
+				pv=pv,
 				source=source
 			)
 			source = ''
@@ -78,7 +77,7 @@ class DbManager:
 			source = '< {source}'.format(source=source)
 
 		if pipe:
-			print('Creating Database...')
+			print('Restoring Database file...')
 
 		command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}'
 		command = command.format(

frappe/database/mariadb/framework_mariadb.sql

@@ -233,7 +233,7 @@ CREATE TABLE `tabDocType` (
 
 DROP TABLE IF EXISTS `tabSeries`;
 CREATE TABLE `tabSeries` (
-  `name` varchar(100) DEFAULT NULL,
+  `name` varchar(100),
   `current` int(10) NOT NULL DEFAULT 0,
   PRIMARY KEY(`name`)
 ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

frappe/database/mariadb/setup_db.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 import frappe
-import os, sys
+import os
 from frappe.database.db_manager import DbManager
 
 expected_settings_10_2_earlier = {
@@ -86,6 +86,8 @@ def drop_user_and_database(db_name, root_login, root_password):
 	dbman.drop_database(db_name)
 
 def bootstrap_database(db_name, verbose, source_sql=None):
+	import sys
+
 	frappe.connect(db_name=db_name)
 	if not check_database_settings():
 		print('Database settings do not match expected values; stopping database setup.')
@@ -94,9 +96,17 @@ def bootstrap_database(db_name, verbose, source_sql=None):
 	import_db_from_sql(source_sql, verbose)
 
 	frappe.connect(db_name=db_name)
-	if not 'tabDefaultValue' in frappe.db.get_tables():
-		print('''Database not installed, this can due to lack of permission, or that the database name exists.
-			Check your mysql root password, or use --force to reinstall''')
+	if 'tabDefaultValue' not in frappe.db.get_tables():
+		from click import secho
+
+		secho(
+			"Table 'tabDefaultValue' missing in the restored site. "
+			"Database not installed correctly, this can due to lack of "
+			"permission, or that the database name exists. Check your mysql"
+			" root password, validity of the backup file or use --force to"
+			" reinstall",
+			fg="red"
+		)
 		sys.exit(1)
 
 def import_db_from_sql(source_sql=None, verbose=False):

frappe/database/postgres/database.py

@@ -140,11 +140,11 @@ class PostgresDatabase(Database):
 
 	@staticmethod
 	def is_table_missing(e):
-		return e.pgcode == '42P01'
+		return getattr(e, 'pgcode', None) == '42P01'
 
 	@staticmethod
 	def is_missing_column(e):
-		return e.pgcode == '42703'
+		return getattr(e, 'pgcode', None) == '42703'
 
 	@staticmethod
 	def is_access_denied(e):

frappe/database/postgres/setup_db.py

@@ -1,5 +1,7 @@
-import frappe, subprocess, os
-from six.moves import input
+import os
+
+import frappe
+
 
 def setup_database(force, source_sql=None, verbose=False):
 	root_conn = get_root_connection()
@@ -10,24 +12,62 @@ def setup_database(force, source_sql=None, verbose=False):
 	root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name,
 		frappe.conf.db_password))
 	root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
+	root_conn.close()
+
+	bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql)
+	frappe.connect()
+
+def bootstrap_database(db_name, verbose, source_sql=None):
+	frappe.connect(db_name=db_name)
+	import_db_from_sql(source_sql, verbose)
+	frappe.connect(db_name=db_name)
+
+	if 'tabDefaultValue' not in frappe.db.get_tables():
+		import sys
+		from click import secho
+
+		secho(
+			"Table 'tabDefaultValue' missing in the restored site. "
+			"This may be due to incorrect permissions or the result of a restore from a bad backup file. "
+			"Database not installed correctly.",
+			fg="red"
+		)
+		sys.exit(1)
+
+def import_db_from_sql(source_sql=None, verbose=False):
+	from shutil import which
+	from subprocess import run, PIPE
 
 	# we can't pass psql password in arguments in postgresql as mysql. So
 	# set password connection parameter in environment variable
 	subprocess_env = os.environ.copy()
 	subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password)
+
 	# bootstrap db
 	if not source_sql:
 		source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql')
 
-	subprocess.check_output([
-		'psql', frappe.conf.db_name,
-		'-h', frappe.conf.db_host or 'localhost',
-		'-p', str(frappe.conf.db_port or '5432'),
-		'-U', frappe.conf.db_name,
-		'-f', source_sql
-	], env=subprocess_env)
+	pv = which('pv')
 
-	frappe.connect()
+	_command = (
+		f"psql {frappe.conf.db_name} "
+		f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} "
+		f"-U {frappe.conf.db_name}"
+	)
+
+	if pv:
+		command = f"{pv} {source_sql} | " + _command
+	else:
+		command = _command + f" -f {source_sql}"
+
+	print("Restoring Database file...")
+	if verbose:
+		print(command)
+
+	restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE)
+
+	if verbose:
+		print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}")
 
 def setup_help_database(help_db_name):
 	root_conn = get_root_connection()
@@ -38,19 +78,20 @@ def setup_help_database(help_db_name):
 	root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name))
 
 def get_root_connection(root_login=None, root_password=None):
-	import getpass
 	if not frappe.local.flags.root_connection:
 		if not root_login:
 			root_login = frappe.conf.get("root_login") or None
 
 		if not root_login:
+			from six.moves import input
 			root_login = input("Enter postgres super user: ")
 
 		if not root_password:
 			root_password = frappe.conf.get("root_password") or None
 
 		if not root_password:
-			root_password = getpass.getpass("Postgres super user password: ")
+			from getpass import getpass
+			root_password = getpass("Postgres super user password: ")
 
 		frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password)
 

frappe/database/schema.py

@@ -186,7 +186,7 @@ class DbColumn:
 			column_def += ' not null default {0}'.format(default_value)
 
 		elif self.default and (self.default not in frappe.db.DEFAULT_SHORTCUTS) \
-			and not self.default.startswith(":") and column_def not in ('text', 'longtext'):
+			and not cstr(self.default).startswith(":") and column_def not in ('text', 'longtext'):
 			column_def += " default {}".format(frappe.db.escape(self.default))
 
 		if self.unique and (column_def not in ('text', 'longtext')):

frappe/desk/doctype/dashboard/dashboard.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 from frappe.model.document import Document
 from frappe.modules.export_file import export_to_files
+from frappe.config import get_modules_from_all_apps_for_user
 import frappe
 from frappe import _
 import json
@@ -42,6 +43,24 @@ class Dashboard(Document):
 			except ValueError as error:
 				frappe.throw(_("Invalid json added in the custom options: {0}").format(error))
 
+
+def get_permission_query_conditions(user):
+	if not user:
+		user = frappe.session.user
+
+	if user == 'Administrator':
+		return
+
+	roles = frappe.get_roles(user)
+	if "System Manager" in roles:
+		return None
+
+	allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()]
+	module_condition =  '`tabDashboard`.`module` in ({allowed_modules}) or `tabDashboard`.`module` is NULL'.format(
+		allowed_modules=','.join(allowed_modules))
+
+	return module_condition
+
 @frappe.whitelist()
 def get_permitted_charts(dashboard_name):
 	permitted_charts = []

frappe/desk/doctype/dashboard_chart/dashboard_chart.py

@@ -7,17 +7,18 @@ import frappe
 from frappe import _
 import datetime
 import json
-from frappe.utils.dashboard import cache_source, get_from_date_from_timespan
-from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate,\
-	get_datetime, cint, now_datetime
+from frappe.utils.dashboard import cache_source
+from frappe.utils import nowdate, getdate, get_datetime, cint, now_datetime
+from frappe.utils.dateutils import\
+	get_period, get_period_beginning, get_from_date_from_timespan, get_dates_from_timegrain
 from frappe.model.naming import append_number_if_name_exists
 from frappe.boot import get_allowed_reports
+from frappe.config import get_modules_from_all_apps_for_user
 from frappe.model.document import Document
 from frappe.modules.export_file import export_to_files
 
 
 def get_permission_query_conditions(user):
-
 	if not user:
 		user = frappe.session.user
 
@@ -30,9 +31,11 @@ def get_permission_query_conditions(user):
 
 	doctype_condition = False
 	report_condition = False
+	module_condition = False
 
 	allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()]
 	allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()]
+	allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()]
 
 	if allowed_doctypes:
 		doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format(
@@ -40,18 +43,24 @@ def get_permission_query_conditions(user):
 	if allowed_reports:
 		report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format(
 			allowed_reports=','.join(allowed_reports))
+	if allowed_modules:
+		module_condition =  '''`tabDashboard Chart`.`module` in ({allowed_modules})
+			or `tabDashboard Chart`.`module` is NULL'''.format(
+				allowed_modules=','.join(allowed_modules))
 
 	return '''
-			(`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average')
-			and {doctype_condition})
-			or
-			(`tabDashboard Chart`.`chart_type` = 'Report'
-			and {report_condition})
-		'''.format(
-			doctype_condition=doctype_condition,
-			report_condition=report_condition
-		)
-
+		((`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average')
+		and {doctype_condition})
+		or
+		(`tabDashboard Chart`.`chart_type` = 'Report'
+		and {report_condition}))
+		and
+		({module_condition})
+	'''.format(
+		doctype_condition=doctype_condition,
+		report_condition=report_condition,
+		module_condition=module_condition
+	)
 
 def has_permission(doc, ptype, user):
 	roles = frappe.get_roles(user)
@@ -156,6 +165,7 @@ def add_chart_to_dashboard(args):
 def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 	if not from_date:
 		from_date = get_from_date_from_timespan(to_date, timespan)
+		from_date = get_period_beginning(from_date, timegrain)
 	if not to_date:
 		to_date = now_datetime()
 
@@ -185,7 +195,7 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
 	result = get_result(data, timegrain, from_date, to_date)
 
 	chart_config = {
-		"labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result],
+		"labels": [get_period(r[0], timegrain) for r in result],
 		"datasets": [{
 			"name": chart.name,
 			"values": [r[1] for r in result]
@@ -279,16 +289,8 @@ def get_aggregate_function(chart_type):
 
 
 def get_result(data, timegrain, from_date, to_date):
-	start_date = getdate(from_date)
-	end_date = getdate(to_date)
-
-	result = [[start_date, 0.0]]
-
-	while start_date < end_date:
-		next_date = get_next_expected_date(start_date, timegrain)
-		result.append([next_date, 0.0])
-		start_date = next_date
-
+	dates = get_dates_from_timegrain(from_date, to_date, timegrain)
+	result = [[date, 0] for date in dates]
 	data_index = 0
 	if data:
 		for i, d in enumerate(result):
@@ -298,65 +300,6 @@ def get_result(data, timegrain, from_date, to_date):
 
 	return result
 
-def get_next_expected_date(date, timegrain):
-	next_date = None
-	# given date is always assumed to be the period ending date
-	next_date = get_period_ending(add_to_date(date, days=1), timegrain)
-	return getdate(next_date)
-
-def get_period_ending(date, timegrain):
-	date = getdate(date)
-	if timegrain == 'Daily':
-		pass
-	elif timegrain == 'Weekly':
-		date = get_week_ending(date)
-	elif timegrain == 'Monthly':
-		date = get_month_ending(date)
-	elif timegrain == 'Quarterly':
-		date = get_quarter_ending(date)
-	elif timegrain == 'Yearly':
-		date = get_year_ending(date)
-
-	return getdate(date)
-
-def get_week_ending(date):
-	# week starts on monday
-	from datetime import timedelta
-	start = date - timedelta(days = date.weekday())
-	end = start + timedelta(days=6)
-
-	return end
-
-def get_month_ending(date):
-	month_of_the_year = int(date.strftime('%m'))
-	# first day of next month (note month starts from 1)
-
-	date = add_to_date('{}-01-01'.format(date.year), months = month_of_the_year)
-	# last day of this month
-	return add_to_date(date, days=-1)
-
-def get_quarter_ending(date):
-	date = getdate(date)
-
-	# find the earliest quarter ending date that is after
-	# the given date
-	for month in (3, 6, 9, 12):
-		quarter_end_month = getdate('{}-{}-01'.format(date.year, month))
-		quarter_end_date = getdate(get_last_day(quarter_end_month))
-		if date <= quarter_end_date:
-			date = quarter_end_date
-			break
-
-	return date
-
-def get_year_ending(date):
-	''' returns year ending of the given date '''
-
-	# first day of next year (note year starts from 1)
-	date = add_to_date('{}-01-01'.format(date.year), months = 12)
-	# last day of this month
-	return add_to_date(date, days=-1)
-
 @frappe.whitelist()
 @frappe.validate_and_sanitize_search_inputs
 def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters):

frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py

@@ -5,8 +5,8 @@ from __future__ import unicode_literals
 
 import unittest, frappe
 from frappe.utils import getdate, formatdate, get_last_day
-from frappe.desk.doctype.dashboard_chart.dashboard_chart import (get,
-	get_period_ending)
+from frappe.utils.dateutils import get_period_ending, get_period
+from frappe.desk.doctype.dashboard_chart.dashboard_chart import get
 
 from datetime import datetime
 from dateutil.relativedelta import relativedelta
@@ -53,15 +53,11 @@ class TestDashboardChart(unittest.TestCase):
 		cur_date = datetime.now() - relativedelta(years=1)
 
 		result = get(chart_name='Test Dashboard Chart', refresh=1)
-		self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
 
-		if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
-			cur_date += relativedelta(months=1)
-
-		for idx in range(1, 13):
+		for idx in range(13):
 			month = get_last_day(cur_date)
 			month = formatdate(month.strftime('%Y-%m-%d'))
-			self.assertEqual(result.get('labels')[idx], month)
+			self.assertEqual(result.get('labels')[idx], get_period(month))
 			cur_date += relativedelta(months=1)
 
 		frappe.db.rollback()
@@ -87,15 +83,11 @@ class TestDashboardChart(unittest.TestCase):
 		cur_date = datetime.now() - relativedelta(years=1)
 
 		result = get(chart_name ='Test Empty Dashboard Chart', refresh=1)
-		self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
 
-		if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
-			cur_date += relativedelta(months=1)
-
-		for idx in range(1, 13):
+		for idx in range(13):
 			month = get_last_day(cur_date)
 			month = formatdate(month.strftime('%Y-%m-%d'))
-			self.assertEqual(result.get('labels')[idx], month)
+			self.assertEqual(result.get('labels')[idx], get_period(month))
 			cur_date += relativedelta(months=1)
 
 		frappe.db.rollback()
@@ -124,15 +116,11 @@ class TestDashboardChart(unittest.TestCase):
 		cur_date = datetime.now() - relativedelta(years=1)
 
 		result = get(chart_name ='Test Empty Dashboard Chart 2', refresh = 1)
-		self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d')))
 
-		if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')):
-			cur_date += relativedelta(months=1)
-
-		for idx in range(1, 13):
+		for idx in range(13):
 			month = get_last_day(cur_date)
 			month = formatdate(month.strftime('%Y-%m-%d'))
-			self.assertEqual(result.get('labels')[idx], month)
+			self.assertEqual(result.get('labels')[idx], get_period(month))
 			cur_date += relativedelta(months=1)
 
 		# only 1 data point with value
@@ -183,13 +171,12 @@ class TestDashboardChart(unittest.TestCase):
 			timeseries = 1
 		)).insert()
 
-		result = get(chart_name ='Test Daily Dashboard Chart', refresh = 1)
+		result = get(chart_name = 'Test Daily Dashboard Chart', refresh = 1)
 
 		self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0])
 		self.assertEqual(
 			result.get('labels'),
-			[formatdate('2019-01-06'), formatdate('2019-01-07'), formatdate('2019-01-08'),\
-			formatdate('2019-01-09'), formatdate('2019-01-10'), formatdate('2019-01-11')]
+			['06-01-19', '07-01-19', '08-01-19', '09-01-19', '10-01-19', '11-01-19']
 		)
 
 		frappe.db.rollback()
@@ -218,7 +205,10 @@ class TestDashboardChart(unittest.TestCase):
 		result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1)
 
 		self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0])
-		self.assertEqual(result.get('labels'), [formatdate('2018-12-30'), formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')])
+		self.assertEqual(
+			result.get('labels'),
+			['30-12-18', '06-01-19', '13-01-19', '20-01-19']
+		)
 
 		frappe.db.rollback()
 

frappe/desk/doctype/notification_settings/notification_settings.js

@@ -2,12 +2,19 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on('Notification Settings', {
-	onload: () => {
+	onload: (frm) => {
 		frappe.breadcrumbs.add({
 			label: __('Settings'),
 			route: '#modules/Settings',
 			type: 'Custom'
 		});
+		frm.set_query('subscribed_documents', () => {
+			return {
+				filters: {
+					istable: 0
+				}
+			};
+		});
 	},
 
 	refresh: (frm) => {

frappe/desk/doctype/notification_settings/notification_settings.json

@@ -22,68 +22,52 @@
    "default": "1",
    "fieldname": "enabled",
    "fieldtype": "Check",
-   "label": "Enabled",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Enabled"
   },
   {
    "fieldname": "subscribed_documents",
    "fieldtype": "Table MultiSelect",
-   "label": "Subscribed Documents",
-   "options": "Notification Subscribed Document",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Open Documents",
+   "options": "Notification Subscribed Document"
   },
   {
    "fieldname": "column_break_3",
    "fieldtype": "Section Break",
-   "label": "Email Settings",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Email Settings"
   },
   {
    "default": "1",
    "fieldname": "enable_email_notifications",
    "fieldtype": "Check",
-   "label": "Enable Email Notifications",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Enable Email Notifications"
   },
   {
    "default": "1",
    "depends_on": "enable_email_notifications",
    "fieldname": "enable_email_mention",
    "fieldtype": "Check",
-   "label": "Mentions",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Mentions"
   },
   {
    "default": "1",
    "depends_on": "enable_email_notifications",
    "fieldname": "enable_email_assignment",
    "fieldtype": "Check",
-   "label": "Assignments",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Assignments"
   },
   {
    "default": "1",
    "depends_on": "enable_email_notifications",
    "fieldname": "enable_email_energy_point",
    "fieldtype": "Check",
-   "label": "Energy Points",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Energy Points"
   },
   {
    "default": "1",
    "depends_on": "enable_email_notifications",
    "fieldname": "enable_email_share",
    "fieldtype": "Check",
-   "label": "Document Share",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Document Share"
   },
   {
    "default": "__user",
@@ -92,23 +76,20 @@
    "hidden": 1,
    "label": "User",
    "options": "User",
-   "read_only": 1,
-   "show_days": 1,
-   "show_seconds": 1
+   "read_only": 1
   },
   {
    "default": "0",
    "fieldname": "seen",
    "fieldtype": "Check",
    "hidden": 1,
-   "label": "Seen",
-   "show_days": 1,
-   "show_seconds": 1
+   "label": "Seen"
   }
  ],
  "in_create": 1,
+ "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-05-31 22:16:40.798019",
+ "modified": "2020-11-04 12:54:57.989317",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Notification Settings",

frappe/desk/doctype/number_card/number_card.py

@@ -8,6 +8,7 @@ from frappe.model.document import Document
 from frappe.utils import cint
 from frappe.model.naming import append_number_if_name_exists
 from frappe.modules.export_file import export_to_files
+from frappe.config import get_modules_from_all_apps_for_user
 
 class NumberCard(Document):
 	def autoname(self):
@@ -33,16 +34,24 @@ def get_permission_query_conditions(user=None):
 		return None
 
 	doctype_condition = False
+	module_condition = False
 
 	allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()]
+	allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()]
 
 	if allowed_doctypes:
 		doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format(
 			allowed_doctypes=','.join(allowed_doctypes))
+	if allowed_modules:
+		module_condition =  '''`tabNumber Card`.`module` in ({allowed_modules})
+			or `tabNumber Card`.`module` is NULL'''.format(
+			allowed_modules=','.join(allowed_modules))
 
 	return '''
-			{doctype_condition}
-		'''.format(doctype_condition=doctype_condition)
+		{doctype_condition}
+		and
+		{module_condition}
+	'''.format(doctype_condition=doctype_condition, module_condition=module_condition)
 
 def has_permission(doc, ptype, user):
 	roles = frappe.get_roles(user)

frappe/desk/form/document_follow.py

@@ -21,7 +21,7 @@ def follow_document(doctype, doc_name, user, force=False):
 		avoided for some doctype
 		follow only if track changes are set to 1
 	'''
-	if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment")
+	if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment", "Email Account", "Email Domain")
 		or doctype in log_types):
 		return
 

frappe/desk/page/user_profile/user_profile.py

@@ -1,17 +1,23 @@
 import frappe
 from datetime import datetime
+from frappe.utils import getdate
 
 @frappe.whitelist()
 def get_energy_points_heatmap_data(user, date):
+	try:
+		date = getdate(date)
+	except Exception:
+		date = getdate()
+
 	return dict(frappe.db.sql("""select unix_timestamp(date(creation)), sum(points)
 		from `tabEnergy Point Log`
 		where
 			date(creation) > subdate('{date}', interval 1 year) and
 			date(creation) < subdate('{date}', interval -1 year) and
-			user = '{user}' and
+			user = %s and
 			type != 'Review'
 		group by date(creation)
-		order by creation asc""".format(user = user, date = date)))
+		order by creation asc""".format(date = date), user))
 
 
 @frappe.whitelist()

frappe/email/doctype/email_group/email_group.js

@@ -3,11 +3,6 @@
 
 frappe.ui.form.on("Email Group", "refresh", function(frm) {
 	if(!frm.is_new()) {
-		frm.add_custom_button(__("View Subscribers"), function() {
-			frappe.route_options = {"email_group": frm.doc.name};
-			frappe.set_route("List", "Email Group Member");
-		}, __("View"));
-
 		frm.add_custom_button(__("Import Subscribers"), function() {
 			frappe.prompt({fieldtype:"Select", options: frm.doc.__onload.import_types,
 				label:__("Import Email From"), fieldname:"doctype", reqd:1},

frappe/email/doctype/email_group/email_group.json

@@ -5,6 +5,7 @@
  "creation": "2015-03-18 06:08:32.729800",
  "doctype": "DocType",
  "document_type": "Setup",
+ "engine": "InnoDB",
  "field_order": [
   "title",
   "total_subscribers",
@@ -41,8 +42,15 @@
    "options": "Email Template"
   }
  ],
- "links": [],
- "modified": "2020-02-21 14:12:48.884738",
+ "index_web_pages_for_search": 1,
+ "links": [
+  {
+   "group": "Members",
+   "link_doctype": "Email Group Member",
+   "link_fieldname": "email_group"
+  }
+ ],
+ "modified": "2020-09-24 16:41:55.286377",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Group",

frappe/email/doctype/email_template/email_template.json

@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "allow_import": 1,
  "allow_rename": 1,
  "autoname": "Prompt",
@@ -8,6 +9,8 @@
  "engine": "InnoDB",
  "field_order": [
   "subject",
+  "use_html",
+  "response_html",
   "response",
   "owner",
   "section_break_4",
@@ -22,11 +25,12 @@
    "reqd": 1
   },
   {
+   "depends_on": "eval:!doc.use_html",
    "fieldname": "response",
    "fieldtype": "Text Editor",
    "in_list_view": 1,
    "label": "Response",
-   "reqd": 1
+   "mandatory_depends_on": "eval:!doc.use_html"
   },
   {
    "default": "user",
@@ -45,10 +49,24 @@
    "fieldtype": "HTML",
    "label": "Email Reply Help",
    "options": "<h4>Email Reply Example</h4>\n\n<pre>Order Overdue\n\nTransaction {{ name }} has exceeded Due Date. Please take necessary action.\n\nDetails\n\n- Customer: {{ customer }}\n- Amount: {{ grand_total }}\n</pre>\n\n<h4>How to get fieldnames</h4>\n\n<p>The fieldnames you can use in your email template are the fields in the document from which you are sending the email. You can find out the fields of any documents via Setup &gt; Customize Form View and selecting the document type (e.g. Sales Invoice)</p>\n\n<h4>Templating</h4>\n\n<p>Templates are compiled using the Jinja Templating Language. To learn more about Jinja, <a class=\"strong\" href=\"http://jinja.pocoo.org/docs/dev/templates/\">read this documentation.</a></p>\n"
+  },
+  {
+   "default": "0",
+   "fieldname": "use_html",
+   "fieldtype": "Check",
+   "label": "Use HTML"
+  },
+  {
+   "depends_on": "eval:doc.use_html",
+   "fieldname": "response_html",
+   "fieldtype": "Code",
+   "label": "Response ",
+   "options": "HTML"
   }
  ],
  "icon": "fa fa-comment",
- "modified": "2019-10-30 14:15:00.956347",
+ "links": [],
+ "modified": "2020-11-30 14:12:50.321633",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Template",

frappe/email/doctype/email_template/email_template.py

@@ -9,7 +9,29 @@ from six import string_types
 
 class EmailTemplate(Document):
 	def validate(self):
-		validate_template(self.response)
+		if self.use_html:
+			validate_template(self.response_html)
+		else:
+			validate_template(self.response)
+
+	def get_formatted_subject(self, doc):
+		return frappe.render_template(self.subject, doc)
+
+	def get_formatted_response(self, doc):
+		if self.use_html:
+			return frappe.render_template(self.response_html, doc)
+
+		return frappe.render_template(self.response, doc)
+
+	def get_formatted_email(self, doc):
+		if isinstance(doc, string_types):
+			doc = json.loads(doc)
+
+		return {
+			"subject" : self.get_formatted_subject(doc),
+			"message" : self.get_formatted_response(doc)
+		}
+
 
 @frappe.whitelist()
 def get_email_template(template_name, doc):
@@ -18,5 +40,4 @@ def get_email_template(template_name, doc):
 		doc = json.loads(doc)
 
 	email_template = frappe.get_doc("Email Template", template_name)
-	return {"subject" : frappe.render_template(email_template.subject, doc),
-			"message" : frappe.render_template(email_template.response, doc)}
+	return email_template.get_formatted_email(doc)

frappe/email/doctype/newsletter/newsletter.py

@@ -85,11 +85,11 @@ class Newsletter(WebsiteGenerator):
 			self.db_set("scheduled_to_send", len(self.recipients))
 
 	def get_message(self):
-
+		if self.content_type == "HTML":
+			return frappe.render_template(self.message_html, {"doc": self.as_dict()})
 		return {
 			'Rich Text': self.message,
-			'Markdown': markdown(self.message_md),
-			'HTML': self.message_html
+			'Markdown': markdown(self.message_md)
 		}[self.content_type or 'Rich Text']
 
 	def get_recipients(self):

frappe/email/doctype/notification/notification.js

@@ -97,14 +97,7 @@ frappe.notification = {
 	},
 	setup_example_message: function(frm) {
 		let template = '';
-		if (frm.doc.channel === 'WhatsApp') {
-			template = `<h5 style='display: inline-block'>Warning:</h5> Only Use Pre-Approved WhatsApp for Business Template
-<h5>Message Example</h5>
-
-<pre>
-Your appointment is coming up on {{ doc.date }} at {{ doc.time }}
-</pre>`;
-		} else if (frm.doc.channel === 'Email') {
+		if (frm.doc.channel === 'Email') {
 			template = `<h5>Message Example</h5>
 
 <pre>&lt;h3&gt;Order Overdue&lt;/h3&gt;
@@ -124,7 +117,7 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 &lt;/ul&gt;
 </pre>
 			`;
-		} else {
+		} else if (in_list(['Slack', 'System Notification', 'SMS'], frm.doc.channel)) {
 			template = `<h5>Message Example</h5>
 
 <pre>*Order Overdue*
@@ -142,7 +135,9 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 • Amount: {{ doc.grand_total }}
 </pre>`;
 		}
-		frm.set_df_property('message_examples', 'options', template);
+		if (template) {
+			frm.set_df_property('message_examples', 'options', template);
+		}
 
 	}
 };

frappe/email/doctype/notification/notification.json

@@ -10,7 +10,6 @@
   "enabled",
   "column_break_2",
   "channel",
-  "twilio_number",
   "slack_webhook_url",
   "filters",
   "subject",
@@ -61,7 +60,7 @@
    "fieldname": "channel",
    "fieldtype": "Select",
    "label": "Channel",
-   "options": "Email\nSlack\nSystem Notification\nWhatsApp\nSMS",
+   "options": "Email\nSlack\nSystem Notification\nSMS",
    "reqd": 1,
    "set_only_once": 1
   },
@@ -80,14 +79,14 @@
    "label": "Filters"
   },
   {
-   "depends_on": "eval: !in_list(['SMS', 'WhatsApp'], doc.channel)",
+   "depends_on": "eval: in_list(['Email', 'Slack', 'System Notification'], doc.channel)",
    "description": "To add dynamic subject, use jinja tags like\n\n<div><pre><code>{{ doc.name }} Delivered</code></pre></div>",
    "fieldname": "subject",
    "fieldtype": "Data",
    "ignore_xss_filter": 1,
    "in_list_view": 1,
    "label": "Subject",
-   "mandatory_depends_on": "eval:!in_list(['SMS', 'WhatsApp'], doc.channel)"
+   "mandatory_depends_on": "eval: in_list(['Email', 'Slack', 'System Notification'], doc.channel)"
   },
   {
    "fieldname": "document_type",
@@ -208,7 +207,7 @@
    "label": "Value To Be Set"
   },
   {
-   "depends_on": "eval:in_list(['Email', 'SMS', 'WhatsApp'], doc.channel)",
+   "depends_on": "eval:doc.channel !=\"Slack\"",
    "fieldname": "column_break_5",
    "fieldtype": "Section Break",
    "label": "Recipients"
@@ -263,15 +262,6 @@
    "label": "Print Format",
    "options": "Print Format"
   },
-  {
-   "depends_on": "eval: doc.channel==='WhatsApp'",
-   "description": "To use WhatsApp for Business, initialize <a href=\"#Form/Twilio Settings\">Twilio Settings</a>.",
-   "fieldname": "twilio_number",
-   "fieldtype": "Link",
-   "label": "Twilio Number",
-   "mandatory_depends_on": "eval: doc.channel==='WhatsApp'",
-   "options": "Twilio Number Group"
-  },
   {
    "default": "0",
    "depends_on": "eval: doc.channel !== 'System Notification'",
@@ -291,7 +281,7 @@
  "icon": "fa fa-envelope",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-09-03 10:33:23.084590",
+ "modified": "2020-11-24 14:25:43.245677",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Notification",

frappe/email/doctype/notification/notification.py

@@ -14,7 +14,6 @@ from frappe.utils.safe_exec import get_safe_globals
 from frappe.modules.utils import export_module_json, get_doc_module
 from six import string_types
 from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
-from frappe.integrations.doctype.twilio_settings.twilio_settings import send_whatsapp_message
 from frappe.core.doctype.sms_settings.sms_settings import send_sms
 from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification
 
@@ -29,7 +28,7 @@ class Notification(Document):
 			self.name = self.subject
 
 	def validate(self):
-		if self.channel not in ('WhatsApp', 'SMS'):
+		if self.channel in ("Email", "Slack", "System Notification"):
 			validate_template(self.subject)
 
 		validate_template(self.message)
@@ -43,7 +42,6 @@ class Notification(Document):
 		self.validate_forbidden_types()
 		self.validate_condition()
 		self.validate_standard()
-		self.validate_twilio_settings()
 		frappe.cache().hdel('notifications', self.document_type)
 
 	def on_update(self):
@@ -70,11 +68,6 @@ def get_context(context):
 		if self.is_standard and not frappe.conf.developer_mode:
 			frappe.throw(_('Cannot edit Standard Notification. To edit, please disable this and duplicate it'))
 
-	def validate_twilio_settings(self):
-		if self.enabled and self.channel == "WhatsApp" \
-			and not frappe.db.get_single_value("Twilio Settings", "enabled"):
-			frappe.throw(_("Please enable Twilio settings to send WhatsApp messages"))
-
 	def validate_condition(self):
 		temp_doc = frappe.new_doc(self.document_type)
 		if self.condition:
@@ -137,9 +130,6 @@ def get_context(context):
 			if self.channel == 'Slack':
 				self.send_a_slack_msg(doc, context)
 
-			if self.channel == 'WhatsApp':
-				self.send_whatsapp_msg(doc, context)
-
 			if self.channel == 'SMS':
 				self.send_sms(doc, context)
 
@@ -191,6 +181,7 @@ def get_context(context):
 			'document_type': doc.doctype,
 			'document_name': doc.name,
 			'subject': subject,
+			'from_user': doc.modified_by or doc.owner,
 			'email_content': frappe.render_template(self.message, context),
 			'attached_file': attachments and json.dumps(attachments[0])
 		}
@@ -230,13 +221,6 @@ def get_context(context):
 			reference_doctype=doc.doctype,
 			reference_name=doc.name)
 
-	def send_whatsapp_msg(self, doc, context):
-		send_whatsapp_message(
-			sender=self.twilio_number,
-			receiver_list=self.get_receiver_list(doc, context),
-			message=frappe.render_template(self.message, context),
-		)
-
 	def send_sms(self, doc, context):
 		send_sms(
 			receiver_list=self.get_receiver_list(doc, context),
@@ -302,7 +286,7 @@ def get_context(context):
 
 			# For sending messages to the owner's mobile phone number
 			if recipient.receiver_by_document_field == 'owner':
-				receiver_list.append(get_user_info(doc.get('owner'), 'mobile_no'))
+				receiver_list += get_user_info([dict(user_name=doc.get('owner'))], 'mobile_no')
 			# For sending messages to the number specified in the receiver field
 			elif recipient.receiver_by_document_field:
 				receiver_list.append(doc.get(recipient.receiver_by_document_field))

frappe/email/receive.py

@@ -536,6 +536,8 @@ class Email:
 			except MaxFileSizeReachedError:
 				# WARNING: bypass max file size exception
 				pass
+			except frappe.FileAlreadyAttachedException:
+				pass
 			except frappe.DuplicateEntryError:
 				# same file attached twice??
 				pass

frappe/email/smtp.py

@@ -210,10 +210,9 @@ class SMTPServer:
 		try:
 			if self.use_ssl:
 				if not self.port:
-					self.smtp_port = 465
+					self.port = 465
 
-				self._sess = smtplib.SMTP_SSL((self.server or "").encode('utf-8'),
-						cint(self.port) or None)
+				self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port))
 			else:
 				if self.use_tls and not self.port:
 					self.port = 587

frappe/email/test_smtp.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# License: The MIT License
+
+import unittest
+from frappe.email.smtp import SMTPServer
+
+class TestSMTP(unittest.TestCase):
+	def test_smtp_ssl_session(self):
+		for port in [None, 0, 465, "465"]:
+			make_server(port, 1, 0)
+
+	def test_smtp_tls_session(self):
+		for port in [None, 0, 587, "587"]:
+			make_server(port, 0, 1)
+
+
+def make_server(port, ssl, tls):
+	server = SMTPServer(
+		server = "smtp.gmail.com",
+		port = port,
+		use_ssl = ssl,
+		use_tls = tls
+	)
+
+	server.sess

frappe/event_streaming/doctype/event_consumer/event_consumer.py

@@ -31,10 +31,12 @@ class EventConsumer(Document):
 			self.update_consumer_status()
 		else:
 			frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0)
-
+		
 		frappe.cache().delete_value('event_consumer_document_type_map')
 
 	def on_trash(self):
+		for i in frappe.get_all('Event Update Log Consumer', {'consumer': self.name}):
+			frappe.delete_doc('Event Update Log Consumer', i.name)
 		frappe.cache().delete_value('event_consumer_document_type_map')
 
 	def update_consumer_status(self):
@@ -88,8 +90,9 @@ def register_consumer(data):
 
 	for entry in consumer_doctypes:
 		consumer.append('consumer_doctypes', {
-			'ref_doctype': entry,
-			'status': 'Pending'
+			'ref_doctype': entry.get('doctype'),
+			'status': 'Pending',
+			'condition': entry.get('condition')
 		})
 
 	consumer.insert()
@@ -153,3 +156,53 @@ def notify(consumer):
 		jobs = get_jobs()
 		if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed:
 			frappe.enqueue(enqueued_method, queue='long', enqueue_after_commit=True, **{'consumer': consumer})
+
+
+def has_consumer_access(consumer, update_log):
+	"""Checks if consumer has completely satisfied all the conditions on the doc"""
+
+	if isinstance(consumer, str):
+		consumer = frappe.get_doc('Event Consumer', consumer)
+
+	if not frappe.db.exists(update_log.ref_doctype, update_log.docname):
+		# Delete Log
+		# Check if the last Update Log of this document was read by this consumer
+		last_update_log = frappe.get_all(
+			'Event Update Log',
+			filters={
+				'ref_doctype': update_log.ref_doctype,
+				'docname': update_log.docname,
+				'creation': ['<', update_log.creation]
+			},
+			order_by='creation desc',
+			limit_page_length=1
+		)
+		if not len(last_update_log):
+			return False
+
+		last_update_log = frappe.get_doc('Event Update Log', last_update_log[0].name)
+		return len([x for x in last_update_log.consumers if x.consumer == consumer.name])
+
+	doc = frappe.get_doc(update_log.ref_doctype, update_log.docname)
+	try:
+		for dt_entry in consumer.consumer_doctypes:
+			if dt_entry.ref_doctype != update_log.ref_doctype:
+				continue
+
+			if not dt_entry.condition:
+				return True
+
+			condition: str = dt_entry.condition
+			if condition.startswith('cmd:'):
+				cmd = condition.split('cmd:')[1].strip()
+				args = {
+					'consumer': consumer,
+					'doc': doc,
+					'update_log': update_log
+				}
+				return frappe.call(cmd, **args)
+			else:
+				return frappe.safe_eval(condition, frappe._dict(doc=doc))
+	except Exception as e:
+		frappe.log_error(title='has_consumer_access error', message=e)
+	return False

frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json

@@ -7,7 +7,8 @@
  "field_order": [
   "ref_doctype",
   "status",
-  "unsubscribed"
+  "unsubscribed",
+  "condition"
  ],
  "fields": [
   {
@@ -37,11 +38,17 @@
    "in_list_view": 1,
    "label": "Unsubscribed",
    "read_only": 1
+  },
+  {
+   "fieldname": "condition",
+   "fieldtype": "Code",
+   "label": "Condition",
+   "read_only": 1
   }
  ],
  "istable": 1,
  "links": [],
- "modified": "2020-08-14 12:38:40.918620",
+ "modified": "2020-11-07 09:26:49.894294",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Consumer Document Type",

frappe/event_streaming/doctype/event_producer/event_producer.json

@@ -13,7 +13,6 @@
   "api_secret",
   "column_break_6",
   "user",
-  "last_update",
   "incoming_change"
  ],
  "fields": [
@@ -25,12 +24,6 @@
    "reqd": 1,
    "unique": 1
   },
-  {
-   "fieldname": "last_update",
-   "fieldtype": "Data",
-   "label": "Last Update",
-   "read_only": 1
-  },
   {
    "description": "API Key of the user(Event Subscriber) on the producer site",
    "fieldname": "api_key",
@@ -77,7 +70,7 @@
   }
  ],
  "links": [],
- "modified": "2020-09-08 18:50:57.687979",
+ "modified": "2020-10-26 13:00:15.361316",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Producer",

frappe/event_streaming/doctype/event_producer/event_producer.py

@@ -79,18 +79,36 @@ class EventProducer(Document):
 			)
 			if response:
 				response = json.loads(response)
-				self.last_update = response['last_update']
+				self.set_last_update(response['last_update'])
 			else:
 				frappe.throw(_('Failed to create an Event Consumer or an Event Consumer for the current site is already registered.'))
 
+	def set_last_update(self, last_update):
+		last_update_doc_name = frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name))
+		if not last_update_doc_name:
+			frappe.get_doc(dict(
+				doctype = 'Event Producer Last Update',
+				event_producer = self.producer_url,
+				last_update = last_update
+			)).insert(ignore_permissions=True)
+		else:
+			frappe.db.set_value('Event Producer Last Update', last_update_doc_name, 'last_update', last_update)
+
+	def get_last_update(self):
+		return frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name), 'last_update')
+
 	def get_request_data(self):
 		consumer_doctypes = []
 		for entry in self.producer_doctypes:
 			if entry.has_mapping:
 				# if mapping, subscribe to remote doctype on consumer's site
-				consumer_doctypes.append(frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype'))
+				dt = frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype')
 			else:
-				consumer_doctypes.append(entry.ref_doctype)
+				dt = entry.ref_doctype
+			consumer_doctypes.append({
+				"doctype": dt,
+				"condition": entry.condition
+			})
 
 		user_key = frappe.db.get_value('User', self.user, 'api_key')
 		user_secret = get_decrypted_password('User', self.user, 'api_secret')
@@ -131,7 +149,8 @@ class EventProducer(Document):
 					event_consumer.consumer_doctypes.append({
 						'ref_doctype': ref_doctype,
 						'status': get_approval_status(config, ref_doctype),
-						'unsubscribed': entry.unsubscribe
+						'unsubscribed': entry.unsubscribe,
+						'condition': entry.condition
 					})
 				event_consumer.user = self.user
 				event_consumer.incoming_change = True
@@ -184,7 +203,7 @@ def pull_from_node(event_producer):
 	"""pull all updates after the last update timestamp from event producer site"""
 	event_producer = frappe.get_doc('Event Producer', event_producer)
 	producer_site = get_producer_site(event_producer.producer_url)
-	last_update = event_producer.last_update
+	last_update = event_producer.get_last_update()
 
 	(doctypes, mapping_config, naming_config) = get_config(event_producer.producer_doctypes)
 
@@ -239,7 +258,7 @@ def sync(update, producer_site, event_producer, in_retry=False):
 			return 'Failed'
 		log_event_sync(update, event_producer.name, 'Failed', frappe.get_traceback())
 
-	event_producer.db_set('last_update', update.creation)
+	event_producer.set_last_update(update.creation)
 	frappe.db.commit()
 
 
@@ -333,13 +352,13 @@ def set_delete(update):
 
 def get_updates(producer_site, last_update, doctypes):
 	"""Get all updates generated after the last update timestamp"""
-	docs = producer_site.get_list(
-		doctype='Event Update Log',
-		filters={'ref_doctype': ('in', doctypes), 'creation': ('>', last_update)},
-		fields=['update_type', 'ref_doctype', 'docname', 'data', 'name', 'creation']
-	)
-	docs.reverse()
-	return [frappe._dict(d) for d in docs]
+	docs = producer_site.post_request({
+			'cmd': 'frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer',
+			'event_consumer': get_url(),
+			'doctypes': frappe.as_json(doctypes),
+			'last_update': last_update
+	})
+	return [frappe._dict(d) for d in (docs or [])]
 
 
 def get_local_doc(update):

frappe/event_streaming/doctype/event_producer/test_event_producer.py

@@ -152,6 +152,82 @@ class TestEventProducer(unittest.TestCase):
 
 		reset_configuration(producer_url)
 
+	def test_conditional_events(self):
+		producer = get_remote_site()
+		
+		# Add Condition
+		event_producer = frappe.get_doc('Event Producer', producer_url)
+		note_producer_entry = [
+			x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note'
+		][0]
+		note_producer_entry.condition = 'doc.public == 1'
+		event_producer.save()
+
+		# Make test doc
+		producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync')
+		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']})
+		producer_note1 = producer.insert(producer_note1)
+
+		# Make Update
+		producer_note1['content'] = 'Test Conditional Sync Content'
+		producer_note1 = producer.update(producer_note1)
+
+		self.pull_producer_data()
+
+		# Check if synced here
+		self.assertFalse(frappe.db.exists('Note', producer_note1.name))
+
+		# Lets satisfy the condition
+		producer_note1['public'] = 1
+		producer_note1 = producer.update(producer_note1)
+
+		self.pull_producer_data()
+
+		# it should sync now
+		self.assertTrue(frappe.db.exists('Note', producer_note1.name))
+		local_note = frappe.get_doc('Note', producer_note1.name)
+		self.assertEqual(local_note.content, producer_note1.content)
+
+		reset_configuration(producer_url)
+
+	def test_conditional_events_with_cmd(self):
+		producer = get_remote_site()
+		
+		# Add Condition
+		event_producer = frappe.get_doc('Event Producer', producer_url)
+		note_producer_entry = [
+			x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note'
+		][0]
+		note_producer_entry.condition = 'cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note'
+		event_producer.save()
+
+		# Make test doc
+		producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync cmd')
+		delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']})
+		producer_note1 = producer.insert(producer_note1)
+
+		# Make Update
+		producer_note1['content'] = 'Test Conditional Sync Content'
+		producer_note1 = producer.update(producer_note1)
+
+		self.pull_producer_data()
+
+		# Check if synced here
+		self.assertFalse(frappe.db.exists('Note', producer_note1.name))
+
+		# Lets satisfy the condition
+		producer_note1['public'] = 1
+		producer_note1 = producer.update(producer_note1)
+
+		self.pull_producer_data()
+
+		# it should sync now
+		self.assertTrue(frappe.db.exists('Note', producer_note1.name))
+		local_note = frappe.get_doc('Note', producer_note1.name)
+		self.assertEqual(local_note.content, producer_note1.content)
+
+		reset_configuration(producer_url)
+
 	def test_update_log(self):
 		producer = get_remote_site()
 		producer_doc = insert_into_producer(producer, 'test update log')
@@ -221,6 +297,8 @@ class TestEventProducer(unittest.TestCase):
 
 		reset_configuration(producer_url)
 
+def can_sync_note(consumer, doc, update_log):
+	return doc.public == 1
 
 def setup_event_producer_for_inner_mapping():
 	event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
@@ -322,6 +400,7 @@ def create_event_producer(producer_url):
 def reset_configuration(producer_url):
 	event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True)
 	event_producer.producer_doctypes = []
+	event_producer.conditions = []
 	event_producer.producer_url = producer_url
 	event_producer.append('producer_doctypes', {
 		'ref_doctype': 'ToDo',

frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json

@@ -10,7 +10,8 @@
   "use_same_name",
   "unsubscribe",
   "has_mapping",
-  "mapping"
+  "mapping",
+  "condition"
  ],
  "fields": [
   {
@@ -63,11 +64,16 @@
    "fieldtype": "Check",
    "in_list_view": 1,
    "label": "Unsubscribe"
+  },
+  {
+   "fieldname": "condition",
+   "fieldtype": "Code",
+   "label": "Condition"
   }
  ],
  "istable": 1,
  "links": [],
- "modified": "2020-08-14 11:38:01.278996",
+ "modified": "2020-11-07 09:26:58.463868",
  "modified_by": "Administrator",
  "module": "Event Streaming",
  "name": "Event Producer Document Type",

frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2020, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Event Producer Last Update', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/custom/doctype/custom_link/custom_link.json → frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json

@@ -1,36 +1,36 @@
 {
  "actions": [],
- "autoname": "field:document_type",
- "creation": "2020-04-08 15:16:44.342509",
+ "autoname": "field:event_producer",
+ "creation": "2020-10-26 12:53:11.940177",
  "doctype": "DocType",
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
-  "document_type",
-  "links"
+  "event_producer",
+  "last_update"
  ],
  "fields": [
   {
-   "fieldname": "document_type",
-   "fieldtype": "Link",
+   "fieldname": "event_producer",
+   "fieldtype": "Data",
    "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
+   "label": "Event Producer",
    "reqd": 1,
    "unique": 1
   },
   {
-   "fieldname": "links",
-   "fieldtype": "Table",
-   "label": "Links",
-   "options": "DocType Link"
+   "fieldname": "last_update",
+   "fieldtype": "Data",
+   "label": "Last Update"
   }
  ],
+ "in_create": 1,
+ "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-04-08 16:42:59.402671",
+ "modified": "2020-10-26 13:22:27.056599",
  "modified_by": "Administrator",
- "module": "Custom",
- "name": "Custom Link",
+ "module": "Event Streaming",
+ "name": "Event Producer Last Update",
  "owner": "Administrator",
  "permissions": [
   {
@@ -46,6 +46,7 @@
    "write": 1
   }
  ],
+ "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1