reactor(scheduler): created "Scheduler Job Type" and cleaned up scheduler

.pylintrc

@@ -0,0 +1,2 @@
+disable=access-member-before-definition
+disable=no-member

frappe/__init__.py

@@ -123,7 +123,6 @@ def init(site, sites_path=None, new_site=False):
 	local.debug_log = []
 	local.realtime_log = []
 	local.flags = _dict({
-		"ran_schedulers": [],
 		"currently_saving": [],
 		"redirect_location": "",
 		"in_install_db": False,
@@ -1504,7 +1503,20 @@ def logger(module=None, with_more_info=True):
 
 def log_error(message=None, title=None):
 	'''Log error to Error Log'''
-	return get_doc(dict(doctype='Error Log', error=as_unicode(message or get_traceback()),
+
+	# AI ALERT:
+	# the title and message may be swapped
+	# the better API for this is log_error(title, message), and used in many cases this way
+	# this hack tries to be smart about whats a title (single line ;-)) and fixes it
+
+	if message:
+		if '\n' not in message:
+			title = message
+			error = get_traceback()
+		else:
+			error = message
+
+	return get_doc(dict(doctype='Error Log', error=as_unicode(error),
 		method=title)).insert(ignore_permissions=True)
 
 def get_desk_link(doctype, name):

frappe/core/doctype/communication/email.py

@@ -10,7 +10,6 @@ from email.utils import formataddr
 from frappe.core.utils import get_parent_doc
 from frappe.utils import (get_url, get_formatted_email, cint,
   validate_email_address, split_emails, time_diff_in_seconds, parse_addr, get_datetime)
-from frappe.utils.scheduler import log
 from frappe.email.email_body import get_message_id
 import frappe.email.smtp
 import time
@@ -509,17 +508,7 @@ def sendmail(communication_name, print_html=None, print_format=None, attachments
 				break
 
 	except:
-		traceback = log("frappe.core.doctype.communication.email.sendmail", frappe.as_json({
-			"communication_name": communication_name,
-			"print_html": print_html,
-			"print_format": print_format,
-			"attachments": attachments,
-			"recipients": recipients,
-			"cc": cc,
-			"bcc": bcc,
-			"lang": lang
-		}))
-		frappe.logger(__name__).error(traceback)
+		traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail")
 		raise
 
 def update_mins_to_first_communication(parent, communication):

frappe/core/doctype/doctype/doctype.json

@@ -57,6 +57,8 @@
   "restrict_to_domain",
   "read_only",
   "in_create",
+  "actions_section",
+  "actions",
   "web_view",
   "has_web_view",
   "allow_guest_to_view",
@@ -454,11 +456,22 @@
    "fieldname": "nsm_parent_field",
    "fieldtype": "Data",
    "label": "Parent Field (Tree)"
+  },
+  {
+   "fieldname": "actions_section",
+   "fieldtype": "Section Break",
+   "label": "Actions"
+  },
+  {
+   "fieldname": "actions",
+   "fieldtype": "Table",
+   "label": "Actions",
+   "options": "DocType Action"
   }
  ],
  "icon": "fa fa-bolt",
  "idx": 6,
- "modified": "2019-09-07 14:28:05.392490",
+ "modified": "2019-09-23 16:29:21.209832",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",

frappe/core/doctype/doctype_action/doctype_action.json

@@ -0,0 +1,54 @@
+{
+ "actions": [],
+ "creation": "2019-09-23 16:28:13.953520",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "label",
+  "action_type",
+  "method",
+  "group"
+ ],
+ "fields": [
+  {
+   "columns": 2,
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label",
+   "reqd": 1
+  },
+  {
+   "columns": 6,
+   "fieldname": "method",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Method",
+   "reqd": 1
+  },
+  {
+   "fieldname": "group",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Group"
+  },
+  {
+   "fieldname": "action_type",
+   "fieldtype": "Data",
+   "label": "Action Type",
+   "options": "Server Action"
+  }
+ ],
+ "istable": 1,
+ "modified": "2019-09-23 21:34:39.971700",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "DocType Action",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/doctype_action/doctype_action.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class DocTypeAction(Document):
+	pass

frappe/core/doctype/scheduled_job_log/scheduled_job_log.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Scheduled Job Log', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/scheduled_job_log/scheduled_job_log.json

@@ -0,0 +1,62 @@
+{
+ "creation": "2019-09-23 14:36:36.935869",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "status",
+  "scheduled_job",
+  "details"
+ ],
+ "fields": [
+  {
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Status",
+   "options": "Scheduled\nSuccess\nFailed",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "scheduled_job",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Scheduled Job",
+   "options": "Scheduled Job Type",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "details",
+   "fieldtype": "Code",
+   "label": "Details",
+   "read_only": 1
+  }
+ ],
+ "modified": "2019-09-23 14:36:36.935869",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Scheduled Job Log",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/scheduled_job_log/scheduled_job_log.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+# import frappe
+from frappe.model.document import Document
+
+class ScheduledJobLog(Document):
+	pass

frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+# import frappe
+import unittest
+
+class TestScheduledJobLog(unittest.TestCase):
+	pass

frappe/core/doctype/scheduled_job_type/scheduled_job_type.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2019, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('Scheduled Job Type', {
+	// refresh: function(frm) {
+
+	// }
+});

frappe/core/doctype/scheduled_job_type/scheduled_job_type.json

@@ -0,0 +1,92 @@
+{
+ "actions": [
+  {
+   "action_path": "frappe.core.doctype.scheduled_job_type.scheduled_job_type.execute_event",
+   "label": "Execute",
+   "method": "frappe.core.doctype.scheduled_job_type.scheduled_job_type.execute_event"
+  }
+ ],
+ "creation": "2019-09-23 14:34:09.205368",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "stopped",
+  "method",
+  "queue",
+  "cron_format",
+  "last_execution",
+  "create_log"
+ ],
+ "fields": [
+  {
+   "fieldname": "method",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Method",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "queue",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Queue",
+   "options": "All\nHourly\nDaily\nDaily Long\nWeekly\nWeekly Long\nMonthly\nMonthly Long\nCron\nYearly\nAnnual",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "stopped",
+   "fieldtype": "Check",
+   "label": "Stopped"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:doc.queue==='All'",
+   "fieldname": "create_log",
+   "fieldtype": "Check",
+   "label": "Create Log"
+  },
+  {
+   "fieldname": "last_execution",
+   "fieldtype": "Datetime",
+   "label": "Last Execution",
+   "read_only": 1
+  },
+  {
+   "allow_in_quick_entry": 1,
+   "depends_on": "eval:doc.queue==='Cron'",
+   "fieldname": "cron_format",
+   "fieldtype": "Data",
+   "label": "Cron Format",
+   "read_only": 1
+  }
+ ],
+ "in_create": 1,
+ "modified": "2019-09-23 22:19:22.594874",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "Scheduled Job Type",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+
+import frappe, json
+from frappe.model.document import Document
+from frappe.utils import now_datetime, get_datetime
+from datetime import datetime
+from croniter import croniter
+from frappe.utils.background_jobs import enqueue
+
+CRON_MAP = {
+	"Yearly": "0 0 1 1 *",
+	"Annual": "0 0 1 1 *",
+	"Monthly": "0 0 1 * *",
+	"Monthly Long": "0 0 1 * *",
+	"Weekly": "0 0 * * 0",
+	"Weekly Long": "0 0 * * 0",
+	"Daily": "0 0 * * *",
+	"Daily Long": "0 0 * * *",
+	"Hourly": "0 * * * *",
+	"Hourly Long": "0 * * * *",
+	"All": "0/" + str((frappe.get_conf().scheduler_interval or 240) // 60) + " * * * *",
+}
+
+class ScheduledJobType(Document):
+	def autoname(self):
+		self.name = '.'.join(self.method.split('.')[-2:])
+
+	def validate(self):
+		if self.queue != 'All':
+			# force logging for all events other than continuous ones (ALL)
+			self.create_log = 1
+
+	def enqueue(self):
+		# enqueue event if last execution is done
+		if self.is_event_due():
+			self.update_last_execution()
+			frappe.flags.enqueued_jobs.append(self.method)
+			enqueue('frappe.core.doctype.scheduled_job_type.scheduled_job_type.run_scheduled_job',
+				job_type=self.method)
+
+	def is_event_due(self, current_time = None):
+		'''Return true if event is due based on time lapsed since last execution'''
+		# save last execution in expected execution time as per cron
+		self.last_execution = self.get_next_execution()
+
+		# if the next scheduled event is before NOW, then its due!
+		return self.last_execution <= (current_time or now_datetime())
+
+	def get_next_execution(self):
+		if not self.cron_format:
+			self.cron_format = CRON_MAP[self.queue]
+
+		return croniter(self.cron_format,
+			get_datetime(self.last_execution)).get_next(datetime)
+
+	def execute(self):
+		try:
+			frappe.logger(__name__).info('Started Scheduled Job: {0} for {1}'.format(self.method, frappe.local.site))
+			frappe.get_attr(self.method)()
+			frappe.db.commit()
+			frappe.logger(__name__).info('Completed Scheduled Job: {0} for {1}'.format(self.method, frappe.local.site))
+		except Exception:
+			frappe.db.rollback()
+			frappe.log_error('{} failed'.format(self.method))
+			frappe.logger(__name__).info('Failed Scheduled Job: {0} for {1}'.format(self.method, frappe.local.site))
+
+
+	def update_last_execution(self):
+		self.db_set('last_execution', self.last_execution, update_modified=False)
+		frappe.db.commit()
+
+	def get_queue_name(self):
+		return self.queue.replace(' ', '_').lower()
+
+@frappe.whitelist()
+def execute_event(doc):
+	frappe.only_for('System Manager')
+	doc = json.loads(doc)
+	frappe.get_doc('Scheduled Job Type', doc.get('name')).execute()
+
+def run_scheduled_job(job_type):
+	'''This is a wrapper function that runs a hooks.scheduler_events method'''
+	frappe.get_doc('Scheduled Job Type', dict(method=job_type)).execute()
+
+def sync_jobs():
+	frappe.reload_doc('core', 'doctype', 'scheduled_job_type')
+	all_events = []
+	scheduler_events = frappe.get_hooks("scheduler_events")
+	insert_events(all_events, scheduler_events)
+	clear_events(all_events, scheduler_events)
+
+def insert_events(all_events, scheduler_events):
+	for event_type in scheduler_events:
+		events = scheduler_events.get(event_type)
+		if isinstance(events, dict):
+			insert_cron_event(events, all_events)
+		else:
+			# hourly, daily etc
+			insert_event_list(events, event_type, all_events)
+
+def insert_cron_event(events, all_events):
+	for cron_format in events:
+		for event in events.get(cron_format):
+			all_events.append(event)
+			insert_single_event('Cron', event, cron_format)
+
+def insert_event_list(events, event_type, all_events):
+	for event in events:
+		all_events.append(event)
+		queue = event_type.replace('_', ' ').title()
+		insert_single_event(queue, event)
+
+def insert_single_event(queue, event, cron_format = None):
+	if not frappe.db.exists('Scheduled Job Type', dict(method=event)):
+		frappe.get_doc(dict(
+			doctype = 'Scheduled Job Type',
+			method = event,
+			cron_format = cron_format,
+			queue = queue
+		)).insert()
+
+def clear_events(all_events, scheduler_events):
+	for event in frappe.get_all('Scheduled Job Type', ('name', 'method')):
+		if event.method not in all_events:
+			frappe.db.delete_doc('Scheduled Job Type', event.name)

frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import frappe
+import unittest
+from frappe.utils import get_datetime
+
+from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
+
+class TestScheduledJobType(unittest.TestCase):
+	def setUp(self):
+		if not frappe.get_all('Scheduled Job Type', limit=1):
+			frappe.db.rollback()
+			frappe.db.sql('truncate `tabScheduled Job Type`')
+			sync_jobs()
+			frappe.db.commit()
+
+	def test_sync_jobs(self):
+		all_job = frappe.get_doc('Scheduled Job Type',
+			dict(method='frappe.email.queue.flush'))
+		self.assertEqual(all_job.queue, 'All')
+
+		daily_job = frappe.get_doc('Scheduled Job Type',
+			dict(method='frappe.email.queue.clear_outbox'))
+		self.assertEqual(daily_job.queue, 'Daily')
+
+		# check if cron jobs are synced
+		cron_job = frappe.get_doc('Scheduled Job Type',
+			dict(method='frappe.oauth.delete_oauth2_data'))
+		self.assertEqual(cron_job.queue, 'Cron')
+		self.assertEqual(cron_job.cron_format, '0/15 * * * *')
+
+	def test_daily_job(self):
+		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.clear_outbox'))
+		job.db_set('last_execution', '2019-01-01 00:00:00')
+		self.assertTrue(job.is_event_due(get_datetime('2019-01-02 00:00:06')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:00:06')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-01 23:59:59')))
+
+	def test_weekly_job(self):
+		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.utils.change_log.check_for_update'))
+		job.db_set('last_execution', '2019-01-01 00:00:00')
+		self.assertTrue(job.is_event_due(get_datetime('2019-01-06 00:00:01')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-02 00:00:06')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-05 23:59:59')))
+
+	def test_monthly_job(self):
+		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.doctype.auto_email_report.auto_email_report.send_monthly'))
+		job.db_set('last_execution', '2019-01-01 00:00:00')
+		self.assertTrue(job.is_event_due(get_datetime('2019-02-01 00:00:01')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-15 00:00:06')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-31 23:59:59')))
+
+	def test_cron_job(self):
+		# runs every 15 mins
+		job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.oauth.delete_oauth2_data'))
+		job.db_set('last_execution', '2019-01-01 00:00:00')
+		self.assertTrue(job.is_event_due(get_datetime('2019-01-01 00:15:01')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:05:06')))
+		self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:14:59')))

frappe/core/doctype/version/version.py

@@ -50,7 +50,7 @@ def get_diff(old, new, for_child=False):
 		if df.fieldtype in no_value_fields and df.fieldtype not in table_fields:
 			continue
 
-		old_value, new_value = old.get(df.fieldname), new.get(df.fieldname)
+		old_value, new_value = old.get(df.fieldname) or [], new.get(df.fieldname) or []
 
 		if df.fieldtype in table_fields:
 			# make maps

frappe/email/doctype/email_account/email_account.py

@@ -21,7 +21,6 @@ from frappe.desk.form import assign_to
 from frappe.utils.user import get_system_managers
 from frappe.utils.background_jobs import enqueue, get_jobs
 from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts
-from frappe.utils.scheduler import log
 from frappe.utils.html_utils import clean_email_html
 from frappe.email.utils import get_port
 
@@ -284,7 +283,7 @@ class EmailAccount(Document):
 
 				except Exception:
 					frappe.db.rollback()
-					log('email_account.receive')
+					frappe.log_error('email_account.receive')
 					if self.use_imap:
 						self.handle_bad_emails(email_server, uid, msg, frappe.get_traceback())
 					exceptions.append(frappe.get_traceback())

frappe/email/doctype/newsletter/newsletter.py

@@ -9,7 +9,6 @@ from frappe import throw, _
 from frappe.website.website_generator import WebsiteGenerator
 from frappe.utils.verified_command import get_signed_params, verify_request
 from frappe.utils.background_jobs import enqueue
-from frappe.utils.scheduler import log
 from frappe.email.queue import send
 from frappe.email.doctype.email_group.email_group import add_subscribers
 from frappe.utils import parse_addr
@@ -213,7 +212,7 @@ def send_newsletter(newsletter):
 		doc.db_set("email_sent", 0)
 		frappe.db.commit()
 
-		log("send_newsletter")
+		frappe.log_error("send_newsletter")
 
 		raise
 

frappe/email/queue.py

@@ -12,7 +12,6 @@ from frappe.utils.verified_command import get_signed_params, verify_request
 from html2text import html2text
 from frappe.utils import get_url, nowdate, encode, now_datetime, add_days, split_emails, cstr, cint
 from rq.timeouts import JobTimeoutException
-from frappe.utils.scheduler import log
 from six import text_type, string_types
 
 class EmailLimitCrossedError(frappe.ValidationError): pass
@@ -469,7 +468,7 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
 
 		else:
 			# log to Error Log
-			log('frappe.email.queue.flush', text_type(e))
+			frappe.log_error('frappe.email.queue.flush')
 
 def prepare_message(email, recipient, recipients_list):
 	message = email.message

frappe/email/receive.py

@@ -12,7 +12,6 @@ import frappe
 from frappe import _, safe_decode, safe_encode
 from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now,
 	cint, cstr, strip, markdown, parse_addr)
-from frappe.utils.scheduler import log
 from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError
 
 class EmailSizeExceededError(frappe.ValidationError): pass
@@ -80,7 +79,7 @@ class EmailServer:
 
 		except _socket.error:
 			# log performs rollback and logs error in Error Log
-			log("receive.connect_pop")
+			frappe.log_error("receive.connect_pop")
 
 			# Invalid mail server -- due to refusing connection
 			frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
@@ -255,7 +254,7 @@ class EmailServer:
 
 			else:
 				# log performs rollback and logs error in Error Log
-				log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
+				frappe.log_error("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
 				self.errors = True
 				frappe.db.rollback()
 

frappe/hooks.py

@@ -76,8 +76,7 @@ leaderboards = "frappe.desk.leaderboard.get_leaderboards"
 
 on_session_creation = [
 	"frappe.core.doctype.activity_log.feed.login_feed",
-	"frappe.core.doctype.user.user.notify_admin_access_to_system_manager",
-	"frappe.utils.scheduler.reset_enabled_scheduler_events",
+	"frappe.core.doctype.user.user.notify_admin_access_to_system_manager"
 ]
 
 on_logout = "frappe.core.doctype.session_default_settings.session_default_settings.clear_session_defaults"
@@ -153,14 +152,18 @@ doc_events = {
 }
 
 scheduler_events = {
+	"cron": {
+		"0/15 * * * *": [
+			"frappe.oauth.delete_oauth2_data",
+			"frappe.website.doctype.web_page.web_page.check_publish_status",
+			"frappe.twofactor.delete_all_barcodes_for_users"
+		]
+	},
 	"all": [
 		"frappe.email.queue.flush",
 		"frappe.email.doctype.email_account.email_account.pull",
 		"frappe.email.doctype.email_account.email_account.notify_unreplied",
-		"frappe.oauth.delete_oauth2_data",
 		"frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment",
-		"frappe.twofactor.delete_all_barcodes_for_users",
-		"frappe.website.doctype.web_page.web_page.check_publish_status",
 		'frappe.utils.global_search.sync_global_search'
 	],
 	"hourly": [

frappe/migrate.py

@@ -15,6 +15,7 @@ from frappe.desk.notifications import clear_notifications
 from frappe.website import render
 from frappe.core.doctype.language.language import sync_languages
 from frappe.modules.utils import sync_customizations
+from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.utils import global_search
 
 def migrate(verbose=True, rebuild_website=False, skip_failing=False):
@@ -46,9 +47,11 @@ def migrate(verbose=True, rebuild_website=False, skip_failing=False):
 
 		# run patches
 		frappe.modules.patch_handler.run_all(skip_failing)
+
 		# sync
 		frappe.model.sync.sync_all(verbose=verbose)
 		frappe.translate.clear_cache()
+		sync_jobs()
 		sync_fixtures()
 		sync_customizations()
 		sync_languages()

frappe/model/__init__.py

@@ -45,7 +45,7 @@ default_fields = ('doctype','name','owner','creation','modified','modified_by',
 	'parent','parentfield','parenttype','idx','docstatus')
 optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen")
 table_fields = ('Table', 'Table MultiSelect')
-core_doctypes_list = ('DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role',
+core_doctypes_list = ('DocType', 'DocField', 'DocPerm', 'DocType Action','User', 'Role', 'Has Role',
 	'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',
 	'Customize Form Field', 'Property Setter', 'Custom Field', 'Custom Script')
 

frappe/model/base_document.py

@@ -22,6 +22,8 @@ max_positive_value = {
 	'bigint': 2 ** 63
 }
 
+DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action')
+
 _classes = {}
 
 def get_controller(doctype):
@@ -255,7 +257,7 @@ class BaseDocument(object):
 
 	def get_valid_columns(self):
 		if self.doctype not in frappe.local.valid_columns:
-			if self.doctype in ("DocField", "DocPerm") and self.parent in ("DocType", "DocField", "DocPerm"):
+			if self.doctype in DOCTYPES_FOR_DOCTYPE:
 				from frappe.model.meta import get_table_columns
 				valid = get_table_columns(self.doctype)
 			else:
@@ -312,7 +314,7 @@ class BaseDocument(object):
 			self.created_by = self.modified_by = frappe.session.user
 
 		# if doctype is "DocType", don't insert null values as we don't know who is valid yet
-		d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in ('DocType', 'DocField', 'DocPerm'))
+		d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in DOCTYPES_FOR_DOCTYPE)
 
 		columns = list(d)
 		try:
@@ -347,7 +349,7 @@ class BaseDocument(object):
 			self.db_insert()
 			return
 
-		d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in ('DocType', 'DocField', 'DocPerm'))
+		d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in DOCTYPES_FOR_DOCTYPE)
 
 		# don't update name, as case might've been changed
 		name = d['name']

frappe/model/document.py

@@ -150,8 +150,8 @@ class Document(BaseDocument):
 			super(Document, self).__init__(d)
 
 		if self.name=="DocType" and self.doctype=="DocType":
-			from frappe.model.meta import doctype_table_fields
-			table_fields = doctype_table_fields
+			from frappe.model.meta import DOCTYPE_TABLE_FIELDS
+			table_fields = DOCTYPE_TABLE_FIELDS
 		else:
 			table_fields = self.meta.get_table_fields()
 

frappe/model/meta.py

@@ -151,7 +151,7 @@ class Meta(Document):
 			if self.name!="DocType":
 				self._table_fields = self.get('fields', {"fieldtype": ['in', table_fields]})
 			else:
-				self._table_fields = doctype_table_fields
+				self._table_fields = DOCTYPE_TABLE_FIELDS
 
 		return self._table_fields
 
@@ -165,7 +165,7 @@ class Meta(Document):
 
 	def get_valid_columns(self):
 		if not hasattr(self, "_valid_columns"):
-			if self.name in ("DocType", "DocField", "DocPerm", "Property Setter"):
+			if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action',"Property Setter"):
 				self._valid_columns = get_table_columns(self.name)
 			else:
 				self._valid_columns = self.default_fields + \
@@ -174,7 +174,7 @@ class Meta(Document):
 		return self._valid_columns
 
 	def get_table_field_doctype(self, fieldname):
-		return { "fields": "DocField", "permissions": "DocPerm"}.get(fieldname)
+		return { "fields": "DocField", "permissions": "DocPerm", "actions": "DocType Action"}.get(fieldname)
 
 	def get_field(self, fieldname):
 		'''Return docfield from meta'''
@@ -441,9 +441,10 @@ class Meta(Document):
 	def is_nested_set(self):
 		return self.has_field('lft') and self.has_field('rgt')
 
-doctype_table_fields = [
+DOCTYPE_TABLE_FIELDS = [
 	frappe._dict({"fieldname": "fields", "options": "DocField"}),
-	frappe._dict({"fieldname": "permissions", "options": "DocPerm"})
+	frappe._dict({"fieldname": "permissions", "options": "DocPerm"}),
+	frappe._dict({"fieldname": "actions", "options": "DocType Action"}),
 ]
 
 #######

frappe/model/sync.py

@@ -29,6 +29,7 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe
 		# these need to go first at time of install
 		for d in (("core", "docfield"),
 			("core", "docperm"),
+			("core", "doctype_action"),
 			("core", "role"),
 			("core", "has_role"),
 			("core", "doctype"),

frappe/patches.txt

@@ -9,6 +9,7 @@ frappe.patches.v7_2.remove_in_filter
 frappe.patches.v11_0.drop_column_apply_user_permissions
 execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) #2017-09-22
 execute:frappe.reload_doc('core', 'doctype', 'docfield', force=True) #2018-02-20
+execute:frappe.reload_doc('core', 'doctype', 'doctype_action', force=True) #2019-09-23
 execute:frappe.reload_doc('core', 'doctype', 'custom_docperm')
 execute:frappe.reload_doc('core', 'doctype', 'docperm') #2018-05-29
 execute:frappe.reload_doc('core', 'doctype', 'comment')

frappe/public/js/frappe/form/form.js

@@ -111,6 +111,7 @@ frappe.ui.form.Form = class FrappeForm {
 			$("body").attr("data-sidebar", 1);
 		}
 		this.setup_file_drop();
+		this.setup_doctype_actions();
 
 		this.setup_done = true;
 	}
@@ -319,6 +320,24 @@ frappe.ui.form.Form = class FrappeForm {
 		}
 	}
 
+	// sets up the refresh event for custom buttons
+	// added via configuration
+	setup_doctype_actions() {
+		if (this.meta.actions) {
+			for (let action of this.meta.actions) {
+				frappe.ui.form.on(this.doctype, 'refresh', () => {
+					if (!this.is_new()) {
+						this.add_custom_button(action.label, () => {
+							frappe.xcall(action.method, {doc: this.doc}).then(() => {
+								frappe.msgprint({message:__('Event Executed'), alert:true});
+							});
+						}, action.group);
+					}
+				});
+			}
+		}
+	}
+
 	switch_doc(docname) {
 		// record switch
 		if(this.docname != docname && (!this.meta.in_dialog || this.in_form) && !this.meta.istable) {

frappe/test_runner.py

@@ -71,7 +71,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(),
 		else:
 			ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast)
 
-		frappe.db.commit()
+		if frappe.db: frappe.db.commit()
 
 		# workaround! since there is no separate test db
 		frappe.clear_cache()

frappe/tests/test_scheduler.py

@@ -2,11 +2,9 @@ from __future__ import unicode_literals
 
 from unittest import TestCase
 from dateutil.relativedelta import relativedelta
-from frappe.utils.scheduler import (enqueue_applicable_events, restrict_scheduler_events_if_dormant,
-	get_enabled_scheduler_events)
-from frappe import _dict
+from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.utils.background_jobs import enqueue
-from frappe.utils import now_datetime, today, add_days, add_to_date
+from frappe.utils.scheduler import enqueue_events
 
 import frappe
 import time
@@ -17,60 +15,19 @@ def test_timeout():
 
 class TestScheduler(TestCase):
 	def setUp(self):
-		frappe.db.set_global('enabled_scheduler_events', "")
-		frappe.flags.ran_schedulers = []
-
-	def test_all_events(self):
-		last = now_datetime() - relativedelta(hours=2)
-		enqueue_applicable_events(frappe.local.site, now_datetime(), last)
-		self.assertTrue("all" in frappe.flags.ran_schedulers)
-
-	def test_enabled_events(self):
-		frappe.flags.enabled_events = ["hourly", "hourly_long", "daily", "daily_long",
-			"weekly", "weekly_long", "monthly", "monthly_long"]
-
-		# maintain last_event and next_event on the same day
-		last_event = now_datetime().replace(hour=0, minute=0, second=0, microsecond=0)
-		next_event = last_event + relativedelta(minutes=30)
-
-		enqueue_applicable_events(frappe.local.site, next_event, last_event)
-		self.assertFalse("cron" in frappe.flags.ran_schedulers)
-
-		# maintain last_event and next_event on the same day
-		last_event = now_datetime().replace(hour=0, minute=0, second=0, microsecond=0)
-		next_event = last_event + relativedelta(hours=2)
-
-		frappe.flags.ran_schedulers = []
-		enqueue_applicable_events(frappe.local.site, next_event, last_event)
-		self.assertTrue("all" in frappe.flags.ran_schedulers)
-		self.assertTrue("hourly" in frappe.flags.ran_schedulers)
-
-		frappe.flags.enabled_events = None
-
-	def test_enabled_events_day_change(self):
-
-		# use flags instead of globals as this test fails intermittently
-		# the root cause has not been identified but the culprit seems cache
-		# since cache is mutable, it maybe be changed by a parallel process
-		frappe.flags.enabled_events = ["daily", "daily_long", "weekly", "weekly_long",
-			"monthly", "monthly_long"]
-
-		# maintain last_event and next_event on different days
-		next_event = now_datetime().replace(hour=0, minute=0, second=0, microsecond=0)
-		last_event = next_event - relativedelta(hours=2)
-
-		frappe.flags.ran_schedulers = []
-		enqueue_applicable_events(frappe.local.site, next_event, last_event)
-		self.assertTrue("all" in frappe.flags.ran_schedulers)
-		self.assertFalse("hourly" in frappe.flags.ran_schedulers)
-
-		frappe.flags.enabled_events = None
-
-
+		if not frappe.get_all('Scheduled Job Type', limit=1):
+			sync_jobs()
 
+	def test_enqueue_jobs(self):
+		frappe.db.sql('update `tabScheduled Job Type` set last_execution = "2010-01-01 00:00:00"')
+		enqueue_events(site = frappe.local.site)
 
+		self.assertTrue('frappe.email.queue.clear_outbox', frappe.flags.enqueued_jobs)
+		self.assertTrue('frappe.utils.change_log.check_for_update', frappe.flags.enqueued_jobs)
+		self.assertTrue('frappe.email.doctype.auto_email_report.auto_email_report.send_monthly', frappe.flags.enqueued_jobs)
 
 	def test_job_timeout(self):
+		return
 		job = enqueue(test_timeout, timeout=10)
 		count = 5
 		while count > 0:
@@ -80,6 +37,3 @@ class TestScheduler(TestCase):
 				break
 
 		self.assertTrue(job.is_failed)
-
-	def tearDown(self):
-		frappe.flags.ran_schedulers = []

frappe/utils/background_jobs.py

@@ -79,8 +79,6 @@ def run_doc_method(doctype, name, doc_method, **kwargs):
 
 def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, retry=0):
 	'''Executes job in a worker, performs commit/rollback and logs if there is any error'''
-	from frappe.utils.scheduler import log
-
 	if is_async:
 		frappe.connect(site)
 		if os.environ.get('CI'):
@@ -115,12 +113,12 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True,
 				is_async=is_async, retry=retry+1)
 
 		else:
-			log(method_name, message=repr(locals()))
+			frappe.log_error(method_name)
 			raise
 
 	except:
 		frappe.db.rollback()
-		log(method_name, message=repr(locals()))
+		frappe.log_error(method_name)
 		raise
 
 	else:

frappe/utils/scheduler.py

@@ -10,38 +10,15 @@ Events:
 
 from __future__ import unicode_literals, print_function
 
-import frappe
-import json
+import frappe, os, time
 import schedule
-import time
-import frappe.utils
-import os
+from frappe.utils import now_datetime, get_datetime
 from frappe.utils import get_sites
-from datetime import datetime
-from frappe.utils.background_jobs import enqueue, get_jobs, queue_timeout
-from frappe.utils.data import get_datetime, now_datetime
 from frappe.core.doctype.user.user import STANDARD_USERS
-from frappe.installer import update_site_config
-from six import string_types
-from croniter import croniter
+from frappe.utils.background_jobs import get_jobs
 
 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
 
-cron_map = {
-	"yearly": "0 0 1 1 *",
-	"annual": "0 0 1 1 *",
-	"monthly": "0 0 1 * *",
-	"monthly_long": "0 0 1 * *",
-	"weekly": "0 0 * * 0",
-	"weekly_long": "0 0 * * 0",
-	"daily": "0 0 * * *",
-	"daily_long": "0 0 * * *",
-	"midnight": "0 0 * * *",
-	"hourly": "0 * * * *",
-	"hourly_long": "0 * * * *",
-	"all": "0/" + str((frappe.get_conf().scheduler_interval or 240) // 60) + " * * * *",
-}
-
 def start_scheduler():
 	'''Run enqueue_events_for_all_sites every 2 minutes (default).
 	Specify scheduler_interval in seconds in common_site_config.json'''
@@ -60,17 +37,16 @@ def enqueue_events_for_all_sites():
 		return
 
 	with frappe.init_site():
-		jobs_per_site = get_jobs()
 		sites = get_sites()
 
 	for site in sites:
 		try:
-			enqueue_events_for_site(site=site, queued_jobs=jobs_per_site[site])
+			enqueue_events_for_site(site=site)
 		except:
 			# it should try to enqueue other sites
 			print(frappe.get_traceback())
 
-def enqueue_events_for_site(site, queued_jobs):
+def enqueue_events_for_site(site):
 	def log_and_raise():
 		frappe.logger(__name__).error('Exception in Enqueue Events for Site {0}'.format(site) +
 			'\n' + frappe.get_traceback())
@@ -82,7 +58,7 @@ def enqueue_events_for_site(site, queued_jobs):
 		if is_scheduler_inactive():
 			return
 
-		enqueue_events(site=site, queued_jobs=queued_jobs)
+		enqueue_events(site=site)
 
 		frappe.logger(__name__).debug('Queued events for site {0}'.format(site))
 	except frappe.db.OperationalError as e:
@@ -96,128 +72,13 @@ def enqueue_events_for_site(site, queued_jobs):
 	finally:
 		frappe.destroy()
 
-def enqueue_events(site, queued_jobs):
-	nowtime = frappe.utils.now_datetime()
-	last = frappe.db.get_value('System Settings', 'System Settings', 'scheduler_last_event')
-
-	# set scheduler last event
-	frappe.db.set_value('System Settings', 'System Settings',
-		'scheduler_last_event', nowtime.strftime(DATETIME_FORMAT),
-		update_modified=False)
-	frappe.db.commit()
-
-	out = []
-	if last:
-		last = datetime.strptime(last, DATETIME_FORMAT)
-		out = enqueue_applicable_events(site, nowtime, last, queued_jobs)
-
-	return '\n'.join(out)
-
-def enqueue_applicable_events(site, nowtime, last, queued_jobs=()):
-	nowtime_str = nowtime.strftime(DATETIME_FORMAT)
-	out = []
-
-	enabled_events = get_enabled_scheduler_events()
-
-	def trigger_if_enabled(site, event, last, queued_jobs):
-		trigger(site, event, last, queued_jobs)
-		_log(event)
-
-	def _log(event):
-		out.append("{time} - {event} - queued".format(time=nowtime_str, event=event))
-
-	for event in enabled_events:
-		trigger_if_enabled(site, event, last, queued_jobs)
-
-	if "all" not in enabled_events:
-		trigger_if_enabled(site, "all", last, queued_jobs)
-
-	return out
-
-def trigger(site, event, last=None, queued_jobs=(), now=False):
-	"""Trigger method in hooks.scheduler_events."""
-
-	queue = 'long' if event.endswith('_long') else 'short'
-	timeout = queue_timeout[queue]
-	if not queued_jobs and not now:
-		queued_jobs = get_jobs(site=site, queue=queue)
-
-	if frappe.flags.in_test:
-		frappe.flags.ran_schedulers.append(event)
-
-	events_from_hooks = get_scheduler_events(event)
-	if not events_from_hooks:
-		return
-
-	events = events_from_hooks
-	if not now:
-		events = []
-		if event == "cron":
-			for e in events_from_hooks:
-				e = cron_map.get(e, e)
-				if croniter.is_valid(e):
-					if croniter(e, last).get_next(datetime) <= frappe.utils.now_datetime():
-						events.extend(events_from_hooks[e])
-				else:
-					frappe.log_error("Cron string " + e + " is not valid", "Error triggering cron job")
-					frappe.logger(__name__).error('Exception in Trigger Events for Site {0}, Cron String {1}'.format(site, e))
-
-		else:
-			if croniter(cron_map[event], last).get_next(datetime) <= frappe.utils.now_datetime():
-				events.extend(events_from_hooks)
-
-	for handler in events:
-		if not now:
-			if handler not in queued_jobs:
-				enqueue(handler, queue, timeout, event)
-		else:
-			scheduler_task(site=site, event=event, handler=handler, now=True)
-
-def get_scheduler_events(event):
-	'''Get scheduler events from hooks and integrations'''
-	scheduler_events = frappe.cache().get_value('scheduler_events')
-	if not scheduler_events:
-		scheduler_events = frappe.get_hooks("scheduler_events")
-		frappe.cache().set_value('scheduler_events', scheduler_events)
-
-	return scheduler_events.get(event) or []
-
-def log(method, message=None):
-	"""log error in patch_log"""
-	message = frappe.utils.cstr(message) + "\n" if message else ""
-	message += frappe.get_traceback()
-
-	if not (frappe.db and frappe.db._conn):
-		frappe.connect()
-
-	frappe.db.rollback()
-	frappe.db.begin()
-
-	d = frappe.new_doc("Error Log")
-	d.method = method
-	d.error = message
-	d.insert(ignore_permissions=True)
-
-	frappe.db.commit()
-
-	return message
-
-def get_enabled_scheduler_events():
-	if 'enabled_events' in frappe.flags and frappe.flags.enabled_events:
-		return frappe.flags.enabled_events
-
-	enabled_events = frappe.db.get_global("enabled_scheduler_events")
-	if frappe.flags.in_test:
-		# TEMP for debug: this test fails randomly
-		print('found enabled_scheduler_events {0}'.format(enabled_events))
-
-	if enabled_events:
-		if isinstance(enabled_events, string_types):
-			enabled_events = json.loads(enabled_events)
-		return enabled_events
-
-	return ["all", "hourly", "hourly_long", "daily", "daily_long",
-		"weekly", "weekly_long", "monthly", "monthly_long", "cron"]
+def enqueue_events(site):
+	frappe.flags.enqueued_jobs = []
+	queued_jobs = get_jobs(key='job_type').get(site) or []
+	for job_type in frappe.get_all('Scheduled Job Type', dict(stopped=0)):
+		if not job_type.method in queued_jobs:
+			# don't add it to queue if still pending
+			frappe.get_doc('Scheduled Job Type', job_type.name).enqueue()
 
 def is_scheduler_inactive():
 	if frappe.local.conf.maintenance_mode:
@@ -229,6 +90,9 @@ def is_scheduler_inactive():
 	if is_scheduler_disabled():
 		return True
 
+	if is_dormant():
+		return True
+
 	return False
 
 def is_scheduler_disabled():
@@ -246,90 +110,15 @@ def enable_scheduler():
 def disable_scheduler():
 	toggle_scheduler(False)
 
-def get_errors(from_date, to_date, limit):
-	errors = frappe.db.sql("""select modified, method, error from `tabError Log`
-		where date(modified) between %s and %s
-		and error not like '%%[Errno 110] Connection timed out%%'
-		order by modified limit %s""", (from_date, to_date, limit), as_dict=True)
-	return ["""<p>Time: {modified}</p><pre><code>Method: {method}\n{error}</code></pre>""".format(**e)
-		for e in errors]
-
-def get_error_report(from_date=None, to_date=None, limit=10):
-	from frappe.utils import get_url, now_datetime, add_days
-
-	if not from_date:
-		from_date = add_days(now_datetime().date(), -1)
-	if not to_date:
-		to_date = add_days(now_datetime().date(), -1)
-
-	errors = get_errors(from_date, to_date, limit)
-
-	if errors:
-		return 1, """<h4>Error Logs (max {limit}):</h4>
-			<p>URL: <a href="{url}" target="_blank">{url}</a></p><hr>{errors}""".format(
-			limit=limit, url=get_url(), errors="<hr>".join(errors))
-	else:
-		return 0, "<p>No error logs</p>"
-
-def scheduler_task(site, event, handler, now=False):
-	'''This is a wrapper function that runs a hooks.scheduler_events method'''
-	frappe.logger(__name__).info('running {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
-	try:
-		if not now:
-			frappe.connect(site=site)
-
-		frappe.flags.in_scheduler = True
-		frappe.get_attr(handler)()
-
-	except Exception:
-		frappe.db.rollback()
-		traceback = log(handler, "Method: {event}, Handler: {handler}".format(event=event, handler=handler))
-		frappe.logger(__name__).error(traceback)
-		raise
-
-	else:
-		frappe.db.commit()
-
-	frappe.logger(__name__).info('ran {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
-
-
-def reset_enabled_scheduler_events(login_manager):
-	if login_manager.info.user_type == "System User":
-		try:
-			if frappe.db.get_global('enabled_scheduler_events'):
-				# clear restricted events, someone logged in!
-				frappe.db.set_global('enabled_scheduler_events', None)
-		except frappe.db.InternalError as e:
-			if frappe.db.is_timedout(e):
-				frappe.log_error(frappe.get_traceback(), "Error in reset_enabled_scheduler_events")
-			else:
-				raise
-		else:
-			is_dormant = frappe.conf.get('dormant')
-			if is_dormant:
-				update_site_config('dormant', 'None')
-
-
-def restrict_scheduler_events_if_dormant():
-	if is_dormant():
-		restrict_scheduler_events()
-		update_site_config('dormant', True)
-
-def restrict_scheduler_events(*args, **kwargs):
-	val = json.dumps(["hourly", "hourly_long", "daily", "daily_long", "weekly", "weekly_long", "monthly", "monthly_long", "cron"])
-	frappe.db.set_global('enabled_scheduler_events', val)
-
 def is_dormant(since = 345600):
 	last_user_activity = get_last_active()
 	if not last_user_activity:
 		# no user has ever logged in, so not yet used
 		return False
-	last_active = get_datetime(last_user_activity)
-	# Get now without tz info
-	now = now_datetime().replace(tzinfo=None)
-	time_since_last_active = now - last_active
-	if time_since_last_active.total_seconds() > since:  # 4 days
+
+	if now_datetime() - get_datetime(last_user_activity) > since:  # 4 days
 		return True
+
 	return False
 
 def get_last_active():