You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

278 regels
7.3 KiB

  1. # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
  2. # MIT License. See license.txt
  3. from __future__ import unicode_literals
  4. import frappe
  5. from frappe.utils.scheduler import enqueue_events
  6. from frappe.celery_app import get_celery, celery_task, task_logger, LONGJOBS_PREFIX, ASYNC_TASKS_PREFIX
  7. from frappe.utils import get_sites, touch_file
  8. from frappe.utils.error import make_error_snapshot
  9. from frappe.utils.file_lock import create_lock, delete_lock
  10. from frappe.handler import execute_cmd
  11. from frappe.async import set_task_status, END_LINE, get_std_streams
  12. from frappe.utils.scheduler import log
  13. import frappe.utils.response
  14. import sys
  15. import time
  16. import json
  17. import os
  18. import MySQLdb
  19. from frappe.utils.file_lock import check_lock, LockTimeoutError
  20. @celery_task()
  21. def sync_queues():
  22. """notifies workers to monitor newly added sites"""
  23. app = get_celery()
  24. shortjob_workers, longjob_workers, async_tasks_workers = get_workers(app)
  25. if shortjob_workers:
  26. for worker in shortjob_workers:
  27. sync_worker(app, worker)
  28. if longjob_workers:
  29. for worker in longjob_workers:
  30. sync_worker(app, worker, prefix=LONGJOBS_PREFIX)
  31. if async_tasks_workers:
  32. for worker in async_tasks_workers:
  33. sync_worker(app, worker, prefix=ASYNC_TASKS_PREFIX)
  34. def get_workers(app):
  35. longjob_workers = []
  36. shortjob_workers = []
  37. async_tasks_workers = []
  38. active_queues = app.control.inspect().active_queues()
  39. for worker in active_queues:
  40. if worker.startswith(LONGJOBS_PREFIX):
  41. longjob_workers.append(worker)
  42. elif worker.startswith(ASYNC_TASKS_PREFIX):
  43. async_tasks_workers.append(worker)
  44. else:
  45. shortjob_workers.append(worker)
  46. return shortjob_workers, longjob_workers, async_tasks_workers
  47. def sync_worker(app, worker, prefix=''):
  48. active_queues = set(get_active_queues(app, worker))
  49. required_queues = set(get_required_queues(app, prefix=prefix))
  50. to_add = required_queues - active_queues
  51. to_remove = active_queues - required_queues
  52. for queue in to_add:
  53. if is_site_in_maintenance_mode(queue, prefix):
  54. continue
  55. app.control.broadcast('add_consumer', arguments={
  56. 'queue': queue
  57. }, reply=True, destination=[worker])
  58. for queue in to_remove:
  59. app.control.broadcast('cancel_consumer', arguments={
  60. 'queue': queue
  61. }, reply=True, destination=[worker])
  62. def get_active_queues(app, worker):
  63. active_queues = app.control.inspect().active_queues()
  64. if not (active_queues and active_queues.get(worker)):
  65. return []
  66. return [queue['name'] for queue in active_queues[worker]]
  67. def get_required_queues(app, prefix=''):
  68. ret = []
  69. for site in get_sites():
  70. ret.append('{}{}'.format(prefix, site))
  71. if not prefix:
  72. # default queue only for shortjob workers
  73. ret.append(app.conf['CELERY_DEFAULT_QUEUE'])
  74. return ret
  75. def is_site_in_maintenance_mode(queue, prefix):
  76. # check if site is in maintenance mode
  77. site = queue.replace(prefix, "")
  78. try:
  79. frappe.init(site=site)
  80. if not frappe.local.conf.db_name or frappe.local.conf.maintenance_mode or frappe.conf.disable_scheduler:
  81. # don't add site if in maintenance mode
  82. return True
  83. except frappe.IncorrectSitePath:
  84. return True
  85. finally:
  86. frappe.destroy()
  87. return False
  88. @celery_task()
  89. def scheduler_task(site, event, handler, now=False):
  90. traceback = ""
  91. task_logger.info('running {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
  92. try:
  93. frappe.init(site=site)
  94. if not create_lock(handler):
  95. return
  96. if not now:
  97. frappe.connect(site=site)
  98. frappe.get_attr(handler)()
  99. except Exception:
  100. frappe.db.rollback()
  101. traceback = log(handler, "Method: {event}, Handler: {handler}".format(event=event, handler=handler))
  102. task_logger.warn(traceback)
  103. raise
  104. else:
  105. frappe.db.commit()
  106. finally:
  107. delete_lock(handler)
  108. if not now:
  109. frappe.destroy()
  110. task_logger.info('ran {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
  111. @celery_task()
  112. def enqueue_scheduler_events():
  113. for site in get_sites():
  114. enqueue_lock = os.path.join(site, 'locks', 'enqueue.lock')
  115. try:
  116. if check_lock(enqueue_lock, timeout=1800):
  117. continue
  118. touch_file(enqueue_lock)
  119. enqueue_events_for_site.delay(site=site)
  120. except LockTimeoutError:
  121. os.remove(enqueue_lock)
  122. @celery_task()
  123. def enqueue_events_for_site(site):
  124. try:
  125. frappe.init(site=site)
  126. if frappe.local.conf.maintenance_mode or frappe.conf.disable_scheduler:
  127. return
  128. frappe.connect(site=site)
  129. enqueue_events(site)
  130. except:
  131. task_logger.error('Exception in Enqueue Events for Site {0}'.format(site))
  132. raise
  133. finally:
  134. delete_lock('enqueue')
  135. frappe.destroy()
  136. @celery_task()
  137. def pull_from_email_account(site, email_account):
  138. try:
  139. frappe.init(site=site)
  140. frappe.connect(site=site)
  141. email_account = frappe.get_doc("Email Account", email_account)
  142. email_account.receive()
  143. frappe.db.commit()
  144. finally:
  145. frappe.destroy()
  146. @celery_task(bind=True)
  147. def run_async_task(self, site=None, user=None, cmd=None, form_dict=None, hijack_std=False):
  148. ret = {}
  149. frappe.init(site)
  150. frappe.connect()
  151. frappe.local.task_id = self.request.id
  152. if hijack_std:
  153. original_stdout, original_stderr = sys.stdout, sys.stderr
  154. sys.stdout, sys.stderr = get_std_streams(self.request.id)
  155. frappe.local.stdout, frappe.local.stderr = sys.stdout, sys.stderr
  156. try:
  157. set_task_status(self.request.id, "Running")
  158. frappe.db.commit()
  159. frappe.set_user(user)
  160. # sleep(60)
  161. frappe.local.form_dict = frappe._dict(form_dict)
  162. execute_cmd(cmd, from_async=True)
  163. ret = frappe.local.response
  164. except Exception, e:
  165. frappe.db.rollback()
  166. ret = frappe.local.response
  167. http_status_code = getattr(e, "http_status_code", 500)
  168. ret['status_code'] = http_status_code
  169. frappe.errprint(frappe.get_traceback())
  170. frappe.utils.response.make_logs()
  171. set_task_status(self.request.id, "Error", response=ret)
  172. task_logger.error('Exception in running {}: {}'.format(cmd, ret['exc']))
  173. else:
  174. set_task_status(self.request.id, "Success", response=ret)
  175. if not frappe.flags.in_test:
  176. frappe.db.commit()
  177. finally:
  178. if not frappe.flags.in_test:
  179. frappe.destroy()
  180. if hijack_std:
  181. sys.stdout.write('\n' + END_LINE)
  182. sys.stderr.write('\n' + END_LINE)
  183. sys.stdout.close()
  184. sys.stderr.close()
  185. sys.stdout, sys.stderr = original_stdout, original_stderr
  186. return ret
  187. @celery_task()
  188. def sendmail(site, communication_name, print_html=None, print_format=None, attachments=None,
  189. recipients=None, cc=None, lang=None, session=None):
  190. try:
  191. frappe.connect(site=site)
  192. if lang:
  193. frappe.local.lang = lang
  194. if session:
  195. # hack to enable access to private files in PDF
  196. session['data'] = frappe._dict(session['data'])
  197. frappe.local.session.update(session)
  198. # upto 3 retries
  199. for i in xrange(3):
  200. try:
  201. communication = frappe.get_doc("Communication", communication_name)
  202. communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
  203. recipients=recipients, cc=cc)
  204. except MySQLdb.OperationalError, e:
  205. # deadlock, try again
  206. if e.args[0]==1213:
  207. frappe.db.rollback()
  208. time.sleep(1)
  209. continue
  210. else:
  211. raise
  212. else:
  213. break
  214. except:
  215. traceback = log("frappe.tasks.sendmail", frappe.as_json({
  216. "site": site,
  217. "communication_name": communication_name,
  218. "print_html": print_html,
  219. "print_format": print_format,
  220. "attachments": attachments,
  221. "recipients": recipients,
  222. "cc": cc,
  223. "lang": lang
  224. }))
  225. task_logger.error(traceback)
  226. raise
  227. else:
  228. frappe.db.commit()
  229. finally:
  230. frappe.destroy()