25'ten fazla konu seçemezsiniz Konular bir harf veya rakamla başlamalı, kısa çizgiler ('-') içerebilir ve en fazla 35 karakter uzunluğunda olabilir.
 
 
 
 
 
 

452 satır
13 KiB

  1. # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
  2. # MIT License. See license.txt
  3. from __future__ import unicode_literals
  4. import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re
  5. from email_reply_parser import EmailReplyParser
  6. from email.header import decode_header
  7. import frappe
  8. from frappe import _
  9. from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now,
  10. cint, cstr, strip, markdown)
  11. from frappe.utils.scheduler import log
  12. from frappe.utils.file_manager import get_random_filename, save_file, MaxFileSizeReachedError
  13. class EmailSizeExceededError(frappe.ValidationError): pass
  14. class EmailTimeoutError(frappe.ValidationError): pass
  15. class TotalSizeExceededError(frappe.ValidationError): pass
  16. class LoginLimitExceeded(frappe.ValidationError): pass
  17. class EmailServer:
  18. """Wrapper for POP server to pull emails."""
  19. def __init__(self, args=None):
  20. self.setup(args)
  21. def setup(self, args=None):
  22. # overrride
  23. self.settings = args or frappe._dict()
  24. def check_mails(self):
  25. # overrride
  26. return True
  27. def process_message(self, mail):
  28. # overrride
  29. pass
  30. def connect(self):
  31. """Connect to **Email Account**."""
  32. if cint(self.settings.use_imap):
  33. return self.connect_imap()
  34. else:
  35. return self.connect_pop()
  36. def connect_imap(self):
  37. """Connect to IMAP"""
  38. try:
  39. if cint(self.settings.use_ssl):
  40. self.imap = Timed_IMAP4_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
  41. else:
  42. self.imap = Timed_IMAP4(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
  43. self.imap.login(self.settings.username, self.settings.password)
  44. # connection established!
  45. return True
  46. except _socket.error:
  47. # Invalid mail server -- due to refusing connection
  48. frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
  49. raise
  50. except Exception, e:
  51. frappe.msgprint(_('Cannot connect: {0}').format(str(e)))
  52. raise
  53. def connect_pop(self):
  54. #this method return pop connection
  55. try:
  56. if cint(self.settings.use_ssl):
  57. self.pop = Timed_POP3_SSL(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
  58. else:
  59. self.pop = Timed_POP3(self.settings.host, timeout=frappe.conf.get("pop_timeout"))
  60. self.pop.user(self.settings.username)
  61. self.pop.pass_(self.settings.password)
  62. # connection established!
  63. return True
  64. except _socket.error:
  65. # log performs rollback and logs error in Error Log
  66. log("receive.connect_pop")
  67. # Invalid mail server -- due to refusing connection
  68. frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.'))
  69. raise
  70. except poplib.error_proto, e:
  71. if self.is_temporary_system_problem(e):
  72. return False
  73. else:
  74. frappe.msgprint(_('Invalid User Name or Support Password. Please rectify and try again.'))
  75. raise
  76. def get_messages(self):
  77. """Returns new email messages in a list."""
  78. if not self.check_mails():
  79. return # nothing to do
  80. frappe.db.commit()
  81. if not self.connect():
  82. return []
  83. try:
  84. # track if errors arised
  85. self.errors = False
  86. self.latest_messages = []
  87. email_list = self.get_new_mails()
  88. num = num_copy = len(email_list)
  89. # WARNING: Hard coded max no. of messages to be popped
  90. if num > 20: num = 20
  91. # size limits
  92. self.total_size = 0
  93. self.max_email_size = cint(frappe.local.conf.get("max_email_size"))
  94. self.max_total_size = 5 * self.max_email_size
  95. for i, message_meta in enumerate(email_list):
  96. # do not pull more than NUM emails
  97. if (i+1) > num:
  98. break
  99. try:
  100. self.retrieve_message(message_meta, i+1)
  101. except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded):
  102. break
  103. # WARNING: Mark as read - message number 101 onwards from the pop list
  104. # This is to avoid having too many messages entering the system
  105. num = num_copy
  106. if not cint(self.settings.use_imap):
  107. if num > 100 and not self.errors:
  108. for m in xrange(101, num+1):
  109. self.pop.dele(m)
  110. except Exception, e:
  111. if self.has_login_limit_exceeded(e):
  112. pass
  113. else:
  114. raise
  115. finally:
  116. # no matter the exception, pop should quit if connected
  117. if cint(self.settings.use_imap):
  118. self.imap.logout()
  119. else:
  120. self.pop.quit()
  121. return self.latest_messages
  122. def get_new_mails(self):
  123. """Return list of new mails"""
  124. if cint(self.settings.use_imap):
  125. self.imap.select("Inbox")
  126. response, message = self.imap.uid('search', None, "UNSEEN")
  127. email_list = message[0].split()
  128. else:
  129. email_list = self.pop.list()[1]
  130. return email_list
  131. def retrieve_message(self, message_meta, msg_num=None):
  132. incoming_mail = None
  133. try:
  134. self.validate_message_limits(message_meta)
  135. if cint(self.settings.use_imap):
  136. status, message = self.imap.uid('fetch', message_meta, '(RFC822)')
  137. self.latest_messages.append(message[0][1])
  138. else:
  139. msg = self.pop.retr(msg_num)
  140. self.latest_messages.append(b'\n'.join(msg[1]))
  141. except (TotalSizeExceededError, EmailTimeoutError):
  142. # propagate this error to break the loop
  143. self.errors = True
  144. raise
  145. except Exception, e:
  146. if self.has_login_limit_exceeded(e):
  147. self.errors = True
  148. raise LoginLimitExceeded, e
  149. else:
  150. # log performs rollback and logs error in Error Log
  151. log("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
  152. self.errors = True
  153. frappe.db.rollback()
  154. if not cint(self.settings.use_imap):
  155. self.pop.dele(msg_num)
  156. else:
  157. # mark as seen
  158. self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)')
  159. else:
  160. if not cint(self.settings.use_imap):
  161. self.pop.dele(msg_num)
  162. else:
  163. # mark as seen
  164. self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)')
  165. def has_login_limit_exceeded(self, e):
  166. return "-ERR Exceeded the login limit" in strip(cstr(e.message))
  167. def is_temporary_system_problem(self, e):
  168. messages = (
  169. "-ERR [SYS/TEMP] Temporary system problem. Please try again later.",
  170. "Connection timed out",
  171. )
  172. for message in messages:
  173. if message in strip(cstr(e.message)) or message in strip(cstr(getattr(e, 'strerror', ''))):
  174. return True
  175. return False
  176. def validate_message_limits(self, message_meta):
  177. # throttle based on email size
  178. if not self.max_email_size:
  179. return
  180. m, size = message_meta.split()
  181. size = cint(size)
  182. if size < self.max_email_size:
  183. self.total_size += size
  184. if self.total_size > self.max_total_size:
  185. raise TotalSizeExceededError
  186. else:
  187. raise EmailSizeExceededError
  188. def make_error_msg(self, msg_num, incoming_mail):
  189. error_msg = "Error in retrieving email."
  190. if not incoming_mail:
  191. try:
  192. # retrieve headers
  193. incoming_mail = Email(b'\n'.join(self.pop.top(msg_num, 5)[1]))
  194. except:
  195. pass
  196. if incoming_mail:
  197. error_msg += "\nDate: {date}\nFrom: {from_email}\nSubject: {subject}\n".format(
  198. date=incoming_mail.date, from_email=incoming_mail.from_email, subject=incoming_mail.subject)
  199. return error_msg
  200. class Email:
  201. """Wrapper for an email."""
  202. def __init__(self, content):
  203. """Parses headers, content, attachments from given raw message.
  204. :param content: Raw message."""
  205. self.raw = content
  206. self.mail = email.message_from_string(self.raw)
  207. self.text_content = ''
  208. self.html_content = ''
  209. self.attachments = []
  210. self.cid_map = {}
  211. self.parse()
  212. self.set_content_and_type()
  213. self.set_subject()
  214. self.set_from()
  215. self.message_id = (self.mail.get('Message-ID') or "").strip(" <>")
  216. if self.mail["Date"]:
  217. utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))
  218. utc_dt = datetime.datetime.utcfromtimestamp(utc)
  219. self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S')
  220. else:
  221. self.date = now()
  222. def parse(self):
  223. """Walk and process multi-part email."""
  224. for part in self.mail.walk():
  225. self.process_part(part)
  226. def set_subject(self):
  227. """Parse and decode `Subject` header."""
  228. _subject = decode_header(self.mail.get("Subject", "No Subject"))
  229. self.subject = _subject[0][0] or ""
  230. if _subject[0][1]:
  231. self.subject = self.subject.decode(_subject[0][1])
  232. else:
  233. # assume that the encoding is utf-8
  234. self.subject = self.subject.decode("utf-8")[:140]
  235. if not self.subject:
  236. self.subject = "No Subject"
  237. def set_from(self):
  238. # gmail mailing-list compatibility
  239. # use X-Original-Sender if available, as gmail sometimes modifies the 'From'
  240. _from_email = self.mail.get("X-Original-From") or self.mail["From"]
  241. _from_email, encoding = decode_header(_from_email)[0]
  242. if encoding:
  243. _from_email = _from_email.decode(encoding)
  244. else:
  245. _from_email = _from_email.decode('utf-8')
  246. self.from_email = extract_email_id(_from_email)
  247. self.from_real_name = email.utils.parseaddr(_from_email)[0]
  248. def set_content_and_type(self):
  249. self.content, self.content_type = '[Blank Email]', 'text/plain'
  250. if self.html_content:
  251. self.content, self.content_type = self.html_content, 'text/html'
  252. else:
  253. self.content, self.content_type = EmailReplyParser.parse_reply(self.text_content), 'text/plain'
  254. def process_part(self, part):
  255. """Parse email `part` and set it to `text_content`, `html_content` or `attachments`."""
  256. content_type = part.get_content_type()
  257. if content_type == 'text/plain':
  258. self.text_content += self.get_payload(part)
  259. elif content_type == 'text/html':
  260. self.html_content += self.get_payload(part)
  261. elif content_type == 'message/rfc822':
  262. # sent by outlook when another email is sent as an attachment to this email
  263. self.show_attached_email_headers_in_content(part)
  264. elif part.get_filename():
  265. self.get_attachment(part)
  266. def show_attached_email_headers_in_content(self, part):
  267. # get the multipart/alternative message
  268. message = list(part.walk())[1]
  269. headers = []
  270. for key in ('From', 'To', 'Subject', 'Date'):
  271. value = cstr(message.get(key))
  272. if value:
  273. headers.append('{label}: {value}'.format(label=_(key), value=value))
  274. self.text_content += '\n'.join(headers)
  275. self.html_content += '<hr>' + '\n'.join('<p>{0}</p>'.format(h) for h in headers)
  276. if not message.is_multipart() and message.get_content_type()=='text/plain':
  277. # email.parser didn't parse it!
  278. text_content = self.get_payload(message)
  279. self.text_content += text_content
  280. self.html_content += markdown(text_content)
  281. def get_charset(self, part):
  282. """Detect chartset."""
  283. charset = part.get_content_charset()
  284. if not charset:
  285. charset = chardet.detect(str(part))['encoding']
  286. return charset
  287. def get_payload(self, part):
  288. charset = self.get_charset(part)
  289. try:
  290. return unicode(part.get_payload(decode=True), str(charset), "ignore")
  291. except LookupError:
  292. return part.get_payload()
  293. def get_attachment(self, part):
  294. #charset = self.get_charset(part)
  295. fcontent = part.get_payload(decode=True)
  296. if fcontent:
  297. content_type = part.get_content_type()
  298. fname = part.get_filename()
  299. if fname:
  300. try:
  301. fname = cstr(decode_header(fname)[0][0])
  302. except:
  303. fname = get_random_filename(content_type=content_type)
  304. else:
  305. fname = get_random_filename(content_type=content_type)
  306. self.attachments.append({
  307. 'content_type': content_type,
  308. 'fname': fname,
  309. 'fcontent': fcontent,
  310. })
  311. cid = (part.get("Content-Id") or "").strip("><")
  312. if cid:
  313. self.cid_map[fname] = cid
  314. def save_attachments_in_doc(self, doc):
  315. """Save email attachments in given document."""
  316. saved_attachments = []
  317. for attachment in self.attachments:
  318. try:
  319. file_data = save_file(attachment['fname'], attachment['fcontent'],
  320. doc.doctype, doc.name, is_private=1)
  321. saved_attachments.append(file_data)
  322. if attachment['fname'] in self.cid_map:
  323. self.cid_map[file_data.name] = self.cid_map[attachment['fname']]
  324. except MaxFileSizeReachedError:
  325. # WARNING: bypass max file size exception
  326. pass
  327. except frappe.DuplicateEntryError:
  328. # same file attached twice??
  329. pass
  330. return saved_attachments
  331. def get_thread_id(self):
  332. """Extract thread ID from `[]`"""
  333. l = re.findall('(?<=\[)[\w/-]+', self.subject)
  334. return l and l[0] or None
  335. # fix due to a python bug in poplib that limits it to 2048
  336. poplib._MAXLINE = 20480
  337. class TimerMixin(object):
  338. def __init__(self, *args, **kwargs):
  339. self.timeout = kwargs.pop('timeout', 0.0)
  340. self.elapsed_time = 0.0
  341. self._super.__init__(self, *args, **kwargs)
  342. if self.timeout:
  343. # set per operation timeout to one-fifth of total pop timeout
  344. self.sock.settimeout(self.timeout / 5.0)
  345. def _getline(self, *args, **kwargs):
  346. start_time = time.time()
  347. ret = self._super._getline(self, *args, **kwargs)
  348. self.elapsed_time += time.time() - start_time
  349. if self.timeout and self.elapsed_time > self.timeout:
  350. raise EmailTimeoutError
  351. return ret
  352. def quit(self, *args, **kwargs):
  353. self.elapsed_time = 0.0
  354. return self._super.quit(self, *args, **kwargs)
  355. class Timed_POP3(TimerMixin, poplib.POP3):
  356. _super = poplib.POP3
  357. class Timed_POP3_SSL(TimerMixin, poplib.POP3_SSL):
  358. _super = poplib.POP3_SSL
  359. class Timed_IMAP4(TimerMixin, imaplib.IMAP4):
  360. _super = imaplib.IMAP4
  361. class Timed_IMAP4_SSL(TimerMixin, imaplib.IMAP4_SSL):
  362. _super = imaplib.IMAP4_SSL