[email] also thread by subject and sender if in-reply-to is missing

frappe/__init__.py

@@ -503,6 +503,10 @@ def get_meta(doctype, cached=True):
 	import frappe.model.meta
 	return frappe.model.meta.get_meta(doctype, cached=cached)
 
+def get_meta_module(doctype):
+	import frappe.modules
+	return frappe.modules.load_doctype_module(doctype)
+
 def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reload=False,
 	ignore_permissions=False, flags=None):
 	"""Delete a document. Calls `frappe.model.delete_doc.delete_doc`.

frappe/desk/doctype/todo/test_todo.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import frappe
+import unittest
+
+# test_records = frappe.get_test_records('ToDo')
+
+class TestToDo(unittest.TestCase):
+	pass

frappe/desk/doctype/todo/todo.json

@@ -160,6 +160,14 @@
    "label": "Assigned By", 
    "options": "User", 
    "permlevel": 0
+  }, 
+  {
+   "fieldname": "sender", 
+   "fieldtype": "Data", 
+   "hidden": 1, 
+   "label": "Sender", 
+   "permlevel": 0, 
+   "precision": ""
   }
  ], 
  "hide_heading": 0, 
@@ -170,7 +178,7 @@
  "in_dialog": 0, 
  "issingle": 0, 
  "max_attachments": 0, 
- "modified": "2015-02-05 05:11:48.421852", 
+ "modified": "2015-05-04 07:44:46.567785", 
  "modified_by": "Administrator", 
  "module": "Desk", 
  "name": "ToDo", 

frappe/desk/doctype/todo/todo.py

@@ -8,11 +8,10 @@ import json
 from frappe.model.document import Document
 from frappe.utils import get_fullname
 
-class ToDo(Document):
-	def set_subject(self, subject):
-		"""Set description (called via incoming email)"""
-		self.description = subject
+subject_field = "description"
+sender_field = "sender"
 
+class ToDo(Document):
 	def validate(self):
 		if self.is_new():
 			self.add_assign_comment(frappe._("Assigned to {0}").format(get_fullname(self.owner)), "Assigned")

frappe/email/doctype/email_account/email_account.py

@@ -5,11 +5,12 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 from frappe.model.document import Document
-from frappe.utils import validate_email_add, cint
+from frappe.utils import validate_email_add, cint, get_datetime, DATE_FORMAT
 from frappe.email.smtp import SMTPServer
 from frappe.email.receive import POP3Server, Email
 from poplib import error_proto
 import markdown2
+from dateutil.relativedelta import relativedelta
 from datetime import datetime, timedelta
 
 class EmailAccount(Document):
@@ -157,6 +158,18 @@ class EmailAccount(Document):
 		it will create a new parent transaction (e.g. Issue)"""
 		in_reply_to = (email.mail.get("In-Reply-To") or "").strip(" <>")
 		parent = None
+
+		if self.append_to:
+			# set subject_field and sender_field
+			meta_module = frappe.get_meta_module(self.append_to)
+			meta = frappe.get_meta(self.append_to)
+			subject_field = getattr(meta_module, "subject_field", "subject")
+			if not meta.get_field(subject_field):
+				subject_field = None
+			sender_field = getattr(meta_module, "sender_field", "sender")
+			if not meta.get_field(sender_field):
+				sender_field = None
+
 		if in_reply_to:
 			if "@" in in_reply_to:
 
@@ -170,22 +183,30 @@ class EmailAccount(Document):
 						parent = frappe.get_doc(parent.reference_doctype,
 							parent.reference_name)
 
+		elif self.append_to and subject_field and sender_field:
+			# try and match by subject and sender
+			# if sent by same sender with same subject,
+			# append it to old coversation
+
+			parent = frappe.db.get_all(self.append_to, filters={
+				sender_field: email.from_email,
+				subject_field: ("like", "%{0}%".format(email.subject)),
+				"creation": (">", (get_datetime() - relativedelta(days=10)).strftime(DATE_FORMAT))
+			}, fields="name")
+
+			if parent:
+				parent = frappe.get_doc(self.append_to, parent[0].name)
+
 		if not parent and self.append_to:
 			# no parent found, but must be tagged
 			# insert parent type doc
 			parent = frappe.new_doc(self.append_to)
 
-			if parent.meta.get_field("subject"):
-				parent.subject = email.subject
-
-			if parent.meta.get_field("sender"):
-				parent.sender = email.from_email
-
-			if hasattr(parent, "set_subject"):
-				parent.set_subject(email.subject)
+			if subject_field:
+				parent.set(subject_field, email.subject)
 
-			if hasattr(parent, "set_sender"):
-				parent.set_sender(email.from_email)
+			if sender_field:
+				parent.set(sender_field, email.from_email)
 
 			parent.flags.ignore_mandatory = True
 

frappe/email/doctype/email_account/test_email_account.py

@@ -108,7 +108,26 @@ class TestEmailAccount(unittest.TestCase):
 		self.assertEquals(comm.reference_doctype, sent.doctype)
 		self.assertEquals(comm.reference_name, sent.name)
 
+	def test_threading_by_subject(self):
+		frappe.db.sql("""delete from tabCommunication
+			where sender in ('test_sender@example.com', 'test@example.com')""")
+
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-2.raw"), "r") as f:
+			test_mails = [f.read()]
+
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-3.raw"), "r") as f:
+			test_mails.append(f.read())
+
+		# parse reply
+		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
+		email_account.receive(test_mails=test_mails)
+
+		comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"},
+			fields=["name", "reference_doctype", "reference_name"])
 
+		# both communications attached to the same reference
+		self.assertEquals(comm_list[0].reference_doctype, comm_list[1].reference_doctype)
+		self.assertEquals(comm_list[0].reference_name, comm_list[1].reference_name)
 
 
 

frappe/email/doctype/email_account/test_mails/reply-2.raw

@@ -0,0 +1,45 @@
+Return-Path: <test_sender@example.com>
+Received: from [192.168.0.100] ([27.106.4.70])
+        by mx.google.com with ESMTPSA id n15sm4041161pdj.34.2014.09.15.23.48.02
+        for <test_sender@example.com>
+        (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128);
+        Mon, 15 Sep 2014 23:48:04 -0700 (PDT)
+From: Rushabh Mehta <test_sender@example.com>
+X-Google-Original-From: Rushabh Mehta <test_receiver@example.com>
+Content-Type: multipart/alternative; boundary="Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040"
+Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\))
+Subject: weird subject
+Date: Tue, 16 Sep 2014 12:17:58 +0530
+References: <54A4EFFA-AD17-456A-9851-9715574DF0C9@gmail.com>
+To: Rushabh Mehta <test_sender@example.com>
+X-Mailer: Apple Mail (2.1878.6)
+
+
+--Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+	charset=us-ascii
+
+test-reply-2
+
+
+On 16-Sep-2014, at 11:57 am, Rushabh Mehta <test_sender@example.com> wrote:
+
+> with attachment
+>
+> <erpnext-conf-14.png>
+>
+>
+
+
+--Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040
+Content-Transfer-Encoding: 7bit
+Content-Type: text/html;
+	charset=us-ascii
+
+<html><head><meta http-equiv="Content-Type" content="text/html charset=us-ascii"></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;">test-reply<br><div apple-content-edited="true">
+<br><br></div><div><div>On 16-Sep-2014, at 11:57 am, Rushabh Mehta &lt;<a href="mailto:test_sender@example.com">test_sender@example.com</a>&gt; wrote:</div><br class="Apple-interchange-newline"><blockquote type="cite"><meta http-equiv="Content-Type" content="text/html charset=us-ascii"><div style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><div>with attachment</div><br><div apple-content-edited="true">
+<div style="letter-spacing: normal; orphans: auto; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: auto; word-spacing: 0px; -webkit-text-stroke-width: 0px; word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><div style="font-family: Helvetica; font-style: normal; font-variant: normal; font-weight: normal; letter-spacing: normal; line-height: normal; orphans: 2; text-align: -webkit-auto; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><span>&lt;erpnext-conf-14.png&gt;</span><br><br></div></div>
+</div>
+<br></div></blockquote></div><br></body></html>
+--Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040--

frappe/email/doctype/email_account/test_mails/reply-3.raw

@@ -0,0 +1,45 @@
+Return-Path: <test_sender@example.com>
+Received: from [192.168.0.100] ([27.106.4.70])
+        by mx.google.com with ESMTPSA id n15sm4041161pdj.34.2014.09.15.23.48.02
+        for <test_sender@example.com>
+        (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128);
+        Mon, 15 Sep 2014 23:48:04 -0700 (PDT)
+From: Rushabh Mehta <test_sender@example.com>
+X-Google-Original-From: Rushabh Mehta <test_receiver@example.com>
+Content-Type: multipart/alternative; boundary="Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040"
+Mime-Version: 1.0 (Mac OS X Mail 7.3 \(1878.6\))
+Subject: Re: weird subject
+Date: Tue, 16 Sep 2014 12:17:58 +0530
+References: <54A4EFFA-AD17-456A-9851-9715574DF0C9@gmail.com>
+To: Rushabh Mehta <test_sender@example.com>
+X-Mailer: Apple Mail (2.1878.6)
+
+
+--Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain;
+	charset=us-ascii
+
+test-reply-3
+
+
+On 16-Sep-2014, at 11:57 am, Rushabh Mehta <test_sender@example.com> wrote:
+
+> with attachment
+>
+> <erpnext-conf-14.png>
+>
+>
+
+
+--Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040
+Content-Transfer-Encoding: 7bit
+Content-Type: text/html;
+	charset=us-ascii
+
+<html><head><meta http-equiv="Content-Type" content="text/html charset=us-ascii"></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;">test-reply<br><div apple-content-edited="true">
+<br><br></div><div><div>On 16-Sep-2014, at 11:57 am, Rushabh Mehta &lt;<a href="mailto:test_sender@example.com">test_sender@example.com</a>&gt; wrote:</div><br class="Apple-interchange-newline"><blockquote type="cite"><meta http-equiv="Content-Type" content="text/html charset=us-ascii"><div style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><div>with attachment</div><br><div apple-content-edited="true">
+<div style="letter-spacing: normal; orphans: auto; text-align: start; text-indent: 0px; text-transform: none; white-space: normal; widows: auto; word-spacing: 0px; -webkit-text-stroke-width: 0px; word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><div style="font-family: Helvetica; font-style: normal; font-variant: normal; font-weight: normal; letter-spacing: normal; line-height: normal; orphans: 2; text-align: -webkit-auto; text-indent: 0px; text-transform: none; white-space: normal; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;"><span>&lt;erpnext-conf-14.png&gt;</span><br><br></div></div>
+</div>
+<br></div></blockquote></div><br></body></html>
+--Apple-Mail=_C996D08F-7A29-4DA2-99B3-17133FA73040--