Email cc in headers if expose recipients (#2441)

* move email cc to header

* [test fixes] move email cc to header

* email cc additional tests

* email cc cleanup and unsubsribe link test

* email cc permit footer based cc with expose_recipient and tests + ui

frappe/__init__.py

@@ -355,11 +355,11 @@ def get_request_header(key, default=None):
 	:param default: Default value."""
 	return request.headers.get(key, default)
 
-def sendmail(recipients=(), sender="", subject="No Subject", message="No Message",
+def sendmail(recipients=[], sender="", subject="No Subject", message="No Message",
 		as_markdown=False, delayed=True, reference_doctype=None, reference_name=None,
 		unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None,
 		attachments=None, content=None, doctype=None, name=None, reply_to=None,
-		cc=(), show_as_cc=(), in_reply_to=None, send_after=None, expose_recipients=False,
+		cc=[], in_reply_to=None, send_after=None, expose_recipients=None,
 		send_priority=1, communication=None, retry=1, now=None):
 	"""Send email using user's default **Email Account** or global default **Email Account**.
 
@@ -396,7 +396,7 @@ def sendmail(recipients=(), sender="", subject="No Subject", message="No Message
 		subject=subject, message=message,
 		reference_doctype = doctype or reference_doctype, reference_name = name or reference_name,
 		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message,
-		attachments=attachments, reply_to=reply_to, cc=cc, show_as_cc=show_as_cc, in_reply_to=in_reply_to,
+		attachments=attachments, reply_to=reply_to, cc=cc, in_reply_to=in_reply_to,
 		send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority,
 		communication=communication, now=now)
 

frappe/core/doctype/communication/email.py

@@ -136,9 +136,9 @@ def _notify(doc, print_html=None, print_format=None, attachments=None,
 		unsubscribe_message = ""
 
 	frappe.sendmail(
-		recipients=(recipients or []) + (cc or []),
-		show_as_cc=(cc or []),
-		expose_recipients=True,
+		recipients=(recipients or []),
+		cc=(cc or []),
+		expose_recipients="header",
 		sender=doc.sender,
 		reply_to=doc.incoming_email_account,
 		subject=doc.subject,

frappe/email/doctype/email_queue/email_queue.js

@@ -3,7 +3,7 @@
 
 frappe.ui.form.on("Email Queue", {
 	refresh: function(frm) {
-		if (frm.doc.status==="Not Sent") {
+		if (["Not Sent","Partially Sent"].indexOf(frm.doc.status)!=-1) {
 			frm.add_custom_button("Send Now", function() {
 				frappe.call({
 					method: 'frappe.email.doctype.email_queue.email_queue.send_now',
@@ -17,7 +17,7 @@ frappe.ui.form.on("Email Queue", {
 			});
 		}
 
-		if (frm.doc.status==="Error") {
+		if (["Error","Partially Errored"].indexOf(frm.doc.status)!=-1) {
 			frm.add_custom_button("Retry Sending", function() {
 				frm.call({
 					method: "retry_sending",

frappe/email/doctype/email_queue/email_queue.json

@@ -48,19 +48,47 @@
    "collapsible": 0, 
    "columns": 0, 
    "fieldname": "recipient", 
-   "fieldtype": "Data", 
+   "fieldtype": "Table", 
    "hidden": 0, 
    "ignore_user_permissions": 0, 
    "ignore_xss_filter": 0, 
    "in_filter": 0, 
    "in_global_search": 0, 
-   "in_list_view": 1, 
+   "in_list_view": 0, 
    "in_standard_filter": 0, 
    "label": "Recipient", 
    "length": 0, 
    "no_copy": 0, 
-   "options": "Email", 
+   "options": "Email Queue Recipient", 
+   "permlevel": 0, 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "show_as_cc", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Show as cc", 
+   "length": 0, 
+   "no_copy": 0, 
    "permlevel": 0, 
+   "precision": "", 
    "print_hide": 0, 
    "print_hide_if_no_value": 0, 
    "read_only": 0, 
@@ -331,6 +359,81 @@
    "search_index": 0, 
    "set_only_once": 0, 
    "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "fieldname": "unsubscribe_param", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_list_view": 0, 
+   "label": "Unsubscribe Param", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 1, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "fieldname": "unsubscribe_method", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_list_view": 0, 
+   "label": "Unsubscribe Method", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "fieldname": "expose_recipients", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_list_view": 0, 
+   "label": "Expose Recipients", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
   }
  ], 
  "hide_heading": 0, 
@@ -344,7 +447,7 @@
  "issingle": 0, 
  "istable": 0, 
  "max_attachments": 0, 
- "modified": "2016-11-17 16:45:02.296617", 
+ "modified": "2016-12-13 20:43:56.976928", 
  "modified_by": "Administrator", 
  "module": "Email", 
  "name": "Email Queue", 

frappe/email/doctype/email_queue/email_queue.py

@@ -22,7 +22,7 @@ class EmailQueue(Document):
 @frappe.whitelist()
 def retry_sending(name):
 	doc = frappe.get_doc("Email Queue", name)
-	if doc and doc.status == "Error":
+	if doc and (doc.status == "Error" or doc.status == "Partially Errored"):
 		doc.status = "Not Sent"
 		doc.save(ignore_permissions=True)
 

frappe/email/doctype/email_queue_recipient/email_queue_recipient.json

@@ -0,0 +1,122 @@
+{
+ "allow_copy": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2016-12-08 12:01:07.993900", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 0, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "recipient", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0,
+   "in_list_view": 1, 
+   "label": "Recipient", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Email", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "default": "Not Sent", 
+   "fieldname": "status", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0,
+   "in_list_view": 1, 
+   "label": "Status", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "\nNot Sent\nSending\nSent\nError\nExpired", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "error", 
+   "fieldtype": "Code", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0,
+   "in_list_view": 0, 
+   "label": "Error", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "hide_heading": 0, 
+ "hide_toolbar": 0, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "in_dialog": 0, 
+ "is_submittable": 0, 
+ "issingle": 0, 
+ "istable": 1, 
+ "max_attachments": 0, 
+ "modified": "2016-12-08 14:05:33.578240", 
+ "modified_by": "Administrator", 
+ "module": "Email", 
+ "name": "Email Queue Recipient", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_seen": 0
+}

frappe/email/doctype/email_queue_recipient/email_queue_recipient.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2015, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe.model.document import Document
+
+class EmailQueueRecipient(Document):
+	pass

frappe/email/doctype/newsletter/test_newsletter.py

@@ -20,21 +20,25 @@ class TestNewsletter(unittest.TestCase):
 
 	def test_send(self):
 		self.send_newsletter()
-		self.assertEquals(len(frappe.get_all("Email Queue")), 3)
+		self.assertEquals(len(frappe.get_all("Email Queue")), 1)
+		self.assertEquals(len(frappe.get_all("Email Queue Recipient")), 3)
 
 	def test_unsubscribe(self):
 		# test unsubscribe
 		self.send_newsletter()
-
+		from frappe.email.queue import flush
+		flush(from_test=True)
 		email = unquote(frappe.local.flags.signed_query_string.split("email=")[1].split("&")[0])
 
 		unsubscribe(email, "_Test Email Group")
 
 		self.send_newsletter()
-		self.assertEquals(len(frappe.get_all("Email Queue")), 2)
+		self.assertEquals(len(frappe.get_all("Email Queue")), 1)
+		self.assertEquals(len(frappe.get_all("Email Queue Recipient")), 2)
 
 	def send_newsletter(self):
 		frappe.db.sql("delete from `tabEmail Queue`")
+		frappe.db.sql("delete from `tabEmail Queue Recipient`")
 		frappe.delete_doc("Newsletter", "_Test Newsletter")
 		newsletter = frappe.get_doc({
 			"doctype": "Newsletter",

frappe/email/email_body.py

@@ -11,10 +11,10 @@ import email.utils
 
 def get_email(recipients, sender='', msg='', subject='[No Subject]',
 	text_content = None, footer=None, print_html=None, formatted=None, attachments=None,
-	content=None, reply_to=None, cc=(), email_account=None):
+	content=None, reply_to=None, cc=[], email_account=None, expose_recipients=None):
 	"""send an html email as multipart with attachments and all"""
 	content = content or msg
-	emailobj = EMail(sender, recipients, subject, reply_to=reply_to, cc=cc, email_account=email_account)
+	emailobj = EMail(sender, recipients, subject, reply_to=reply_to, cc=cc, email_account=email_account, expose_recipients=expose_recipients)
 
 	if not content.strip().startswith("<"):
 		content = markdown(content)
@@ -35,7 +35,7 @@ class EMail:
 	Also provides a clean way to add binary `FileData` attachments
 	Also sets all messages as multipart/alternative for cleaner reading in text-only clients
 	"""
-	def __init__(self, sender='', recipients=(), subject='', alternative=0, reply_to=None, cc=(), email_account=None):
+	def __init__(self, sender='', recipients=(), subject='', alternative=0, reply_to=None, cc=(), email_account=None, expose_recipients=None):
 		from email.mime.multipart import MIMEMultipart
 		from email import Charset
 		Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8')
@@ -51,6 +51,7 @@ class EMail:
 		self.reply_to = reply_to or sender
 		self.recipients = recipients
 		self.subject = subject
+		self.expose_recipients = expose_recipients
 
 		self.msg_root = MIMEMultipart('mixed')
 		self.msg_multipart = MIMEMultipart('alternative')
@@ -195,10 +196,10 @@ class EMail:
 		headers = {
 			"Subject":        strip(self.subject),
 			"From":           self.sender,
-			"To":             ', '.join(self.recipients),
+			"To":             ', '.join(self.recipients) if self.expose_recipients=="header" else "<!--recipient-->",
 			"Date":           email.utils.formatdate(),
 			"Reply-To":       self.reply_to if self.reply_to else None,
-			"CC":             ', '.join(self.cc) if self.cc else None,
+			"CC":             ', '.join(self.cc) if self.cc and self.expose_recipients=="header" else None,
 			'X-Frappe-Site':  get_url(),
 		}
 

frappe/email/queue.py

@@ -18,8 +18,8 @@ class EmailLimitCrossedError(frappe.ValidationError): pass
 
 def send(recipients=None, sender=None, subject=None, message=None, reference_doctype=None,
 		reference_name=None, unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None,
-		attachments=None, reply_to=None, cc=(), show_as_cc=(), in_reply_to=None, send_after=None,
-		expose_recipients=False, send_priority=1, communication=None, now=False):
+		attachments=None, reply_to=None, cc=[], in_reply_to=None, send_after=None,
+		expose_recipients=None, send_priority=1, communication=None, now=False):
 	"""Add email to sending queue (Email Queue)
 
 	:param recipients: List of recipients.
@@ -41,7 +41,7 @@ def send(recipients=None, sender=None, subject=None, message=None, reference_doc
 	if not unsubscribe_method:
 		unsubscribe_method = "/api/method/frappe.email.queue.unsubscribe"
 
-	if not recipients:
+	if not recipients and not cc:
 		return
 
 	if isinstance(recipients, basestring):
@@ -74,54 +74,34 @@ def send(recipients=None, sender=None, subject=None, message=None, reference_doc
 
 	recipients = [r for r in list(set(recipients)) if r and r not in unsubscribed]
 
-	for email in recipients:
-		email_content = formatted
-		email_text_context = text_content
-
-		if reference_doctype and (unsubscribe_message or reference_doctype=="Newsletter"):
-			unsubscribe_link = get_unsubscribe_link(
-				reference_doctype=reference_doctype,
-				reference_name=reference_name,
-				email=email,
-				recipients=recipients,
-				expose_recipients=expose_recipients,
-				unsubscribe_method=unsubscribe_method,
-				unsubscribe_params=unsubscribe_params,
-				unsubscribe_message=unsubscribe_message,
-				show_as_cc=show_as_cc
-			)
-
-			email_content = email_content.replace("<!--unsubscribe link here-->", unsubscribe_link.html)
-			email_text_context += unsubscribe_link.text
-
-			# show as cc
-			cc_message = ""
-			if email in show_as_cc:
-				cc_message = _("This email was sent to you as CC")
-
-			email_content = email_content.replace("<!-- cc message -->", cc_message)
-			email_text_context = cc_message + "\n" + email_text_context
-		# add to queue
-		email_queue = add(email, sender, subject, email_content, email_text_context, reference_doctype,
-			reference_name, attachments, reply_to, cc, in_reply_to, send_after, send_priority, email_account=email_account, communication=communication)
+	email_content = formatted
+	email_text_context = text_content
 
-		if now:
-			send_one(email_queue.name, now=True)
+	if reference_doctype and (unsubscribe_message or reference_doctype=="Newsletter"):
+		unsubscribe_link = get_unsubscribe_message(unsubscribe_message, expose_recipients)
+		email_content = email_content.replace("<!--unsubscribe link here-->", unsubscribe_link.html)
+		email_text_context += unsubscribe_link.text
+
+	# add to queue
+	email_queue = add(recipients, sender, subject, email_content, email_text_context, reference_doctype,
+		reference_name, attachments, reply_to, cc, in_reply_to, send_after, send_priority, email_account=email_account, communication=communication,
+		unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, expose_recipients=expose_recipients)
+	if now:
+		send_one(email_queue.name, now=True)
 
 
-def add(email, sender, subject, formatted, text_content=None,
+def add(recipients, sender, subject, formatted, text_content=None,
 	reference_doctype=None, reference_name=None, attachments=None, reply_to=None,
-	cc=(), in_reply_to=None, send_after=None, send_priority=1, email_account=None,
-	communication=None):
+	cc=[], in_reply_to=None, send_after=None, send_priority=1, email_account=None,
+	communication=None, unsubscribe_method=None, unsubscribe_params=None, expose_recipients=None):
 	"""Add to Email Queue"""
 	e = frappe.new_doc('Email Queue')
-	e.recipient = email
 	e.priority = send_priority
 
 	try:
-		mail = get_email(email, sender=sender, formatted=formatted, subject=subject,
+		mail = get_email(recipients, sender=sender, formatted=formatted, subject=subject,
 			text_content=text_content, attachments=attachments, reply_to=reply_to,
-			cc=cc, email_account=email_account)
+			cc=cc, email_account=email_account, expose_recipients=expose_recipients)
 
 		if in_reply_to:
 			mail.set_in_reply_to(in_reply_to)
@@ -134,11 +114,18 @@ def add(email, sender, subject, formatted, text_content=None,
 		# bad email id - don't add to queue
 		return
 
+	e.set("recipient", [])
+	for r in recipients + cc:
+		e.append("recipient",{"recipient":r})
 	e.reference_doctype = reference_doctype
 	e.reference_name = reference_name
+	e.unsubscribe_method = unsubscribe_method
+	e.unsubscribe_params = unsubscribe_params
+	e.expose_recipients = expose_recipients
 	e.communication = communication
 	e.send_after = send_after
-	e.db_insert()
+	e.show_as_cc = ",".join(cc)
+	e.insert(ignore_permissions=True)
 
 	return e
 
@@ -170,43 +157,23 @@ def get_emails_sent_this_month():
 	return frappe.db.sql("""select count(name) from `tabEmail Queue` where
 		status='Sent' and MONTH(creation)=MONTH(CURDATE())""")[0][0]
 
-def get_unsubscribe_link(reference_doctype, reference_name,
-	email, recipients, expose_recipients, show_as_cc,
-	unsubscribe_method, unsubscribe_params, unsubscribe_message):
-
-	email_sent_to = recipients if expose_recipients else [email]
-	email_sent_cc = ", ".join([e for e in email_sent_to if e in show_as_cc])
-	email_sent_to = ", ".join([e for e in email_sent_to if e not in show_as_cc])
-
-	if email_sent_cc:
-		email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to, email_sent_cc)
-	else:
-		email_sent_message = _("This email was sent to {0}").format(email_sent_to)
-
+def get_unsubscribe_message(unsubscribe_message, expose_recipients):
 	if not unsubscribe_message:
 		unsubscribe_message = _("Unsubscribe from this list")
 
-	unsubscribe_url = get_unsubcribed_url(reference_doctype, reference_name, email,
-		unsubscribe_method, unsubscribe_params)
-
 	html = """<div style="margin: 15px auto; padding: 0px 7px; text-align: center; color: #8d99a6;">
-			{email}
+			<!--cc message-->
 			<p style="margin: 15px auto;">
-				<a href="{unsubscribe_url}" style="color: #8d99a6; text-decoration: underline;
+				<a href="<!--unsubscribe url-->" style="color: #8d99a6; text-decoration: underline;
 					target="_blank">{unsubscribe_message}
 				</a>
 			</p>
-		</div>""".format(
-			unsubscribe_url = unsubscribe_url,
-			email=email_sent_message,
-			unsubscribe_message=unsubscribe_message
-		)
-
-	text = "\n{email}\n\n{unsubscribe_message}: {unsubscribe_url}".format(
-		email=email_sent_message,
-		unsubscribe_message=unsubscribe_message,
-		unsubscribe_url=unsubscribe_url
-	)
+		</div>""".format(unsubscribe_message=unsubscribe_message)
+	if expose_recipients == "footer":
+		text = "\n<!--cc message-->"
+	else:
+		text = ""
+	text += "\n\n{unsubscribe_message}: <!--unsubscribe url-->".format(unsubscribe_message=unsubscribe_message)
 
 	return frappe._dict({
 		"html": html,
@@ -281,7 +248,7 @@ def make_cache_queue():
 	cache = frappe.cache()
 
 	emails = frappe.db.sql('''select name from `tabEmail Queue`
-		where status='Not Sent' and (send_after is null or send_after < %(now)s)
+		where (status='Not Sent' or status='Partially Sent') and (send_after is null or send_after < %(now)s)
 		order by priority desc, creation asc
 		limit 500''', { 'now': now_datetime() })
 
@@ -294,28 +261,21 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
 	'''Send Email Queue with given smtpserver'''
 
 	email = frappe.db.sql('''select name, status, communication,
-		message, sender, recipient, reference_doctype
+		message, sender, reference_doctype, reference_name, unsubscribe_param, unsubscribe_method, expose_recipients, show_as_cc
 		from `tabEmail Queue` where name=%s for update''', email, as_dict=True)[0]
 
-	if from_test:
-		# called from specific test, just set it as sent
-		frappe.db.set_value('Email Queue', email.name, 'status', 'Sent')
-		return
-
-	if frappe.flags.in_test:
-		# call form general test, add the sent email to flags and quit
-		frappe.flags.sent_mail = email.message
-		return
+	recipients_list = frappe.db.sql('''select name, recipient, status from `tabEmail Queue Recipient` where parent=%s''',email.name,as_dict=1)
 
 	if frappe.are_emails_muted():
 		frappe.msgprint(_("Emails are muted"))
 		return
 
-	if email.status != 'Not Sent':
+	if email.status not in ('Not Sent','Partially Sent') :
 		# rollback to release lock and return
 		frappe.db.rollback()
 		return
 
+
 	frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""",
 		(now_datetime(), email.name), auto_commit=auto_commit)
 
@@ -323,14 +283,32 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
 		frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
 
 	try:
-		if auto_commit:
+		if not frappe.flags.in_test:
 			if not smtpserver: smtpserver = SMTPServer()
 			smtpserver.setup_email_account(email.reference_doctype)
-			smtpserver.sess.sendmail(email.sender, email.recipient, encode(email.message))
 
-		frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
-			(now_datetime(), email.name), auto_commit=auto_commit)
+		for recipient in recipients_list:
+			if recipient.status != "Not Sent":
+				continue
 
+			message = prepare_message(email, recipient.recipient, recipients_list)
+			if not frappe.flags.in_test:
+				smtpserver.sess.sendmail(email.sender, recipient.recipient, encode(message))
+
+			recipient.status = "Sent"
+			frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""",
+				(now_datetime(), recipient.name), auto_commit=auto_commit)
+
+		#if all are sent set status
+		if any("Sent" == s.status for s in recipients_list):
+			frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""",
+				(now_datetime(), email.name), auto_commit=auto_commit)
+		else:
+			frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
+				where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit)
+		if frappe.flags.in_test:
+			frappe.flags.sent_mail = message
+			return
 		if email.communication:
 			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
 
@@ -341,8 +319,13 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
 			JobTimeoutException):
 
 		# bad connection/timeout, retry later
-		frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
-			(now_datetime(), email.name), auto_commit=auto_commit)
+
+		if any("Sent" == s.status for s in recipients_list):
+			frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""",
+				(now_datetime(), email.name), auto_commit=auto_commit)
+		else:
+			frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""",
+				(now_datetime(), email.name), auto_commit=auto_commit)
 
 		if email.communication:
 			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
@@ -353,8 +336,12 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
 	except Exception, e:
 		frappe.db.rollback()
 
-		frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
-			where name=%s""", (unicode(e), email.name), auto_commit=auto_commit)
+		if any("Sent" == s.status for s in recipients_list):
+			frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""",
+				(unicode(e), email.name), auto_commit=auto_commit)
+		else:
+			frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s
+where name=%s""", (unicode(e), email.name), auto_commit=auto_commit)
 
 		if email.communication:
 			frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit)
@@ -366,12 +353,38 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False, from_test=Fals
 			# log to Error Log
 			log('frappe.email.queue.flush', unicode(e))
 
+def prepare_message(email, recipient, recipients_list):
+	message = email.message
+	if email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url
+		unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient,
+		email.unsubscribe_method, email.unsubscribe_params)
+		message = message.replace("<!--unsubscribe url-->", unsubscribe_url)
+
+	if email.expose_recipients == "header":
+		pass
+	else:
+		if email.expose_recipients == "footer":
+			if isinstance(email.show_as_cc, basestring):
+				email.show_as_cc = email.show_as_cc.split(",")
+			email_sent_to = [r.recipient for r in recipients_list]
+			email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc])
+			email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc])
+
+			if email_sent_cc:
+				email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc)
+			else:
+				email_sent_message = _("This email was sent to {0}").format(email_sent_to)
+			message = message.replace("<!--cc message-->", email_sent_message)
+
+		message = message.replace("<!--recipient-->", recipient)
+	return message
+
 def clear_outbox():
 	"""Remove low priority older than 31 days in Outbox and expire mails not sent for 7 days.
 
 	Called daily via scheduler."""
-	frappe.db.sql("""delete from `tabEmail Queue` where priority=0 and
-		datediff(now(), modified) > 31""")
+	frappe.db.sql("""delete q, r from `tabEmail Queue` as q, `tabEmail Queue Recipient` as r where q.name = r.parent and q.priority=0 and
+		datediff(now(), q.modified) > 31""")
 
-	frappe.db.sql("""update `tabEmail Queue` set status='Expired'
-		where datediff(curdate(), modified) > 7 and status='Not Sent'""")
+	frappe.db.sql("""update `tabEmail Queue` as q, `tabEmail Queue Recipient` as r set q.status='Expired', r.status='Expired'
+		where q.name = r.parent and datediff(curdate(), q.modified) > 7 and q.status='Not Sent' and r.status='Not Sent'""")

frappe/tests/test_email.py

@@ -3,7 +3,7 @@
 
 from __future__ import unicode_literals
 
-import unittest, frappe
+import unittest, frappe, re
 
 from frappe.test_runner import make_test_records
 make_test_records("User")
@@ -13,6 +13,7 @@ class TestEmail(unittest.TestCase):
 	def setUp(self):
 		frappe.db.sql("""delete from `tabEmail Unsubscribe`""")
 		frappe.db.sql("""delete from `tabEmail Queue`""")
+		frappe.db.sql("""delete from `tabEmail Queue Recipient`""")
 
 	def test_email_queue(self, send_after=None):
 		frappe.sendmail(recipients = ['test@example.com', 'test1@example.com'],
@@ -21,37 +22,103 @@ class TestEmail(unittest.TestCase):
 			subject='Testing Queue', message='This mail is queued!',
 			unsubscribe_message="Unsubscribe", send_after=send_after)
 
-		email_queue = frappe.db.sql("""select * from `tabEmail Queue` where status='Not Sent'""", as_dict=1)
-		self.assertEquals(len(email_queue), 2)
-		self.assertTrue('test@example.com' in [d['recipient'] for d in email_queue])
-		self.assertTrue('test1@example.com' in [d['recipient'] for d in email_queue])
-		self.assertTrue('Unsubscribe' in email_queue[0]['message'])
+		email_queue = frappe.db.sql("""select name,message from `tabEmail Queue` where status='Not Sent'""", as_dict=1)
+		self.assertEquals(len(email_queue), 1)
+		queue_recipients = [r.recipient for r in frappe.db.sql("""SELECT recipient FROM `tabEmail Queue Recipient` 
+			WHERE status='Not Sent'""", as_dict=1)]
+		self.assertTrue('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+		self.assertEquals(len(queue_recipients), 2)
+		self.assertTrue('<!--unsubscribe url-->' in email_queue[0]['message'])
 
 	def test_send_after(self):
 		self.test_email_queue(send_after = 1)
 		from frappe.email.queue import flush
 		flush(from_test=True)
-		email_queue = frappe.db.sql("""select * from `tabEmail Queue` where status='Sent'""", as_dict=1)
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1)
 		self.assertEquals(len(email_queue), 0)
 
 	def test_flush(self):
 		self.test_email_queue()
 		from frappe.email.queue import flush
 		flush(from_test=True)
-		email_queue = frappe.db.sql("""select * from `tabEmail Queue` where status='Sent'""", as_dict=1)
-		self.assertEquals(len(email_queue), 2)
-		self.assertTrue('test@example.com' in [d['recipient'] for d in email_queue])
-		self.assertTrue('test1@example.com' in [d['recipient'] for d in email_queue])
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1)
+		self.assertEquals(len(email_queue), 1)
+		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` 
+			where status='Sent'""", as_dict=1)]
+		self.assertTrue('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+		self.assertEquals(len(queue_recipients), 2)
+		self.assertTrue('Unsubscribe' in frappe.flags.sent_mail)
+
+	def test_cc_header(self):
+		#test if sending with cc's makes it into header
+		frappe.sendmail(recipients=['test@example.com'],
+			cc=['test1@example.com'],
+			sender="admin@example.com",
+			reference_doctype='User', reference_name="Administrator",
+			subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe", expose_recipients="header")
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""", as_dict=1)
+		self.assertEquals(len(email_queue), 1)
+		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` 
+			where status='Not Sent'""", as_dict=1)]
+		self.assertTrue('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+
+		message = frappe.db.sql("""select message from `tabEmail Queue` 
+			where status='Not Sent'""", as_dict=1)[0].message
+		self.assertTrue('To: test@example.com' in message)
+		self.assertTrue('CC: test1@example.com' in message)
+
+	def test_cc_footer(self):
+		#test if sending with cc's makes it into header
+		frappe.sendmail(recipients=['test@example.com'],
+			cc=['test1@example.com'],
+			sender="admin@example.com",
+			reference_doctype='User', reference_name="Administrator",
+			subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe", expose_recipients="footer", now=True)
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1)
+		self.assertEquals(len(email_queue), 1)
+		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` 
+			where status='Sent'""", as_dict=1)]
+		self.assertTrue('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+
+		self.assertTrue('This email was sent to test@example.com and copied to test1@example.com' in frappe.flags.sent_mail)
+
+	def test_expose(self):
+		frappe.sendmail(recipients=['test@example.com'],
+			cc=['test1@example.com'],
+			sender="admin@example.com",
+			reference_doctype='User', reference_name="Administrator",
+			subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe", now=True)
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1)
+		self.assertEquals(len(email_queue), 1)
+		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` 
+			where status='Sent'""", as_dict=1)]
+		self.assertTrue('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+		
+		message = frappe.db.sql("""select message from `tabEmail Queue` 
+			where status='Sent'""", as_dict=1)[0].message
+		self.assertTrue('<!--recipient-->' in message)
+
+		frappe.local.flags.signed_query_string = re.search('(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=\n)', frappe.flags.sent_mail).group(0)
+		from frappe.utils.verified_command import verify_request
+		self.assertTrue(verify_request())
 
 	def test_expired(self):
 		self.test_email_queue()
 		frappe.db.sql("update `tabEmail Queue` set modified=DATE_SUB(curdate(), interval 8 day)")
 		from frappe.email.queue import clear_outbox
 		clear_outbox()
-		email_queue = frappe.db.sql("""select * from `tabEmail Queue` where status='Expired'""", as_dict=1)
-		self.assertEquals(len(email_queue), 2)
-		self.assertTrue('test@example.com' in [d['recipient'] for d in email_queue])
-		self.assertTrue('test1@example.com' in [d['recipient'] for d in email_queue])
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Expired'""", as_dict=1)
+		self.assertEquals(len(email_queue), 1)
+		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` 
+			where parent = %s""",email_queue[0].name, as_dict=1)]
+		self.assertTrue('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+		self.assertEquals(len(queue_recipients), 2)
 
 	def test_unsubscribe(self):
 		from frappe.email.queue import unsubscribe, send
@@ -69,12 +136,15 @@ class TestEmail(unittest.TestCase):
 
 		# this is sent async (?)
 
-		email_queue = frappe.db.sql("""select * from `tabEmail Queue` where status='Not Sent'""",
+		email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""",
 			as_dict=1)
 		self.assertEquals(len(email_queue), before + 1)
-		self.assertFalse('test@example.com' in [d['recipient'] for d in email_queue])
-		self.assertTrue('test1@example.com' in [d['recipient'] for d in email_queue])
-		self.assertTrue('Unsubscribe' in email_queue[0]['message'])
+		queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` 
+			where status='Not Sent'""", as_dict=1)]
+		self.assertFalse('test@example.com' in queue_recipients)
+		self.assertTrue('test1@example.com' in queue_recipients)
+		self.assertEquals(len(queue_recipients), 1)
+		self.assertTrue('Unsubscribe' in frappe.flags.sent_mail)
 
 	def test_email_queue_limit(self):
 		from frappe.email.queue import send, EmailLimitCrossedError