[major] AWS S3 Integration for Frappé (#4272)

* Added S3 Integration, need to add backup limit

* Fix validation in s3 doctype

* Added auto-deletion of old backups and backup_limit usage

* Fixes for codacy PR review

* Improved exception handling.

* Update s3_backup_settings.py

frappe/config/integrations.py

@@ -32,6 +32,11 @@ def get_data():
 					"name": "Dropbox Settings",
 					"description": _("Dropbox backup settings"),
 				},
+				{
+					"type": "doctype",
+					"name": "S3 Backup Settings",
+					"description": _("S3 Backup Settings"),
+				},
 			]
 		},
 		{

frappe/hooks.py

@@ -156,15 +156,19 @@ scheduler_events = {
 		"frappe.core.doctype.authentication_log.authentication_log.clear_authentication_logs"
 	],
 	"daily_long": [
-		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily"
+		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
+		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily"
 	],
 	"weekly_long": [
-		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly"
+		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
+		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly"
 	],
 	"monthly": [
 		"frappe.email.doctype.auto_email_report.auto_email_report.send_monthly"
+	],
+	"monthly_long": [
+		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_monthly"
 	]
-
 }
 
 get_translated_dict = {

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js

@@ -0,0 +1,26 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('S3 Backup Settings', {
+	refresh: function(frm) {
+		frm.clear_custom_buttons();
+		frm.events.take_backup(frm);
+	},
+
+	take_backup: function(frm) {
+		if (frm.doc.access_key_id && frm.doc.secret_access_key) {
+			frm.add_custom_button(__("Take Backup Now"), function(){
+				frm.dashboard.set_headline_alert("S3 Backup Started!");
+				frappe.call({
+					method: "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
+					callback: function(r) {
+						if(!r.exc) {
+							frappe.msgprint(__("S3 Backup complete!"));
+							frm.dashboard.clear_headline();
+						}
+					}
+				});
+			}).addClass("btn-primary");
+		}
+	}
+});

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json

@@ -0,0 +1,273 @@
+{
+ "allow_copy": 0, 
+ "allow_guest_to_view": 0, 
+ "allow_import": 0, 
+ "allow_rename": 0, 
+ "beta": 0, 
+ "creation": "2017-09-04 20:57:20.129205", 
+ "custom": 0, 
+ "docstatus": 0, 
+ "doctype": "DocType", 
+ "document_type": "", 
+ "editable_grid": 1, 
+ "engine": "InnoDB", 
+ "fields": [
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "enabled", 
+   "fieldtype": "Check", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Enable Automatic Backup", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 0, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "notify_email", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Send Notifications To", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "frequency", 
+   "fieldtype": "Select", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Backup Frequency", 
+   "length": 0, 
+   "no_copy": 0, 
+   "options": "Daily\nWeekly\nMonthly\nNone", 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "access_key_id", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Access Key ID", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "secret_access_key", 
+   "fieldtype": "Password", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 1, 
+   "in_standard_filter": 0, 
+   "label": "Secret Access Key", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "bucket", 
+   "fieldtype": "Data", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Bucket", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }, 
+  {
+   "allow_bulk_edit": 0, 
+   "allow_on_submit": 0, 
+   "bold": 0, 
+   "collapsible": 0, 
+   "columns": 0, 
+   "fieldname": "backup_limit", 
+   "fieldtype": "Int", 
+   "hidden": 0, 
+   "ignore_user_permissions": 0, 
+   "ignore_xss_filter": 0, 
+   "in_filter": 0, 
+   "in_global_search": 0, 
+   "in_list_view": 0, 
+   "in_standard_filter": 0, 
+   "label": "Backup Limit", 
+   "length": 0, 
+   "no_copy": 0, 
+   "permlevel": 0, 
+   "precision": "", 
+   "print_hide": 0, 
+   "print_hide_if_no_value": 0, 
+   "read_only": 0, 
+   "remember_last_selected_value": 0, 
+   "report_hide": 0, 
+   "reqd": 1, 
+   "search_index": 0, 
+   "set_only_once": 0, 
+   "unique": 0
+  }
+ ], 
+ "has_web_view": 0, 
+ "hide_heading": 0, 
+ "hide_toolbar": 1, 
+ "idx": 0, 
+ "image_view": 0, 
+ "in_create": 0, 
+ "is_submittable": 0, 
+ "issingle": 1, 
+ "istable": 0, 
+ "max_attachments": 0, 
+ "modified": "2017-10-06 18:27:09.022674", 
+ "modified_by": "Administrator", 
+ "module": "Integrations", 
+ "name": "S3 Backup Settings", 
+ "name_case": "", 
+ "owner": "Administrator", 
+ "permissions": [
+  {
+   "amend": 0, 
+   "apply_user_permissions": 0, 
+   "cancel": 0, 
+   "create": 1, 
+   "delete": 1, 
+   "email": 1, 
+   "export": 0, 
+   "if_owner": 0, 
+   "import": 0, 
+   "permlevel": 0, 
+   "print": 1, 
+   "read": 1, 
+   "report": 0, 
+   "role": "System Manager", 
+   "set_user_permissions": 0, 
+   "share": 1, 
+   "submit": 0, 
+   "write": 1
+  }
+ ], 
+ "quick_entry": 1, 
+ "read_only": 0, 
+ "read_only_onload": 0, 
+ "show_name_in_global_search": 0, 
+ "sort_field": "modified", 
+ "sort_order": "DESC", 
+ "track_changes": 1, 
+ "track_seen": 0
+}

frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import os
+import os.path
+import frappe
+import boto3
+from frappe import _
+from frappe.model.document import Document
+from frappe.utils import cint, split_emails
+from frappe.utils.background_jobs import enqueue
+from botocore.exceptions import ClientError
+
+class S3BackupSettings(Document):
+
+	def validate(self):
+		conn = boto3.client(
+			's3',
+			aws_access_key_id=self.access_key_id,
+			aws_secret_access_key=self.get_password('secret_access_key'),
+		)
+
+		bucket_lower = str(self.bucket).lower()
+
+		try:
+			conn.list_buckets()
+
+		except ClientError:
+			frappe.throw(_("Invalid Access Key ID or Secret Access Key."))
+
+		try:
+			conn.create_bucket(Bucket=bucket_lower)
+		except ClientError:
+			frappe.throw(_("Unable to create bucket: {0}. Change it to a more unique name.").format(bucket_lower))
+
+
+@frappe.whitelist()
+def take_backup():
+	"Enqueue longjob for taking backup to s3"
+	enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500)
+	frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
+
+
+def take_backups_daily():
+	take_backups_if("Daily")
+
+
+def take_backups_weekly():
+	take_backups_if("Weekly")
+
+
+def take_backups_monthly():
+	take_backups_if("Monthly")
+
+
+def take_backups_if(freq):
+	if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
+		if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
+			take_backups_s3()
+
+
+@frappe.whitelist()
+def take_backups_s3():
+	try:
+		backup_to_s3()
+		send_email(True, "S3 Backup Settings")
+	except Exception:
+		error_message = frappe.get_traceback()
+		frappe.errprint(error_message)
+		send_email(False, "S3 Backup Settings", error_message)
+
+
+def send_email(success, service_name, error_status=None):
+	if success:
+		subject = "Backup Upload Successful"
+		message = """<h3>Backup Uploaded Successfully! </h3><p>Hi there, this is just to inform you
+		that your backup was successfully uploaded to your Amazon S3 bucket. So relax!</p> """
+
+	else:
+		subject = "[Warning] Backup Upload Failed"
+		message = """<h3>Backup Upload Failed! </h3><p>Oops, your automated backup to Amazon S3 failed.
+		</p> <p>Error message: %s</p> <p>Please contact your system manager
+		for more information.</p>""" % error_status
+
+	if not frappe.db:
+		frappe.connect()
+
+	if frappe.db.get_value("S3 Backup Settings", None, "notification_email"):
+		recipients = split_emails(frappe.db.get_value("S3 Backup Settings", None, "notification_email"))
+		frappe.sendmail(recipients=recipients, subject=subject, message=message)
+
+
+def backup_to_s3():
+	from frappe.utils.backups import new_backup
+	from frappe.utils import get_backups_path
+
+	doc = frappe.get_single("S3 Backup Settings")
+	bucket = doc.bucket
+
+	conn = boto3.client(
+			's3',
+			aws_access_key_id=doc.access_key_id,
+			aws_secret_access_key=doc.get_password('secret_access_key'),
+			)
+
+	backup = new_backup(ignore_files=False, backup_path_db=None,
+						backup_path_files=None, backup_path_private_files=None, force=True)
+	db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
+	files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
+	private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))
+	folder = os.path.basename(db_filename)[:15] + '/'
+	# for adding datetime to folder name
+
+	upload_file_to_s3(db_filename, folder, conn, bucket)
+	upload_file_to_s3(private_files, folder, conn, bucket)
+	upload_file_to_s3(files_filename, folder, conn, bucket)
+	delete_old_backups(doc.backup_limit, bucket)
+
+def upload_file_to_s3(filename, folder, conn, bucket):
+
+	destpath = os.path.join(folder, os.path.basename(filename))
+	try:
+		print "Uploading file:", filename
+		conn.upload_file(filename, bucket, destpath)
+
+	except Exception as e:
+		print "Error uploading: %s" % (e)
+
+
+def delete_old_backups(limit, bucket):
+	all_backups = list()
+	doc = frappe.get_single("S3 Backup Settings")
+	backup_limit = int(limit)
+
+	s3 = boto3.resource(
+			's3',
+			aws_access_key_id=doc.access_key_id,
+			aws_secret_access_key=doc.get_password('secret_access_key'),
+			)
+	bucket = s3.Bucket(bucket)
+	objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter='/')
+	for obj in objects.get('CommonPrefixes'):
+		all_backups.append(obj.get('Prefix'))
+
+	oldest_backup = sorted(all_backups)[0]
+
+	if len(all_backups) > backup_limit:
+		print "Deleting Backup: {0}".format(oldest_backup)
+		for obj in bucket.objects.filter(Prefix=oldest_backup):
+			# delete all keys that are inside the oldest_backup
+			s3.Object(bucket.name, obj.key).delete()

frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.js

@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: S3 Backup Settings", function (assert) {
+	let done = assert.async();
+
+	// number of asserts
+	assert.expect(1);
+
+	frappe.run_serially([
+		// insert a new S3 Backup Settings
+		() => frappe.tests.make('S3 Backup Settings', [
+			// values to be set
+			{key: 'value'}
+		]),
+		() => {
+			assert.equal(cur_frm.doc.key, 'value');
+		},
+		() => done()
+	]);
+
+});

frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import unittest
+
+class TestS3BackupSettings(unittest.TestCase):
+	pass

requirements.txt

@@ -1,3 +1,4 @@
+boto3
 chardet
 cssmin
 dropbox==7.3.1