diff --git a/frappe/config/integrations.py b/frappe/config/integrations.py
index cb69cb2a6d..7c28372382 100644
--- a/frappe/config/integrations.py
+++ b/frappe/config/integrations.py
@@ -32,6 +32,11 @@ def get_data():
"name": "Dropbox Settings",
"description": _("Dropbox backup settings"),
},
+ {
+ "type": "doctype",
+ "name": "S3 Backup Settings",
+ "description": _("S3 Backup Settings"),
+ },
]
},
{
diff --git a/frappe/hooks.py b/frappe/hooks.py
index 749132cba8..503e60f811 100755
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -156,15 +156,19 @@ scheduler_events = {
"frappe.core.doctype.authentication_log.authentication_log.clear_authentication_logs"
],
"daily_long": [
- "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily"
+ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
+ "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily"
],
"weekly_long": [
- "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly"
+ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
+ "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly"
],
"monthly": [
"frappe.email.doctype.auto_email_report.auto_email_report.send_monthly"
+ ],
+ "monthly_long": [
+ "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_monthly"
]
-
}
get_translated_dict = {
diff --git a/frappe/integrations/doctype/s3_backup_settings/__init__.py b/frappe/integrations/doctype/s3_backup_settings/__init__.py
new file mode 100755
index 0000000000..e69de29bb2
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
new file mode 100755
index 0000000000..1a1b8a7c67
--- /dev/null
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
@@ -0,0 +1,26 @@
+// Copyright (c) 2017, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on('S3 Backup Settings', {
+ refresh: function(frm) {
+ frm.clear_custom_buttons();
+ frm.events.take_backup(frm);
+ },
+
+ take_backup: function(frm) {
+ if (frm.doc.access_key_id && frm.doc.secret_access_key) {
+ frm.add_custom_button(__("Take Backup Now"), function(){
+ frm.dashboard.set_headline_alert("S3 Backup Started!");
+ frappe.call({
+ method: "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
+ callback: function(r) {
+ if(!r.exc) {
+ frappe.msgprint(__("S3 Backup complete!"));
+ frm.dashboard.clear_headline();
+ }
+ }
+ });
+ }).addClass("btn-primary");
+ }
+ }
+});
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
new file mode 100755
index 0000000000..0cdc8e1dd6
--- /dev/null
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
@@ -0,0 +1,273 @@
+{
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 0,
+ "allow_rename": 0,
+ "beta": 0,
+ "creation": "2017-09-04 20:57:20.129205",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "fields": [
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "enabled",
+ "fieldtype": "Check",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Enable Automatic Backup",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 0,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "notify_email",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Send Notifications To",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "frequency",
+ "fieldtype": "Select",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Backup Frequency",
+ "length": 0,
+ "no_copy": 0,
+ "options": "Daily\nWeekly\nMonthly\nNone",
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "access_key_id",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Access Key ID",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "secret_access_key",
+ "fieldtype": "Password",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 1,
+ "in_standard_filter": 0,
+ "label": "Secret Access Key",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "bucket",
+ "fieldtype": "Data",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Bucket",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ },
+ {
+ "allow_bulk_edit": 0,
+ "allow_on_submit": 0,
+ "bold": 0,
+ "collapsible": 0,
+ "columns": 0,
+ "fieldname": "backup_limit",
+ "fieldtype": "Int",
+ "hidden": 0,
+ "ignore_user_permissions": 0,
+ "ignore_xss_filter": 0,
+ "in_filter": 0,
+ "in_global_search": 0,
+ "in_list_view": 0,
+ "in_standard_filter": 0,
+ "label": "Backup Limit",
+ "length": 0,
+ "no_copy": 0,
+ "permlevel": 0,
+ "precision": "",
+ "print_hide": 0,
+ "print_hide_if_no_value": 0,
+ "read_only": 0,
+ "remember_last_selected_value": 0,
+ "report_hide": 0,
+ "reqd": 1,
+ "search_index": 0,
+ "set_only_once": 0,
+ "unique": 0
+ }
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 1,
+ "idx": 0,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 1,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2017-10-06 18:27:09.022674",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "S3 Backup Settings",
+ "name_case": "",
+ "owner": "Administrator",
+ "permissions": [
+ {
+ "amend": 0,
+ "apply_user_permissions": 0,
+ "cancel": 0,
+ "create": 1,
+ "delete": 1,
+ "email": 1,
+ "export": 0,
+ "if_owner": 0,
+ "import": 0,
+ "permlevel": 0,
+ "print": 1,
+ "read": 1,
+ "report": 0,
+ "role": "System Manager",
+ "set_user_permissions": 0,
+ "share": 1,
+ "submit": 0,
+ "write": 1
+ }
+ ],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "show_name_in_global_search": 0,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1,
+ "track_seen": 0
+}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
new file mode 100755
index 0000000000..c557365e6b
--- /dev/null
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import os
+import os.path
+import frappe
+import boto3
+from frappe import _
+from frappe.model.document import Document
+from frappe.utils import cint, split_emails
+from frappe.utils.background_jobs import enqueue
+from botocore.exceptions import ClientError
+
+class S3BackupSettings(Document):
+
+ def validate(self):
+ conn = boto3.client(
+ 's3',
+ aws_access_key_id=self.access_key_id,
+ aws_secret_access_key=self.get_password('secret_access_key'),
+ )
+
+ bucket_lower = str(self.bucket).lower()
+
+ try:
+ conn.list_buckets()
+
+ except ClientError:
+ frappe.throw(_("Invalid Access Key ID or Secret Access Key."))
+
+ try:
+ conn.create_bucket(Bucket=bucket_lower)
+ except ClientError:
+ frappe.throw(_("Unable to create bucket: {0}. Change it to a more unique name.").format(bucket_lower))
+
+
+@frappe.whitelist()
+def take_backup():
+ "Enqueue longjob for taking backup to s3"
+ enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500)
+ frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour."))
+
+
+def take_backups_daily():
+ take_backups_if("Daily")
+
+
+def take_backups_weekly():
+ take_backups_if("Weekly")
+
+
+def take_backups_monthly():
+ take_backups_if("Monthly")
+
+
+def take_backups_if(freq):
+ if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
+ if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
+ take_backups_s3()
+
+
+@frappe.whitelist()
+def take_backups_s3():
+ try:
+ backup_to_s3()
+ send_email(True, "S3 Backup Settings")
+ except Exception:
+ error_message = frappe.get_traceback()
+ frappe.errprint(error_message)
+ send_email(False, "S3 Backup Settings", error_message)
+
+
+def send_email(success, service_name, error_status=None):
+ if success:
+ subject = "Backup Upload Successful"
+ message = """Backup Uploaded Successfully!
Hi there, this is just to inform you
+ that your backup was successfully uploaded to your Amazon S3 bucket. So relax!
"""
+
+ else:
+ subject = "[Warning] Backup Upload Failed"
+ message = """Backup Upload Failed!
Oops, your automated backup to Amazon S3 failed.
+
Error message: %s
Please contact your system manager
+ for more information.
""" % error_status
+
+ if not frappe.db:
+ frappe.connect()
+
+ if frappe.db.get_value("S3 Backup Settings", None, "notification_email"):
+ recipients = split_emails(frappe.db.get_value("S3 Backup Settings", None, "notification_email"))
+ frappe.sendmail(recipients=recipients, subject=subject, message=message)
+
+
+def backup_to_s3():
+ from frappe.utils.backups import new_backup
+ from frappe.utils import get_backups_path
+
+ doc = frappe.get_single("S3 Backup Settings")
+ bucket = doc.bucket
+
+ conn = boto3.client(
+ 's3',
+ aws_access_key_id=doc.access_key_id,
+ aws_secret_access_key=doc.get_password('secret_access_key'),
+ )
+
+ backup = new_backup(ignore_files=False, backup_path_db=None,
+ backup_path_files=None, backup_path_private_files=None, force=True)
+ db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
+ files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
+ private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))
+ folder = os.path.basename(db_filename)[:15] + '/'
+ # for adding datetime to folder name
+
+ upload_file_to_s3(db_filename, folder, conn, bucket)
+ upload_file_to_s3(private_files, folder, conn, bucket)
+ upload_file_to_s3(files_filename, folder, conn, bucket)
+ delete_old_backups(doc.backup_limit, bucket)
+
+def upload_file_to_s3(filename, folder, conn, bucket):
+
+ destpath = os.path.join(folder, os.path.basename(filename))
+ try:
+ print "Uploading file:", filename
+ conn.upload_file(filename, bucket, destpath)
+
+ except Exception as e:
+ print "Error uploading: %s" % (e)
+
+
+def delete_old_backups(limit, bucket):
+ all_backups = list()
+ doc = frappe.get_single("S3 Backup Settings")
+ backup_limit = int(limit)
+
+ s3 = boto3.resource(
+ 's3',
+ aws_access_key_id=doc.access_key_id,
+ aws_secret_access_key=doc.get_password('secret_access_key'),
+ )
+ bucket = s3.Bucket(bucket)
+ objects = bucket.meta.client.list_objects_v2(Bucket=bucket.name, Delimiter='/')
+ for obj in objects.get('CommonPrefixes'):
+ all_backups.append(obj.get('Prefix'))
+
+ oldest_backup = sorted(all_backups)[0]
+
+ if len(all_backups) > backup_limit:
+ print "Deleting Backup: {0}".format(oldest_backup)
+ for obj in bucket.objects.filter(Prefix=oldest_backup):
+ # delete all keys that are inside the oldest_backup
+ s3.Object(bucket.name, obj.key).delete()
diff --git a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.js b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.js
new file mode 100755
index 0000000000..27e36661f0
--- /dev/null
+++ b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.js
@@ -0,0 +1,23 @@
+/* eslint-disable */
+// rename this file from _test_[name] to test_[name] to activate
+// and remove above this line
+
+QUnit.test("test: S3 Backup Settings", function (assert) {
+ let done = assert.async();
+
+ // number of asserts
+ assert.expect(1);
+
+ frappe.run_serially([
+ // insert a new S3 Backup Settings
+ () => frappe.tests.make('S3 Backup Settings', [
+ // values to be set
+ {key: 'value'}
+ ]),
+ () => {
+ assert.equal(cur_frm.doc.key, 'value');
+ },
+ () => done()
+ ]);
+
+});
diff --git a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
new file mode 100755
index 0000000000..04d90f9b44
--- /dev/null
+++ b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2017, Frappe Technologies and Contributors
+# See license.txt
+from __future__ import unicode_literals
+
+import unittest
+
+class TestS3BackupSettings(unittest.TestCase):
+ pass
diff --git a/requirements.txt b/requirements.txt
index 57fae9d6b3..cf72c3109d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+boto3
chardet
cssmin
dropbox==7.3.1