frappe/frappe/utils/backups.py

# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

# imports - standard imports
import gzip
import os
from calendar import timegm
from datetime import datetime
from glob import glob
from shutil import which

# imports - third party imports
import click

# imports - module imports
import frappe
from frappe import _, conf
from frappe.utils import get_file_size, get_url, now, now_datetime

# backup variable for backwards compatibility
verbose = False
compress = False
_verbose = verbose
base_tables = ["__Auth", "__global_search", "__UserSettings"]


class BackupGenerator:
	"""
	This class contains methods to perform On Demand Backup

	To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
	If specifying db_file_name, also append ".sql.gz"
	"""

	def __init__(
		self,
		db_name,
		user,
		password,
		backup_path=None,
		backup_path_db=None,
		backup_path_files=None,
		backup_path_private_files=None,
		db_host="localhost",
		db_port=None,
		db_type="mariadb",
		backup_path_conf=None,
		ignore_conf=False,
		compress_files=False,
		include_doctypes="",
		exclude_doctypes="",
		verbose=False,
	):
		global _verbose
		self.compress_files = compress_files or compress
		self.db_host = db_host
		self.db_port = db_port
		self.db_name = db_name
		self.db_type = db_type
		self.user = user
		self.password = password
		self.backup_path = backup_path
		self.backup_path_conf = backup_path_conf
		self.backup_path_db = backup_path_db
		self.backup_path_files = backup_path_files
		self.backup_path_private_files = backup_path_private_files
		self.ignore_conf = ignore_conf
		self.include_doctypes = include_doctypes
		self.exclude_doctypes = exclude_doctypes
		self.partial = False

		if not self.db_type:
			self.db_type = "mariadb"

		if not self.db_port:
			if self.db_type == "mariadb":
				self.db_port = 3306
			if self.db_type == "postgres":
				self.db_port = 5432

		site = frappe.local.site or frappe.generate_hash(length=8)
		self.site_slug = site.replace(".", "_")
		self.verbose = verbose
		self.setup_backup_directory()
		self.setup_backup_tables()
		_verbose = verbose

	def setup_backup_directory(self):
		specified = (
			self.backup_path
			or self.backup_path_db
			or self.backup_path_files
			or self.backup_path_private_files
			or self.backup_path_conf
		)

		if not specified:
			backups_folder = get_backup_path()
			if not os.path.exists(backups_folder):
				os.makedirs(backups_folder, exist_ok=True)
		else:
			if self.backup_path:
				os.makedirs(self.backup_path, exist_ok=True)

			for file_path in set(
				[
					self.backup_path_files,
					self.backup_path_db,
					self.backup_path_private_files,
					self.backup_path_conf,
				]
			):
				if file_path:
					dir = os.path.dirname(file_path)
					os.makedirs(dir, exist_ok=True)

	def setup_backup_tables(self):
		"""Sets self.backup_includes, self.backup_excludes based on passed args"""
		existing_doctypes = set([x.name for x in frappe.get_all("DocType")])

		def get_tables(doctypes):
			tables = []
			for doctype in doctypes:
				if doctype and doctype in existing_doctypes:
					if doctype.startswith("tab"):
						tables.append(doctype)
					else:
						tables.append("tab" + doctype)
			return tables

		passed_tables = {
			"include": get_tables(self.include_doctypes.strip().split(",")),
			"exclude": get_tables(self.exclude_doctypes.strip().split(",")),
		}
		specified_tables = get_tables(frappe.conf.get("backup", {}).get("includes", []))
		include_tables = (specified_tables + base_tables) if specified_tables else []

		conf_tables = {
			"include": include_tables,
			"exclude": get_tables(frappe.conf.get("backup", {}).get("excludes", [])),
		}

		self.backup_includes = passed_tables["include"]
		self.backup_excludes = passed_tables["exclude"]

		if not (self.backup_includes or self.backup_excludes) and not self.ignore_conf:
			self.backup_includes = self.backup_includes or conf_tables["include"]
			self.backup_excludes = self.backup_excludes or conf_tables["exclude"]

		self.partial = (self.backup_includes or self.backup_excludes) and not self.ignore_conf

	@property
	def site_config_backup_path(self):
		# For backwards compatibility
		click.secho(
			"BackupGenerator.site_config_backup_path has been deprecated in favour of"
			" BackupGenerator.backup_path_conf",
			fg="yellow",
		)
		return getattr(self, "backup_path_conf", None)

	def get_backup(self, older_than=24, ignore_files=False, force=False):
		"""
		Takes a new dump if existing file is old
		and sends the link to the file as email
		"""
		# Check if file exists and is less than a day old
		# If not Take Dump
		if not force:
			(
				last_db,
				last_file,
				last_private_file,
				site_config_backup_path,
			) = self.get_recent_backup(older_than)
		else:
			last_db, last_file, last_private_file, site_config_backup_path = (
				False,
				False,
				False,
				False,
			)

		self.todays_date = now_datetime().strftime("%Y%m%d_%H%M%S")

		if not (
			self.backup_path_conf
			and self.backup_path_db
			and self.backup_path_files
			and self.backup_path_private_files
		):
			self.set_backup_file_name()

		if not (last_db and last_file and last_private_file and site_config_backup_path):
			self.take_dump()
			self.copy_site_config()
			if not ignore_files:
				self.backup_files()

		else:
			self.backup_path_files = last_file
			self.backup_path_db = last_db
			self.backup_path_private_files = last_private_file
			self.backup_path_conf = site_config_backup_path

	def set_backup_file_name(self):
		partial = "-partial" if self.partial else ""
		ext = "tgz" if self.compress_files else "tar"

		for_conf = f"{self.todays_date}-{self.site_slug}-site_config_backup.json"
		for_db = f"{self.todays_date}-{self.site_slug}{partial}-database.sql.gz"
		for_public_files = f"{self.todays_date}-{self.site_slug}-files.{ext}"
		for_private_files = f"{self.todays_date}-{self.site_slug}-private-files.{ext}"
		backup_path = self.backup_path or get_backup_path()

		if not self.backup_path_conf:
			self.backup_path_conf = os.path.join(backup_path, for_conf)
		if not self.backup_path_db:
			self.backup_path_db = os.path.join(backup_path, for_db)
		if not self.backup_path_files:
			self.backup_path_files = os.path.join(backup_path, for_public_files)
		if not self.backup_path_private_files:
			self.backup_path_private_files = os.path.join(backup_path, for_private_files)

	def get_recent_backup(self, older_than, partial=False):
		backup_path = get_backup_path()

		file_type_slugs = {
			"database": "*-{{}}-{}database.sql.gz".format('*' if partial else ''),
			"public": "*-{}-files.tar",
			"private": "*-{}-private-files.tar",
			"config": "*-{}-site_config_backup.json",
		}

		def backup_time(file_path):
			file_name = file_path.split(os.sep)[-1]
			file_timestamp = file_name.split("-")[0]
			return timegm(datetime.strptime(file_timestamp, "%Y%m%d_%H%M%S").utctimetuple())

		def get_latest(file_pattern):
			file_pattern = os.path.join(backup_path, file_pattern.format(self.site_slug))
			file_list = glob(file_pattern)
			if file_list:
				return max(file_list, key=backup_time)

		def old_enough(file_path):
			if file_path:
				if not os.path.isfile(file_path) or is_file_old(file_path, older_than):
					return None
				return file_path

		latest_backups = {
			file_type: get_latest(pattern) for file_type, pattern in file_type_slugs.items()
		}

		recent_backups = {
			file_type: old_enough(file_name) for file_type, file_name in latest_backups.items()
		}

		return (
			recent_backups.get("database"),
			recent_backups.get("public"),
			recent_backups.get("private"),
			recent_backups.get("config"),
		)

	def zip_files(self):
		# For backwards compatibility - pre v13
		click.secho(
			"BackupGenerator.zip_files has been deprecated in favour of"
			" BackupGenerator.backup_files",
			fg="yellow",
		)
		return self.backup_files()

	def get_summary(self):
		summary = {
			"config": {
				"path": self.backup_path_conf,
				"size": get_file_size(self.backup_path_conf, format=True),
			},
			"database": {
				"path": self.backup_path_db,
				"size": get_file_size(self.backup_path_db, format=True),
			},
		}

		if os.path.exists(self.backup_path_files) and os.path.exists(
			self.backup_path_private_files
		):
			summary.update(
				{
					"public": {
						"path": self.backup_path_files,
						"size": get_file_size(self.backup_path_files, format=True),
					},
					"private": {
						"path": self.backup_path_private_files,
						"size": get_file_size(self.backup_path_private_files, format=True),
					},
				}
			)

		return summary

	def print_summary(self):
		backup_summary = self.get_summary()
		print("Backup Summary for {0} at {1}".format(frappe.local.site, now()))

		title = max([len(x) for x in backup_summary])
		path = max([len(x["path"]) for x in backup_summary.values()])

		for _type, info in backup_summary.items():
			template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path)
			print(template.format(_type.title(), info["path"], info["size"]))

	def backup_files(self):
		import subprocess

		for folder in ("public", "private"):
			files_path = frappe.get_site_path(folder, "files")
			backup_path = (
				self.backup_path_files if folder == "public" else self.backup_path_private_files
			)

			if self.compress_files:
				cmd_string = "tar cf - {1} | gzip > {0}"
			else:
				cmd_string = "tar -cf {0} {1}"
			output = subprocess.check_output(
				cmd_string.format(backup_path, files_path), shell=True
			)

			if self.verbose and output:
				print(output.decode("utf8"))

	def copy_site_config(self):
		site_config_backup_path = self.backup_path_conf
		site_config_path = os.path.join(frappe.get_site_path(), "site_config.json")

		with open(site_config_backup_path, "w") as n, open(site_config_path) as c:
			n.write(c.read())

	def take_dump(self):
		import frappe.utils
		from frappe.utils.change_log import get_app_branch

		db_exc = {
			"mariadb": ("mysqldump", which("mysqldump")),
			"postgres": ("pg_dump", which("pg_dump")),
		}[self.db_type]
		gzip_exc = which("gzip")

		if not (gzip_exc and db_exc[1]):
			_exc = "gzip" if not gzip_exc else db_exc[0]
			frappe.throw(
				f"{_exc} not found in PATH! This is required to take a backup.",
				exc=frappe.ExecutableNotFound
			)
		db_exc = db_exc[0]

		database_header_content = [
			f"Backup generated by Frappe {frappe.__version__} on branch {get_app_branch('frappe') or 'N/A'}",
			"",
		]

		# escape reserved characters
		args = frappe._dict(
			[item[0], frappe.utils.esc(str(item[1]), "$ ")]
			for item in self.__dict__.copy().items()
		)

		if self.backup_includes:
			backup_info = ("Backing Up Tables: ", ", ".join(self.backup_includes))
		elif self.backup_excludes:
			backup_info = ("Skipping Tables: ", ", ".join(self.backup_excludes))

		if self.partial:
			print(''.join(backup_info), "\n")
			database_header_content.extend([
				f"Partial Backup of Frappe Site {frappe.local.site}",
				("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1],
				"",
			])

		generated_header = "\n".join([f"-- {x}" for x in database_header_content]) + "\n"

		with gzip.open(args.backup_path_db, "wt") as f:
			f.write(generated_header)

		if self.db_type == "postgres":
			if self.backup_includes:
				args["include"] = " ".join(
					["--table='public.\"{0}\"'".format(table) for table in self.backup_includes]
				)
			elif self.backup_excludes:
				args["exclude"] = " ".join(
					["--exclude-table-data='public.\"{0}\"'".format(table) for table in self.backup_excludes]
				)

			cmd_string = (
				"{db_exc} postgres://{user}:{password}@{db_host}:{db_port}/{db_name}"
				" {include} {exclude} | {gzip} >> {backup_path_db}"
			)

		else:
			if self.backup_includes:
				args["include"] = " ".join(["'{0}'".format(x) for x in self.backup_includes])
			elif self.backup_excludes:
				args["exclude"] = " ".join(
					[
						"--ignore-table='{0}.{1}'".format(frappe.conf.db_name, table)
						for table in self.backup_excludes
					]
				)

			cmd_string = (
				"{db_exc} --single-transaction --quick --lock-tables=false -u {user}"
				" -p{password} {db_name} -h {db_host} -P {db_port} {include} {exclude}"
				" | {gzip} >> {backup_path_db}"
			)

		command = cmd_string.format(
			user=args.user,
			password=args.password,
			db_exc=db_exc,
			db_host=args.db_host,
			db_port=args.db_port,
			db_name=args.db_name,
			backup_path_db=args.backup_path_db,
			exclude=args.get("exclude", ""),
			include=args.get("include", ""),
			gzip=gzip_exc,
		)

		if self.verbose:
			print(command + "\n")

		err, out = frappe.utils.execute_in_shell(command)

	def send_email(self):
		"""
		Sends the link to backup file located at erpnext/backups
		"""
		from frappe.email import get_system_managers

		recipient_list = get_system_managers()
		db_backup_url = get_url(
			os.path.join("backups", os.path.basename(self.backup_path_db))
		)
		files_backup_url = get_url(
			os.path.join("backups", os.path.basename(self.backup_path_files))
		)

		msg = """Hello,

Your backups are ready to be downloaded.

1. [Click here to download the database backup](%(db_backup_url)s)
2. [Click here to download the files backup](%(files_backup_url)s)

This link will be valid for 24 hours. A new backup will be available for
download only after 24 hours.""" % {
			"db_backup_url": db_backup_url,
			"files_backup_url": files_backup_url,
		}

		datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime)
		subject = (
			datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded"""
		)

		frappe.sendmail(recipients=recipient_list, msg=msg, subject=subject)
		return recipient_list


@frappe.whitelist()
def get_backup():
	"""
	This function is executed when the user clicks on
	Toos > Download Backup
	"""
	delete_temp_backups()
	odb = BackupGenerator(
		frappe.conf.db_name,
		frappe.conf.db_name,
		frappe.conf.db_password,
		db_host=frappe.db.host,
		db_type=frappe.conf.db_type,
		db_port=frappe.conf.db_port,
	)
	odb.get_backup()
	recipient_list = odb.send_email()
	frappe.msgprint(
		_(
			"Download link for your backup will be emailed on the following email address: {0}"
		).format(", ".join(recipient_list))
	)

@frappe.whitelist()
def fetch_latest_backups(partial=False):
	"""Fetches paths of the latest backup taken in the last 30 days
	Only for: System Managers

	Returns:
		dict: relative Backup Paths
	"""
	frappe.only_for("System Manager")
	odb = BackupGenerator(
		frappe.conf.db_name,
		frappe.conf.db_name,
		frappe.conf.db_password,
		db_host=frappe.db.host,
		db_type=frappe.conf.db_type,
		db_port=frappe.conf.db_port,
	)
	database, public, private, config = odb.get_recent_backup(older_than=24 * 30, partial=partial)

	return {"database": database, "public": public, "private": private, "config": config}


def scheduled_backup(
	older_than=6,
	ignore_files=False,
	backup_path=None,
	backup_path_db=None,
	backup_path_files=None,
	backup_path_private_files=None,
	backup_path_conf=None,
	ignore_conf=False,
	include_doctypes="",
	exclude_doctypes="",
	compress=False,
	force=False,
	verbose=False,
):
	"""this function is called from scheduler
	deletes backups older than 7 days
	takes backup"""
	odb = new_backup(
		older_than=older_than,
		ignore_files=ignore_files,
		backup_path=backup_path,
		backup_path_db=backup_path_db,
		backup_path_files=backup_path_files,
		backup_path_private_files=backup_path_private_files,
		backup_path_conf=backup_path_conf,
		ignore_conf=ignore_conf,
		include_doctypes=include_doctypes,
		exclude_doctypes=exclude_doctypes,
		compress=compress,
		force=force,
		verbose=verbose,
	)
	return odb


def new_backup(
	older_than=6,
	ignore_files=False,
	backup_path=None,
	backup_path_db=None,
	backup_path_files=None,
	backup_path_private_files=None,
	backup_path_conf=None,
	ignore_conf=False,
	include_doctypes="",
	exclude_doctypes="",
	compress=False,
	force=False,
	verbose=False,
):
	delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24)
	odb = BackupGenerator(
		frappe.conf.db_name,
		frappe.conf.db_name,
		frappe.conf.db_password,
		db_host=frappe.db.host,
		db_port=frappe.db.port,
		db_type=frappe.conf.db_type,
		backup_path=backup_path,
		backup_path_db=backup_path_db,
		backup_path_files=backup_path_files,
		backup_path_private_files=backup_path_private_files,
		backup_path_conf=backup_path_conf,
		ignore_conf=ignore_conf,
		include_doctypes=include_doctypes,
		exclude_doctypes=exclude_doctypes,
		verbose=verbose,
		compress_files=compress,
	)
	odb.get_backup(older_than, ignore_files, force=force)
	return odb


def delete_temp_backups(older_than=24):
	"""
	Cleans up the backup_link_path directory by deleting files older than 24 hours
	"""
	backup_path = get_backup_path()
	if os.path.exists(backup_path):
		file_list = os.listdir(get_backup_path())
		for this_file in file_list:
			this_file_path = os.path.join(get_backup_path(), this_file)
			if is_file_old(this_file_path, older_than):
				os.remove(this_file_path)


def is_file_old(file_path, older_than=24):
	"""
	Checks if file exists and is older than specified hours
	Returns ->
	True: file does not exist or file is old
	False: file is new
	"""
	if os.path.isfile(file_path):
		from datetime import timedelta

		# Get timestamp of the file
		file_datetime = datetime.fromtimestamp(os.stat(file_path).st_ctime)
		if datetime.today() - file_datetime >= timedelta(hours=older_than):
			if _verbose:
				print(f"File {file_path} is older than {older_than} hours")
			return True
		else:
			if _verbose:
				print(f"File {file_path} is recent")
			return False
	else:
		if _verbose:
			print(f"File {file_path} does not exist")
		return True


def get_backup_path():
	backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups"))
	return backup_path


def backup(
	with_files=False,
	backup_path_db=None,
	backup_path_files=None,
	backup_path_private_files=None,
	backup_path_conf=None,
	quiet=False,
):
	"Backup"
	odb = scheduled_backup(
		ignore_files=not with_files,
		backup_path_db=backup_path_db,
		backup_path_files=backup_path_files,
		backup_path_private_files=backup_path_private_files,
		backup_path_conf=backup_path_conf,
		force=True,
	)
	return {
		"backup_path_db": odb.backup_path_db,
		"backup_path_files": odb.backup_path_files,
		"backup_path_private_files": odb.backup_path_private_files,
	}


if __name__ == "__main__":
	import sys

	cmd = sys.argv[1]

	db_type = "mariadb"
	try:
		db_type = sys.argv[6]
	except IndexError:
		pass

	db_port = 3306
	try:
		db_port = int(sys.argv[7])
	except IndexError:
		pass

	if cmd == "is_file_old":
		odb = BackupGenerator(
			sys.argv[2],
			sys.argv[3],
			sys.argv[4],
			sys.argv[5] or "localhost",
			db_type=db_type,
			db_port=db_port,
		)
		is_file_old(odb.db_file_name)

	if cmd == "get_backup":
		odb = BackupGenerator(
			sys.argv[2],
			sys.argv[3],
			sys.argv[4],
			sys.argv[5] or "localhost",
			db_type=db_type,
			db_port=db_port,
		)
		odb.get_backup()

	if cmd == "take_dump":
		odb = BackupGenerator(
			sys.argv[2],
			sys.argv[3],
			sys.argv[4],
			sys.argv[5] or "localhost",
			db_type=db_type,
			db_port=db_port,
		)
		odb.take_dump()

	if cmd == "send_email":
		odb = BackupGenerator(
			sys.argv[2],
			sys.argv[3],
			sys.argv[4],
			sys.argv[5] or "localhost",
			db_type=db_type,
			db_port=db_port,
		)
		odb.send_email("abc.sql.gz")

	if cmd == "delete_temp_backups":
		delete_temp_backups()