feat: Zero* downtime migrations (backport #18050) (#18107)

* feat: db.begin(read_only=True)

You can now start read only transaction by passing read only flag. Read
only transactions prevent any query that is of "WRITE" type like
insert/delete/update.

(cherry picked from commit 5e86e1f192)

* feat: allow reads during maintenance_mode

To reduce downtime reading from main db server during maintenance_mode
can be allowed. This lets users browse desk, static sites or any other
pages while ensuring that no writes happen to DB.

refactor: use read replica if available
(cherry picked from commit 5beccd8802)

* fix: always explicitly start a new transaction

Active read only transaction can be aborted by doing a commit and then
issuing queries. This prevents such edge cases.

(cherry picked from commit 4389447148)

* fix: dont renew session during read only mode

(cherry picked from commit 5922c0ea35)

* feat: wrap read only mode SQL errors

(cherry picked from commit f96505fae0)

* fix: defer logging during read only mode

Deferred:
- Error log
- view log
- web page view

Disable:
- "_seen" tracking used on list view to highlight unseen docs.
- "seen" on error log.
- dashboard chart last ts caching

(cherry picked from commit 55617b9e86)

* fix: ensure deferred insert are flushed during update

(cherry picked from commit 98b57f6a1a)

* fix: remove ad-hoc maintenance mode implementation

(cherry picked from commit e1253e8299)

* feat(UX): Disable write actions in read-only Desk

I won't be covering each and every aspect of desk that shouldn't work in
read only mode. This just handles major interactions and assumes that
user will get a hint about why other things aren't working.

Changes:
- Add read only badge on navbar.
- Disable forms
- Disable new doc creation

(cherry picked from commit 1ec03dacff)

* test: add api tests for read only mode

(cherry picked from commit 7f316fa427)

* feat(ux): `no-indicator-dot` for  indicator pills

Adding this class will disable indicator's tiny dot added before text.

(cherry picked from commit f6c548c7b9)

* refactor: remove dead flag db.read_only

This was added in last DB refactor but it does nothing, it was probably
supposed to do something with the connection pool but to best of my
knowledge "read only" is not a property of a connection.

It can be achieved with users who only have read access, that however
isn't implemented anywhere.

Removing this for now.

(cherry picked from commit ea7fbb2c10)

# Conflicts:
#	frappe/database/__init__.py

* fix(UX): show read only mode warning on web pages

(cherry picked from commit 06d888126b)

* perf: duplicate database initialization (#18049)

* fix: don't attempt to delete session during read only session

* fix: error handling without user set

* chore: conflicts

Co-authored-by: Ankush Menat <ankush@frappe.io>

frappe/__init__.py

@@ -203,6 +203,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None:
 			"mute_emails": False,
 			"has_dataurl": False,
 			"new_site": new_site,
+			"read_only": False,
 		}
 	)
 	local.rollback_observers = []
@@ -284,9 +285,7 @@ def connect_replica():
 		user = local.conf.replica_db_name
 		password = local.conf.replica_db_password
 
-	local.replica_db = get_db(
-		host=local.conf.replica_host, user=user, password=password, port=port, read_only=True
-	)
+	local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port)
 
 	# swap db connections
 	local.primary_db = local.db
@@ -2225,13 +2224,18 @@ def log_error(title=None, message=None, reference_doctype=None, reference_name=N
 	title = title or "Error"
 	traceback = as_unicode(traceback or get_traceback(with_context=True))
 
-	return get_doc(
+	error_log = get_doc(
 		doctype="Error Log",
 		error=traceback,
 		method=title,
 		reference_doctype=reference_doctype,
 		reference_name=reference_name,
-	).insert(ignore_permissions=True)
+	)
+
+	if flags.read_only:
+		error_log.deferred_insert()
+	else:
+		return error_log.insert(ignore_permissions=True)
 
 
 def get_desk_link(doctype, name):

frappe/app.py

@@ -115,9 +115,12 @@ def init_request(request):
 		# site does not exist
 		raise NotFound
 
-	if frappe.local.conf.get("maintenance_mode"):
+	if frappe.local.conf.maintenance_mode:
 		frappe.connect()
-		raise frappe.SessionStopped("Session Stopped")
+		if frappe.local.conf.allow_reads_during_maintenance:
+			setup_read_only_mode()
+		else:
+			raise frappe.SessionStopped("Session Stopped")
 	else:
 		frappe.connect(set_admin_as_user=False)
 
@@ -129,6 +132,24 @@ def init_request(request):
 		frappe.local.http_request = frappe.auth.HTTPRequest()
 
 
+def setup_read_only_mode():
+	"""During maintenance_mode reads to DB can still be performed to reduce downtime. This
+	function sets up read only mode
+
+	- Setting global flag so other pages, desk and database can know that we are in read only mode.
+	- Setup read only database access either by:
+	    - Connecting to read replica if one exists
+	    - Or setting up read only SQL transactions.
+	"""
+	frappe.flags.read_only = True
+
+	# If replica is available then just connect replica, else setup read only transaction.
+	if frappe.conf.read_from_replica:
+		frappe.connect_replica()
+	else:
+		frappe.db.begin(read_only=True)
+
+
 def log_request(request, response):
 	if hasattr(frappe.local, "conf") and frappe.local.conf.enable_frappe_logger:
 		frappe.logger("frappe.web", allow_site=frappe.local.site).info(
@@ -230,6 +251,12 @@ def handle_exception(e):
 		or (frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text"))
 	)
 
+	if not frappe.session.user:
+		# If session creation fails then user won't be unset. This causes a lot of code that
+		# assumes presence of this to fail. Session creation fails => guest or expired login
+		# usually.
+		frappe.session.user = "Guest"
+
 	if respond_as_json:
 		# handle ajax responses first
 		# if the request is ajax, send back the trace or error message

frappe/auth.py

@@ -6,9 +6,8 @@ import frappe
 import frappe.database
 import frappe.utils
 import frappe.utils.user
-from frappe import _, conf
+from frappe import _
 from frappe.core.doctype.activity_log.activity_log import add_authentication_log
-from frappe.modules.patch_handler import check_session_stopped
 from frappe.sessions import Session, clear_sessions, delete_session
 from frappe.translate import get_language
 from frappe.twofactor import (
@@ -30,9 +29,6 @@ class HTTPRequest:
 		# load cookies
 		self.set_cookies()
 
-		# set frappe.local.db
-		self.connect()
-
 		# login and start/resume user session
 		self.set_session()
 
@@ -45,9 +41,6 @@ class HTTPRequest:
 		# write out latest cookies
 		frappe.local.cookie_manager.init_cookies()
 
-		# check session status
-		check_session_stopped()
-
 	@property
 	def domain(self):
 		if not getattr(self, "_domain", None):
@@ -97,16 +90,6 @@ class HTTPRequest:
 	def set_lang(self):
 		frappe.local.lang = get_language()
 
-	def get_db_name(self):
-		"""get database name from conf"""
-		return conf.db_name
-
-	def connect(self):
-		"""connect to db, from ac_name or db_name"""
-		frappe.local.db = frappe.database.get_db(
-			user=self.get_db_name(), password=getattr(conf, "db_password", "")
-		)
-
 
 class LoginManager:
 

frappe/core/doctype/error_log/error_log.py

@@ -9,7 +9,7 @@ from frappe.query_builder.functions import Now
 
 class ErrorLog(Document):
 	def onload(self):
-		if not self.seen:
+		if not self.seen and not frappe.flags.read_only:
 			self.db_set("seen", 1, update_modified=0)
 			frappe.db.commit()
 

frappe/database/__init__.py

@@ -39,21 +39,17 @@ def drop_user_and_database(db_name, root_login=None, root_password=None):
 		)
 
 
-def get_db(host=None, user=None, password=None, port=None, read_only=False):
+def get_db(host=None, user=None, password=None, port=None):
 	import frappe
 
 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.database
 
-		return frappe.database.postgres.database.PostgresDatabase(
-			host, user, password, port=port, read_only=read_only
-		)
+		return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port)
 	else:
 		import frappe.database.mariadb.database
 
-		return frappe.database.mariadb.database.MariaDBDatabase(
-			host, user, password, port=port, read_only=read_only
-		)
+		return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port)
 
 
 def setup_help_database(help_db_name):

frappe/database/database.py

@@ -82,14 +82,12 @@ class Database:
 		ac_name=None,
 		use_default=0,
 		port=None,
-		read_only=False,
 	):
 		self.setup_type_map()
 		self.host = host or frappe.conf.db_host or "127.0.0.1"
 		self.port = port or frappe.conf.db_port or ""
 		self.user = user or frappe.conf.db_name
 		self.db_name = frappe.conf.db_name
-		self.read_only = read_only  # Uses READ ONLY connection if set
 		self._conn = None
 
 		if ac_name:
@@ -217,6 +215,15 @@ class Database:
 			elif self.is_timedout(e):
 				raise frappe.QueryTimeoutError(e) from e
 
+			elif self.is_read_only_mode_error(e):
+				frappe.throw(
+					_(
+						"Site is running in read only mode, this action can not be performed right now. Please try again later."
+					),
+					title=_("In Read Only Mode"),
+					exc=frappe.InReadOnlyMode,
+				)
+
 			# TODO: added temporarily
 			elif self.db_type == "postgres":
 				traceback.print_stack()
@@ -957,8 +964,10 @@ class Database:
 
 		return defaults.get(frappe.scrub(key))
 
-	def begin(self):
-		self.sql("START TRANSACTION")
+	def begin(self, *, read_only=False):
+		read_only = read_only or frappe.flags.read_only
+		mode = "READ ONLY" if read_only else ""
+		self.sql(f"START TRANSACTION {mode}")
 
 	def commit(self):
 		"""Commit current transaction. Calls SQL `COMMIT`."""
@@ -966,9 +975,7 @@ class Database:
 			frappe.call(method[0], *(method[1] or []), **(method[2] or {}))
 
 		self.sql("commit")
-		if self.db_type == "postgres":
-			# Postgres requires explicitly starting new transaction
-			self.begin()
+		self.begin()  # explicitly start a new transaction
 
 		frappe.local.rollback_observers = []
 		self.flush_realtime_log()

frappe/database/mariadb/database.py

@@ -32,6 +32,10 @@ class MariaDBExceptionUtil:
 	def is_timedout(e: pymysql.Error) -> bool:
 		return e.args[0] == ER.LOCK_WAIT_TIMEOUT
 
+	@staticmethod
+	def is_read_only_mode_error(e: pymysql.Error) -> bool:
+		return e.args[0] == 1792
+
 	@staticmethod
 	def is_table_missing(e: pymysql.Error) -> bool:
 		return e.args[0] == ER.NO_SUCH_TABLE

frappe/database/postgres/database.py

@@ -12,7 +12,7 @@ from psycopg2.errorcodes import (
 	UNDEFINED_TABLE,
 	UNIQUE_VIOLATION,
 )
-from psycopg2.errors import SequenceGeneratorLimitExceeded, SyntaxError
+from psycopg2.errors import ReadOnlySqlTransaction, SequenceGeneratorLimitExceeded, SyntaxError
 from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
 
 import frappe
@@ -55,6 +55,10 @@ class PostgresExceptionUtil:
 		# http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError
 		return isinstance(e, psycopg2.extensions.QueryCanceledError)
 
+	@staticmethod
+	def is_read_only_mode_error(e) -> bool:
+		return isinstance(e, ReadOnlySqlTransaction)
+
 	@staticmethod
 	def is_syntax_error(e):
 		return isinstance(e, SyntaxError)

frappe/exceptions.py

@@ -236,6 +236,10 @@ class QueryDeadlockError(Exception):
 	pass
 
 
+class InReadOnlyMode(ValidationError):
+	http_status_code = 503  # temporarily not available
+
+
 class TooManyWritesError(Exception):
 	pass
 

frappe/migrate.py

@@ -13,6 +13,7 @@ from frappe.cache_manager import clear_global_cache
 from frappe.core.doctype.language.language import sync_languages
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.database.schema import add_column
+from frappe.deferred_insert import save_to_db as flush_deferred_inserts
 from frappe.desk.notifications import clear_notifications
 from frappe.modules.patch_handler import PatchType
 from frappe.modules.utils import sync_customizations
@@ -123,6 +124,7 @@ class SiteMigration:
 		* Sync in-Desk Module Dashboards
 		* Sync customizations: Custom Fields, Property Setters, Custom Permissions
 		* Sync Frappe's internal language master
+		* Flush deferred inserts made during maintenance mode.
 		* Sync Portal Menu Items
 		* Sync Installed Applications Version History
 		* Execute `after_migrate` hooks
@@ -132,6 +134,7 @@ class SiteMigration:
 		sync_dashboards()
 		sync_customizations()
 		sync_languages()
+		flush_deferred_inserts()
 
 		frappe.get_single("Portal Settings").sync_menu()
 		frappe.get_single("Installed Applications").update_versions()

frappe/model/document.py

@@ -1365,7 +1365,7 @@ class Document(BaseDocument):
 		if not user:
 			user = frappe.session.user
 
-		if self.meta.track_seen:
+		if self.meta.track_seen and not frappe.flags.read_only:
 			_seen = self.get("_seen") or []
 			_seen = frappe.parse_json(_seen)
 
@@ -1380,15 +1380,19 @@ class Document(BaseDocument):
 			user = frappe.session.user
 
 		if hasattr(self.meta, "track_views") and self.meta.track_views:
-			frappe.get_doc(
+			view_log = frappe.get_doc(
 				{
 					"doctype": "View Log",
 					"viewed_by": frappe.session.user,
 					"reference_doctype": self.doctype,
 					"reference_name": self.name,
 				}
-			).insert(ignore_permissions=True)
-			frappe.local.flags.commit = True
+			)
+			if frappe.flags.read_only:
+				view_log.deferred_insert()
+			else:
+				view_log.insert(ignore_permissions=True)
+				frappe.local.flags.commit = True
 
 	def log_error(self, title=None, message=None):
 		"""Helper function to create an Error Log"""
@@ -1535,6 +1539,20 @@ class Document(BaseDocument):
 
 		return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
 
+	def deferred_insert(self) -> None:
+		"""Push the document to redis temporarily and insert later.
+
+		WARN: This doesn't guarantee insertion as redis can be restarted
+		before data is flushed to database.
+		"""
+
+		from frappe.deferred_insert import deferred_insert
+
+		self.set_user_and_timestamp()
+
+		doc = self.get_valid_dict(convert_dates_to_str=True, ignore_virtual=True)
+		deferred_insert(doctype=self.doctype, records=doc)
+
 	def __repr__(self):
 		name = self.name or "unsaved"
 		doctype = self.__class__.__name__

frappe/model/sync.py

@@ -8,17 +8,17 @@ import os
 
 import frappe
 from frappe.modules.import_file import import_file_by_path
-from frappe.modules.patch_handler import block_user
+from frappe.modules.patch_handler import _patch_mode
 from frappe.utils import update_progress_bar
 
 
 def sync_all(force=0, reset_permissions=False):
-	block_user(True)
+	_patch_mode(True)
 
 	for app in frappe.get_installed_apps():
 		sync_for(app, force, reset_permissions=reset_permissions)
 
-	block_user(False)
+	_patch_mode(False)
 
 	frappe.clear_cache()
 

frappe/modules/patch_handler.py

@@ -154,7 +154,7 @@ def run_single(patchmodule=None, method=None, methodargs=None, force=False):
 
 def execute_patch(patchmodule, method=None, methodargs=None):
 	"""execute the patch"""
-	block_user(True)
+	_patch_mode(True)
 
 	if patchmodule.startswith("execute:"):
 		has_patch_file = False
@@ -197,7 +197,7 @@ def execute_patch(patchmodule, method=None, methodargs=None):
 	else:
 		frappe.db.commit()
 		end_time = time.time()
-		block_user(False)
+		_patch_mode(False)
 		print(f"Success: Done in {round(end_time - start_time, 3)}s")
 
 	return True
@@ -216,18 +216,7 @@ def executed(patchmodule):
 	return frappe.db.get_value("Patch Log", {"patch": patchmodule})
 
 
-def block_user(block, msg=None):
+def _patch_mode(enable):
 	"""stop/start execution till patch is run"""
-	frappe.local.flags.in_patch = block
-	frappe.db.begin()
-	if not msg:
-		msg = "Patches are being executed in the system. Please try again in a few moments."
-	frappe.db.set_global("__session_status", block and "stop" or None)
-	frappe.db.set_global("__session_status_message", block and msg or None)
+	frappe.local.flags.in_patch = enable
 	frappe.db.commit()
-
-
-def check_session_stopped():
-	if frappe.db.get_global("__session_status") == "stop":
-		frappe.msgprint(frappe.db.get_global("__session_status_message"))
-		raise frappe.SessionStopped("Session Stopped")

frappe/public/js/frappe/form/form.js

@@ -445,6 +445,10 @@ frappe.ui.form.Form = class FrappeForm {
 				.toggleClass("cancelled-form", this.doc.docstatus === 2);
 
 			this.show_conflict_message();
+
+			if (frappe.boot.read_only) {
+				this.disable_form();
+			}
 		}
 	}
 

frappe/public/js/frappe/list/list_view.js

@@ -215,7 +215,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
 	}
 
 	set_primary_action() {
-		if (this.can_create) {
+		if (this.can_create && !frappe.boot.read_only) {
 			const doctype_name = __(frappe.router.doctype_layout) || __(this.doctype);
 
 			// Better style would be __("Add {0}", [doctype_name], "Primary action in list view")

frappe/public/js/frappe/ui/toolbar/navbar.html

@@ -6,6 +6,11 @@
 		<ul class="nav navbar-nav d-none d-sm-flex" id="navbar-breadcrumbs"></ul>
 		<div class="collapse navbar-collapse justify-content-end">
 			<form class="form-inline fill-width justify-content-end" role="search" onsubmit="return false;">
+				{% if (frappe.boot.read_only) { %}
+					<span class="indicator-pill yellow no-indicator-dot" title="{%= __("Your site is getting upgraded.") %}">
+						{%= __("Read Only Mode") %}
+					</span>
+				{% } %}
 				<div class="input-group search-bar text-muted hidden">
 					<input
 						id="navbar-search"

frappe/public/scss/common/indicator.scss

@@ -48,7 +48,7 @@
 	height: 24px;
 }
 
-.indicator-pill::before,
+.indicator-pill:not(.no-indicator-dot)::before,
 .indicator-pill-right::after {
 	content:'';
 	display: inline-table;
@@ -179,4 +179,4 @@
 
 @keyframes blink {
 	50% { opacity: 0.5; }
-}
+}

frappe/sessions.py

@@ -90,6 +90,11 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None):
 def delete_session(sid=None, user=None, reason="Session Expired"):
 	from frappe.core.doctype.activity_log.feed import logout_feed
 
+	if frappe.flags.read_only:
+		# This isn't manually initated logout, most likely user's cookies were expired in such case
+		# we should just ignore it till database is back up again.
+		return
+
 	frappe.cache().hdel("session", sid)
 	frappe.cache().hdel("last_db_session_update", sid)
 	if sid and not user:
@@ -179,6 +184,7 @@ def get():
 
 	bootinfo.notes = get_unseen_notes()
 	bootinfo.assets_json = get_assets_json()
+	bootinfo.read_only = bool(frappe.flags.read_only)
 
 	for hook in frappe.get_hooks("extend_bootinfo"):
 		frappe.get_attr(hook)(bootinfo=bootinfo)
@@ -407,7 +413,7 @@ class Session:
 
 		# database persistence is secondary, don't update it too often
 		updated_in_db = False
-		if force or (time_diff is None) or (time_diff > 600):
+		if (force or (time_diff is None) or (time_diff > 600)) and not frappe.flags.read_only:
 			# update sessions table
 			frappe.db.sql(
 				"""update `tabSessions` set sessiondata=%s,
@@ -426,7 +432,6 @@ class Session:
 
 			updated_in_db = True
 
-		# set in memcache
 		frappe.cache().hset("session", self.sid, self.data)
 
 		return updated_in_db

frappe/templates/includes/navbar/navbar_items.html

@@ -89,6 +89,15 @@
 	</div>
 	{% endif %}
 
+	{% if read_only_mode %}
+		<div
+			class="indicator-pill yellow no-indicator-dot align-self-center"
+			title="{{ _("This site is in read only mode, full functionality will be restored soon.") }}"
+		>
+			{{ _("Read Only Mode") }}
+		</div>
+	{% endif %}
+
 	{% include "templates/includes/navbar/navbar_login.html" %}
 
 </ul>

frappe/tests/test_api.py

@@ -9,6 +9,7 @@ from semantic_version import Version
 from werkzeug.test import TestResponse
 
 import frappe
+from frappe.installer import update_site_config
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils import get_site_url, get_test_client
 
@@ -269,3 +270,29 @@ class TestMethodAPI(FrappeAPITestCase):
 		self.assertEqual(response.json["message"], "Administrator")
 
 		authorization_token = None
+
+
+class TestReadOnlyMode(FrappeAPITestCase):
+	"""During migration if read only mode can be enabled.
+	Test if reads work well and writes are blocked"""
+
+	REQ_PATH = "/api/resource/ToDo"
+
+	@classmethod
+	def setUpClass(cls):
+		super().setUpClass()
+		update_site_config("allow_reads_during_maintenance", 1)
+		cls.addClassCleanup(update_site_config, "maintenance_mode", 0)
+		# XXX: this has potential to crumble rest of the test suite.
+		update_site_config("maintenance_mode", 1)
+
+	def test_reads(self):
+		response = self.get(self.REQ_PATH, {"sid": self.sid})
+		self.assertEqual(response.status_code, 200)
+		self.assertIsInstance(response.json, dict)
+		self.assertIsInstance(response.json["data"], list)
+
+	def test_blocked_writes(self):
+		response = self.post(self.REQ_PATH, {"description": frappe.mock("paragraph"), "sid": self.sid})
+		self.assertEqual(response.status_code, 503)
+		self.assertEqual(response.json["exc_type"], "InReadOnlyMode")

frappe/tests/test_db.py

@@ -460,6 +460,14 @@ class TestDB(FrappeTestCase):
 			# recover transaction to continue other tests
 			raise Exception
 
+	def test_read_only_errors(self):
+		frappe.db.rollback()
+		frappe.db.begin(read_only=True)
+		self.addCleanup(frappe.db.rollback)
+
+		with self.assertRaises(frappe.InReadOnlyMode):
+			frappe.db.set_value("User", "Administrator", "full_name", "Haxor")
+
 	def test_exists(self):
 		dt, dn = "User", "Administrator"
 		self.assertEqual(frappe.db.exists(dt, dn, cache=True), dn)

frappe/utils/dashboard.py

@@ -64,9 +64,10 @@ def generate_and_cache_results(args, function, cache_key, chart):
 		else:
 			raise
 
-	frappe.db.set_value(
-		"Dashboard Chart", args.chart_name, "last_synced_on", frappe.utils.now(), update_modified=False
-	)
+	if not frappe.flags.read_only:
+		frappe.db.set_value(
+			"Dashboard Chart", args.chart_name, "last_synced_on", frappe.utils.now(), update_modified=False
+		)
 	return results
 
 

frappe/website/doctype/web_page_view/web_page_view.py

@@ -37,7 +37,10 @@ def make_view_log(path, referrer=None, browser=None, version=None, url=None, use
 	view.is_unique = is_unique
 
 	try:
-		view.insert(ignore_permissions=True)
+		if frappe.flags.read_only:
+			view.deferred_insert()
+		else:
+			view.insert(ignore_permissions=True)
 	except Exception:
 		if frappe.message_log:
 			frappe.message_log.pop()

frappe/website/doctype/website_settings/website_settings.py

@@ -191,6 +191,7 @@ def get_website_settings(context=None):
 	if settings.splash_image:
 		context["splash_image"] = settings.splash_image
 
+	context.read_only_mode = frappe.flags.read_only
 	context.boot = get_boot_data()
 
 	return context