From f0cc541a716ca892f4772c022e8f4ac87fb38da0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Aug 2021 12:36:54 +0530 Subject: [PATCH 1/9] feat: Add util to convert compressed tables to DYNAMIC NOTE: This shouldn't be considered as a finished command. Only run this if you understand what you're doing --- frappe/commands/utils.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index f2395ae490..54ee559cf5 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -512,6 +512,29 @@ def console(context): IPython.embed(display_banner="", header="", colors="neutral") +@click.command('convert-database') +@pass_context +def convert_database(context): + "convert row_formats to DNAMIC from older formats -- innodb mariadb v10.6.3" + site = get_site(context) + frappe.init(site=site) + frappe.connect() + + information_schema = frappe.qb.Schema("information_schema") + queried_tables = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + information_schema.tables.row_format=="Compressed" + ).run() + tables = [x[0] for x in queried_tables] + + for table in tables: + frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT=DYNAMIC") + + frappe.db.commit() + frappe.destroy() + + @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") @@ -796,6 +819,7 @@ commands = [ build, clear_cache, clear_website_cache, + convert_database, jupyter, console, destroy_all_sessions, From f8826750d4016c606eebb55ea7bf5fbde5b734d5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 17 Aug 2021 12:38:19 +0530 Subject: [PATCH 2/9] chore: Use DYNAMIC row_format instead of deprecated COMPRESSED refs: * MariaDB 10.6 gets rid of the COMPRESSED row_format * https://dev.mysql.com/worklog/task/?id=8307 * https://stackoverflow.com/questions/24321896/mysql-row-format-compressed-vs-dynamic --- frappe/database/mariadb/database.py | 2 +- frappe/database/mariadb/framework_mariadb.sql | 22 +++++++++---------- frappe/database/mariadb/schema.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index d4a119804b..9fc485e133 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -195,7 +195,7 @@ class MariaDBDatabase(Database): `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) - ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") def create_global_search_table(self): if not '__global_search' in self.get_tables(): diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index f8841e9417..426fae6d3e 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -71,7 +71,7 @@ CREATE TABLE `tabDocField` ( KEY `label` (`label`), KEY `fieldtype` (`fieldtype`), KEY `fieldname` (`fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -108,7 +108,7 @@ CREATE TABLE `tabDocPerm` ( `email` int(1) NOT NULL DEFAULT 1, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDocType Action` @@ -132,7 +132,7 @@ CREATE TABLE `tabDocType Action` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType Action` @@ -155,7 +155,7 @@ CREATE TABLE `tabDocType Link` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType` @@ -226,7 +226,7 @@ CREATE TABLE `tabDocType` ( `sender_field` varchar(255) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabSeries` @@ -237,7 +237,7 @@ CREATE TABLE `tabSeries` ( `name` varchar(100), `current` int(10) NOT NULL DEFAULT 0, PRIMARY KEY(`name`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -254,7 +254,7 @@ CREATE TABLE `tabSessions` ( `device` varchar(255) DEFAULT 'desktop', `status` varchar(20) DEFAULT NULL, KEY `sid` (`sid`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -267,7 +267,7 @@ CREATE TABLE `tabSingles` ( `field` varchar(255) DEFAULT NULL, `value` text, KEY `singles_doctype_field_index` (`doctype`, `field`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `__Auth` @@ -281,7 +281,7 @@ CREATE TABLE `__Auth` ( `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabFile` @@ -309,7 +309,7 @@ CREATE TABLE `tabFile` ( KEY `parent` (`parent`), KEY `attached_to_name` (`attached_to_name`), KEY `attached_to_doctype` (`attached_to_doctype`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDefaultValue` @@ -332,4 +332,4 @@ CREATE TABLE `tabDefaultValue` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index b40af59286..c5091cfe8f 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -29,7 +29,7 @@ class MariaDBTable(DBTable): %sindex parent(parent), index modified(modified)) ENGINE={engine} - ROW_FORMAT=COMPRESSED + ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN, engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text)) From 41b30b7442015e76f3996b94fcc61669a9a319d0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 30 Aug 2021 12:26:18 +0530 Subject: [PATCH 3/9] feat: Handle site restores to MariaDB 10.6 --- frappe/installer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/frappe/installer.py b/frappe/installer.py index d4d8117fcb..41fd7b1cf0 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -445,9 +445,21 @@ def extract_sql_from_archive(sql_file_path): else: decompressed_file_name = sql_file_path + # convert archive sql to latest compatible + convert_archive_content(decompressed_file_name) + return decompressed_file_name +def convert_archive_content(sql_file_path): + if frappe.conf.db_type == "mariadb": + # ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed + # this step is added to ease restoring sites depending on older mariaDB servers + contents = open(sql_file_path).read() + with open(sql_file_path, "w") as f: + f.write(contents.replace("ROW_FORMAT=COMPRESSED", "ROW_FORMAT=DYNAMIC")) + + def extract_sql_gzip(sql_gz_path): import subprocess @@ -457,7 +469,7 @@ def extract_sql_gzip(sql_gz_path): decompressed_file = original_file.rstrip(".gz") cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) - except: + except Exception: raise return decompressed_file From 3f2d0ac0cfc4b7ded6fad83c82d35e009d6cce8b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 1 Sep 2021 13:37:01 +0530 Subject: [PATCH 4/9] feat(cli): New command 'db-console' Don't remember what db console you have to open for what site! How to use: `bench --site framework.io db-console` --- frappe/commands/utils.py | 44 +++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 54ee559cf5..490671b66a 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -408,20 +408,47 @@ def bulk_rename(context, doctype, path): frappe.destroy() +@click.command('db-console') +@pass_context +def database(context): + """ + Enter into the Database console for given site. + """ + site = get_site(context) + if not site: + raise SiteNotSpecifiedError + frappe.init(site=site) + if not frappe.conf.db_type or frappe.conf.db_type == "mariadb": + _mariadb() + elif frappe.conf.db_type == "postgres": + _psql() + + @click.command('mariadb') @pass_context def mariadb(context): """ Enter into mariadb console for a given site. """ - import os - site = get_site(context) if not site: raise SiteNotSpecifiedError frappe.init(site=site) + _mariadb() - # This is assuming you're within the bench instance. + +@click.command('postgres') +@pass_context +def postgres(context): + """ + Enter into postgres console for a given site. + """ + site = get_site(context) + frappe.init(site=site) + _psql() + + +def _mariadb(): mysql = find_executable('mysql') os.execv(mysql, [ mysql, @@ -434,15 +461,7 @@ def mariadb(context): "-A"]) -@click.command('postgres') -@pass_context -def postgres(context): - """ - Enter into postgres console for a given site. - """ - site = get_site(context) - frappe.init(site=site) - # This is assuming you're within the bench instance. +def _psql(): psql = find_executable('psql') subprocess.run([ psql, '-d', frappe.conf.db_name]) @@ -819,6 +838,7 @@ commands = [ build, clear_cache, clear_website_cache, + database, convert_database, jupyter, console, From c472fd359f0c5f402a284c7dcc4c299e8eb5b893 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 12:53:07 +0530 Subject: [PATCH 5/9] feat: Transform tables for given site * Ability to update ROW_FORMAT for mentioned tables in --table option * Failfast to stop on first error occured * Show progressbar for overall conversion progress (remaining tales) --- frappe/commands/utils.py | 61 +++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 490671b66a..5ca6bd6d3f 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -531,26 +531,59 @@ def console(context): IPython.embed(display_banner="", header="", colors="neutral") -@click.command('convert-database') +@click.command('transform-database') +@click.option('--table', default="all") +@click.option('--row_format', default="DYNAMIC", type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) +@click.option('--failfast', is_flag=True, default=False) @pass_context -def convert_database(context): - "convert row_formats to DNAMIC from older formats -- innodb mariadb v10.6.3" +def transform_database(context, table, row_format, failfast): + "Transform site database through given parameters" site = get_site(context) + check_table = [] + add_line = False + skipped = 0 frappe.init(site=site) + + if frappe.conf.db_type and frappe.conf.db_type != "mariadb": + click.secho("This command only has support for MariaDB databases at this point", fg="yellow") + sys.exit(1) + frappe.connect() - information_schema = frappe.qb.Schema("information_schema") - queried_tables = frappe.qb.from_( - information_schema.tables - ).select("table_name").where( - information_schema.tables.row_format=="Compressed" - ).run() - tables = [x[0] for x in queried_tables] + if table == "all": + information_schema = frappe.qb.Schema("information_schema") + queried_tables = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + (information_schema.tables.row_format != row_format) + & (information_schema.tables.table_schema == frappe.conf.db_name) + ).run() + tables = [x[0] for x in queried_tables] + else: + tables = [x.strip() for x in table.split(",")] + + total = len(tables) + + for current, table in enumerate(tables): + try: + frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + update_progress_bar("Updating table schema", current - skipped, total) + add_line = True + except Exception as e: + check_table.append([table, e.args]) + skipped += 1 + + if failfast: + break + + if add_line: + print() - for table in tables: - frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT=DYNAMIC") + for errored_table in check_table: + table, err = errored_table + err_msg = f"{table}: ERROR {err[0]}: {err[1]}" + click.secho(err_msg, fg="yellow") - frappe.db.commit() frappe.destroy() @@ -839,7 +872,7 @@ commands = [ clear_cache, clear_website_cache, database, - convert_database, + transform_database, jupyter, console, destroy_all_sessions, From 6ccf73026fc23ee874e1e3a9cc02436871179517 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 15:33:27 +0530 Subject: [PATCH 6/9] feat(minor): transform-tables - Update table engine --- frappe/commands/utils.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 5ca6bd6d3f..19bc355034 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -532,11 +532,12 @@ def console(context): @click.command('transform-database') -@click.option('--table', default="all") -@click.option('--row_format', default="DYNAMIC", type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) +@click.option('--table', required=True) +@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"])) +@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) @click.option('--failfast', is_flag=True, default=False) @pass_context -def transform_database(context, table, row_format, failfast): +def transform_database(context, table, engine, row_format, failfast): "Transform site database through given parameters" site = get_site(context) check_table = [] @@ -548,6 +549,10 @@ def transform_database(context, table, row_format, failfast): click.secho("This command only has support for MariaDB databases at this point", fg="yellow") sys.exit(1) + if not (engine or row_format): + click.secho("Values for `--engine` or `--row_format` must be set") + sys.exit(1) + frappe.connect() if table == "all": @@ -566,9 +571,15 @@ def transform_database(context, table, row_format, failfast): for current, table in enumerate(tables): try: - frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + if engine: + frappe.db.sql(f"ALTER TABLE `{table}` ENGINE={engine}") + + if row_format: + frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + update_progress_bar("Updating table schema", current - skipped, total) add_line = True + except Exception as e: check_table.append([table, e.args]) skipped += 1 From 71dc7f474ff695839de253d41b95bef11fa98ad2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 15:51:49 +0530 Subject: [PATCH 7/9] perf: Single query to update a table --- frappe/commands/utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 2dba0dd678..bad26944b6 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -583,13 +583,14 @@ def transform_database(context, table, engine, row_format, failfast): total = len(tables) for current, table in enumerate(tables): - try: - if engine: - frappe.db.sql(f"ALTER TABLE `{table}` ENGINE={engine}") - - if row_format: - frappe.db.sql(f"ALTER TABLE `{table}` ROW_FORMAT={row_format}") + values_to_set = "" + if engine: + values_to_set += f" ENGINE={engine}" + if row_format: + values_to_set += f" ROW_FORMAT={row_format}" + try: + frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}") update_progress_bar("Updating table schema", current - skipped, total) add_line = True From a1c8a7b5f2866a76ba62351711fa402f55e03622 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 2 Sep 2021 16:00:07 +0530 Subject: [PATCH 8/9] fix(restore): Initialize site before decompressing file --- frappe/commands/site.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9098e31738..9b0ecee896 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas validate_database_sql ) + site = get_site(context) + frappe.init(site=site) + force = context.force or force decompressed_file_name = extract_sql_from_archive(sql_file_path) @@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas # check if valid SQL file validate_database_sql(decompressed_file_name, _raise=not force) - site = get_site(context) - frappe.init(site=site) - # dont allow downgrading to older versions of frappe without force if not force and is_downgrade(decompressed_file_name, verbose=True): warn_message = ( From 8b881d01d5d108d80b89738fd444112ed57935c4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 14 Sep 2021 21:07:46 +0530 Subject: [PATCH 9/9] chore: Add help for bench transform-database --- frappe/commands/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index bad26944b6..90cd60c6ec 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -544,11 +544,11 @@ def console(context, autoreload=False): terminal() -@click.command('transform-database') -@click.option('--table', required=True) -@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"])) -@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"])) -@click.option('--failfast', is_flag=True, default=False) +@click.command('transform-database', help="Change tables' internal settings changing engine and row formats") +@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'") +@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)") +@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)") +@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred") @pass_context def transform_database(context, table, engine, row_format, failfast): "Transform site database through given parameters"