diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 9098e31738..9b0ecee896 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas validate_database_sql ) + site = get_site(context) + frappe.init(site=site) + force = context.force or force decompressed_file_name = extract_sql_from_archive(sql_file_path) @@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas # check if valid SQL file validate_database_sql(decompressed_file_name, _raise=not force) - site = get_site(context) - frappe.init(site=site) - # dont allow downgrading to older versions of frappe without force if not force and is_downgrade(decompressed_file_name, verbose=True): warn_message = ( diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index b0151106db..90cd60c6ec 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -408,20 +408,47 @@ def bulk_rename(context, doctype, path): frappe.destroy() +@click.command('db-console') +@pass_context +def database(context): + """ + Enter into the Database console for given site. + """ + site = get_site(context) + if not site: + raise SiteNotSpecifiedError + frappe.init(site=site) + if not frappe.conf.db_type or frappe.conf.db_type == "mariadb": + _mariadb() + elif frappe.conf.db_type == "postgres": + _psql() + + @click.command('mariadb') @pass_context def mariadb(context): """ Enter into mariadb console for a given site. """ - import os - site = get_site(context) if not site: raise SiteNotSpecifiedError frappe.init(site=site) + _mariadb() + + +@click.command('postgres') +@pass_context +def postgres(context): + """ + Enter into postgres console for a given site. + """ + site = get_site(context) + frappe.init(site=site) + _psql() - # This is assuming you're within the bench instance. + +def _mariadb(): mysql = find_executable('mysql') os.execv(mysql, [ mysql, @@ -434,15 +461,7 @@ def mariadb(context): "-A"]) -@click.command('postgres') -@pass_context -def postgres(context): - """ - Enter into postgres console for a given site. - """ - site = get_site(context) - frappe.init(site=site) - # This is assuming you're within the bench instance. +def _psql(): psql = find_executable('psql') subprocess.run([ psql, '-d', frappe.conf.db_name]) @@ -525,6 +544,74 @@ def console(context, autoreload=False): terminal() +@click.command('transform-database', help="Change tables' internal settings changing engine and row formats") +@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'") +@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)") +@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)") +@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred") +@pass_context +def transform_database(context, table, engine, row_format, failfast): + "Transform site database through given parameters" + site = get_site(context) + check_table = [] + add_line = False + skipped = 0 + frappe.init(site=site) + + if frappe.conf.db_type and frappe.conf.db_type != "mariadb": + click.secho("This command only has support for MariaDB databases at this point", fg="yellow") + sys.exit(1) + + if not (engine or row_format): + click.secho("Values for `--engine` or `--row_format` must be set") + sys.exit(1) + + frappe.connect() + + if table == "all": + information_schema = frappe.qb.Schema("information_schema") + queried_tables = frappe.qb.from_( + information_schema.tables + ).select("table_name").where( + (information_schema.tables.row_format != row_format) + & (information_schema.tables.table_schema == frappe.conf.db_name) + ).run() + tables = [x[0] for x in queried_tables] + else: + tables = [x.strip() for x in table.split(",")] + + total = len(tables) + + for current, table in enumerate(tables): + values_to_set = "" + if engine: + values_to_set += f" ENGINE={engine}" + if row_format: + values_to_set += f" ROW_FORMAT={row_format}" + + try: + frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}") + update_progress_bar("Updating table schema", current - skipped, total) + add_line = True + + except Exception as e: + check_table.append([table, e.args]) + skipped += 1 + + if failfast: + break + + if add_line: + print() + + for errored_table in check_table: + table, err = errored_table + err_msg = f"{table}: ERROR {err[0]}: {err[1]}" + click.secho(err_msg, fg="yellow") + + frappe.destroy() + + @click.command('run-tests') @click.option('--app', help="For App") @click.option('--doctype', help="For DocType") @@ -811,6 +898,8 @@ commands = [ build, clear_cache, clear_website_cache, + database, + transform_database, jupyter, console, destroy_all_sessions, diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 71acefe17c..54cf349d19 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -195,7 +195,7 @@ class MariaDBDatabase(Database): `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) - ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") def create_global_search_table(self): if not '__global_search' in self.get_tables(): diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index 777e036049..670fb71aa2 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -72,7 +72,7 @@ CREATE TABLE `tabDocField` ( KEY `label` (`label`), KEY `fieldtype` (`fieldtype`), KEY `fieldname` (`fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -109,7 +109,7 @@ CREATE TABLE `tabDocPerm` ( `email` int(1) NOT NULL DEFAULT 1, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDocType Action` @@ -133,7 +133,7 @@ CREATE TABLE `tabDocType Action` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType Action` @@ -156,7 +156,7 @@ CREATE TABLE `tabDocType Link` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `modified` (`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- -- Table structure for table `tabDocType` @@ -228,7 +228,7 @@ CREATE TABLE `tabDocType` ( `sender_field` varchar(255) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabSeries` @@ -239,7 +239,7 @@ CREATE TABLE `tabSeries` ( `name` varchar(100), `current` int(10) NOT NULL DEFAULT 0, PRIMARY KEY(`name`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -256,7 +256,7 @@ CREATE TABLE `tabSessions` ( `device` varchar(255) DEFAULT 'desktop', `status` varchar(20) DEFAULT NULL, KEY `sid` (`sid`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -269,7 +269,7 @@ CREATE TABLE `tabSingles` ( `field` varchar(255) DEFAULT NULL, `value` text, KEY `singles_doctype_field_index` (`doctype`, `field`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `__Auth` @@ -283,7 +283,7 @@ CREATE TABLE `__Auth` ( `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabFile` @@ -311,7 +311,7 @@ CREATE TABLE `tabFile` ( KEY `parent` (`parent`), KEY `attached_to_name` (`attached_to_name`), KEY `attached_to_doctype` (`attached_to_doctype`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDefaultValue` @@ -334,4 +334,4 @@ CREATE TABLE `tabDefaultValue` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`) -) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index b40af59286..c5091cfe8f 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -29,7 +29,7 @@ class MariaDBTable(DBTable): %sindex parent(parent), index modified(modified)) ENGINE={engine} - ROW_FORMAT=COMPRESSED + ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN, engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text)) diff --git a/frappe/installer.py b/frappe/installer.py index 23247046f6..f0bf0cb51c 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -445,9 +445,21 @@ def extract_sql_from_archive(sql_file_path): else: decompressed_file_name = sql_file_path + # convert archive sql to latest compatible + convert_archive_content(decompressed_file_name) + return decompressed_file_name +def convert_archive_content(sql_file_path): + if frappe.conf.db_type == "mariadb": + # ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed + # this step is added to ease restoring sites depending on older mariaDB servers + contents = open(sql_file_path).read() + with open(sql_file_path, "w") as f: + f.write(contents.replace("ROW_FORMAT=COMPRESSED", "ROW_FORMAT=DYNAMIC")) + + def extract_sql_gzip(sql_gz_path): import subprocess @@ -457,7 +469,7 @@ def extract_sql_gzip(sql_gz_path): decompressed_file = original_file.rstrip(".gz") cmd = 'gzip -dvf < {0} > {1}'.format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) - except: + except Exception: raise return decompressed_file