From a24393f12d5e33fb89a2d58f82b47d75ba0baf41 Mon Sep 17 00:00:00 2001 From: Anand Doshi Date: Wed, 25 Mar 2015 19:01:14 +0530 Subject: [PATCH] Convert to barracuda and utf8mb4, trim tables --- frappe/data/Framework.sql | 18 ++++---- frappe/model/__init__.py | 13 +++--- frappe/model/db_query.py | 5 ++- frappe/model/db_schema.py | 4 +- frappe/model/meta.py | 19 ++++++++- frappe/patches.txt | 2 + .../v5_0/convert_to_barracuda_and_utf8mb4.py | 41 +++++++++++++++++++ 7 files changed, 82 insertions(+), 20 deletions(-) create mode 100644 frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py diff --git a/frappe/data/Framework.sql b/frappe/data/Framework.sql index ea8339e6e9..92d90a3da7 100644 --- a/frappe/data/Framework.sql +++ b/frappe/data/Framework.sql @@ -51,7 +51,7 @@ CREATE TABLE `tabDocField` ( KEY `label` (`label`), KEY `fieldtype` (`fieldtype`), KEY `fieldname` (`fieldname`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -89,7 +89,7 @@ CREATE TABLE `tabDocPerm` ( `restrict` int(1) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDocType` @@ -151,7 +151,7 @@ CREATE TABLE `tabDocType` ( `custom` int(1) DEFAULT NULL, PRIMARY KEY (`name`), KEY `parent` (`parent`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabSeries` @@ -161,7 +161,7 @@ DROP TABLE IF EXISTS `tabSeries`; CREATE TABLE `tabSeries` ( `name` varchar(100) DEFAULT NULL, `current` int(10) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -177,7 +177,7 @@ CREATE TABLE `tabSessions` ( `lastupdate` datetime(6) DEFAULT NULL, `status` varchar(20) DEFAULT NULL, KEY `sid` (`sid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- @@ -190,7 +190,7 @@ CREATE TABLE `tabSingles` ( `field` varchar(255) DEFAULT NULL, `value` text, KEY `singles_doctype_field_index` (`doctype`, `field`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `__Auth` @@ -201,7 +201,7 @@ CREATE TABLE `__Auth` ( `user` VARCHAR(255) NOT NULL PRIMARY KEY, `password` VARCHAR(255) NOT NULL, KEY `user` (`user`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabFile Data` @@ -229,7 +229,7 @@ CREATE TABLE `tabFile Data` ( KEY `parent` (`parent`), KEY `attached_to_name` (`attached_to_name`), KEY `attached_to_doctype` (`attached_to_doctype`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -- -- Table structure for table `tabDefaultValue` @@ -252,4 +252,4 @@ CREATE TABLE `tabDefaultValue` ( PRIMARY KEY (`name`), KEY `parent` (`parent`), KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index caccd2d488..bad531c8d6 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -7,13 +7,14 @@ import frappe import json -no_value_fields = ['Section Break', 'Column Break', 'HTML', 'Table', 'Button', 'Image', 'Fold', 'Heading'] -display_fieldtypes = ['Section Break', 'Column Break', 'HTML', 'Button', 'Image', 'Fold', 'Heading'] -default_fields = ['doctype','name','owner','creation','modified','modified_by', - 'parent','parentfield','parenttype','idx','docstatus'] -integer_docfield_properties = ["reqd", "search_index", "in_list_view", "permlevel", +no_value_fields = ('Section Break', 'Column Break', 'HTML', 'Table', 'Button', 'Image', 'Fold', 'Heading') +display_fieldtypes = ('Section Break', 'Column Break', 'HTML', 'Button', 'Image', 'Fold', 'Heading') +default_fields = ('doctype','name','owner','creation','modified','modified_by', + 'parent','parentfield','parenttype','idx','docstatus') +integer_docfield_properties = ("reqd", "search_index", "in_list_view", "permlevel", "hidden", "read_only", "ignore_user_permissions", "allow_on_submit", "report_hide", - "in_filter", "no_copy", "print_hide", "unique"] + "in_filter", "no_copy", "print_hide", "unique") +optional_fields = ("_user_tags", "_comments", "_assign", "_starred_by") def rename(doctype, old, new, debug=False): import frappe.model.rename_doc diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 55a0499fe3..7ea3fb43c1 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -10,6 +10,7 @@ import frappe.share import frappe.permissions from frappe.utils import flt, cint, getdate, get_datetime, get_time from frappe import _ +from frappe.model import optional_fields class DatabaseQuery(object): def __init__(self, doctype): @@ -161,7 +162,7 @@ class DatabaseQuery(object): # remove from fields to_remove = [] for fld in self.fields: - for f in ("_user_tags", "_comments", "_assign", "_starred_by"): + for f in optional_fields: if f in fld and not f in columns: to_remove.append(fld) @@ -175,7 +176,7 @@ class DatabaseQuery(object): each = [each] for element in each: - if element in ("_user_tags", "_comments", "_assign", "_starred_by") and element not in columns: + if element in optional_fields and element not in columns: to_remove.append(each) for each in to_remove: diff --git a/frappe/model/db_schema.py b/frappe/model/db_schema.py index d2f55611ef..a54509b275 100644 --- a/frappe/model/db_schema.py +++ b/frappe/model/db_schema.py @@ -112,7 +112,9 @@ class DbTable: idx int(8), %sindex parent(parent)) ENGINE=InnoDB - CHARACTER SET=utf8""" % (self.name, add_text)) + ROW_FORMAT=COMPRESSED + CHARACTER SET=utf8mb4 + COLLATE=utf8mb4_unicode_ci""" % (self.name, add_text)) def get_column_definitions(self): column_list = [] + default_columns diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 89859a1a97..6d223e1846 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals import frappe, json from frappe.utils import cstr, cint -from frappe.model import integer_docfield_properties, default_fields +from frappe.model import integer_docfield_properties, default_fields, no_value_fields, optional_fields from frappe.model.document import Document from frappe.model.base_document import BaseDocument from frappe.model.db_schema import type_map @@ -41,7 +41,7 @@ def load_doctype_from_file(doctype): class Meta(Document): _metaclass = True - default_fields = default_fields[1:] + default_fields = list(default_fields)[1:] special_doctypes = ("DocField", "DocPerm", "Role", "DocType", "Module Def") def __init__(self, doctype): @@ -322,3 +322,18 @@ def get_default_df(fieldname): fieldname = fieldname, fieldtype = "Data" ) + +def trim_tables(): + """Use this to remove columns that don't exist in meta""" + ignore_fields = default_fields + optional_fields + + for doctype in frappe.db.get_all("DocType", filters={"issingle": 0}): + doctype = doctype.name + columns = frappe.db.get_table_columns(doctype) + fields = [df.fieldname for df in frappe.get_meta(doctype).fields if df.fieldtype not in no_value_fields] + columns_to_remove = [f for f in list(set(columns) - set(fields)) if f not in ignore_fields] + if columns_to_remove: + columns_to_remove = ", ".join(["drop `{0}`".format(c) for c in columns_to_remove]) + query = """alter table `tab{doctype}` {columns}""".format( + doctype=doctype, columns=columns_to_remove) + frappe.db.sql_ddl(query) diff --git a/frappe/patches.txt b/frappe/patches.txt index 8f90c7f946..5706045be3 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -1,4 +1,5 @@ execute:frappe.db.sql("""update `tabPatch Log` set patch=replace(patch, '.4_0.', '.v4_0.')""") #2014-05-12 +frappe.patches.v5_0.convert_to_barracuda_and_utf8mb4 execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) #2014-01-24 execute:frappe.reload_doc('core', 'doctype', 'docfield', force=True) #2015-01-01 execute:frappe.reload_doc('core', 'doctype', 'docperm') #2014-06-04 @@ -68,3 +69,4 @@ frappe.patches.v5_0.bookmarks_to_stars frappe.patches.v5_0.style_settings_to_website_theme frappe.patches.v5_0.rename_ref_type_fieldnames frappe.patches.v5_0.fix_email_alert + diff --git a/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py b/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py new file mode 100644 index 0000000000..3fd7d16b2f --- /dev/null +++ b/frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py @@ -0,0 +1,41 @@ +from __future__ import unicode_literals +import frappe +import sys + +def execute(): + check_if_ready_for_barracuda() + + for table in frappe.db.get_tables(): + frappe.db.sql_ddl("""alter table `{0}` ENGINE=InnoDB ROW_FORMAT=COMPRESSED""".format(table)) + frappe.db.sql_ddl("""alter table `{0}` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci""".format(table)) + +def check_if_ready_for_barracuda(): + mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) + for key, value in { + "innodb_file_format": "Barracuda", + "innodb_file_per_table": "ON", + "innodb_large_prefix": "ON", + "character_set_server": "utf8mb4", + "collation_server": "utf8mb4_unicode_ci" + }.items(): + + if mariadb_variables.get(key) != value: + print "="*80 + print "Please add this to MariaDB's my.cnf and restart MariaDB before proceeding" + print + print expected + print "="*80 + sys.exit(1) + # raise Exception, "MariaDB needs to be configured!" + +expected = """[mysqld] +innodb-file-format=barracuda +innodb-file-per-table=1 +innodb-large-prefix=1 +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci + +[mysql] +default-character-set = utf8mb4 +"""