Convert to barracuda and utf8mb4, trim tables

frappe/data/Framework.sql

@@ -51,7 +51,7 @@ CREATE TABLE `tabDocField` (
   KEY `label` (`label`),
   KEY `fieldtype` (`fieldtype`),
   KEY `fieldname` (`fieldname`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
 --
@@ -89,7 +89,7 @@ CREATE TABLE `tabDocPerm` (
   `restrict` int(1) DEFAULT NULL,
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabDocType`
@@ -151,7 +151,7 @@ CREATE TABLE `tabDocType` (
   `custom` int(1) DEFAULT NULL,
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabSeries`
@@ -161,7 +161,7 @@ DROP TABLE IF EXISTS `tabSeries`;
 CREATE TABLE `tabSeries` (
   `name` varchar(100) DEFAULT NULL,
   `current` int(10) DEFAULT NULL
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
 --
@@ -177,7 +177,7 @@ CREATE TABLE `tabSessions` (
   `lastupdate` datetime(6) DEFAULT NULL,
   `status` varchar(20) DEFAULT NULL,
   KEY `sid` (`sid`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
 --
@@ -190,7 +190,7 @@ CREATE TABLE `tabSingles` (
   `field` varchar(255) DEFAULT NULL,
   `value` text,
   KEY `singles_doctype_field_index` (`doctype`, `field`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `__Auth`
@@ -201,7 +201,7 @@ CREATE TABLE `__Auth` (
   `user` VARCHAR(255) NOT NULL PRIMARY KEY,
   `password` VARCHAR(255) NOT NULL,
   KEY `user` (`user`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabFile Data`
@@ -229,7 +229,7 @@ CREATE TABLE `tabFile Data` (
   KEY `parent` (`parent`),
   KEY `attached_to_name` (`attached_to_name`),
   KEY `attached_to_doctype` (`attached_to_doctype`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabDefaultValue`
@@ -252,4 +252,4 @@ CREATE TABLE `tabDefaultValue` (
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`),
   KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

frappe/model/__init__.py

@@ -7,13 +7,14 @@ import frappe
 import json
 
 
-no_value_fields = ['Section Break', 'Column Break', 'HTML', 'Table', 'Button', 'Image', 'Fold', 'Heading']
-display_fieldtypes = ['Section Break', 'Column Break', 'HTML', 'Button', 'Image', 'Fold', 'Heading']
-default_fields = ['doctype','name','owner','creation','modified','modified_by',
-	'parent','parentfield','parenttype','idx','docstatus']
-integer_docfield_properties = ["reqd", "search_index", "in_list_view", "permlevel",
+no_value_fields = ('Section Break', 'Column Break', 'HTML', 'Table', 'Button', 'Image', 'Fold', 'Heading')
+display_fieldtypes = ('Section Break', 'Column Break', 'HTML', 'Button', 'Image', 'Fold', 'Heading')
+default_fields = ('doctype','name','owner','creation','modified','modified_by',
+	'parent','parentfield','parenttype','idx','docstatus')
+integer_docfield_properties = ("reqd", "search_index", "in_list_view", "permlevel",
 	"hidden", "read_only", "ignore_user_permissions", "allow_on_submit", "report_hide",
-	"in_filter", "no_copy", "print_hide", "unique"]
+	"in_filter", "no_copy", "print_hide", "unique")
+optional_fields = ("_user_tags", "_comments", "_assign", "_starred_by")
 
 def rename(doctype, old, new, debug=False):
 	import frappe.model.rename_doc

frappe/model/db_query.py

@@ -10,6 +10,7 @@ import frappe.share
 import frappe.permissions
 from frappe.utils import flt, cint, getdate, get_datetime, get_time
 from frappe import _
+from frappe.model import optional_fields
 
 class DatabaseQuery(object):
 	def __init__(self, doctype):
@@ -161,7 +162,7 @@ class DatabaseQuery(object):
 		# remove from fields
 		to_remove = []
 		for fld in self.fields:
-			for f in ("_user_tags", "_comments", "_assign", "_starred_by"):
+			for f in optional_fields:
 				if f in fld and not f in columns:
 					to_remove.append(fld)
 
@@ -175,7 +176,7 @@ class DatabaseQuery(object):
 				each = [each]
 
 			for element in each:
-				if element in ("_user_tags", "_comments", "_assign", "_starred_by") and element not in columns:
+				if element in optional_fields and element not in columns:
 					to_remove.append(each)
 
 		for each in to_remove:

frappe/model/db_schema.py

@@ -112,7 +112,9 @@ class DbTable:
 			idx int(8),
 			%sindex parent(parent))
 			ENGINE=InnoDB
-			CHARACTER SET=utf8""" % (self.name, add_text))
+			ROW_FORMAT=COMPRESSED
+			CHARACTER SET=utf8mb4
+			COLLATE=utf8mb4_unicode_ci""" % (self.name, add_text))
 
 	def get_column_definitions(self):
 		column_list = [] + default_columns

frappe/model/meta.py

@@ -6,7 +6,7 @@
 from __future__ import unicode_literals
 import frappe, json
 from frappe.utils import cstr, cint
-from frappe.model import integer_docfield_properties, default_fields
+from frappe.model import integer_docfield_properties, default_fields, no_value_fields, optional_fields
 from frappe.model.document import Document
 from frappe.model.base_document import BaseDocument
 from frappe.model.db_schema import type_map
@@ -41,7 +41,7 @@ def load_doctype_from_file(doctype):
 
 class Meta(Document):
 	_metaclass = True
-	default_fields = default_fields[1:]
+	default_fields = list(default_fields)[1:]
 	special_doctypes = ("DocField", "DocPerm", "Role", "DocType", "Module Def")
 
 	def __init__(self, doctype):
@@ -322,3 +322,18 @@ def get_default_df(fieldname):
 				fieldname = fieldname,
 				fieldtype = "Data"
 			)
+
+def trim_tables():
+	"""Use this to remove columns that don't exist in meta"""
+	ignore_fields = default_fields + optional_fields
+
+	for doctype in frappe.db.get_all("DocType", filters={"issingle": 0}):
+		doctype = doctype.name
+		columns = frappe.db.get_table_columns(doctype)
+		fields = [df.fieldname for df in frappe.get_meta(doctype).fields if df.fieldtype not in no_value_fields]
+		columns_to_remove = [f for f in list(set(columns) - set(fields)) if f not in ignore_fields]
+		if columns_to_remove:
+			columns_to_remove = ", ".join(["drop `{0}`".format(c) for c in columns_to_remove])
+			query = """alter table `tab{doctype}` {columns}""".format(
+				doctype=doctype, columns=columns_to_remove)
+			frappe.db.sql_ddl(query)

frappe/patches.txt

@@ -1,4 +1,5 @@
 execute:frappe.db.sql("""update `tabPatch Log` set patch=replace(patch, '.4_0.', '.v4_0.')""") #2014-05-12
+frappe.patches.v5_0.convert_to_barracuda_and_utf8mb4
 execute:frappe.reload_doc('core', 'doctype', 'doctype', force=True) #2014-01-24
 execute:frappe.reload_doc('core', 'doctype', 'docfield', force=True) #2015-01-01
 execute:frappe.reload_doc('core', 'doctype', 'docperm') #2014-06-04
@@ -68,3 +69,4 @@ frappe.patches.v5_0.bookmarks_to_stars
 frappe.patches.v5_0.style_settings_to_website_theme
 frappe.patches.v5_0.rename_ref_type_fieldnames
 frappe.patches.v5_0.fix_email_alert
+

frappe/patches/v5_0/convert_to_barracuda_and_utf8mb4.py

@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+import frappe
+import sys
+
+def execute():
+	check_if_ready_for_barracuda()
+
+	for table in frappe.db.get_tables():
+		frappe.db.sql_ddl("""alter table `{0}` ENGINE=InnoDB ROW_FORMAT=COMPRESSED""".format(table))
+		frappe.db.sql_ddl("""alter table `{0}` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci""".format(table))
+
+def check_if_ready_for_barracuda():
+	mariadb_variables = frappe._dict(frappe.db.sql("""show variables"""))
+	for key, value in {
+			"innodb_file_format": "Barracuda",
+			"innodb_file_per_table": "ON",
+			"innodb_large_prefix": "ON",
+			"character_set_server": "utf8mb4",
+			"collation_server": "utf8mb4_unicode_ci"
+		}.items():
+
+		if mariadb_variables.get(key) != value:
+			print "="*80
+			print "Please add this to MariaDB's my.cnf and restart MariaDB before proceeding"
+			print
+			print expected
+			print "="*80
+			sys.exit(1)
+			# raise Exception, "MariaDB needs to be configured!"
+
+expected = """[mysqld]
+innodb-file-format=barracuda
+innodb-file-per-table=1
+innodb-large-prefix=1
+character-set-client-handshake = FALSE
+character-set-server = utf8mb4
+collation-server = utf8mb4_unicode_ci
+
+[mysql]
+default-character-set = utf8mb4
+"""