@@ -13,9 +13,13 @@ import os
import frappe
from frappe import _
from frappe.utils import cstr, cint
import MySQLdb
class InvalidColumnName(frappe.ValidationError): pass
varchar_len = '140'
standard_varchar_columns = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype')
type_map = {
'Currency': ('decimal', '18,6')
,'Int': ('int', '11')
@@ -30,14 +34,14 @@ type_map = {
,'Datetime': ('datetime', '6')
,'Time': ('time', '6')
,'Text': ('text', '')
,'Data': ('varchar', '255' )
,'Link': ('varchar', '255' )
,'Dynamic Link':('varchar', '255' )
,'Password': ('varchar', '255' )
,'Select': ('varchar', '255' )
,'Read Only': ('varchar', '255' )
,'Attach': ('varchar', '255' )
,'Attach Image':('varchar', '255' )
,'Data': ('varchar', varchar_len )
,'Link': ('varchar', varchar_len )
,'Dynamic Link':('varchar', varchar_len )
,'Password': ('varchar', varchar_len )
,'Select': ('varchar', varchar_len )
,'Read Only': ('varchar', varchar_len )
,'Attach': ('varchar', varchar_len )
,'Attach Image':('varchar', varchar_len )
}
default_columns = ['name', 'creation', 'modified', 'modified_by', 'owner',
@@ -57,8 +61,10 @@ def updatedb(dt):
raise Exception, 'Wrong doctype "%s" in updatedb' % dt
if not res[0][0]:
frappe.db.commit()
tab = DbTable(dt, 'tab')
tab.validate()
frappe.db.commit()
tab.sync()
frappe.db.begin()
@@ -80,12 +86,57 @@ class DbTable:
# load
self.get_columns_from_docfields()
def validate(self):
"""Check if change in varchar length isn't truncating the columns"""
if self.is_new():
return
self.get_columns_from_db()
columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in standard_varchar_columns]
columns += self.columns.values()
for col in columns:
if col.fieldtype in type_map and type_map[col.fieldtype][0]=="varchar":
# validate length range
new_length = cint(col.length) or cint(varchar_len)
if not (1 <= new_length <= 255):
frappe.throw(_("Length of {0} should be between 1 and 255").format(col.fieldname))
try:
# check for truncation
max_length = frappe.db.sql("""select max(length(`{fieldname}`)) from `tab{doctype}`"""\
.format(fieldname=col.fieldname, doctype=self.doctype))
except MySQLdb.OperationalError, e:
if e.args[0]==1054:
# Unknown column 'column_name' in 'field list'
continue
else:
raise
if max_length and max_length[0][0] > new_length:
current_type = self.current_columns[col.fieldname]["type"]
current_length = re.findall('varchar\(([\d]+)\)', current_type)[0]
if col.fieldname in self.columns:
self.columns[col.fieldname].length = current_length
frappe.msgprint(_("Reverting length to {0} for '{1}' in '{2}'; Setting the length as {3} will cause truncation of data.")\
.format(current_length, col.fieldname, self.doctype, new_length))
def sync(self):
if not self.name in DbManager(frappe.db).get_tables_list(frappe.db.cur_db_name):
if self.is_new( ):
self.create()
else:
self.alter()
def is_new(self):
return self.name not in DbManager(frappe.db).get_tables_list(frappe.db.cur_db_name)
def create(self):
add_text = ''
@@ -99,21 +150,21 @@ class DbTable:
# create table
frappe.db.sql("""create table `%s` (
name varchar(255 ) not null primary key,
name varchar({varchar_len} ) not null primary key,
creation datetime(6),
modified datetime(6),
modified_by varchar(255 ),
owner varchar(255 ),
modified_by varchar({varchar_len} ),
owner varchar({varchar_len} ),
docstatus int(1) default '0',
parent varchar(255 ),
parentfield varchar(255 ),
parenttype varchar(255 ),
parent varchar({varchar_len} ),
parentfield varchar({varchar_len} ),
parenttype varchar({varchar_len} ),
idx int(8),
%sindex parent(parent))
ENGINE=InnoDB
ROW_FORMAT=COMPRESSED
CHARACTER SET=utf8mb4
COLLATE=utf8mb4_unicode_ci""" % (self.name, add_text))
COLLATE=utf8mb4_unicode_ci""".format(varchar_len=varchar_len) % (self.name, add_text))
def get_column_definitions(self):
column_list = [] + default_columns
@@ -139,6 +190,7 @@ class DbTable:
get columns from docfields and custom fields
"""
fl = frappe.db.sql("SELECT * FROM tabDocField WHERE parent = %s", self.doctype, as_dict = 1)
lengths = {}
precisions = {}
uniques = {}
@@ -148,19 +200,26 @@ class DbTable:
WHERE dt = %s AND docstatus < 2""", (self.doctype,), as_dict=1)
if custom_fl: fl += custom_fl
# get precision from property setters
for ps in frappe.get_all("Property Setter", fields=["field_name", "value"],
filters={"doc_type": self.doctype, "doctype_or_field": "DocField", "property": "precision"}):
precisions[ps.field_name] = ps.value
# apply length, precision and unique from property setters
for ps in frappe.get_all("Property Setter", fields=["field_name", "property", "value"],
filters={
"doc_type": self.doctype,
"doctype_or_field": "DocField",
"property": ["in", ["precision", "length", "unique"]]
}):
# apply unique from property setters
for ps in frappe.get_all("Property Setter", fields=["field_name", "value"],
filters={"doc_type": self.doctype, "doctype_or_field": "DocField", "property": "unique"}):
if ps.property=="length":
lengths[ps.field_name] = cint(ps.value)
elif ps.property=="precision":
precisions[ps.field_name] = cint(ps.value)
elif ps.property=="unique":
uniques[ps.field_name] = cint(ps.value)
for f in fl:
self.columns[f['fieldname']] = DbColumn(self, f['fieldname'],
f['fieldtype'], f.get('length'), f.get('default'), f.get('search_index'),
f['fieldtype'], lengths.get(f["fieldname"]) or f.get('length'), f.get('default'), f.get('search_index'),
f.get('options'), uniques.get(f["fieldname"], f.get('unique')), precisions.get(f['fieldname']) or f.get('precision'))
def get_columns_from_db(self):
@@ -201,7 +260,6 @@ class DbTable:
frappe.db.sql("set foreign_key_checks=1")
def alter(self):
self.get_columns_from_db()
for col in self.columns.values():
col.build_for_alter_table(self.current_columns.get(col.fieldname, None))
@@ -268,7 +326,7 @@ class DbColumn:
self.precision = precision
def get_definition(self, with_default=1):
column_def = get_definition(self.fieldtype, self.precision)
column_def = get_definition(self.fieldtype, precision= self.precision, length=self.length )
if not column_def:
return column_def
@@ -287,7 +345,7 @@ class DbColumn:
return column_def
def build_for_alter_table(self, current_def):
column_def = get_definition(self.fieldtype)
column_def = get_definition(self.fieldtype, self.precision, self.length )
# no columns
if not column_def:
@@ -471,20 +529,28 @@ def remove_all_foreign_keys():
for f in fklist:
frappe.db.sql("alter table `tab%s` drop foreign key `%s`" % (t[0], f[1]))
def get_definition(fieldtype, precision=None):
def get_definition(fieldtype, precision=None, length=None ):
d = type_map.get(fieldtype)
if not d:
return
ret = d[0]
coltype = d[0]
size = None
if d[1]:
length = d[1]
size = d[1]
if size:
if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6:
length = '18,9'
ret += '(' + length + ')'
size = '18,9'
if coltype == "varchar" and length:
size = length
if size is not None:
coltype = "{coltype}({size})".format(coltype=coltype, size=size)
return ret
return coltype
def add_column(doctype, column_name, fieldtype, precision=None):
frappe.db.commit()