From 0b1821b10a5a7053c8c9a807835e9d6d8fab50d5 Mon Sep 17 00:00:00 2001 From: Don-Leopardo Date: Wed, 22 Dec 2021 13:10:22 -0300 Subject: [PATCH 01/14] fix: Module path for external print formats --- frappe/www/printview.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 569ebe27d6..e9be76373f 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -2,7 +2,7 @@ # License: MIT. See LICENSE import frappe, os, copy, json, re -from frappe import _ +from frappe import _, get_module_path from frappe.modules import get_doc_path from frappe.core.doctype.access_log.access_log import make_access_log @@ -251,8 +251,9 @@ def get_print_format(doctype, print_format): frappe.DoesNotExistError) # server, find template - path = os.path.join(get_doc_path(frappe.db.get_value("DocType", doctype, "module"), - "Print Format", print_format.name), frappe.scrub(print_format.name) + ".html") + module = print_format.module or frappe.db.get_value("DocType", doctype, "module") + path = os.path.join(get_module_path(module, "Print Format",print_format.name), + frappe.scrub(print_format.name) + ".html") if os.path.exists(path): with open(path, "r") as pffile: From c9ce941554fccfcafc7d07501c587993a1c0d7b5 Mon Sep 17 00:00:00 2001 From: Don-Leopardo Date: Wed, 22 Dec 2021 14:19:08 -0300 Subject: [PATCH 02/14] fix: Unused imports --- frappe/www/printview.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index e9be76373f..088c9dc403 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -4,7 +4,6 @@ import frappe, os, copy, json, re from frappe import _, get_module_path -from frappe.modules import get_doc_path from frappe.core.doctype.access_log.access_log import make_access_log from frappe.utils import cint, sanitize_html, strip_html from frappe.utils.jinja_globals import is_rtl From abaa2dc35e8355af9a13d7887cf24dcb6f0f89c8 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 18 Jan 2022 11:57:38 +0530 Subject: [PATCH 03/14] style: Fix formatting issue --- frappe/www/printview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 088c9dc403..bea1300764 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -251,7 +251,7 @@ def get_print_format(doctype, print_format): # server, find template module = print_format.module or frappe.db.get_value("DocType", doctype, "module") - path = os.path.join(get_module_path(module, "Print Format",print_format.name), + path = os.path.join(get_module_path(module, "Print Format", print_format.name), frappe.scrub(print_format.name) + ".html") if os.path.exists(path): From c0fe37b94eb21cc756c53d5de91073177a1bd474 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 21 Jan 2022 21:18:16 +0530 Subject: [PATCH 04/14] fix: Sync "index" & "Unique" constraint changes in DB --- frappe/database/mariadb/database.py | 11 +++++++++-- frappe/database/mariadb/schema.py | 30 +++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 6b827a4e89..de28dad900 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -245,9 +245,16 @@ class MariaDBDatabase(Database): column_name as 'name', column_type as 'type', column_default as 'default', - column_key = 'MUL' as 'index', + COALESCE( + (select 1 + from information_schema.statistics + where table_name="{table_name}" + and column_name=columns.column_name + and NON_UNIQUE=1 + limit 1 + ), 0) as 'index', column_key = 'UNI' as 'unique' - from information_schema.columns + from information_schema.columns as columns where table_name = '{table_name}' '''.format(table_name=table_name), as_dict=1) def has_index(self, table_name, index_name): diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 5768a2f23d..f1691765b5 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -58,18 +58,32 @@ class MariaDBTable(DBTable): modify_column_query.append("MODIFY `{}` {}".format(col.fieldname, col.get_definition())) for col in self.add_index: - # if index key not exists - if not frappe.db.sql("SHOW INDEX FROM `%s` WHERE key_name = %s" % - (self.table_name, '%s'), col.fieldname): - add_index_query.append("ADD INDEX `{}`(`{}`)".format(col.fieldname, col.fieldname)) + # if index key does not exists + if not frappe.db.has_index(self.table_name, col.fieldname + '_index'): + add_index_query.append("ADD INDEX `{}_index`(`{}`)".format(col.fieldname, col.fieldname)) for col in self.drop_index: if col.fieldname != 'name': # primary key + current_column = self.current_columns.get(col.fieldname.lower()) + unique_constraint_changed = current_column.unique != col.unique + if unique_constraint_changed and not col.unique: + unique_index_record = frappe.db.sql(""" + SHOW INDEX FROM `{0}` + WHERE Column_name=%s + AND Non_unique=0 + """.format(self.table_name), (col.fieldname), as_dict=1) + if unique_index_record: + drop_index_query.append("DROP INDEX `{}`".format(unique_index_record[0].Key_name)) + index_constraint_changed = current_column.index != col.set_index # if index key exists - if frappe.db.sql("""SHOW INDEX FROM `{0}` - WHERE key_name=%s - AND Non_unique=%s""".format(self.table_name), (col.fieldname, col.unique)): - drop_index_query.append("drop index `{}`".format(col.fieldname)) + if index_constraint_changed and not col.set_index: + index_record = frappe.db.sql(""" + SHOW INDEX FROM `{0}` + WHERE Column_name=%s + AND Non_unique=1 + """.format(self.table_name), (col.fieldname), as_dict=1) + if index_record: + drop_index_query.append("DROP INDEX `{}`".format(index_record[0].Key_name)) try: for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]: From b3f9e8e2cf00e85488dfbaef0fd5ba60af80a130 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 21 Jan 2022 21:20:47 +0530 Subject: [PATCH 05/14] test: Add cases to validate index and unique constraints --- frappe/tests/test_db_update.py | 52 +++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 4ae33a2fab..7831ecd5f4 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -34,6 +34,52 @@ class TestDBUpdate(unittest.TestCase): self.assertEqual(fieldtype, table_column.type) self.assertIn(cstr(table_column.default) or 'NULL', [cstr(default), "'{}'".format(default)]) + def test_index_and_unique_constraints(self): + doctype = "User" + frappe.reload_doctype('User', force=True) + frappe.model.meta.trim_tables('User') + + make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.unique, 1) + + make_property_setter(doctype, 'restrict_ip', 'unique', '0', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.unique, 0) + + make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.index, 1) + + make_property_setter(doctype, 'restrict_ip', 'search_index', '0', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.index, 0) + + make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') + make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.index, 1) + self.assertEqual(restrict_ip_in_table.unique, 1) + + make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') + make_property_setter(doctype, 'restrict_ip', 'unique', '0', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.index, 1) + self.assertEqual(restrict_ip_in_table.unique, 0) + + make_property_setter(doctype, 'restrict_ip', 'search_index', '0', 'Int') + make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') + frappe.db.updatedb(doctype) + restrict_ip_in_table = get_table_column("User", "restrict_ip") + self.assertEqual(restrict_ip_in_table.index, 0) + self.assertEqual(restrict_ip_in_table.unique, 1) + def get_fieldtype_from_def(field_def): fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) fieldtype = fieldtuple[0] @@ -69,4 +115,8 @@ def get_other_fields_meta(meta): fields = dict(default_fields_map, **optional_fields_map) field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()] - return field_map \ No newline at end of file + return field_map + +def get_table_column(doctype, fieldname): + table_columns = frappe.db.get_table_columns_description('tab{}'.format(doctype)) + return find(table_columns, lambda d: d.get('name') == fieldname) \ No newline at end of file From e1497e1aadf81e26c65793d592a3e2109027310a Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 21 Jan 2022 17:03:54 +0100 Subject: [PATCH 06/14] fix: translate fieldname in report dialog and alert --- frappe/public/js/frappe/views/reports/report_view.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 6d8e281793..f5d9f3e110 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -340,7 +340,7 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { options: columns_in_picker }, { - label: __('Insert Column Before {0}', [datatabe_col.docfield.label.bold()]), + label: __('Insert Column Before {0}', [__(datatabe_col.docfield.label).bold()]), fieldname: 'insert_before', fieldtype: 'Check' } @@ -789,7 +789,10 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { } else { this.fields.splice(col_index, 0, field); } - frappe.show_alert(__('Also adding the dependent currency field {0}', [field[0].bold()])); + const field_label = frappe.meta.get_label(doctype, field[0]); + frappe.show_alert( + __('Also adding the dependent currency field {0}', [__(field_label).bold()]) + ); } } @@ -799,7 +802,10 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { const field = [col, doctype]; this.fields.push(field); this.refresh(); - frappe.show_alert(__('Also adding the status dependency field {0}', [field[0].bold()])); + const field_label = frappe.meta.get_label(doctype, field[0]); + frappe.show_alert( + __('Also adding the status dependency field {0}', [__(field_label).bold()]) + ); } } From 39031d18ed5ff063d50a045b3f3a947fc642fa86 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 23 Jan 2022 17:44:07 +0530 Subject: [PATCH 07/14] fix: Check if index exists with specific key_name --- frappe/database/mariadb/schema.py | 6 +++--- frappe/database/schema.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index f1691765b5..f7abac6547 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -69,7 +69,7 @@ class MariaDBTable(DBTable): if unique_constraint_changed and not col.unique: unique_index_record = frappe.db.sql(""" SHOW INDEX FROM `{0}` - WHERE Column_name=%s + WHERE Key_name=%s AND Non_unique=0 """.format(self.table_name), (col.fieldname), as_dict=1) if unique_index_record: @@ -79,9 +79,9 @@ class MariaDBTable(DBTable): if index_constraint_changed and not col.set_index: index_record = frappe.db.sql(""" SHOW INDEX FROM `{0}` - WHERE Column_name=%s + WHERE Key_name=%s AND Non_unique=1 - """.format(self.table_name), (col.fieldname), as_dict=1) + """.format(self.table_name), (col.fieldname + '_index'), as_dict=1) if index_record: drop_index_query.append("DROP INDEX `{}`".format(index_record[0].Key_name)) diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 10582eff8f..5b4788a9c2 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -230,7 +230,7 @@ class DbColumn: self.table.set_default.append(self) # index should be applied or dropped irrespective of type change - if ((current_def['index'] and not self.set_index and not self.unique) + if ((current_def['index'] and not self.set_index) or (current_def['unique'] and not self.unique)): # to drop unique you have to drop index self.table.drop_index.append(self) From 4d384f308b888f8b3c6fad197afb4b180a03dc61 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 23 Jan 2022 22:57:33 +0530 Subject: [PATCH 08/14] fix: Unique and Index constraints --- frappe/database/mariadb/schema.py | 2 +- frappe/database/postgres/database.py | 16 ++++---- frappe/database/postgres/schema.py | 58 +++++++++++++++++----------- frappe/database/schema.py | 9 +++-- 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index f7abac6547..d5a1096ba4 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -62,7 +62,7 @@ class MariaDBTable(DBTable): if not frappe.db.has_index(self.table_name, col.fieldname + '_index'): add_index_query.append("ADD INDEX `{}_index`(`{}`)".format(col.fieldname, col.fieldname)) - for col in self.drop_index: + for col in self.drop_index + self.drop_unique: if col.fieldname != 'name': # primary key current_column = self.current_columns.get(col.fieldname.lower()) unique_constraint_changed = current_column.unique != col.unique diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 33f07990af..3547907e58 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -77,11 +77,11 @@ class PostgresDatabase(Database): """Escape quotes and percent in given string.""" if isinstance(s, bytes): s = s.decode('utf-8') - + # MariaDB's driver treats None as an empty string # So Postgres should do the same - if s is None: + if s is None: s = '' if percent: @@ -308,18 +308,20 @@ class PostgresDatabase(Database): WHEN 'timestamp without time zone' THEN 'timestamp' ELSE a.data_type END AS type, - COUNT(b.indexdef) AS Index, + BOOL_OR(b.index) AS index, SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default, BOOL_OR(b.unique) AS unique FROM information_schema.columns a LEFT JOIN - (SELECT indexdef, tablename, indexdef LIKE '%UNIQUE INDEX%' AS unique + (SELECT indexdef, tablename, + indexdef LIKE '%UNIQUE INDEX%' AS unique, + indexdef NOT LIKE '%UNIQUE INDEX%' AS index FROM pg_indexes WHERE tablename='{table_name}') b - ON SUBSTRING(b.indexdef, '\(.*\)') LIKE CONCAT('%', a.column_name, '%') + ON SUBSTRING(b.indexdef, '\(.*\)') LIKE CONCAT('%', a.column_name, '%') WHERE a.table_name = '{table_name}' - GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length;''' - .format(table_name=table_name), as_dict=1) + GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length; + '''.format(table_name=table_name), as_dict=1) def get_database_list(self, target): return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 58153ca6ce..a7c96a5532 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -52,8 +52,8 @@ class PostgresTable(DBTable): query.append("ALTER COLUMN `{0}` TYPE {1} {2}".format( col.fieldname, get_definition(col.fieldtype, precision=col.precision, length=col.length), - using_clause) - ) + using_clause + )) for col in self.set_default: if col.fieldname=="name": @@ -73,37 +73,49 @@ class PostgresTable(DBTable): query.append("ALTER COLUMN `{}` SET DEFAULT {}".format(col.fieldname, col_default)) - create_index_query = "" + create_contraint_query = "" for col in self.add_index: # if index key not exists - create_index_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + create_contraint_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( index_name=col.fieldname, table_name=self.table_name, field=col.fieldname) - drop_index_query = "" + for col in self.add_unique: + # if index key not exists + create_contraint_query += 'CREATE UNIQUE INDEX IF NOT EXISTS "unique_{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, + table_name=self.table_name, + field=col.fieldname + ) + + drop_contraint_query = "" for col in self.drop_index: # primary key if col.fieldname != 'name': # if index key exists - if not frappe.db.has_index(self.table_name, col.fieldname): - drop_index_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) + drop_contraint_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) - if query: - try: + for col in self.drop_unique: + # primary key + if col.fieldname != 'name': + # if index key exists + drop_contraint_query += 'DROP INDEX IF EXISTS "unique_{}" ;'.format(col.fieldname) + try: + if query: final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) if final_alter_query: frappe.db.sql(final_alter_query) - if create_index_query: frappe.db.sql(create_index_query) - if drop_index_query: frappe.db.sql(drop_index_query) - except Exception as e: - # sanitize - if frappe.db.is_duplicate_fieldname(e): - frappe.throw(str(e)) - elif frappe.db.is_duplicate_entry(e): - fieldname = str(e).split("'")[-2] - frappe.throw(_("""{0} field cannot be set as unique in {1}, - as there are non-unique existing values""".format( - fieldname, self.table_name))) - raise e - else: - raise e + if create_contraint_query: frappe.db.sql(create_contraint_query) + if drop_contraint_query: frappe.db.sql(drop_contraint_query) + except Exception as e: + # sanitize + if frappe.db.is_duplicate_fieldname(e): + frappe.throw(str(e)) + elif frappe.db.is_duplicate_entry(e): + fieldname = str(e).split("'")[-2] + frappe.throw(_("""{0} field cannot be set as unique in {1}, + as there are non-unique existing values""".format( + fieldname, self.table_name))) + raise e + else: + raise e diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 5b4788a9c2..1767f90af0 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -21,6 +21,7 @@ class DBTable: self.change_name = [] self.add_unique = [] self.add_index = [] + self.drop_unique = [] self.drop_index = [] self.set_default = [] @@ -219,8 +220,10 @@ class DbColumn: self.table.change_type.append(self) # unique - if((self.unique and not current_def['unique']) and column_type not in ('text', 'longtext')): + if ((self.unique and not current_def['unique']) and column_type not in ('text', 'longtext')): self.table.add_unique.append(self) + elif (current_def['unique'] and not self.unique): + self.table.drop_unique.append(self) # default if (self.default_changed(current_def) @@ -230,9 +233,7 @@ class DbColumn: self.table.set_default.append(self) # index should be applied or dropped irrespective of type change - if ((current_def['index'] and not self.set_index) - or (current_def['unique'] and not self.unique)): - # to drop unique you have to drop index + if (current_def['index'] and not self.set_index): self.table.drop_index.append(self) elif (not current_def['index'] and self.set_index) and not (column_type in ('text', 'longtext')): From 72af9c7366dab2259813a45268f394e408c8f223 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 23 Jan 2022 22:59:13 +0530 Subject: [PATCH 09/14] test: Fix test_index_and_unique_constraints --- frappe/tests/test_db_update.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 7831ecd5f4..74254718e6 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -42,43 +42,43 @@ class TestDBUpdate(unittest.TestCase): make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.unique, 1) + self.assertTrue(restrict_ip_in_table.unique) make_property_setter(doctype, 'restrict_ip', 'unique', '0', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.unique, 0) + self.assertFalse(restrict_ip_in_table.unique) make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.index, 1) + self.assertTrue(restrict_ip_in_table.index) make_property_setter(doctype, 'restrict_ip', 'search_index', '0', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.index, 0) + self.assertFalse(restrict_ip_in_table.index) make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.index, 1) - self.assertEqual(restrict_ip_in_table.unique, 1) + self.assertTrue(restrict_ip_in_table.index) + self.assertTrue(restrict_ip_in_table.unique) make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') make_property_setter(doctype, 'restrict_ip', 'unique', '0', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.index, 1) - self.assertEqual(restrict_ip_in_table.unique, 0) + self.assertTrue(restrict_ip_in_table.index) + self.assertFalse(restrict_ip_in_table.unique) make_property_setter(doctype, 'restrict_ip', 'search_index', '0', 'Int') make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") - self.assertEqual(restrict_ip_in_table.index, 0) - self.assertEqual(restrict_ip_in_table.unique, 1) + self.assertFalse(restrict_ip_in_table.index) + self.assertTrue(restrict_ip_in_table.unique) def get_fieldtype_from_def(field_def): fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) From 47393bcb85b5f3a53a9799d5a4db842ef6a14d04 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 24 Jan 2022 09:44:25 +0530 Subject: [PATCH 10/14] style: Fix formatting issues --- frappe/database/postgres/database.py | 2 +- frappe/database/postgres/schema.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 3547907e58..d5495c6879 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -318,7 +318,7 @@ class PostgresDatabase(Database): indexdef NOT LIKE '%UNIQUE INDEX%' AS index FROM pg_indexes WHERE tablename='{table_name}') b - ON SUBSTRING(b.indexdef, '\(.*\)') LIKE CONCAT('%', a.column_name, '%') + ON SUBSTRING(b.indexdef, '(.*)') LIKE CONCAT('%', a.column_name, '%') WHERE a.table_name = '{table_name}' GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length; '''.format(table_name=table_name), as_dict=1) diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index a7c96a5532..6c40a4e91d 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -104,18 +104,20 @@ class PostgresTable(DBTable): try: if query: final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) - if final_alter_query: frappe.db.sql(final_alter_query) - if create_contraint_query: frappe.db.sql(create_contraint_query) - if drop_contraint_query: frappe.db.sql(drop_contraint_query) + frappe.db.sql(final_alter_query) + if create_contraint_query: + frappe.db.sql(create_contraint_query) + if drop_contraint_query: + frappe.db.sql(drop_contraint_query) except Exception as e: # sanitize if frappe.db.is_duplicate_fieldname(e): frappe.throw(str(e)) elif frappe.db.is_duplicate_entry(e): fieldname = str(e).split("'")[-2] - frappe.throw(_("""{0} field cannot be set as unique in {1}, - as there are non-unique existing values""".format( - fieldname, self.table_name))) - raise e + frappe.throw( + _("{0} field cannot be set as unique in {1}, as there are non-unique existing values") + .format(fieldname, self.table_name) + ) else: raise e From 9a5409892d929d54e4cc101c916d39014eec5880 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 24 Jan 2022 10:26:38 +0530 Subject: [PATCH 11/14] fix(postgres): Create indexes while creating table Previously, indexes were not getting created based on "search_index" configuration while creating new table --- frappe/database/postgres/schema.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 6c40a4e91d..99e64ce08d 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -11,8 +11,6 @@ class PostgresTable(DBTable): column_defs = self.get_column_definitions() if column_defs: add_text += ',\n'.join(column_defs) - # index - # index_defs = self.get_index_definitions() # TODO: set docstatus length # create table frappe.db.sql("""create table `%s` ( @@ -28,8 +26,24 @@ class PostgresTable(DBTable): idx bigint not null default '0', %s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text)) + self.create_indexes() frappe.db.commit() + def create_indexes(self): + create_index_query = "" + for key, col in self.columns.items(): + if (col.set_index + and col.fieldtype in frappe.db.type_map + and frappe.db.type_map.get(col.fieldtype)[0] + not in ('text', 'longtext')): + create_index_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, + table_name=self.table_name, + field=col.fieldname + ) + if create_index_query: + frappe.db.sql(create_index_query) + def alter(self): for col in self.columns.values(): col.build_for_alter_table(self.current_columns.get(col.fieldname.lower())) From 490b2f61cd0136b8e97e15fbaf6f0f3a00cc7231 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 24 Jan 2022 11:00:18 +0530 Subject: [PATCH 12/14] chore: Add nosemgrep comment to avoid semgrep warnings --- frappe/database/mariadb/schema.py | 2 ++ frappe/database/postgres/schema.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index d5a1096ba4..07bb4d5d7c 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -67,6 +67,7 @@ class MariaDBTable(DBTable): current_column = self.current_columns.get(col.fieldname.lower()) unique_constraint_changed = current_column.unique != col.unique if unique_constraint_changed and not col.unique: + # nosemgrep unique_index_record = frappe.db.sql(""" SHOW INDEX FROM `{0}` WHERE Key_name=%s @@ -77,6 +78,7 @@ class MariaDBTable(DBTable): index_constraint_changed = current_column.index != col.set_index # if index key exists if index_constraint_changed and not col.set_index: + # nosemgrep index_record = frappe.db.sql(""" SHOW INDEX FROM `{0}` WHERE Key_name=%s diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 99e64ce08d..a2d5be0b70 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -42,6 +42,7 @@ class PostgresTable(DBTable): field=col.fieldname ) if create_index_query: + # nosemgrep frappe.db.sql(create_index_query) def alter(self): @@ -118,10 +119,13 @@ class PostgresTable(DBTable): try: if query: final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) + # nosemgrep frappe.db.sql(final_alter_query) if create_contraint_query: + # nosemgrep frappe.db.sql(create_contraint_query) if drop_contraint_query: + # nosemgrep frappe.db.sql(drop_contraint_query) except Exception as e: # sanitize From 1b6f94dfe63133a62b7db06cd99227d6943c0b04 Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 24 Jan 2022 21:18:48 +0530 Subject: [PATCH 13/14] fix: permissions for payment gateway doctype --- .../payment_gateway/payment_gateway.json | 191 +++++------------- 1 file changed, 47 insertions(+), 144 deletions(-) diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.json b/frappe/core/doctype/payment_gateway/payment_gateway.json index b97d72c771..f548b0dffb 100644 --- a/frappe/core/doctype/payment_gateway/payment_gateway.json +++ b/frappe/core/doctype/payment_gateway/payment_gateway.json @@ -1,154 +1,57 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:gateway", - "beta": 0, - "creation": "2015-12-15 22:26:45.221162", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, + "actions": [], + "autoname": "field:gateway", + "creation": "2022-01-24 21:09:47.229371", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "gateway", + "gateway_settings", + "gateway_controller" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "gateway", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Gateway", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "gateway", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Gateway", + "reqd": 1, + "unique": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "gateway_settings", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Gateway Settings", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "gateway_settings", + "fieldtype": "Link", + "label": "Gateway Settings", + "options": "DocType", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "gateway_controller", - "fieldtype": "Dynamic Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Gateway Controller", - "length": 0, - "no_copy": 0, - "options": "gateway_settings", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "gateway_controller", + "fieldtype": "Dynamic Link", + "label": "Gateway Controller", + "options": "gateway_settings", + "reqd": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2018-02-05 14:24:33.526645", - "modified_by": "Administrator", - "module": "Core", - "name": "Payment Gateway", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2022-01-24 21:17:03.864719", + "modified_by": "Administrator", + "module": "Core", + "name": "Payment Gateway", + "naming_rule": "By fieldname", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "create": 1, + "delete": 1, + "read": 1, + "role": "System Manager", + "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file From b550096161006b17890e430705ca2f779f08662d Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 24 Jan 2022 20:51:55 +0000 Subject: [PATCH 14/14] feat: time field in webform --- frappe/website/doctype/web_form_field/web_form_field.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/website/doctype/web_form_field/web_form_field.json b/frappe/website/doctype/web_form_field/web_form_field.json index 2770f03e80..94a8851739 100644 --- a/frappe/website/doctype/web_form_field/web_form_field.json +++ b/frappe/website/doctype/web_form_field/web_form_field.json @@ -39,7 +39,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Fieldtype", - "options": "Attach\nAttach Image\nCheck\nCurrency\nData\nDate\nDatetime\nDuration\nFloat\nHTML\nInt\nLink\nPassword\nRating\nSelect\nSmall Text\nText\nText Editor\nTable\nSection Break\nColumn Break" + "options": "Attach\nAttach Image\nCheck\nCurrency\nData\nDate\nDatetime\nDuration\nFloat\nHTML\nInt\nLink\nPassword\nRating\nSelect\nSmall Text\nText\nText Editor\nTable\nTime\nSection Break\nColumn Break" }, { "fieldname": "label", @@ -146,7 +146,7 @@ ], "istable": 1, "links": [], - "modified": "2021-04-30 12:02:25.422345", + "modified": "2022-01-24 20:43:25.422345", "modified_by": "Administrator", "module": "Website", "name": "Web Form Field",