From 9bfc97a823328163d7c632072528c163e66c9e23 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Tue, 23 Feb 2021 18:53:24 +0530 Subject: [PATCH 01/42] fix: 'Not Saved' even after saving/submitting a doctype --- frappe/core/doctype/user_permission/user_permission.js | 2 +- frappe/public/js/frappe/form/form.js | 4 ++-- frappe/public/js/frappe/model/model.js | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/user_permission/user_permission.js b/frappe/core/doctype/user_permission/user_permission.js index 4c3f5b4eb8..6c6b74c5df 100644 --- a/frappe/core/doctype/user_permission/user_permission.js +++ b/frappe/core/doctype/user_permission/user_permission.js @@ -45,7 +45,7 @@ frappe.ui.form.on('User Permission', { set_applicable_for_constraint: frm => { frm.toggle_reqd('applicable_for', !frm.doc.apply_to_all_doctypes); if (frm.doc.apply_to_all_doctypes) { - frm.set_value('applicable_for', null); + frm.set_value('applicable_for', null, null, true); } }, diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8d96054d16..a0f546b42c 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1320,7 +1320,7 @@ frappe.ui.form.Form = class FrappeForm { return doc; } - set_value(field, value, if_missing) { + set_value(field, value, if_missing, avoid_dirty=false) { var me = this; var _set = function(f, v) { var fieldobj = me.fields_dict[f]; @@ -1340,7 +1340,7 @@ frappe.ui.form.Form = class FrappeForm { me.refresh_field(f); return Promise.resolve(); } else { - return frappe.model.set_value(me.doctype, me.doc.name, f, v); + return frappe.model.set_value(me.doctype, me.doc.name, f, v, me.fieldtype, avoid_dirty); } } } else { diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 9ec7b0e931..f93f712740 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -401,7 +401,7 @@ $.extend(frappe.model, { } }, - set_value: function(doctype, docname, fieldname, value, fieldtype) { + set_value: function(doctype, docname, fieldname, value, fieldtype, avoid_dirty=false) { /* help: Set a value locally (if changed) and execute triggers */ var doc; @@ -427,7 +427,7 @@ $.extend(frappe.model, { } doc[key] = value; - tasks.push(() => frappe.model.trigger(key, value, doc)); + if (!avoid_dirty) tasks.push(() => frappe.model.trigger(key, value, doc)); } else { // execute link triggers (want to reselect to execute triggers) if(in_list(["Link", "Dynamic Link"], fieldtype) && doc) { From 97387b3dbc1a15cbfe8d6c07a14b5299e2f63055 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 22 Feb 2022 13:06:21 +0530 Subject: [PATCH 02/42] feat: grid search --- frappe/public/js/frappe/form/grid.js | 129 +++++++++++++++++--- frappe/public/js/frappe/form/grid_row.js | 144 +++++++++++++++++++++-- frappe/public/js/frappe/utils/utils.js | 12 +- frappe/public/scss/common/grid.scss | 24 ++++ 4 files changed, 285 insertions(+), 24 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 8b615f3c59..e806e46ebc 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -35,7 +35,7 @@ export default class Grid { && this.frm.meta.__form_grid_templates[this.df.fieldname]) { this.template = this.frm.meta.__form_grid_templates[this.df.fieldname]; } - + this.filter = {}; this.is_grid = true; this.debounced_refresh = this.refresh.bind(this); this.debounced_refresh = frappe.utils.debounce(this.debounced_refresh, 100); @@ -274,6 +274,8 @@ export default class Grid { } make_head() { + if (this.prevent_build) return; + // labels if (this.header_row) { $(this.parent).find(".grid-heading-row .grid-row").remove(); @@ -286,12 +288,42 @@ export default class Grid { grid: this, configure_columns: true }); + + this.header_search = new GridRow({ + parent: $(this.parent).find(".grid-heading-row"), + parent_df: this.df, + docfields: this.docfields, + frm: this.frm, + grid: this, + show_search: true + }); + + Object.keys(this.filter).length !== 0 && + this.update_search_columns(); + } + + update_search_columns() { + for (const field in this.filter) { + if (this.filter[field] && !this.header_search.search_columns[field]) { + delete this.filter[field]; + this.data = this.get_data(Object.keys(this.filter).length !== 0); + break; + } + + if (this.filter[field] && this.filter[field].value) { + let $input = this.header_search.row_index.find('input'); + if (field && field !== 'row-index') { + $input = this.header_search.search_columns[field].find('input'); + } + $input.val(this.filter[field].value); + } + }; } - refresh(force) { + refresh() { if (this.frm && this.frm.setting_dependency) return; - this.data = this.get_data(); + this.data = this.get_data(Object.keys(this.filter).length !== 0); !this.wrapper && this.make(); let $rows = $(this.parent).find('.rows'); @@ -453,7 +485,7 @@ export default class Grid { } make_sortable($rows) { - new Sortable($rows.get(0), { + this.grid_sortable = new Sortable($rows.get(0), { group: { name: this.df.fieldname }, handle: '.sortable-handle', draggable: '.grid-row', @@ -484,14 +516,74 @@ export default class Grid { $(this.frm.wrapper).trigger("grid-make-sortable", [this.frm]); } - get_data() { - var data = this.frm ? - this.frm.doc[this.df.fieldname] || [] - : this.df.data || this.get_modal_data(); - // data.sort(function(a, b) { return a.idx - b.idx}); + get_data(filter_field) { + let data = []; + if (filter_field) { + data = this.get_filtered_data(); + } else { + data = this.frm ? + this.frm.doc[this.df.fieldname] || [] + : this.df.data || this.get_modal_data(); + } return data; } + get_filtered_data() { + if (!this.frm) return; + + let all_data = this.frm.doc[this.df.fieldname]; + + for (const field in this.filter) { + all_data = all_data.filter(data => { + let {df, value} = this.filter[field]; + + if (["Check"].includes(df.fieldtype)) { + return (data[df.fieldname] === parseInt(value || 0)) && data; + } else if (df.fieldtype === "Sr No" && data.idx.toString().indexOf(value) > -1) { + return data; + } else if (["Currency", "Float", "Int", "Percent", "Rating"].includes(df.fieldtype)) { + let num = data[df.fieldname] || 0; + + if (df.fieldtype === "Rating") { + let out_of_rating = parseInt(df.options) || 5; + num = data[df.fieldname] * out_of_rating; + } + + if (num.toString().indexOf(value) > -1) { + return data; + } + } else if (["Datetime", "Date"].includes(df.fieldtype) && data[df.fieldname]) { + let user_formatted_date = frappe.datetime.str_to_user(data[df.fieldname]); + + if (user_formatted_date.includes(value)) { + return data; + } + } else if (df.fieldtype === "Duration" && data[df.fieldname]) { + let formatted_duration = frappe.utils.get_formatted_duration(data[df.fieldname]); + + if (formatted_duration.includes(value.toLowerCase())) { + return data; + } + } else if (df.fieldtype === "Barcode" && data[df.fieldname]) { + let svg = data[df.fieldname]; + + if (svg.startsWith(' { if (!this.deleted_docs || !in_list(this.deleted_docs, data.name)) { @@ -701,7 +793,7 @@ export default class Grid { if (this.visible_columns && this.visible_columns.length > 0) return; this.user_defined_columns = []; - this.setup_user_defined_columns(); + this.setup_user_settings(); var total_colsize = 1, fields = (this.user_defined_columns && this.user_defined_columns.length > 0) ? this.user_defined_columns : this.editable_fields || this.docfields; @@ -775,12 +867,16 @@ export default class Grid { df.colsize = colsize; } - setup_user_defined_columns() { - if (this.frm) { - let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); - if (user_settings && user_settings[this.doctype] && user_settings[this.doctype].length) { - this.user_defined_columns = user_settings[this.doctype].map(row => { + setup_user_settings() { + if (!this.frm) return; + + let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); + + if (user_settings && user_settings[this.doctype] && user_settings[this.doctype]) { + if (user_settings[this.doctype]['columns'] && user_settings[this.doctype]['columns'].length) { + this.user_defined_columns = user_settings[this.doctype]['columns'].map(row => { let column = frappe.meta.get_docfield(this.doctype, row.fieldname); + if (column) { column.in_list_view = 1; column.columns = row.columns; @@ -788,6 +884,9 @@ export default class Grid { } }); } + + this.show_search = this.frm.doc[this.df.fieldname].length >= + (user_settings[this.doctype]['enable_search_count'] || 15); } } diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index a40f428969..97263d2529 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -12,7 +12,7 @@ export default class GridRow { } this.columns = {}; this.columns_list = []; - this.row_check_html = ''; + this.row_check_html = ''; this.make(); } make() { @@ -192,23 +192,67 @@ export default class GridRow { this.set_row_index(); // index (1, 2, 3 etc) - if(!this.row_index) { + if(!this.row_index && !this.show_search) { // REDESIGN-TODO: Make translation contextual, this No is Number var txt = (this.doc ? this.doc.idx : __("No.")); - this.row_index = $( - `
+ + this.row_check = $( + `
${this.row_check_html} -
`) +
`) + .appendTo(this.row); + + this.row_index = $( + ``) .appendTo(this.row) .on('click', function(e) { if(!$(e.target).hasClass('grid-row-check')) { me.toggle_view(); } }); + } else if (this.show_search) { + let timer = null; + this.row_check = $( + `
` + ).appendTo(this.row); + + this.row_index = $( + `` + ).appendTo(this.row); + + this.row_index.find('input').on('keyup', (e) => { + clearTimeout(timer); + timer = setTimeout(() => { + let df = { + fieldtype: "Sr No" + }; + + this.grid.filter['row-index'] = { + df: df, + value: e.target.value + } + + if(e.target.value == "") { + delete this.grid.filter['row-index']; + } + + this.grid.grid_sortable + .option('disabled', Object.keys(this.grid.filter).length !== 0); + + this.grid.prevent_build = true; + me.grid.refresh(); + this.grid.prevent_build = false; + }, 500); + }); + frappe.utils.only_allow_num_decimal(this.row_index.find('input')); } else { this.row_index.find('span').html(txt); } - + this.show_search && this.show_search_columns(); this.setup_columns(); this.add_open_form_button(); this.add_column_configure_button(); @@ -266,14 +310,26 @@ export default class GridRow { } configure_dialog_for_columns_selector() { + let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); + let enable_search_count = user_settings[this.grid.doctype] && + user_settings[this.grid.doctype]["enable_search_count"] || 15; + this.grid_settings_dialog = new frappe.ui.Dialog({ title: __("Configure Columns"), fields: [{ 'fieldtype': 'HTML', 'fieldname': 'fields_html' + }, + { + 'label': 'Enable Grid Search Count', + 'fieldtype': 'Data', + 'fieldname': 'enable_search', + 'default': enable_search_count, + 'description': __("Enable grid search if the grid row's are greater than or equal to the entered number") }] }); + this.enable_search_count = this.grid_settings_dialog.fields_dict.enable_search; this.grid.setup_visible_columns(); this.setup_columns_for_dialog(); this.prepare_wrapper_for_columns(); @@ -512,7 +568,10 @@ export default class GridRow { } let value = {}; - value[this.grid.doctype] = this.selected_columns_for_grid; + value[this.grid.doctype] = {}; + value[this.grid.doctype]['columns'] = this.selected_columns_for_grid; + value[this.grid.doctype]['enable_search_count'] = this.enable_search_count.get_value(); + frappe.model.user_settings.save(this.frm.doctype, 'GridView', value) .then((r) => { frappe.model.user_settings[this.frm.doctype] = r.message || r; @@ -530,6 +589,7 @@ export default class GridRow { setup_columns() { this.focus_set = false; + this.search_columns = {}; this.grid.setup_visible_columns(); this.grid.visible_columns.forEach((col, ci) => { @@ -545,8 +605,10 @@ export default class GridRow { txt = __(txt); } let column; - if (!this.columns[df.fieldname]) { + if (!this.columns[df.fieldname] && !this.show_search) { column = this.make_column(df, colsize, txt, ci); + } else if (!this.columns[df.fieldname] && this.show_search) { + column = this.make_search_column(df, colsize); } else { column = this.columns[df.fieldname]; this.refresh_field(df.fieldname, txt); @@ -564,6 +626,72 @@ export default class GridRow { } } }); + + if (this.show_search) { + // last empty column + $(`
`) + .appendTo(this.row) + } + } + + show_search_columns() { + // show or remove search columns based on Grid Search Count + this.grid.setup_user_settings(); + !this.grid.show_search && this.wrapper.remove(); + } + + make_search_column(df, colsize) { + let timer = null; + let title = ""; + let input_class = ""; + let is_disabled = ""; + + if (["Text", "Small Text"].includes(df.fieldtype)) { + input_class = "grid-overflow-no-ellipsis"; + } else if (["Int", "Currency", "Float", "Percent"].includes(df.fieldtype)) { + input_class = "text-right"; + } else if (df.fieldtype === "Check") { + title = __("1 = True & 0 = False"); + input_class = "text-center"; + } else if (df.fieldtype === 'Password') { + is_disabled = 'disabled' + title = __('Password cannot be filtered') + } + + let $col = $('
') + .appendTo(this.row); + + let $search_input = $(` + + `).appendTo($col); + + this.search_columns[df.fieldname] = $col; + + $search_input.on('keyup', (e) => { + clearTimeout(timer); + timer = setTimeout(() => { + this.grid.filter[df.fieldname] = { + df: df, + value: e.target.value + } + + if(e.target.value == '') { + delete this.grid.filter[df.fieldname]; + } + + this.grid.grid_sortable + .option('disabled', Object.keys(this.grid.filter).length !== 0); + + this.grid.prevent_build = true; + this.grid.refresh(); + this.grid.prevent_build = false; + }, 500); + }); + + ["Currency", "Float", "Int", "Percent", "Rating", "Check"].includes(df.fieldtype) && + frappe.utils.only_allow_num_decimal($search_input); + + return $col; } make_column(df, colsize, txt, ci) { diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index dc75239ed5..ae63f79e82 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1093,7 +1093,7 @@ Object.assign(frappe.utils, { seconds: round(seconds % 60) }; - if (duration_options.hide_days) { + if (duration_options && duration_options.hide_days) { total_duration.hours = round(seconds / 3600); total_duration.days = 0; } @@ -1453,5 +1453,15 @@ Object.assign(frappe.utils, { console.log(error); // eslint-disable-line return Promise.resolve(name); } + }, + + only_allow_num_decimal(input) { + input.on('input', (e) => { + let self = $(e.target); + self.val(self.val().replace(/[^0-9\.]/g, '')); + if ((e.which != 46 || self.val().indexOf('.') != -1) && (e.which < 48 || e.which > 57)) { + e.preventDefault(); + } + }); } }); diff --git a/frappe/public/scss/common/grid.scss b/frappe/public/scss/common/grid.scss index 1903413fbb..d5c9ae8d6b 100644 --- a/frappe/public/scss/common/grid.scss +++ b/frappe/public/scss/common/grid.scss @@ -82,6 +82,29 @@ height: 34px; padding: 8px; max-height: 200px; + + &.search { + padding: 7px !important; + + input { + height: -webkit-fill-available; + padding: 3px 7px; + } + } +} + +.row-check { + height: 34px; + padding: 8px 3px !important; + text-align: center; + + input { + margin-right: 0 !important; + } + + &.search { + padding: 0 !important; + } } .grid-row-check { @@ -409,6 +432,7 @@ } .page-number { + background-color: var(--fg-color); padding: 0 3px; } From cf4f35e8deb7a5e576e2b45ae86482ee46a960de Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 22 Feb 2022 15:51:54 +0530 Subject: [PATCH 03/42] fix(sider): missing semicolons --- frappe/public/js/frappe/form/grid.js | 2 +- frappe/public/js/frappe/form/grid_row.js | 16 ++++++++-------- frappe/public/js/frappe/utils/utils.js | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index e806e46ebc..adbaa5bcad 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -317,7 +317,7 @@ export default class Grid { } $input.val(this.filter[field].value); } - }; + } } refresh() { diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 97263d2529..789114572b 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -192,7 +192,7 @@ export default class GridRow { this.set_row_index(); // index (1, 2, 3 etc) - if(!this.row_index && !this.show_search) { + if (!this.row_index && !this.show_search) { // REDESIGN-TODO: Make translation contextual, this No is Number var txt = (this.doc ? this.doc.idx : __("No.")); @@ -234,9 +234,9 @@ export default class GridRow { this.grid.filter['row-index'] = { df: df, value: e.target.value - } + }; - if(e.target.value == "") { + if (e.target.value == "") { delete this.grid.filter['row-index']; } @@ -630,7 +630,7 @@ export default class GridRow { if (this.show_search) { // last empty column $(`
`) - .appendTo(this.row) + .appendTo(this.row); } } @@ -654,8 +654,8 @@ export default class GridRow { title = __("1 = True & 0 = False"); input_class = "text-center"; } else if (df.fieldtype === 'Password') { - is_disabled = 'disabled' - title = __('Password cannot be filtered') + is_disabled = 'disabled'; + title = __('Password cannot be filtered'); } let $col = $('
') @@ -673,9 +673,9 @@ export default class GridRow { this.grid.filter[df.fieldname] = { df: df, value: e.target.value - } + }; - if(e.target.value == '') { + if (e.target.value == '') { delete this.grid.filter[df.fieldname]; } diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index ae63f79e82..759b7b5499 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1458,7 +1458,7 @@ Object.assign(frappe.utils, { only_allow_num_decimal(input) { input.on('input', (e) => { let self = $(e.target); - self.val(self.val().replace(/[^0-9\.]/g, '')); + self.val(self.val().replace(/[^0-9.]/g, '')); if ((e.which != 46 || self.val().indexOf('.') != -1) && (e.which < 48 || e.which > 57)) { e.preventDefault(); } From 058d89312b63cc39535eded7f638e2d061e68b01 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 24 Feb 2022 12:15:02 +0530 Subject: [PATCH 04/42] revert: search row visiblitity is customizable (made it hard coded) --- frappe/public/js/frappe/form/grid.js | 18 +++++-------- frappe/public/js/frappe/form/grid_row.js | 34 ++++++++---------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index adbaa5bcad..61a5309b39 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -793,7 +793,7 @@ export default class Grid { if (this.visible_columns && this.visible_columns.length > 0) return; this.user_defined_columns = []; - this.setup_user_settings(); + this.setup_user_defined_columns(); var total_colsize = 1, fields = (this.user_defined_columns && this.user_defined_columns.length > 0) ? this.user_defined_columns : this.editable_fields || this.docfields; @@ -867,14 +867,11 @@ export default class Grid { df.colsize = colsize; } - setup_user_settings() { - if (!this.frm) return; - - let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); - - if (user_settings && user_settings[this.doctype] && user_settings[this.doctype]) { - if (user_settings[this.doctype]['columns'] && user_settings[this.doctype]['columns'].length) { - this.user_defined_columns = user_settings[this.doctype]['columns'].map(row => { + setup_user_defined_columns() { + if (this.frm) { + let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); + if (user_settings && user_settings[this.doctype] && user_settings[this.doctype].length) { + this.user_defined_columns = user_settings[this.doctype].map(row => { let column = frappe.meta.get_docfield(this.doctype, row.fieldname); if (column) { @@ -884,9 +881,6 @@ export default class Grid { } }); } - - this.show_search = this.frm.doc[this.df.fieldname].length >= - (user_settings[this.doctype]['enable_search_count'] || 15); } } diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 789114572b..554af0c436 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -188,7 +188,9 @@ export default class GridRow { })); } render_row(refresh) { - var me = this; + if (this.show_search && !this.show_search_row()) return; + + let me = this; this.set_row_index(); // index (1, 2, 3 etc) @@ -252,7 +254,7 @@ export default class GridRow { } else { this.row_index.find('span').html(txt); } - this.show_search && this.show_search_columns(); + this.setup_columns(); this.add_open_form_button(); this.add_column_configure_button(); @@ -310,26 +312,14 @@ export default class GridRow { } configure_dialog_for_columns_selector() { - let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); - let enable_search_count = user_settings[this.grid.doctype] && - user_settings[this.grid.doctype]["enable_search_count"] || 15; - this.grid_settings_dialog = new frappe.ui.Dialog({ title: __("Configure Columns"), fields: [{ 'fieldtype': 'HTML', 'fieldname': 'fields_html' - }, - { - 'label': 'Enable Grid Search Count', - 'fieldtype': 'Data', - 'fieldname': 'enable_search', - 'default': enable_search_count, - 'description': __("Enable grid search if the grid row's are greater than or equal to the entered number") }] }); - this.enable_search_count = this.grid_settings_dialog.fields_dict.enable_search; this.grid.setup_visible_columns(); this.setup_columns_for_dialog(); this.prepare_wrapper_for_columns(); @@ -568,10 +558,7 @@ export default class GridRow { } let value = {}; - value[this.grid.doctype] = {}; - value[this.grid.doctype]['columns'] = this.selected_columns_for_grid; - value[this.grid.doctype]['enable_search_count'] = this.enable_search_count.get_value(); - + value[this.grid.doctype] = this.selected_columns_for_grid; frappe.model.user_settings.save(this.frm.doctype, 'GridView', value) .then((r) => { frappe.model.user_settings[this.frm.doctype] = r.message || r; @@ -634,10 +621,11 @@ export default class GridRow { } } - show_search_columns() { - // show or remove search columns based on Grid Search Count - this.grid.setup_user_settings(); - !this.grid.show_search && this.wrapper.remove(); + show_search_row() { + // show or remove search columns based on grid rows + this.show_search = this.frm.doc[this.grid.df.fieldname].length >= 15; + !this.show_search && this.wrapper.remove(); + return this.show_search; } make_search_column(df, colsize) { @@ -668,7 +656,7 @@ export default class GridRow { this.search_columns[df.fieldname] = $col; $search_input.on('keyup', (e) => { - clearTimeout(timer); + clearTimeout(timer); timer = setTimeout(() => { this.grid.filter[df.fieldname] = { df: df, From 3dda9aeb8354ac81e6c89f1c5066da794ee8c402 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 24 Feb 2022 15:43:43 +0530 Subject: [PATCH 05/42] chore: created function for code readability --- frappe/public/js/frappe/form/grid.js | 106 ++++++++++++++------------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 61a5309b39..0a6211767e 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -536,52 +536,58 @@ export default class Grid { for (const field in this.filter) { all_data = all_data.filter(data => { let {df, value} = this.filter[field]; + return this.get_data_based_on_fieldtype(df, data, value.toLowerCase()); + }); + } - if (["Check"].includes(df.fieldtype)) { - return (data[df.fieldname] === parseInt(value || 0)) && data; - } else if (df.fieldtype === "Sr No" && data.idx.toString().indexOf(value) > -1) { - return data; - } else if (["Currency", "Float", "Int", "Percent", "Rating"].includes(df.fieldtype)) { - let num = data[df.fieldname] || 0; - - if (df.fieldtype === "Rating") { - let out_of_rating = parseInt(df.options) || 5; - num = data[df.fieldname] * out_of_rating; - } + return all_data; + } - if (num.toString().indexOf(value) > -1) { - return data; - } - } else if (["Datetime", "Date"].includes(df.fieldtype) && data[df.fieldname]) { - let user_formatted_date = frappe.datetime.str_to_user(data[df.fieldname]); + get_data_based_on_fieldtype(df, data, value) { + let fieldname = df.fieldname; + let fieldtype = df.fieldtype; + let fieldvalue = data[fieldname]; - if (user_formatted_date.includes(value)) { - return data; - } - } else if (df.fieldtype === "Duration" && data[df.fieldname]) { - let formatted_duration = frappe.utils.get_formatted_duration(data[df.fieldname]); + if (fieldtype === "Check") { + return (fieldvalue === parseInt(value || 0)) && data; + } else if (fieldtype === "Sr No" && data.idx.toString().includes(value)) { + return data; + } else if (fieldtype === "Duration" && fieldvalue) { + let formatted_duration = frappe.utils.get_formatted_duration(fieldvalue); - if (formatted_duration.includes(value.toLowerCase())) { - return data; - } - } else if (df.fieldtype === "Barcode" && data[df.fieldname]) { - let svg = data[df.fieldname]; + if (formatted_duration.includes(value)) { + return data; + } + } else if (fieldtype === "Barcode" && fieldvalue) { + let svg = fieldvalue; - if (svg.startsWith(' -1) { + return data; + } + } else if (fieldvalue && fieldvalue.toLowerCase().includes(value)) { + return data; + } } get_modal_data() { @@ -868,19 +874,19 @@ export default class Grid { } setup_user_defined_columns() { - if (this.frm) { - let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); - if (user_settings && user_settings[this.doctype] && user_settings[this.doctype].length) { - this.user_defined_columns = user_settings[this.doctype].map(row => { - let column = frappe.meta.get_docfield(this.doctype, row.fieldname); - - if (column) { - column.in_list_view = 1; - column.columns = row.columns; - return column; - } - }); - } + if (!this.frm) return; + + let user_settings = frappe.get_user_settings(this.frm.doctype, 'GridView'); + if (user_settings && user_settings[this.doctype] && user_settings[this.doctype].length) { + this.user_defined_columns = user_settings[this.doctype].map(row => { + let column = frappe.meta.get_docfield(this.doctype, row.fieldname); + + if (column) { + column.in_list_view = 1; + column.columns = row.columns; + return column; + } + }); } } From e1ada33cc56b0243c7266580e0573a75a3e306f7 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 24 Feb 2022 16:40:26 +0530 Subject: [PATCH 06/42] fix: show grid search row if rows are >= 20 --- frappe/public/js/frappe/form/grid_row.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 9b270fb8b0..e40fc3ba64 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -640,7 +640,7 @@ export default class GridRow { show_search_row() { // show or remove search columns based on grid rows - this.show_search = this.frm.doc[this.grid.df.fieldname].length >= 15; + this.show_search = this.frm.doc[this.grid.df.fieldname].length >= 20; !this.show_search && this.wrapper.remove(); return this.show_search; } From d800495810f813462172559a232a8ff37c9c8ddc Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 24 Feb 2022 19:36:59 +0530 Subject: [PATCH 07/42] fix: added fieldtype attribute on search field --- frappe/public/js/frappe/form/grid_row.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index e40fc3ba64..509875d8eb 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -667,7 +667,13 @@ export default class GridRow { .appendTo(this.row); let $search_input = $(` - + `).appendTo($col); this.search_columns[df.fieldname] = $col; From 1e68cca66330c9ace32102382c42f04a97d985de Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 24 Feb 2022 19:37:52 +0530 Subject: [PATCH 08/42] fix: allow both svg and text in barcode --- frappe/public/js/frappe/form/grid.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 385bd15cdf..99d9bbe0fc 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -559,14 +559,11 @@ export default class Grid { return data; } } else if (fieldtype === "Barcode" && fieldvalue) { - let svg = fieldvalue; + let barcode = fieldvalue.startsWith(' Date: Thu, 24 Feb 2022 19:44:19 +0530 Subject: [PATCH 09/42] test: UI test for grid search --- cypress/fixtures/child_table_doctype_1.js | 59 +++++++++++ cypress/fixtures/doctype_with_child_table.js | 6 ++ cypress/integration/grid_search.js | 104 +++++++++++++++++++ frappe/tests/ui_test_helpers.py | 45 +++++++- 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 cypress/fixtures/child_table_doctype_1.js create mode 100644 cypress/integration/grid_search.js diff --git a/cypress/fixtures/child_table_doctype_1.js b/cypress/fixtures/child_table_doctype_1.js new file mode 100644 index 0000000000..4657d63e2e --- /dev/null +++ b/cypress/fixtures/child_table_doctype_1.js @@ -0,0 +1,59 @@ +export default { + name: "Child Table Doctype 1", + actions: [], + custom: 1, + autoname: "format: Test-{####}", + creation: "2022-02-09 20:15:21.242213", + doctype: "DocType", + editable_grid: 1, + engine: "InnoDB", + fields: [ + { + fieldname: "data", + fieldtype: "Data", + in_list_view: 1, + label: "Data" + }, + { + fieldname: "barcode", + fieldtype: "Barcode", + in_list_view: 1, + label: "Barcode" + }, + { + fieldname: "check", + fieldtype: "Check", + in_list_view: 1, + label: "Check" + }, + { + fieldname: "rating", + fieldtype: "Rating", + in_list_view: 1, + label: "Rating" + }, + { + fieldname: "duration", + fieldtype: "Duration", + in_list_view: 1, + label: "Duration" + }, + { + fieldname: "date", + fieldtype: "Date", + in_list_view: 1, + label: "Date" + } + ], + links: [], + istable: 1, + modified: "2022-02-10 12:03:12.603763", + modified_by: "Administrator", + module: "Custom", + naming_rule: "By fieldname", + owner: "Administrator", + permissions: [], + sort_field: 'modified', + sort_order: 'ASC', + track_changes: 1 +}; \ No newline at end of file diff --git a/cypress/fixtures/doctype_with_child_table.js b/cypress/fixtures/doctype_with_child_table.js index bbb2127448..014074b0b5 100644 --- a/cypress/fixtures/doctype_with_child_table.js +++ b/cypress/fixtures/doctype_with_child_table.js @@ -20,6 +20,12 @@ export default { label: "Child Table", options: "Child Table Doctype", reqd: 1 + }, + { + fieldname: "child_table_1", + fieldtype: "Table", + label: "Child Table 1", + options: "Child Table Doctype 1" } ], links: [], diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js new file mode 100644 index 0000000000..6ccf3e37f0 --- /dev/null +++ b/cypress/integration/grid_search.js @@ -0,0 +1,104 @@ +import doctype_with_child_table from '../fixtures/doctype_with_child_table'; +import child_table_doctype from '../fixtures/child_table_doctype'; +import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; +const doctype_with_child_table_name = doctype_with_child_table.name; + +context('Grid Search', () => { + before(() => { + cy.visit('/login'); + cy.login(); + cy.insert_doc('DocType', child_table_doctype, true); + cy.insert_doc('DocType', child_table_doctype_1, true); + cy.insert_doc('DocType', doctype_with_child_table, true); + return cy.window().its('frappe').then(frappe => { + frappe.model.user_settings.save('Doctype With Child Table', 'GridView', { + 'Child Table Doctype 1': [ + {'fieldname': 'data', 'columns': 2}, + {'fieldname': 'barcode', 'columns': 1}, + {'fieldname': 'check', 'columns': 1}, + {'fieldname': 'rating', 'columns': 2}, + {'fieldname': 'duration', 'columns': 2}, + {'fieldname': 'date', 'columns': 2} + ] + }); + + return frappe.xcall("frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", { + name: doctype_with_child_table_name + }); + }); + }); + + it('Test search row visibility', () => { + cy.visit(`/app/doctype-with-child-table/Test Grid Search`); + + cy.get('[title="child_table_1"]').as('table'); + cy.get('@table').find('.grid-row-check:last').click(); + cy.get('@table').find('.grid-footer').contains('Delete').click(); + cy.get('.grid-heading-row .grid-row .search').should('not.exist'); + }); + + it('test search field for different fieldtypes', () => { + cy.visit(`/app/doctype-with-child-table/Test Grid Search`); + + cy.get('[title="child_table_1"]').as('table'); + + // Index Column + cy.get('@table').find('.grid-heading-row .row-index.search input').type('3'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); + cy.get('@table').find('.grid-heading-row .row-index.search input').clear(); + + // Data Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('Data'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 1); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').clear(); + + // Barcode Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').type('092'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 4); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').clear(); + + // Check Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').type('1'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 9); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); + + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').type('0'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 11); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); + + // Rating Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Rating"]').type('3'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 3); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Rating"]').clear(); + + // Duration Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').type('3d'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 3); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').clear(); + + // Date Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('2022'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 4); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').clear(); + }); + + it('test with multiple filter', () => { + cy.get('[title="child_table_1"]').as('table'); + + // Data Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('a'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 10); + + // Barcode Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').type('0'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 8); + + // Duration Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').type('d'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 5); + + // Date Column + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('-02'); + cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); + }) +}); \ No newline at end of file diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 26c20f3d18..ca41615ca1 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -268,4 +268,47 @@ def update_child_table(name): 'options': 'Doctype to Link' }) - doc.save() \ No newline at end of file + doc.save() + + +@frappe.whitelist() +def insert_doctype_with_child_table_record(name): + if frappe.db.get_all(name, {'title': 'Test Grid Search'}): + return + + def insert_child(doc, data, barcode, check, rating, duration, date): + doc.append('child_table_1', { + 'data': data, + 'barcode': barcode, + 'check': check, + 'rating': rating, + 'duration': duration, + 'date': date, + }) + + doc = frappe.new_doc(name) + doc.title = 'Test Grid Search' + doc.append('child_table', {'title': 'Test Grid Search'}) + + insert_child(doc, 'Data', '09709KJKKH2432', 1, 0.5, 266851, "2022-02-21") + insert_child(doc, 'Test', '09209KJHKH2432', 1, 0.8, 547877, "2021-05-27") + insert_child(doc, 'New', '09709KJHYH1132', 0, 0.1, 3, "2019-03-02") + insert_child(doc, 'Old', '09701KJHKH8750', 0, 0, 127455, "2022-01-11") + insert_child(doc, 'Alpha', '09204KJHKH2432', 0, 0.6, 364, "2019-12-31") + insert_child(doc, 'Delta', '09709KSPIO2432', 1, 0.9, 1242000, "2020-04-21") + insert_child(doc, 'Update', '76989KJLVA2432', 0, 1, 183845, "2022-02-10") + insert_child(doc, 'Delete', '29189KLHVA1432', 0, 0, 365647, "2021-05-07") + insert_child(doc, 'Make', '09689KJHAA2431', 0, 0.3, 24, "2020-11-11") + insert_child(doc, 'Create', '09709KLKKH2432', 1, 0.3, 264851, "2021-02-21") + insert_child(doc, 'Group', '09209KJLKH2432', 1, 0.8, 537877, "2020-03-15") + insert_child(doc, 'Slide', '01909KJHYH1132', 0, 0.5, 9, "2018-03-02") + insert_child(doc, 'Drop', '09701KJHKH8750', 1, 0, 127255, "2018-01-01") + insert_child(doc, 'Beta', '09204QJHKN2432', 0, 0.6, 354, "2017-12-30") + insert_child(doc, 'Flag', '09709KXPIP2432', 1, 0, 1241000, "2021-04-21") + insert_child(doc, 'Upgrade', '75989ZJLVA2432', 0.8, 1, 183645, "2020-08-13") + insert_child(doc, 'Down', '28189KLHRA1432', 1, 0, 362647, "2020-06-17") + insert_child(doc, 'Note', '09689DJHAA2431', 0, 0.1, 29, "2021-09-11") + insert_child(doc, 'Click', '08189DJHAA2431', 1, 0.3, 209, "2020-07-04") + insert_child(doc, 'Drag', '08189DIHAA2981', 0, 0.7, 342628, "2022-05-04") + + doc.insert() From daa2b7921c2dfff5996e7607565e031d1888000e Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 24 Feb 2022 19:48:10 +0530 Subject: [PATCH 10/42] fix(sider): missing semicolon --- cypress/integration/grid_search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js index 6ccf3e37f0..dfb153f67b 100644 --- a/cypress/integration/grid_search.js +++ b/cypress/integration/grid_search.js @@ -100,5 +100,5 @@ context('Grid Search', () => { // Date Column cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('-02'); cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); - }) + }); }); \ No newline at end of file From 839f488c9185fb7a963102abcc9d36b5c873d0be Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 25 Feb 2022 11:50:55 +0530 Subject: [PATCH 11/42] fix: failing grid search UI test --- cypress/integration/grid_search.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js index dfb153f67b..10444b2d2a 100644 --- a/cypress/integration/grid_search.js +++ b/cypress/integration/grid_search.js @@ -7,10 +7,19 @@ context('Grid Search', () => { before(() => { cy.visit('/login'); cy.login(); + cy.visit('/app/website'); cy.insert_doc('DocType', child_table_doctype, true); cy.insert_doc('DocType', child_table_doctype_1, true); cy.insert_doc('DocType', doctype_with_child_table, true); return cy.window().its('frappe').then(frappe => { + return frappe.xcall("frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", { + name: doctype_with_child_table_name + }); + }); + }); + + it('Test search row visibility', () => { + cy.window().its('frappe').then(frappe => { frappe.model.user_settings.save('Doctype With Child Table', 'GridView', { 'Child Table Doctype 1': [ {'fieldname': 'data', 'columns': 2}, @@ -21,14 +30,8 @@ context('Grid Search', () => { {'fieldname': 'date', 'columns': 2} ] }); - - return frappe.xcall("frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", { - name: doctype_with_child_table_name - }); }); - }); - it('Test search row visibility', () => { cy.visit(`/app/doctype-with-child-table/Test Grid Search`); cy.get('[title="child_table_1"]').as('table'); From 5a0dc5d7c80d3977a706482f4e36160d0488a2a0 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 16:08:03 +0530 Subject: [PATCH 12/42] fix(linter): Trim Trailing Whitespace --- frappe/public/js/frappe/form/grid.js | 2 +- frappe/public/js/frappe/form/grid_row.js | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 99d9bbe0fc..36461fb671 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -559,7 +559,7 @@ export default class Grid { return data; } } else if (fieldtype === "Barcode" && fieldvalue) { - let barcode = fieldvalue.startsWith(' { - clearTimeout(timer); + clearTimeout(timer); timer = setTimeout(() => { let df = { fieldtype: "Sr No" @@ -666,12 +666,12 @@ export default class GridRow { .appendTo(this.row); let $search_input = $(` - `).appendTo($col); From d2e656eff52967952b5a95eb21ef1e6edffc05e4 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 16:11:23 +0530 Subject: [PATCH 13/42] fix(linter): Trim Trailing Whitespace --- frappe/public/js/frappe/form/grid_row.js | 4 ++-- package.json | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 7f039ebb06..1054c863d9 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -205,7 +205,7 @@ export default class GridRow { } render_row(refresh) { if (this.show_search && !this.show_search_row()) return; - + let me = this; this.set_row_index(); @@ -257,7 +257,7 @@ export default class GridRow { if (e.target.value == "") { delete this.grid.filter['row-index']; } - + this.grid.grid_sortable .option('disabled', Object.keys(this.grid.filter).length !== 0); diff --git a/package.json b/package.json index 259a157311..4326ddb226 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ }, "homepage": "https://frappeframework.com", "dependencies": { + "@cypress/code-coverage": "^3", "@editorjs/editorjs": "2.20.0", + "@testing-library/cypress": "^8", "ace-builds": "^1.4.8", "air-datepicker": "github:frappe/air-datepicker", "autoprefixer": "^9.8.6", @@ -31,6 +33,8 @@ "cookie": "^0.4.0", "cropperjs": "^1.5.12", "cssnano": "^5.0.0", + "cypress": "^6", + "cypress-file-upload": "^5", "driver.js": "^0.9.8", "editorjs-undo": "0.1.6", "express": "^4.17.1", From 9c2feb802378d51f3d0fefbe4e2a47e95add5395 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 18:03:50 +0530 Subject: [PATCH 14/42] test: fix failing grid search UI test: --- cypress/integration/grid_search.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js index 10444b2d2a..45473bdd5c 100644 --- a/cypress/integration/grid_search.js +++ b/cypress/integration/grid_search.js @@ -34,7 +34,7 @@ context('Grid Search', () => { cy.visit(`/app/doctype-with-child-table/Test Grid Search`); - cy.get('[title="child_table_1"]').as('table'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); cy.get('@table').find('.grid-row-check:last').click(); cy.get('@table').find('.grid-footer').contains('Delete').click(); cy.get('.grid-heading-row .grid-row .search').should('not.exist'); @@ -43,7 +43,7 @@ context('Grid Search', () => { it('test search field for different fieldtypes', () => { cy.visit(`/app/doctype-with-child-table/Test Grid Search`); - cy.get('[title="child_table_1"]').as('table'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); // Index Column cy.get('@table').find('.grid-heading-row .row-index.search input').type('3'); @@ -86,7 +86,7 @@ context('Grid Search', () => { }); it('test with multiple filter', () => { - cy.get('[title="child_table_1"]').as('table'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); // Data Column cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('a'); From eb0c01e5085d26722691974bae28c212253ef68b Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 18:24:47 +0530 Subject: [PATCH 15/42] fix: don't show search row if table is empty --- frappe/public/js/frappe/form/grid_row.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 1054c863d9..c07c52678b 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -639,7 +639,8 @@ export default class GridRow { show_search_row() { // show or remove search columns based on grid rows - this.show_search = this.frm.doc[this.grid.df.fieldname].length >= 20; + this.show_search = this.frm.doc[this.grid.df.fieldname] && + this.frm.doc[this.grid.df.fieldname].length >= 20; !this.show_search && this.wrapper.remove(); return this.show_search; } From d9a2b57c44d2c2797351060020131e835a855907 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 18:47:30 +0530 Subject: [PATCH 16/42] test: fixed test with multiple filter UI test --- cypress/integration/dashboard_links.js | 2 ++ cypress/integration/grid_search.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cypress/integration/dashboard_links.js b/cypress/integration/dashboard_links.js index 93d10cf1fd..019de1991d 100644 --- a/cypress/integration/dashboard_links.js +++ b/cypress/integration/dashboard_links.js @@ -1,5 +1,6 @@ import doctype_with_child_table from '../fixtures/doctype_with_child_table'; import child_table_doctype from '../fixtures/child_table_doctype'; +import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; import doctype_to_link from '../fixtures/doctype_to_link'; const doctype_to_link_name = doctype_to_link.name; const child_table_doctype_name = child_table_doctype.name; @@ -9,6 +10,7 @@ context('Dashboard links', () => { cy.visit('/login'); cy.login(); cy.insert_doc('DocType', child_table_doctype, true); + cy.insert_doc('DocType', child_table_doctype_1, true); cy.insert_doc('DocType', doctype_with_child_table, true); cy.insert_doc('DocType', doctype_to_link, true); return cy.window().its('frappe').then(frappe => { diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js index 45473bdd5c..d30545a2e1 100644 --- a/cypress/integration/grid_search.js +++ b/cypress/integration/grid_search.js @@ -101,7 +101,7 @@ context('Grid Search', () => { cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 5); // Date Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('-02'); + cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('02-'); cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); }); }); \ No newline at end of file From 7d14c26a9571fd6f68c252dbb8a0ee9623e7e9d2 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 18:54:33 +0530 Subject: [PATCH 17/42] revert: dashboard_links UI test fix --- cypress/integration/dashboard_links.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/cypress/integration/dashboard_links.js b/cypress/integration/dashboard_links.js index 019de1991d..93d10cf1fd 100644 --- a/cypress/integration/dashboard_links.js +++ b/cypress/integration/dashboard_links.js @@ -1,6 +1,5 @@ import doctype_with_child_table from '../fixtures/doctype_with_child_table'; import child_table_doctype from '../fixtures/child_table_doctype'; -import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; import doctype_to_link from '../fixtures/doctype_to_link'; const doctype_to_link_name = doctype_to_link.name; const child_table_doctype_name = child_table_doctype.name; @@ -10,7 +9,6 @@ context('Dashboard links', () => { cy.visit('/login'); cy.login(); cy.insert_doc('DocType', child_table_doctype, true); - cy.insert_doc('DocType', child_table_doctype_1, true); cy.insert_doc('DocType', doctype_with_child_table, true); cy.insert_doc('DocType', doctype_to_link, true); return cy.window().its('frappe').then(frappe => { From 6526e48e0a2a2081d0d7980de6632f8e99d45c3b Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 18:57:23 +0530 Subject: [PATCH 18/42] revert: package.json file updated with cypress --- package.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/package.json b/package.json index 4326ddb226..259a157311 100644 --- a/package.json +++ b/package.json @@ -21,9 +21,7 @@ }, "homepage": "https://frappeframework.com", "dependencies": { - "@cypress/code-coverage": "^3", "@editorjs/editorjs": "2.20.0", - "@testing-library/cypress": "^8", "ace-builds": "^1.4.8", "air-datepicker": "github:frappe/air-datepicker", "autoprefixer": "^9.8.6", @@ -33,8 +31,6 @@ "cookie": "^0.4.0", "cropperjs": "^1.5.12", "cssnano": "^5.0.0", - "cypress": "^6", - "cypress-file-upload": "^5", "driver.js": "^0.9.8", "editorjs-undo": "0.1.6", "express": "^4.17.1", From 9ec61293c0c11a06fc6ea909b908f6e9226010d8 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 7 Mar 2022 19:24:18 +0530 Subject: [PATCH 19/42] test: failing dashboard_links UI test fix --- cypress/integration/dashboard_links.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cypress/integration/dashboard_links.js b/cypress/integration/dashboard_links.js index 93d10cf1fd..019de1991d 100644 --- a/cypress/integration/dashboard_links.js +++ b/cypress/integration/dashboard_links.js @@ -1,5 +1,6 @@ import doctype_with_child_table from '../fixtures/doctype_with_child_table'; import child_table_doctype from '../fixtures/child_table_doctype'; +import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; import doctype_to_link from '../fixtures/doctype_to_link'; const doctype_to_link_name = doctype_to_link.name; const child_table_doctype_name = child_table_doctype.name; @@ -9,6 +10,7 @@ context('Dashboard links', () => { cy.visit('/login'); cy.login(); cy.insert_doc('DocType', child_table_doctype, true); + cy.insert_doc('DocType', child_table_doctype_1, true); cy.insert_doc('DocType', doctype_with_child_table, true); cy.insert_doc('DocType', doctype_to_link, true); return cy.window().its('frappe').then(frappe => { From 533434d1aec82ba54cf6b71888a82856f502b558 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 10 Mar 2022 14:31:32 +0530 Subject: [PATCH 20/42] fix: Last grid column has unnecessary margin --- frappe/public/js/frappe/form/grid_row.js | 3 ++- frappe/public/scss/common/grid.scss | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index c07c52678b..be14bcdff7 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -639,7 +639,8 @@ export default class GridRow { show_search_row() { // show or remove search columns based on grid rows - this.show_search = this.frm.doc[this.grid.df.fieldname] && + this.show_search = this.frm && this.frm.doc && + this.frm.doc[this.grid.df.fieldname] && this.frm.doc[this.grid.df.fieldname].length >= 20; !this.show_search && this.wrapper.remove(); return this.show_search; diff --git a/frappe/public/scss/common/grid.scss b/frappe/public/scss/common/grid.scss index 926dcc8923..d1f89abbcd 100644 --- a/frappe/public/scss/common/grid.scss +++ b/frappe/public/scss/common/grid.scss @@ -147,7 +147,6 @@ .grid-row > .row { .col:last-child { - margin-right: calc(-1 * var(--margin-sm)); border-right: none; } From 54c0fee9a228374359d2dde1f7d83308894f1301 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 10 Mar 2022 14:35:05 +0530 Subject: [PATCH 21/42] fix(linter): Trim Trailing Whitespace --- frappe/public/js/frappe/form/grid_row.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index be14bcdff7..403807eaff 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -639,7 +639,7 @@ export default class GridRow { show_search_row() { // show or remove search columns based on grid rows - this.show_search = this.frm && this.frm.doc && + this.show_search = this.frm && this.frm.doc && this.frm.doc[this.grid.df.fieldname] && this.frm.doc[this.grid.df.fieldname].length >= 20; !this.show_search && this.wrapper.remove(); From bebc8058b626d53f7bbb0e45edc98ddb255819df Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 7 Feb 2022 18:35:47 +0530 Subject: [PATCH 22/42] feat: integer primary keys --- frappe/__init__.py | 4 +- frappe/core/doctype/data_export/exporter.py | 2 +- frappe/core/doctype/doctype/doctype.js | 14 +++ frappe/core/doctype/doctype/doctype.json | 11 ++- frappe/core/doctype/doctype/doctype.py | 14 +++ frappe/core/doctype/doctype/test_doctype.py | 21 ++++- .../server_script/test_server_script.py | 5 +- frappe/database/database.py | 2 - frappe/database/mariadb/schema.py | 24 ++++- frappe/database/postgres/database.py | 46 +++++++++- frappe/database/postgres/schema.py | 18 +++- frappe/database/sequence.py | 76 ++++++++++++++++ frappe/desk/form/load.py | 3 +- frappe/desk/search.py | 2 +- .../doctype/newsletter/test_newsletter.py | 2 +- .../event_update_log/event_update_log.py | 9 +- frappe/frappeclient.py | 5 +- frappe/model/base_document.py | 2 +- frappe/model/db_query.py | 88 ++++++++++++++----- frappe/model/delete_doc.py | 2 +- frappe/model/naming.py | 35 +++++++- frappe/public/js/frappe/form/form.js | 4 +- frappe/public/js/frappe/list/list_view.js | 2 +- frappe/realtime.py | 3 +- frappe/tests/test_db.py | 47 ++++++++++ frappe/tests/test_db_query.py | 21 +++++ frappe/tests/test_naming.py | 11 +++ frappe/utils/data.py | 2 +- frappe/utils/diff.py | 11 +-- frappe/utils/global_search.py | 4 +- 30 files changed, 431 insertions(+), 59 deletions(-) create mode 100644 frappe/database/sequence.py diff --git a/frappe/__init__.py b/frappe/__init__.py index 8a8b70afe3..3ca2082ddb 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -35,6 +35,7 @@ from frappe.query_builder import ( patch_query_execute, patch_query_aggregation, ) +from frappe.utils.data import cstr __version__ = '14.0.0-dev' @@ -214,6 +215,7 @@ def init(site, sites_path=None, new_site=False): local.cache = {} local.document_cache = {} local.meta_cache = {} + local.autoincremented_doctypes = set() local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server @@ -1001,7 +1003,7 @@ def get_module(modulename): def scrub(txt): """Returns sluggified string. e.g. `Sales Order` becomes `sales_order`.""" - return txt.replace(' ', '_').replace('-', '_').lower() + return cstr(txt).replace(' ', '_').replace('-', '_').lower() def unscrub(txt): """Returns titlified string. e.g. `sales_order` becomes `Sales Order`.""" diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index 79570d5048..9f1492af19 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -324,7 +324,7 @@ class DataExporter: d = doc.copy() meta = frappe.get_meta(dt) if self.all_doctypes: - d.name = '"'+ d.name+'"' + d.name = f'"{d.name}"' if len(rows) < rowidx + 1: rows.append([""] * (len(self.columns) + 1)) diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index f250a6a109..88cc5577a6 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -61,6 +61,13 @@ frappe.ui.form.on('DocType', { frm.events.set_naming_rule_description(frm); }, + istable: (frm) => { + if (frm.doc.istable && frm.is_new()) { + frm.set_value('autoname', 'autoincrement'); + frm.set_value('allow_rename', 0); + } + }, + naming_rule: function(frm) { // set the "autoname" property based on naming_rule if (frm.doc.naming_rule && !frm.__from_autoname) { @@ -70,6 +77,10 @@ frappe.ui.form.on('DocType', { if (frm.doc.naming_rule=='Set by user') { frm.set_value('autoname', 'Prompt'); + } else if (frm.doc.naming_rule === 'Autoincrement') { + frm.set_value('autoname', 'autoincrement'); + // set allow rename to be false when using autoincrement + frm.set_value('allow_rename', 0); } else if (frm.doc.naming_rule=='By fieldname') { frm.set_value('autoname', 'field:'); } else if (frm.doc.naming_rule=='By "Naming Series" field') { @@ -91,6 +102,7 @@ frappe.ui.form.on('DocType', { set_naming_rule_description(frm) { let naming_rule_description = { 'Set by user': '', + 'Autoincrement': 'Uses Auto Increment feature of database.
WARNING: After using this option, any other naming option will not be accessible.', 'By fieldname': 'Format: field:[fieldname]. Valid fieldname must exist', 'By "Naming Series" field': 'Format: naming_series:[fieldname]. Fieldname called naming_series must exist', 'Expression': 'Format: format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.', @@ -111,6 +123,8 @@ frappe.ui.form.on('DocType', { frm.__from_autoname = true; if (frm.doc.autoname.toLowerCase() === 'prompt') { frm.set_value('naming_rule', 'Set by user'); + } else if (frm.doc.autoname.toLowerCase() === 'autoincrement') { + frm.set_value('naming_rule', 'Autoincrement'); } else if (frm.doc.autoname.startsWith('field:')) { frm.set_value('naming_rule', 'By fieldname'); } else if (frm.doc.autoname.startsWith('naming_series:')) { diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 2bba4127bb..8169a59566 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -208,7 +208,7 @@ "label": "Naming" }, { - "description": "Naming Options:\n
  1. field:[fieldname] - By Field
  2. naming_series: - By Naming Series (field called naming_series must be present
  3. Prompt - Prompt user for a name
  4. [series] - Series by prefix (separated by a dot); for example PRE.#####
    5. \n
  5. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
", + "description": "Naming Options:\n
  1. field:[fieldname] - By Field
  2. autoincrement - Uses Databases' Auto Increment feature
  3. naming_series: - By Naming Series (field called naming_series must be present
  4. Prompt - Prompt user for a name
  5. [series] - Series by prefix (separated by a dot); for example PRE.#####
    6. \n
  6. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
", "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name", @@ -216,6 +216,7 @@ "oldfieldtype": "Data" }, { + "depends_on": "eval:doc.naming_rule !== \"Autoincrement\"", "fieldname": "name_case", "fieldtype": "Select", "label": "Name Case", @@ -282,6 +283,7 @@ }, { "default": "1", + "depends_on": "eval:doc.naming_rule !== \"Autoincrement\"", "fieldname": "allow_rename", "fieldtype": "Check", "label": "Allow Rename", @@ -565,7 +567,7 @@ "fieldtype": "Select", "label": "Naming Rule", "length": 40, - "options": "\nSet by user\nBy fieldname\nBy \"Naming Series\" field\nExpression\nExpression (old style)\nRandom\nBy script" + "options": "\nSet by user\nAutoincrement\nBy fieldname\nBy \"Naming Series\" field\nExpression\nExpression (old style)\nRandom\nBy script" }, { "fieldname": "migration_hash", @@ -593,6 +595,7 @@ ], "icon": "fa fa-bolt", "idx": 6, + "index_web_pages_for_search": 1, "links": [ { "group": "Views", @@ -670,10 +673,11 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-01-07 16:07:06.196534", + "modified": "2022-02-15 21:47:16.467217", "modified_by": "Administrator", "module": "Core", "name": "DocType", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -703,5 +707,6 @@ "show_name_in_global_search": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 5f82abac1f..d1401cc91e 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -60,6 +60,7 @@ class DocType(Document): self.check_developer_mode() + self.validate_autoname() self.validate_name() self.set_defaults_for_single_and_table() @@ -714,6 +715,19 @@ class DocType(Document): self.name) return max_idx and max_idx[0][0] or 0 + def validate_autoname(self): + if not self.is_new(): + doc_before_save = self.get_doc_before_save() + if doc_before_save: + if (self.autoname == "autoincrement" and doc_before_save.autoname != "autoincrement") \ + or (self.autoname != "autoincrement" and doc_before_save.autoname == "autoincrement"): + frappe.throw(_("Cannot change to/from Autoincrement naming rule")) + + else: + if self.autoname == "autoincrement": + self.allow_rename = 0 + frappe.local.autoincremented_doctypes.add(self.name) + def validate_name(self, name=None): if not name: name = self.name diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index cb22f581c6..dc6d14b451 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -505,7 +505,23 @@ class TestDocType(unittest.TestCase): dt.delete() -def new_doctype(name, unique=0, depends_on='', fields=None): + def test_autoincremented_doctype_transition(self): + frappe.delete_doc("testy_autoinc_dt") + dt = new_doctype("testy_autoinc_dt", autoincremented=True).insert(ignore_permissions=True) + dt.autoname = "hash" + + try: + dt.save(ignore_permissions=True) + except frappe.ValidationError as e: + self.assertEqual(e.args[0], "Cannot change to/from Autoincrement naming rule") + else: + self.fail("Shouldnt be possible to transition autoincremented doctype to any other naming rule") + finally: + # cleanup + dt.delete(ignore_permissions=True) + + +def new_doctype(name, unique=0, depends_on='', fields=None, autoincremented=False): doc = frappe.get_doc({ "doctype": "DocType", "module": "Core", @@ -521,7 +537,8 @@ def new_doctype(name, unique=0, depends_on='', fields=None): "role": "System Manager", "read": 1, }], - "name": name + "name": name, + "autoname": "autoincrement" if autoincremented else "" }) if fields: diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index d9381bcd16..aa4507b858 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -112,7 +112,10 @@ class TestServerScript(unittest.TestCase): self.assertEqual(frappe.get_doc('Server Script', 'test_return_value').execute_method(), 'hello') def test_permission_query(self): - self.assertTrue('where (1 = 1)' in frappe.db.get_list('ToDo', run=False)) + if frappe.conf.db_type == "mariadb": + self.assertTrue('where (1 = 1)' in frappe.db.get_list('ToDo', run=False)) + else: + self.assertTrue('where (1 = \'1\')' in frappe.db.get_list('ToDo', run=False)) self.assertTrue(isinstance(frappe.db.get_list('ToDo'), list)) def test_attribute_error(self): diff --git a/frappe/database/database.py b/frappe/database/database.py index a4eca64d8d..1251a323d3 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -142,8 +142,6 @@ class Database(object): self.log_query(query, values, debug, explain) if values!=(): - if isinstance(values, dict): - values = dict(values) # MySQL-python==1.2.5 hack! if not isinstance(values, (dict, tuple, list)): diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index fd4bfc6dd0..3b7aa443f2 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -1,12 +1,16 @@ import frappe from frappe import _ from frappe.database.schema import DBTable +from frappe.database.sequence import create_sequence +from frappe.model import log_types + class MariaDBTable(DBTable): def create(self): additional_definitions = "" engine = self.meta.get("engine") or "InnoDB" varchar_len = frappe.db.VARCHAR_LEN + name_column = f"name varchar({varchar_len}) primary key" # columns column_defs = self.get_column_definitions() @@ -29,9 +33,27 @@ class MariaDBTable(DBTable): ) ) + ',\n' + # creating sequence(s) + if (not self.meta.issingle and self.meta.autoname == "autoincrement")\ + or self.doctype in log_types: + + # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, + # it drops the cache and uses the next non cached value in setval func and + # puts that in the backup file, which will start the counter + # from that value when inserting any new record in the doctype. + # By default the cache is 1000 which will mess up the sequence when + # using the system after a restore. + # issue link: https://jira.mariadb.org/browse/MDEV-21786 + create_sequence(self.doctype, check_not_exists=True, cache=50) + + # NOTE: not used nextval func as default as the ability to restore + # database with sequences has bugs in mariadb and gives a scary error. + # issue link: https://jira.mariadb.org/browse/MDEV-21786 + name_column = "name bigint primary key" + # create table query = f"""create table `{self.table_name}` ( - name varchar({varchar_len}) not null primary key, + {name_column}, creation datetime(6), modified datetime(6), modified_by varchar({varchar_len}), diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index a20ffe17a5..eb3e33d39c 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -99,8 +99,13 @@ class PostgresDatabase(Database): return db_size[0].get('database_size') # pylint: disable=W0221 - def sql(self, query, *args, **kwargs): - return super(PostgresDatabase, self).sql(modify_query(query), *args, **kwargs) + def sql(self, query, values=(), *args, **kwargs): + return super(PostgresDatabase, self).sql( + modify_query(query), + modify_values(values), + *args, + **kwargs + ) def get_tables(self, cached=True): return [d[0] for d in self.sql("""select table_name @@ -333,10 +338,45 @@ def modify_query(query): if re.search('from tab', query, flags=re.IGNORECASE): query = re.sub(r'from tab([\w-]*)', r'from "tab\1"', query, flags=re.IGNORECASE) + # only find int (with/without signs), ignore decimals (with/without signs), ignore hashes (which start with numbers), + # drop .0 from decimals and add quotes around them + # + # >>> query = "c='abcd' , a >= 45, b = -45.0, c = 40, d=4500.0, e=3500.53, f=40psdfsd, g=9092094312, h=12.00023" + # >>> re.sub(r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query) + # "c='abcd' , a >= '45', b = '-45', c = '40', d= '4500', e=3500.53, f=40psdfsd, g= '9092094312', h=12.00023 + + query = re.sub(r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query) return query +def modify_values(values): + def stringify_value(value): + if isinstance(value, int): + value = str(value) + elif isinstance(value, float): + truncated_float = int(value) + if value == truncated_float: + value = str(truncated_float) + + return value + + if not values: + return values + + if isinstance(values, dict): + for k, v in values.items(): + values[k] = stringify_value(v) + elif isinstance(values, (tuple, list)): + new_values = [] + for val in values: + new_values.append(stringify_value(val)) + values = new_values + else: + values = stringify_value(values) + + return values + def replace_locate_with_strpos(query): # strpos is the locate equivalent in postgres if re.search(r'locate\(', query, flags=re.IGNORECASE): - query = re.sub(r'locate\(([^,]+),([^)]+)\)', r'strpos(\2, \1)', query, flags=re.IGNORECASE) + query = re.sub(r'locate\(([^,]+),([^)]+)(\)?)\)', r'strpos(\2\3, \1)', query, flags=re.IGNORECASE) return query diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index bb7ff20a26..b09f73300e 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -2,10 +2,14 @@ import frappe from frappe import _ from frappe.utils import cint, flt from frappe.database.schema import DBTable, get_definition +from frappe.database.sequence import create_sequence +from frappe.model import log_types + class PostgresTable(DBTable): def create(self): varchar_len = frappe.db.VARCHAR_LEN + name_column = f"name varchar({varchar_len}) primary key" additional_definitions = "" # columns @@ -26,9 +30,21 @@ class PostgresTable(DBTable): ) ) + # creating sequence(s) + if (not self.meta.issingle and self.meta.autoname == "autoincrement")\ + or self.doctype in log_types: + + # The sequence cache is per connection. + # Since we're opening and closing connections for every transaction this results in skipping the cache + # to the next non-cached value hence not using cache in postgres. + # ref: https://stackoverflow.com/questions/21356375/postgres-9-0-4-sequence-skipping-numbers + create_sequence(self.doctype, check_not_exists=True) + name_column = "name bigint primary key" + + # TODO: set docstatus length # create table frappe.db.sql(f"""create table `{self.table_name}` ( - name varchar({varchar_len}) not null primary key, + {name_column}, creation timestamp(6), modified timestamp(6), modified_by varchar({varchar_len}), diff --git a/frappe/database/sequence.py b/frappe/database/sequence.py new file mode 100644 index 0000000000..c89ba468bc --- /dev/null +++ b/frappe/database/sequence.py @@ -0,0 +1,76 @@ +from frappe import db, scrub + + +def create_sequence( + doctype_name: str, + *, + slug: str = "_id_seq", + check_not_exists: bool = False, + cycle: bool = False, + cache: int = 0, + start_value: int = 0, + increment_by: int = 0, + min_value: int = 0, + max_value: int = 0 +) -> str: + + query = "create sequence" + sequence_name = scrub(doctype_name + slug) + + if check_not_exists: + query += " if not exists" + + query += f" {sequence_name}" + + if cache: + query += f" cache {cache}" + else: + # in postgres, the default is cache 1 + if db.db_type == "mariadb": + query += " nocache" + + if start_value: + # default is 1 + query += f" start with {start_value}" + + if increment_by: + # default is 1 + query += f" increment by {increment_by}" + + if min_value: + # default is 1 + query += f" min value {min_value}" + + if max_value: + query += f" max value {max_value}" + + if not cycle: + if db.db_type == "mariadb": + query += " nocycle" + else: + query += " cycle" + + db.sql(query) + + return sequence_name + + +def get_next_val(doctype_name: str, slug: str = "_id_seq") -> int: + if db.db_type == "postgres": + return db.sql(f"select nextval(\'\"{scrub(doctype_name + slug)}\"\')")[0][0] + return db.sql(f"select nextval(`{scrub(doctype_name + slug)}`)")[0][0] + + +def set_next_val( + doctype_name: str, + next_val: int, + *, + slug: str = "_id_seq", + is_val_used :bool = False +) -> None: + + is_val_used = 0 if not is_val_used else 1 + if db.db_type == "postgres": + db.sql(f"SELECT SETVAL(\'\"{scrub(doctype_name + slug)}\"\', {next_val}, {is_val_used})") + else: + db.sql(f"SELECT SETVAL(`{scrub(doctype_name + slug)}`, {next_val}, {is_val_used})") diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index b5dfacb1d6..901e7a3d5e 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -10,6 +10,7 @@ import frappe.desk.form.meta from frappe.model.utils.user_settings import get_user_settings from frappe.permissions import get_doc_permissions from frappe.desk.form.document_follow import is_document_followed +from frappe.utils.data import cstr from frappe import _ from frappe import _dict from urllib.parse import quote @@ -356,7 +357,7 @@ def get_document_email(doctype, name): return None email = email.split("@") - return "{0}+{1}+{2}@{3}".format(email[0], quote(doctype), quote(name), email[1]) + return "{0}+{1}+{2}@{3}".format(email[0], quote(doctype), quote(cstr(name)), email[1]) def get_automatic_email_link(): return frappe.db.get_value("Email Account", {"enable_incoming": 1, "enable_automatic_linking": 1}, "email_id") diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 3b76953ed1..b54ea46268 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -257,7 +257,7 @@ def scrub_custom_query(query, key, txt): def relevance_sorter(key, query, as_dict): value = _(key.name if as_dict else key[0]) return ( - value.lower().startswith(query.lower()) is not True, + cstr(value).lower().startswith(query.lower()) is not True, value ) diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index 8c1f803a46..b091c31c74 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -51,7 +51,7 @@ class TestNewsletterMixin: "reference_name": newsletter, }) frappe.delete_doc("Newsletter", newsletter) - frappe.db.delete("Newsletter Email Group", newsletter) + frappe.db.delete("Newsletter Email Group", {"parent": newsletter}) newsletters.remove(newsletter) def setup_email_group(self): diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py index f4871be312..cd5100623c 100644 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py +++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py @@ -203,12 +203,17 @@ def get_unread_update_logs(consumer_name, dt, dn): SELECT update_log.name FROM `tabEvent Update Log` update_log - JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = update_log.name + JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = %(log_name)s WHERE consumer.consumer = %(consumer)s AND update_log.ref_doctype = %(dt)s AND update_log.docname = %(dn)s - """, {'consumer': consumer_name, "dt": dt, "dn": dn}, as_dict=0)] + """, { + "consumer": consumer_name, + "dt": dt, + "dn": dn, + "log_name": "update_log.name" if frappe.conf.db_type == "mariadb" else "CAST(update_log.name AS VARCHAR)" + }, as_dict=0)] logs = frappe.get_all( 'Event Update Log', diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index 59db38584c..7a1587aae0 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -7,6 +7,7 @@ import json import requests import frappe +from frappe.utils.data import cstr class AuthError(Exception): @@ -122,7 +123,7 @@ class FrappeClient(object): '''Update a remote document :param doc: dict or Document object to be updated remotely. `name` is mandatory for this''' - url = self.url + "/api/resource/" + doc.get("doctype") + "/" + doc.get("name") + url = self.url + "/api/resource/" + doc.get("doctype") + "/" + cstr(doc.get("name")) res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers) return frappe._dict(self.post_process(res)) @@ -207,7 +208,7 @@ class FrappeClient(object): if fields: params["fields"] = json.dumps(fields) - res = self.session.get(self.url + "/api/resource/" + doctype + "/" + name, + res = self.session.get(self.url + "/api/resource/" + doctype + "/" + cstr(name), params=params, verify=self.verify, headers=self.headers) return self.post_process(res) diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 8a81aa5610..3564b1ae11 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -475,7 +475,7 @@ class BaseDocument(object): d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in DOCTYPES_FOR_DOCTYPE) # don't update name, as case might've been changed - name = d['name'] + name = cstr(d['name']) del d['name'] columns = list(d) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index a6b96e8fb5..ba1f157607 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -164,7 +164,8 @@ class DatabaseQuery(object): # left join parent, child tables for child in self.tables[1:]: - args.tables += f" {self.join} {child} on ({child}.parent = {self.tables[0]}.name)" + parent_name = self.cast_autoincremented_name(f"{self.tables[0]}.name") + args.tables += f" {self.join} {child} on ({child}.parent = {parent_name})" if self.grouped_or_conditions: self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})") @@ -318,21 +319,63 @@ class DatabaseQuery(object): ] # add tables from fields if self.fields: - for field in self.fields: - if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field): + for i, field in enumerate(self.fields): + if not ("tab" in field and "." in field): + continue + + # add cast in locate/strpos + func_found = False + for func in sql_functions: + if func in field.lower(): + self.fields[i] = self.cast_autoincremented_name(field, func) + func_found = True + break + + if func_found: continue table_name = field.split('.')[0] if table_name.lower().startswith('group_concat('): table_name = table_name[13:] - if table_name.lower().startswith('ifnull('): - table_name = table_name[7:] if not table_name[0]=='`': table_name = f"`{table_name}`" if table_name not in self.tables: self.append_table(table_name) + def cast_autoincremented_name(self, column: str, sql_function: str = "",) -> str: + if frappe.db.db_type == "postgres": + if "name" in column.lower(): + if "cast(" not in column.lower() or "::" not in column: + if not sql_function: + return f"cast({column} as varchar)" + + elif sql_function == "locate(": + return re.sub( + r'locate\(([^,]+),([^)]+)\)', + r'locate(\1, cast(\2 as varchar))', + column, + flags=re.IGNORECASE + ) + + elif sql_function == "strpos(": + return re.sub( + r'strpos\(([^,]+),([^)]+)\)', + r'strpos(cast(\1 as varchar), \2)', + column, + flags=re.IGNORECASE + ) + + elif sql_function == "ifnull(": + return re.sub( + r"ifnull\(([^,]+)", + r"ifnull(cast(\1 as varchar)", + column, + flags=re.IGNORECASE + ) + + return column + def append_table(self, table_name): self.tables.append(table_name) doctype = table_name[4:-1] @@ -423,6 +466,8 @@ class DatabaseQuery(object): ifnull(`tabDocType`.`fieldname`, fallback) operator "value" """ + # TODO: refactor + from frappe.boot import get_additional_filters_from_hooks additional_filters_config = get_additional_filters_from_hooks() f = get_filter(self.doctype, f, additional_filters_config) @@ -432,15 +477,16 @@ class DatabaseQuery(object): self.append_table(tname) if 'ifnull(' in f.fieldname: - column_name = f.fieldname + column_name = self.cast_autoincremented_name(f.fieldname, "ifnull(") else: - column_name = f"{tname}.{f.fieldname}" - - can_be_null = True + column_name = self.cast_autoincremented_name(f"{tname}.{f.fieldname}") if f.operator.lower() in additional_filters_config: f.update(get_additional_filter_field(additional_filters_config, f, f.value)) + meta = frappe.get_meta(f.doctype) + can_be_null = True + # prepare in condition if f.operator.lower() in ('ancestors of', 'descendants of', 'not ancestors of', 'not descendants of'): values = f.value or '' @@ -449,12 +495,8 @@ class DatabaseQuery(object): # if not isinstance(values, (list, tuple)): # values = values.split(",") - ref_doctype = f.doctype - - if frappe.get_meta(f.doctype).get_field(f.fieldname) is not None : - ref_doctype = frappe.get_meta(f.doctype).get_field(f.fieldname).options - - result=[] + field = meta.get_field(f.fieldname) + ref_doctype = field.options if field else f.doctype lft, rgt = '', '' if f.value: @@ -474,29 +516,30 @@ class DatabaseQuery(object): }, order_by='`lft` DESC') fallback = "''" - value = [frappe.db.escape((v.name or '').strip(), percent=False) for v in result] + value = [frappe.db.escape((cstr(v.name) or '').strip(), percent=False) for v in result] if len(value): value = f"({', '.join(value)})" else: value = "('')" + # changing operator to IN as the above code fetches all the parent / child values and convert into tuple # which can be directly used with IN operator to query. f.operator = 'not in' if f.operator.lower() in ('not ancestors of', 'not descendants of') else 'in' - elif f.operator.lower() in ('in', 'not in'): values = f.value or '' if isinstance(values, str): values = values.split(",") fallback = "''" - value = [frappe.db.escape((v or '').strip(), percent=False) for v in values] + value = [frappe.db.escape((cstr(v) or '').strip(), percent=False) for v in values] if len(value): value = f"({', '.join(value)})" else: value = "('')" + else: - df = frappe.get_meta(f.doctype).get("fields", {"fieldname": f.fieldname}) + df = meta.get("fields", {"fieldname": f.fieldname}) df = df[0] if df else None if df and df.fieldtype in ("Check", "Float", "Int", "Currency", "Percent"): @@ -513,7 +556,8 @@ class DatabaseQuery(object): fallback = "'0001-01-01 00:00:00'" elif f.operator.lower() in ('between') and \ - (f.fieldname in ('creation', 'modified') or (df and (df.fieldtype=="Date" or df.fieldtype=="Datetime"))): + (f.fieldname in ('creation', 'modified') or + (df and (df.fieldtype=="Date" or df.fieldtype=="Datetime"))): value = get_between_date_filter(f.value, df) fallback = "'0001-01-01 00:00:00'" @@ -528,7 +572,7 @@ class DatabaseQuery(object): fallback = "''" can_be_null = True - if 'ifnull' not in column_name: + if 'ifnull' not in column_name.lower(): column_name = f'ifnull({column_name}, {fallback})' elif df and df.fieldtype=="Date": @@ -570,7 +614,7 @@ class DatabaseQuery(object): value = f"{tname}.{quote}{f.value.name}{quote}" # escape value - elif isinstance(value, str) and not f.operator.lower() == 'between': + elif isinstance(value, str) and f.operator.lower() != 'between': value = f"{frappe.db.escape(value, percent=False)}" if ( diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index ef73a349cc..f055cd79d0 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -158,7 +158,7 @@ def update_naming_series(doc): and getattr(doc, "naming_series", None): revert_series_if_last(doc.naming_series, doc.name, doc) - elif doc.meta.autoname.split(":")[0] not in ("Prompt", "field", "hash"): + elif doc.meta.autoname.split(":")[0] not in ("Prompt", "field", "hash", "autoincrement"): revert_series_if_last(doc.meta.autoname, doc.name, doc) def delete_from_table(doctype, name, ignore_doctypes, doc): diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 9024b3d7b4..9ba7f11563 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,14 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from typing import Optional +from typing import Optional, TYPE_CHECKING import frappe from frappe import _ +from frappe.database.sequence import get_next_val from frappe.utils import now_datetime, cint, cstr import re from frappe.model import log_types from frappe.query_builder import DocType +if TYPE_CHECKING: + from frappe.model.meta import Meta + def set_new_name(doc): """ @@ -24,7 +28,8 @@ def set_new_name(doc): doc.run_method("before_naming") - autoname = frappe.get_meta(doc.doctype).autoname or "" + meta = frappe.get_meta(doc.doctype) + autoname = meta.autoname or "" if autoname.lower() != "prompt" and not frappe.flags.in_import: doc.name = None @@ -36,6 +41,10 @@ def set_new_name(doc): elif getattr(doc.meta, "issingle", False): doc.name = doc.doctype + elif is_autoincremented(doc.doctype, meta): + doc.name = get_next_val(doc.doctype) + return + elif getattr(doc.meta, "istable", False): doc.name = make_autoname("hash", doc.doctype) @@ -67,6 +76,28 @@ def set_new_name(doc): frappe.get_meta(doc.doctype).get_field("name_case") ) +def is_autoincremented(doctype: str, meta: "Meta" = None): + if doctype in frappe.local.autoincremented_doctypes: + return True + + elif doctype in log_types: + if frappe.db.sql( + f"""select data_type FROM information_schema.columns + where column_name = 'name' and table_name = 'tab{doctype}'""" + )[0][0] == "bigint": + frappe.local.autoincremented_doctypes.add(doctype) + return True + + else: + if not meta: + meta = frappe.get_meta(doctype) + + if meta.autoname == "autoincrement": + frappe.local.autoincremented_doctypes.add(doctype) + return True + + return False + def set_name_from_naming_options(autoname, doc): """ Get a name based on the autoname field option diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 56e909dd0c..6eed16bd28 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -248,7 +248,7 @@ frappe.ui.form.Form = class FrappeForm { // on main doc frappe.model.on(me.doctype, "*", function(fieldname, value, doc) { // set input - if(doc.name===me.docname) { + if (cstr(doc.name) === me.docname) { me.dirty(); let field = me.fields_dict[fieldname]; @@ -1215,7 +1215,7 @@ frappe.ui.form.Form = class FrappeForm { } is_dirty() { - return !!this.doc.__unsaved; + return this.doc.__unsaved; } is_new() { diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 98553a4a3e..d7502d4e96 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -915,7 +915,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { return this.settings.get_form_link(doc); } - const docname = doc.name.match(/[%'"#\s]/) + const docname = cstr(doc.name).match(/[%'"#\s]/) ? encodeURIComponent(doc.name) : doc.name; diff --git a/frappe/realtime.py b/frappe/realtime.py index e0f64d32fb..940a3220a4 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import frappe +from frappe.utils.data import cstr import os import redis @@ -118,7 +119,7 @@ def get_user_info(): } def get_doc_room(doctype, docname): - return ''.join([frappe.local.site, ':doc:', doctype, '/', docname]) + return ''.join([frappe.local.site, ':doc:', doctype, '/', cstr(docname)]) def get_user_room(user): return ''.join([frappe.local.site, ':user:', user]) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index bbd09590be..ab85f28af3 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -562,3 +562,50 @@ class TestDDLCommandsPost(unittest.TestCase): """, ) self.assertEquals(len(indexs_in_table), 1) + + @run_only_if(db_type_is.POSTGRES) + def test_modify_query(self): + from frappe.database.postgres.database import modify_query + + query = "select * from `tabtree b` where lft > 13 and rgt <= 16 and name =1.0 and parent = 4134qrsdc and isgroup = 1.00045" + self.assertEqual( + "select * from \"tabtree b\" where lft > \'13\' and rgt <= '16' and name = '1' and parent = 4134qrsdc and isgroup = 1.00045", + modify_query(query) + ) + + query = "select locate(\".io\", \"frappe.io\"), locate(\"3\", cast(3 as varchar)), locate(\"3\", 3::varchar)" + self.assertEqual( + "select strpos( \"frappe.io\", \".io\"), strpos( cast(3 as varchar), \"3\"), strpos( 3::varchar, \"3\")", + modify_query(query) + ) + + @run_only_if(db_type_is.POSTGRES) + def test_modify_values(self): + from frappe.database.postgres.database import modify_values + + self.assertEqual( + {"abcd": "23", "efgh": "23", "ijkl": 23.0345, "mnop": "wow"}, + modify_values({"abcd": 23, "efgh": 23.0, "ijkl": 23.0345, "mnop": "wow"}) + ) + self.assertEqual( + ["23", "23", 23.00004345, "wow"], + modify_values((23, 23.0, 23.00004345, "wow")) + ) + + def test_sequence_table_creation(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype + + dt = new_doctype("autoinc_dt_seq_test", autoincremented=True).insert(ignore_permissions=True) + + if frappe.db.db_type == "postgres": + self.assertTrue( + frappe.db.sql("""select sequence_name FROM information_schema.sequences + where sequence_name ilike 'autoinc_dt_seq_test%'""")[0][0] + ) + else: + self.assertTrue( + frappe.db.sql("""select data_type FROM information_schema.tables + where table_type = 'SEQUENCE' and table_name like 'autoinc_dt_seq_test%'""")[0][0] + ) + + dt.delete(ignore_permissions=True) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index a53134064e..2d5edd4025 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -494,6 +494,27 @@ class TestReportview(unittest.TestCase): response = execute_cmd("frappe.desk.reportview.get") self.assertListEqual(response["keys"], ["field_label", "field_name", "_aggregate_column", 'columns']) + def test_cast_autoincremented_name(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype + + dt = new_doctype("autoinc_dt_test", autoincremented=True).insert(ignore_permissions=True) + + query = DatabaseQuery("autoinc_dt_test").execute( + fields=["locate('1', `tabautoinc_dt_test`.`name`)", "`tabautoinc_dt_test`.`name`"], + filters={"name": 1}, + run=False + ) + + if frappe.db.db_type == "postgres": + self.assertTrue("strpos( cast( \"tabautoinc_dt_test\".\"name\" as varchar), \'1\')" in query) + self.assertTrue("where cast(\"tabautoinc_dt_test\".name as varchar) = \'1\'" in query) + else: + self.assertTrue("locate(\'1\', `tabautoinc_dt_test`.`name`)" in query) + self.assertTrue("where `tabautoinc_dt_test`.name = 1" in query) + + dt.delete(ignore_permissions=True) + + def add_child_table_to_blog_post(): child_table = frappe.get_doc({ 'doctype': 'DocType', diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 3e1120dc79..0c5387ccf2 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -245,6 +245,17 @@ class TestNaming(unittest.TestCase): }) self.assertRaises(frappe.ValidationError, tag.insert) + def test_autoincremented_naming(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype + + doctype = "autoinc_doctype" + frappe.generate_hash(length=5) + dt = new_doctype(doctype, autoincremented=True).insert(ignore_permissions=True) + + for i in range(1, 20): + self.assertEqual(frappe.new_doc(doctype).save(ignore_permissions=True).name, i) + + dt.delete(ignore_permissions=True) + def make_invalid_todo(): frappe.get_doc({ diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 50c71bdc2e..212ae8eba6 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1494,7 +1494,7 @@ def expand_relative_urls(html): return html def quoted(url): - return cstr(quote(encode(url), safe=b"~@#$&()*!+=:;,.?/'")) + return cstr(quote(encode(cstr(url)), safe=b"~@#$&()*!+=:;,.?/'")) def quote_urls(html): def _quote_url(match): diff --git a/frappe/utils/diff.py b/frappe/utils/diff.py index ac0e1b7439..2574f47fbd 100644 --- a/frappe/utils/diff.py +++ b/frappe/utils/diff.py @@ -1,14 +1,15 @@ import json from difflib import unified_diff -from typing import List +from typing import List, Union import frappe from frappe.utils import pretty_date +from frappe.utils.data import cstr @frappe.whitelist() def get_version_diff( - from_version: str, to_version: str, fieldname: str = "script" + from_version: Union[int, str], to_version: Union[int, str], fieldname: str = "script" ) -> List[str]: before, before_timestamp = _get_value_from_version(from_version, fieldname) @@ -23,15 +24,15 @@ def get_version_diff( diff = unified_diff( before, after, - fromfile=from_version, - tofile=to_version, + fromfile=cstr(from_version), + tofile=cstr(to_version), fromfiledate=before_timestamp, tofiledate=after_timestamp, ) return list(diff) -def _get_value_from_version(version_name: str, fieldname: str): +def _get_value_from_version(version_name: Union[int, str], fieldname: str): version = frappe.get_list( "Version", fields=["data", "modified"], filters={"name": version_name} ) diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 7b591dff45..22938671a6 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -9,6 +9,8 @@ import os from frappe.utils import cint, strip_html_tags from frappe.utils.html_utils import unescape_html from frappe.model.base_document import get_controller +from frappe.utils.data import cstr + def setup_global_search_table(): """ @@ -251,7 +253,7 @@ def update_global_search(doc): if hasattr(doc, 'is_website_published') and doc.meta.allow_guest_to_view: published = 1 if doc.is_website_published() else 0 - title = (doc.get_title() or '')[:int(frappe.db.VARCHAR_LEN)] + title = (cstr(doc.get_title()) or '')[:int(frappe.db.VARCHAR_LEN)] route = doc.get('route') if doc else '' value = dict( From b876520ef00812c370b64721d713f38cca77f325 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 3 Mar 2022 16:53:50 +0530 Subject: [PATCH 23/42] fix: logic for is_autoincremented --- frappe/__init__.py | 2 +- frappe/core/doctype/doctype/doctype.py | 1 - frappe/model/naming.py | 23 +++++++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 3ca2082ddb..774f157961 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -215,7 +215,7 @@ def init(site, sites_path=None, new_site=False): local.cache = {} local.document_cache = {} local.meta_cache = {} - local.autoincremented_doctypes = set() + local.autoincremented_status_map = {site: -1} local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index d1401cc91e..29b56fbff6 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -726,7 +726,6 @@ class DocType(Document): else: if self.autoname == "autoincrement": self.allow_rename = 0 - frappe.local.autoincremented_doctypes.add(self.name) def validate_name(self, name=None): if not name: diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 9ba7f11563..91ed469ac5 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -77,15 +77,19 @@ def set_new_name(doc): ) def is_autoincremented(doctype: str, meta: "Meta" = None): - if doctype in frappe.local.autoincremented_doctypes: - return True - - elif doctype in log_types: - if frappe.db.sql( - f"""select data_type FROM information_schema.columns - where column_name = 'name' and table_name = 'tab{doctype}'""" - )[0][0] == "bigint": - frappe.local.autoincremented_doctypes.add(doctype) + if doctype in log_types: + if frappe.local.autoincremented_status_map.get(frappe.local.site) is None or \ + frappe.local.autoincremented_status_map[frappe.local.site] == -1: + if frappe.db.sql( + f"""select data_type FROM information_schema.columns + where column_name = 'name' and table_name = 'tab{doctype}'""" + )[0][0] == "bigint": + frappe.local.autoincremented_status_map[frappe.local.site] = 1 + return True + else: + frappe.local.autoincremented_status_map[frappe.local.site] = 0 + + elif frappe.local.autoincremented_status_map[frappe.local.site]: return True else: @@ -93,7 +97,6 @@ def is_autoincremented(doctype: str, meta: "Meta" = None): meta = frappe.get_meta(doctype) if meta.autoname == "autoincrement": - frappe.local.autoincremented_doctypes.add(doctype) return True return False From 4ea87fd9cc57cb11591ca5749ead8f4032956b58 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 3 Mar 2022 17:15:20 +0530 Subject: [PATCH 24/42] chore: rename cast_autoincremented_name to cast_name --- frappe/model/db_query.py | 10 +++++----- frappe/tests/test_db_query.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index ba1f157607..7a7d278a56 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -164,7 +164,7 @@ class DatabaseQuery(object): # left join parent, child tables for child in self.tables[1:]: - parent_name = self.cast_autoincremented_name(f"{self.tables[0]}.name") + parent_name = self.cast_name(f"{self.tables[0]}.name") args.tables += f" {self.join} {child} on ({child}.parent = {parent_name})" if self.grouped_or_conditions: @@ -327,7 +327,7 @@ class DatabaseQuery(object): func_found = False for func in sql_functions: if func in field.lower(): - self.fields[i] = self.cast_autoincremented_name(field, func) + self.fields[i] = self.cast_name(field, func) func_found = True break @@ -343,7 +343,7 @@ class DatabaseQuery(object): if table_name not in self.tables: self.append_table(table_name) - def cast_autoincremented_name(self, column: str, sql_function: str = "",) -> str: + def cast_name(self, column: str, sql_function: str = "",) -> str: if frappe.db.db_type == "postgres": if "name" in column.lower(): if "cast(" not in column.lower() or "::" not in column: @@ -477,9 +477,9 @@ class DatabaseQuery(object): self.append_table(tname) if 'ifnull(' in f.fieldname: - column_name = self.cast_autoincremented_name(f.fieldname, "ifnull(") + column_name = self.cast_name(f.fieldname, "ifnull(") else: - column_name = self.cast_autoincremented_name(f"{tname}.{f.fieldname}") + column_name = self.cast_name(f"{tname}.{f.fieldname}") if f.operator.lower() in additional_filters_config: f.update(get_additional_filter_field(additional_filters_config, f, f.value)) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 2d5edd4025..b4c7c7cce7 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -494,7 +494,7 @@ class TestReportview(unittest.TestCase): response = execute_cmd("frappe.desk.reportview.get") self.assertListEqual(response["keys"], ["field_label", "field_name", "_aggregate_column", 'columns']) - def test_cast_autoincremented_name(self): + def test_cast_name(self): from frappe.core.doctype.doctype.test_doctype import new_doctype dt = new_doctype("autoinc_dt_test", autoincremented=True).insert(ignore_permissions=True) From 906f671d9c47ab612ba011319a78ab436c8c05aa Mon Sep 17 00:00:00 2001 From: phot0n Date: Fri, 4 Mar 2022 18:28:20 +0530 Subject: [PATCH 25/42] fix(db_query): ordering in extract_tables this will allow the fields to be casted first --- frappe/model/db_query.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 7a7d278a56..16056d382a 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -320,9 +320,6 @@ class DatabaseQuery(object): # add tables from fields if self.fields: for i, field in enumerate(self.fields): - if not ("tab" in field and "." in field): - continue - # add cast in locate/strpos func_found = False for func in sql_functions: @@ -331,7 +328,7 @@ class DatabaseQuery(object): func_found = True break - if func_found: + if func_found or not ("tab" in field and "." in field): continue table_name = field.split('.')[0] From 93fbace3319a3090cf34425fd366b359381be51f Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 5 Mar 2022 15:50:49 +0530 Subject: [PATCH 26/42] fix: set next val of sequence when name is of int type in validate_name --- frappe/model/naming.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 91ed469ac5..25b548cac3 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,10 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from typing import Optional, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING, Union import frappe from frappe import _ -from frappe.database.sequence import get_next_val +from frappe.database.sequence import get_next_val, set_next_val from frappe.utils import now_datetime, cint, cstr import re from frappe.model import log_types @@ -318,9 +318,19 @@ def get_default_naming_series(doctype): return None -def validate_name(doctype: str, name: str, case: Optional[str] = None): +def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = None): if not name: frappe.throw(_("No Name Specified for {0}").format(doctype)) + + if isinstance(name, int): + if is_autoincremented(doctype): + # this will set the sequence val to be the provided name and set it to be used + # so that the sequence will start from the next val of the setted val(name) + set_next_val(doctype, name, is_val_used=True) + return name + + frappe.throw("Invalid name type (integer) for varchar name column") + if name.startswith("New "+doctype): frappe.throw(_("There were some errors setting the name, please contact the administrator"), frappe.NameError) if case == "Title Case": From 15ae019b124bbbe360f93a2344f4d7e9f2b889fa Mon Sep 17 00:00:00 2001 From: phot0n Date: Sat, 5 Mar 2022 23:30:08 +0530 Subject: [PATCH 27/42] fix: set_next_val function for sequences postgres uses t & f, mariadb uses 0 & 1 for is_value_used --- frappe/database/sequence.py | 8 ++++++-- frappe/model/naming.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frappe/database/sequence.py b/frappe/database/sequence.py index c89ba468bc..334fd3d71e 100644 --- a/frappe/database/sequence.py +++ b/frappe/database/sequence.py @@ -69,8 +69,12 @@ def set_next_val( is_val_used :bool = False ) -> None: - is_val_used = 0 if not is_val_used else 1 + if not is_val_used: + is_val_used = 0 if db.db_type == "mariadb" else "f" + else: + is_val_used = 1 if db.db_type == "mariadb" else "t" + if db.db_type == "postgres": - db.sql(f"SELECT SETVAL(\'\"{scrub(doctype_name + slug)}\"\', {next_val}, {is_val_used})") + db.sql(f"SELECT SETVAL('\"{scrub(doctype_name + slug)}\"', {next_val}, '{is_val_used}')") else: db.sql(f"SELECT SETVAL(`{scrub(doctype_name + slug)}`, {next_val}, {is_val_used})") diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 25b548cac3..7e807f424e 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -329,7 +329,7 @@ def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = Non set_next_val(doctype, name, is_val_used=True) return name - frappe.throw("Invalid name type (integer) for varchar name column") + frappe.throw(_("Invalid name type (integer) for varchar name column"), frappe.NameError) if name.startswith("New "+doctype): frappe.throw(_("There were some errors setting the name, please contact the administrator"), frappe.NameError) From 86c9791813dfbdee3274d43573a2809f2cd8b505 Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 15 Mar 2022 15:43:34 +0530 Subject: [PATCH 28/42] fix: use next sequence number for ammended doc * chore: use meta --- frappe/model/naming.py | 13 ++++++++----- frappe/public/js/frappe/form/form.js | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 7e807f424e..dd4587fdb0 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -34,6 +34,10 @@ def set_new_name(doc): if autoname.lower() != "prompt" and not frappe.flags.in_import: doc.name = None + if is_autoincremented(doc.doctype, meta): + doc.name = get_next_val(doc.doctype) + return + if getattr(doc, "amended_from", None): _set_amended_name(doc) return @@ -41,10 +45,6 @@ def set_new_name(doc): elif getattr(doc.meta, "issingle", False): doc.name = doc.doctype - elif is_autoincremented(doc.doctype, meta): - doc.name = get_next_val(doc.doctype) - return - elif getattr(doc.meta, "istable", False): doc.name = make_autoname("hash", doc.doctype) @@ -73,7 +73,7 @@ def set_new_name(doc): doc.name = validate_name( doc.doctype, doc.name, - frappe.get_meta(doc.doctype).get_field("name_case") + meta.get_field("name_case") ) def is_autoincremented(doctype: str, meta: "Meta" = None): @@ -96,6 +96,9 @@ def is_autoincremented(doctype: str, meta: "Meta" = None): if not meta: meta = frappe.get_meta(doctype) + if getattr(meta, "issingle", False): + return False + if meta.autoname == "autoincrement": return True diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 6eed16bd28..269c54631f 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1215,7 +1215,7 @@ frappe.ui.form.Form = class FrappeForm { } is_dirty() { - return this.doc.__unsaved; + return !!this.doc.__unsaved; } is_new() { From f21f526ae6b0cb1932d380327e78f4619801fddf Mon Sep 17 00:00:00 2001 From: phot0n Date: Tue, 15 Mar 2022 16:49:33 +0530 Subject: [PATCH 29/42] fix: sider --- frappe/model/naming.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index dd4587fdb0..013e5a19db 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -35,8 +35,8 @@ def set_new_name(doc): doc.name = None if is_autoincremented(doc.doctype, meta): - doc.name = get_next_val(doc.doctype) - return + doc.name = get_next_val(doc.doctype) + return if getattr(doc, "amended_from", None): _set_amended_name(doc) From cd3950401c06fba274f96025608cb09fc889c128 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 15 Mar 2022 17:22:03 +0530 Subject: [PATCH 30/42] fix: using debounce instead of setTimeout() --- frappe/public/js/frappe/form/grid_row.js | 72 +++++++++++------------- 1 file changed, 32 insertions(+), 40 deletions(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 403807eaff..c12ac23319 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -231,7 +231,6 @@ export default class GridRow { } }); } else if (this.show_search) { - let timer = null; this.row_check = $( `
` ).appendTo(this.row); @@ -242,30 +241,27 @@ export default class GridRow { ` ).appendTo(this.row); - this.row_index.find('input').on('keyup', (e) => { - clearTimeout(timer); - timer = setTimeout(() => { - let df = { - fieldtype: "Sr No" - }; + this.row_index.find('input').on('keyup', frappe.utils.debounce((e) => { + let df = { + fieldtype: "Sr No" + }; - this.grid.filter['row-index'] = { - df: df, - value: e.target.value - }; + this.grid.filter['row-index'] = { + df: df, + value: e.target.value + }; - if (e.target.value == "") { - delete this.grid.filter['row-index']; - } + if (e.target.value == "") { + delete this.grid.filter['row-index']; + } - this.grid.grid_sortable - .option('disabled', Object.keys(this.grid.filter).length !== 0); + this.grid.grid_sortable + .option('disabled', Object.keys(this.grid.filter).length !== 0); - this.grid.prevent_build = true; - me.grid.refresh(); - this.grid.prevent_build = false; - }, 500); - }); + this.grid.prevent_build = true; + me.grid.refresh(); + this.grid.prevent_build = false; + }, 500)); frappe.utils.only_allow_num_decimal(this.row_index.find('input')); } else { this.row_index.find('span').html(txt); @@ -647,7 +643,6 @@ export default class GridRow { } make_search_column(df, colsize) { - let timer = null; let title = ""; let input_class = ""; let is_disabled = ""; @@ -679,28 +674,25 @@ export default class GridRow { this.search_columns[df.fieldname] = $col; - $search_input.on('keyup', (e) => { - clearTimeout(timer); - timer = setTimeout(() => { - this.grid.filter[df.fieldname] = { - df: df, - value: e.target.value - }; + $search_input.on('keyup', frappe.utils.debounce((e) => { + this.grid.filter[df.fieldname] = { + df: df, + value: e.target.value + }; - if (e.target.value == '') { - delete this.grid.filter[df.fieldname]; - } + if (e.target.value == '') { + delete this.grid.filter[df.fieldname]; + } - this.grid.grid_sortable - .option('disabled', Object.keys(this.grid.filter).length !== 0); + this.grid.grid_sortable + .option('disabled', Object.keys(this.grid.filter).length !== 0); - this.grid.prevent_build = true; - this.grid.refresh(); - this.grid.prevent_build = false; - }, 500); - }); + this.grid.prevent_build = true; + this.grid.refresh(); + this.grid.prevent_build = false; + }, 500)); - ["Currency", "Float", "Int", "Percent", "Rating", "Check"].includes(df.fieldtype) && + ["Currency", "Float", "Int", "Percent", "Rating"].includes(df.fieldtype) && frappe.utils.only_allow_num_decimal($search_input); return $col; From 5a1bc4b1d61db755b2196da98bd9cea7144a198e Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 15 Mar 2022 17:23:46 +0530 Subject: [PATCH 31/42] fix: use boolean strings (T, F, true, false, 1, 0 etc) for Check fieldtype --- frappe/public/js/frappe/form/grid.js | 3 ++- frappe/public/js/frappe/utils/utils.js | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 36461fb671..be20676183 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -549,7 +549,8 @@ export default class Grid { let fieldvalue = data[fieldname]; if (fieldtype === "Check") { - return (fieldvalue === parseInt(value || 0)) && data; + value = frappe.utils.string_to_boolean(value); + return (Boolean(fieldvalue) === value) && data; } else if (fieldtype === "Sr No" && data.idx.toString().includes(value)) { return data; } else if (fieldtype === "Duration" && fieldvalue) { diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 339917ed77..b253f4da54 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1463,5 +1463,13 @@ Object.assign(frappe.utils, { e.preventDefault(); } }); + }, + + string_to_boolean(string) { + switch(string.toLowerCase().trim()){ + case "t": case "true": case "y": case "yes": case "1": return true; + case "f": case "false": case "n": case "no": case "0": case null: return false; + default: return string; + } } }); From 9268405d62aedc1846d4f9302a081b397b03566c Mon Sep 17 00:00:00 2001 From: shadrak gurupnor Date: Tue, 15 Mar 2022 17:36:33 +0530 Subject: [PATCH 32/42] feat: added redirect for support portal --- frappe/patches.txt | 2 +- frappe/utils/install.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/patches.txt b/frappe/patches.txt index a666480c90..82b1f497c2 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -146,7 +146,7 @@ frappe.patches.v13_0.update_duration_options frappe.patches.v13_0.replace_old_data_import # 2020-06-24 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart -frappe.patches.v13_0.add_standard_navbar_items # 2020-12-15 +frappe.patches.v13_0.add_standard_navbar_items # 2022-03-15 frappe.patches.v13_0.generate_theme_files_in_public_folder frappe.patches.v13_0.increase_password_length frappe.patches.v12_0.fix_email_id_formatting diff --git a/frappe/utils/install.py b/frappe/utils/install.py index a5fd39994f..3af77b885f 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -255,6 +255,12 @@ def add_standard_navbar_items(): 'item_type': 'Action', 'action': 'frappe.ui.toolbar.show_shortcuts(event)', 'is_standard': 1 + }, + { + 'item_label': 'Frappe Support', + 'item_type': 'Route', + 'route': 'https://frappe.io/support', + 'is_standard': 1 } ] From cac1fd40d4d74508294896743603cd883235dbf2 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Tue, 15 Mar 2022 17:39:02 +0530 Subject: [PATCH 33/42] fix(sider): expected space(s) --- frappe/public/js/frappe/utils/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index b253f4da54..03f3204abb 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1466,7 +1466,7 @@ Object.assign(frappe.utils, { }, string_to_boolean(string) { - switch(string.toLowerCase().trim()){ + switch (string.toLowerCase().trim()) { case "t": case "true": case "y": case "yes": case "1": return true; case "f": case "false": case "n": case "no": case "0": case null: return false; default: return string; From 2558c6bee0b2d7254a78d77919302d5bc2af84b9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 16 Mar 2022 18:30:22 +0530 Subject: [PATCH 34/42] feat: Drop site support for postgres --- frappe/database/__init__.py | 3 ++- frappe/database/postgres/setup_db.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index 7b26ac31b3..5db0537ed7 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -18,7 +18,8 @@ def setup_database(force, source_sql=None, verbose=None, no_mariadb_socket=False def drop_user_and_database(db_name, root_login=None, root_password=None): import frappe if frappe.conf.db_type == 'postgres': - pass + import frappe.database.postgres.setup_db + return frappe.database.postgres.setup_db.drop_user_and_database(db_name, root_login, root_password) else: import frappe.database.mariadb.setup_db return frappe.database.mariadb.setup_db.drop_user_and_database(db_name, root_login, root_password) diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index b3b2e0fd41..4b265e7660 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -95,3 +95,11 @@ def get_root_connection(root_login=None, root_password=None): frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password) return frappe.local.flags.root_connection + + +def drop_user_and_database(db_name, root_login, root_password): + root_conn = get_root_connection(frappe.flags.root_login or root_login, frappe.flags.root_password or root_password) + root_conn.commit() + root_conn.sql(f"SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = %s", (db_name, )) + root_conn.sql(f"DROP DATABASE IF EXISTS {db_name}") + root_conn.sql(f"DROP USER IF EXISTS {db_name}") From 776ba30a4d836469776e37af28dc48d5883376df Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 16 Mar 2022 19:06:41 +0530 Subject: [PATCH 35/42] fix: Add more verbosity for drop-site command --- frappe/commands/site.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index b54f369e34..63da4db093 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -677,7 +677,9 @@ def _drop_site(site, db_root_username=None, db_root_password=None, archived_site try: if not no_backup: - scheduled_backup(ignore_files=False, force=True) + click.secho(f"Taking backup of {site}", fg="green") + odb = scheduled_backup(ignore_files=False, force=True, verbose=True) + odb.print_summary() except Exception as err: if force: pass @@ -692,6 +694,7 @@ def _drop_site(site, db_root_username=None, db_root_password=None, archived_site click.echo("\n".join(messages)) sys.exit(1) + click.secho("Dropping site database and user", fg="green") drop_user_and_database(frappe.conf.db_name, db_root_username, db_root_password) archived_sites_path = archived_sites_path or os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived', 'sites') From 14a4e35d8d9eb5917fa680160fd5ca5b328ed2d2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 17 Mar 2022 09:50:14 +0530 Subject: [PATCH 36/42] fix: Typo in is_downgrade's user warning Fixes https://github.com/frappe/frappe/issues/16312 --- frappe/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/installer.py b/frappe/installer.py index 6ebab95a7d..d10dc78286 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -611,7 +611,7 @@ def is_downgrade(sql_file_path, verbose=False): downgrade = backup_version > current_version if verbose and downgrade: - print("Your site will be downgraded from Frappe {0} to {1}".format(current_version, backup_version)) + print(f"Your site will be downgraded from Frappe {backup_version} to {current_version}") return downgrade From bc48c03da78e98557afe2560055fbeadd7e5ccc6 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 17 Mar 2022 11:18:12 +0530 Subject: [PATCH 37/42] test: failing list_paging UI test --- cypress/integration/list_paging.js | 3 +++ frappe/tests/ui_test_helpers.py | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cypress/integration/list_paging.js b/cypress/integration/list_paging.js index b6832f5a53..4a59024a7b 100644 --- a/cypress/integration/list_paging.js +++ b/cypress/integration/list_paging.js @@ -31,5 +31,8 @@ context('List Paging', () => { cy.get('.list-paging-area .btn-group .btn-paging[data-value="500"]').click(); cy.get('.list-paging-area .list-count').should('contain.text', '500 of'); + cy.get('.list-paging-area .btn-more').click(); + + cy.get('.list-paging-area .list-count').should('contain.text', '1000 of'); }); }); diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 473f9b22d3..42a7f48d79 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -138,11 +138,12 @@ def create_contact_records(): def create_multiple_todo_records(): if frappe.db.get_all('ToDo', {'description': 'Multiple ToDo 1'}): return - for index in range(501): - frappe.get_doc({ - 'doctype': 'ToDo', - 'description': 'Multiple ToDo {}'.format(index+1) - }).insert() + + query = "INSERT INTO `tabToDo` (`name`, `description`) VALUES ('1001', 'Multiple ToDo 1')" + for index in range(1000): + query = query + ", ('100{}', 'Multiple ToDo {}')".format(index+2,index+2) + + frappe.db.sql(query) def insert_contact(first_name, phone_number): doc = frappe.get_doc({ From 631e6f32604a8df389a89079f57e77213a52e52c Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 17 Mar 2022 12:03:46 +0530 Subject: [PATCH 38/42] fix: using bulk_insert instead of sql query --- frappe/tests/ui_test_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 42a7f48d79..75c28a8cd7 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -136,14 +136,14 @@ def create_contact_records(): @frappe.whitelist() def create_multiple_todo_records(): + values = [] if frappe.db.get_all('ToDo', {'description': 'Multiple ToDo 1'}): return - query = "INSERT INTO `tabToDo` (`name`, `description`) VALUES ('1001', 'Multiple ToDo 1')" - for index in range(1000): - query = query + ", ('100{}', 'Multiple ToDo {}')".format(index+2,index+2) + for index in range(1, 1002): + values.append(('100{}'.format(index), 'Multiple ToDo {}'.format(index))) - frappe.db.sql(query) + frappe.db.bulk_insert('ToDo', fields=['name', 'description'], values=set(values)) def insert_contact(first_name, phone_number): doc = frappe.get_doc({ From 0b8a2edee70cedb62059a29039eb06d3809efd19 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Thu, 17 Mar 2022 14:54:17 +0530 Subject: [PATCH 39/42] fix(build): separate assets.json and assets-rtl.json to fix concurrency issue --- esbuild/esbuild.js | 22 ++++++++++++---------- frappe/utils/__init__.py | 37 ++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 43c01e88fb..ff31aa4b74 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -286,7 +286,7 @@ function get_watch_config() { notify_redis({ error }); } else { let { - assets_json, + new_assets_json, prev_assets_json } = await write_assets_json(result.metafile); @@ -294,7 +294,7 @@ function get_watch_config() { if (prev_assets_json) { changed_files = get_rebuilt_assets( prev_assets_json, - assets_json + new_assets_json ); let timestamp = new Date().toLocaleTimeString(); @@ -384,6 +384,7 @@ let prev_assets_json; let curr_assets_json; async function write_assets_json(metafile) { + let rtl = false; prev_assets_json = curr_assets_json; let out = {}; for (let output in metafile.outputs) { @@ -392,13 +393,14 @@ async function write_assets_json(metafile) { if (info.entryPoint) { let key = path.basename(info.entryPoint); if (key.endsWith('.css') && asset_path.includes('/css-rtl/')) { + rtl = true; key = `rtl_${key}`; } out[key] = asset_path; } } - let assets_json_path = path.resolve(assets_path, "assets.json"); + let assets_json_path = path.resolve(assets_path, `assets${rtl?'-rtl':''}.json`); let assets_json; try { assets_json = await fs.promises.readFile(assets_json_path, "utf-8"); @@ -407,21 +409,21 @@ async function write_assets_json(metafile) { } assets_json = JSON.parse(assets_json); // update with new values - assets_json = Object.assign({}, assets_json, out); - curr_assets_json = assets_json; + let new_assets_json = Object.assign({}, assets_json, out); + curr_assets_json = new_assets_json; await fs.promises.writeFile( assets_json_path, - JSON.stringify(assets_json, null, 4) + JSON.stringify(new_assets_json, null, 4) ); - await update_assets_json_in_cache(assets_json); + await update_assets_json_in_cache(); return { - assets_json, + new_assets_json, prev_assets_json }; } -function update_assets_json_in_cache(assets_json) { +function update_assets_json_in_cache() { // update assets_json cache in redis, so that it can be read directly by python return new Promise(resolve => { let client = get_redis_subscriber("redis_cache"); @@ -429,7 +431,7 @@ function update_assets_json_in_cache(assets_json) { client.on("error", _ => { log_warn("Cannot connect to redis_cache to update assets_json"); }); - client.set("assets_json", JSON.stringify(assets_json), err => { + client.del("assets_json", err => { client.unref(); resolve(); }); diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 1233bcd30f..c361b5b430 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -796,22 +796,33 @@ def get_assets_json(): # using .get instead of .get_value to avoid pickle.loads try: - assets_json = cache.get("assets_json") - except ConnectionError: - assets_json = None - - # if value found, decode it - if assets_json is not None: - try: - assets_json = assets_json.decode('utf-8') - except (UnicodeDecodeError, AttributeError): + if not frappe.conf.developer_mode: + assets_json = cache.get("assets_json").decode('utf-8') + else: assets_json = None + except (UnicodeDecodeError, AttributeError, ConnectionError): + assets_json = None if not assets_json: - assets_json = frappe.read_file("assets/assets.json") - cache.set_value("assets_json", assets_json, shared=True) - - frappe.local.assets_json = frappe.safe_decode(assets_json) + # get merged assets.json and assets-rtl.json + assets_dict = frappe.parse_json( + frappe.read_file("assets/assets.json") + ) + + assets_rtl = frappe.read_file("assets/assets-rtl.json") + if assets_rtl: + assets_dict.update( + frappe.parse_json(assets_rtl) + ) + frappe.local.assets_json = frappe.as_json(assets_dict) + # save in cache + cache.set_value("assets_json", frappe.local.assets_json, + shared=True) + + return assets_dict + else: + # from cache, decode and send + frappe.local.assets_json = frappe.safe_decode(assets_json) return frappe.parse_json(frappe.local.assets_json) From c4be72c2d404f337ba10d747e27525feb66aa9b7 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 18 Mar 2022 12:18:15 +0530 Subject: [PATCH 40/42] fix: Pass skip_dirty_trigger flag to the set_value and model trigger Also, renamed avoid_dirty to skip_dirty_trigger to be more explicit --- frappe/public/js/frappe/model/model.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index d0b1c729c6..3b95a4b3f1 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -412,7 +412,7 @@ $.extend(frappe.model, { } }, - set_value: function(doctype, docname, fieldname, value, fieldtype, avoid_dirty=false) { + set_value: function(doctype, docname, fieldname, value, fieldtype, skip_dirty_trigger=false) { /* help: Set a value locally (if changed) and execute triggers */ var doc; @@ -438,11 +438,11 @@ $.extend(frappe.model, { } doc[key] = value; - if (!avoid_dirty) tasks.push(() => frappe.model.trigger(key, value, doc)); + tasks.push(() => frappe.model.trigger(key, value, doc, skip_dirty_trigger)); } else { // execute link triggers (want to reselect to execute triggers) if(in_list(["Link", "Dynamic Link"], fieldtype) && doc) { - tasks.push(() => frappe.model.trigger(key, value, doc)); + tasks.push(() => frappe.model.trigger(key, value, doc, skip_dirty_trigger)); } } }); @@ -467,7 +467,7 @@ $.extend(frappe.model, { frappe.model.events[doctype][fieldname].push(fn); }, - trigger: function(fieldname, value, doc) { + trigger: function(fieldname, value, doc, skip_dirty_trigger=false) { const tasks = []; function enqueue_events(events) { @@ -477,7 +477,7 @@ $.extend(frappe.model, { if (!fn) continue; tasks.push(() => { - const return_value = fn(fieldname, value, doc); + const return_value = fn(fieldname, value, doc, skip_dirty_trigger); // if the trigger returns a promise, return it, // or use the default promise frappe.after_ajax From adb989fff2458cc92678ef6b6f5c224904be1864 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 18 Mar 2022 12:19:36 +0530 Subject: [PATCH 41/42] feat: Option to set form dirty even if form save is disabled --- frappe/public/js/frappe/form/form.js | 14 +++++++++----- frappe/public/js/frappe/form/toolbar.js | 6 +++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 494e3af705..7ec6677c7f 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -246,10 +246,12 @@ frappe.ui.form.Form = class FrappeForm { var me = this; // on main doc - frappe.model.on(me.doctype, "*", function(fieldname, value, doc) { + frappe.model.on(me.doctype, "*", function(fieldname, value, doc, skip_dirty_trigger=false) { // set input if (cstr(doc.name) === me.docname) { - me.dirty(); + if (!skip_dirty_trigger) { + me.dirty(); + } let field = me.fields_dict[fieldname]; field && field.refresh(fieldname); @@ -953,10 +955,12 @@ frappe.ui.form.Form = class FrappeForm { this.toolbar.set_primary_action(); } - disable_save() { + disable_save(set_dirty=false) { // IMPORTANT: this function should be called in refresh event this.save_disabled = true; this.toolbar.current_status = null; + // field changes should make form dirty + this.set_dirty = set_dirty; this.page.clear_primary_action(); } @@ -1447,7 +1451,7 @@ frappe.ui.form.Form = class FrappeForm { return doc; } - set_value(field, value, if_missing, avoid_dirty=false) { + set_value(field, value, if_missing, skip_dirty_trigger=false) { var me = this; var _set = function(f, v) { var fieldobj = me.fields_dict[f]; @@ -1467,7 +1471,7 @@ frappe.ui.form.Form = class FrappeForm { me.refresh_field(f); return Promise.resolve(); } else { - return frappe.model.set_value(me.doctype, me.doc.name, f, v, me.fieldtype, avoid_dirty); + return frappe.model.set_value(me.doctype, me.doc.name, f, v, me.fieldtype, skip_dirty_trigger); } } } else { diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index 016390a4e1..e55eb9fdeb 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -534,14 +534,14 @@ frappe.ui.form.Toolbar = class Toolbar { }); } show_title_as_dirty() { - if(this.frm.save_disabled) + if (this.frm.save_disabled && !this.frm.set_dirty) return; - if(this.frm.doc.__unsaved) { + if (this.frm.is_dirty()) { this.page.set_indicator(__("Not Saved"), "orange"); } - $(this.frm.wrapper).attr("data-state", this.frm.doc.__unsaved ? "dirty" : "clean"); + $(this.frm.wrapper).attr("data-state", this.frm.is_dirty() ? "dirty" : "clean"); } show_jump_to_field_dialog() { From fedcf48ada2bd17ce01f2238af9749b68a837437 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 18 Mar 2022 12:20:51 +0530 Subject: [PATCH 42/42] fix: Customize form issue where it remains "Not Saved" even after update fixes: https://github.com/frappe/frappe/issues/16068 --- frappe/custom/doctype/customize_form/customize_form.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 4862185b99..9cfe315e44 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -14,7 +14,6 @@ frappe.ui.form.on("Customize Form", { }, onload: function(frm) { - frm.disable_save(); frm.set_query("doc_type", function() { return { translate_values: false, @@ -110,7 +109,7 @@ frappe.ui.form.on("Customize Form", { }, refresh: function(frm) { - frm.disable_save(); + frm.disable_save(true); frm.page.clear_icons(); if (frm.doc.doc_type) { @@ -169,7 +168,7 @@ frappe.ui.form.on("Customize Form", { doc_type = localStorage.getItem("customize_doctype"); } if (doc_type) { - setTimeout(() => frm.set_value("doc_type", doc_type), 1000); + setTimeout(() => frm.set_value("doc_type", doc_type, false, true), 1000); } }, @@ -341,11 +340,11 @@ frappe.customize_form.confirm = function(msg, frm) { } frappe.customize_form.clear_locals_and_refresh = function(frm) { + delete frm.doc.__unsaved; // clear doctype from locals frappe.model.clear_doc("DocType", frm.doc.doc_type); delete frappe.meta.docfield_copy[frm.doc.doc_type]; - frm.refresh(); -} +}; extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm}));