From f4b53f424c4c88c41e04c1511b3a3599802c482b Mon Sep 17 00:00:00 2001 From: lapardnemihk1099 Date: Wed, 9 Mar 2022 11:34:54 +0530 Subject: [PATCH 01/55] fix: theme switcher shortcut issue --- frappe/public/js/frappe/desk.js | 10 +++++++++- frappe/public/js/frappe/ui/dialog.js | 1 + frappe/public/js/frappe/ui/theme_switcher.js | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 51ada70948..36df5f450e 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -55,12 +55,20 @@ frappe.Application = class Application { frappe.ui.keys.setup(); + let is_dialog_open = false; frappe.ui.keys.add_shortcut({ shortcut: 'shift+ctrl+g', description: __('Switch Theme'), action: () => { frappe.theme_switcher = new frappe.ui.ThemeSwitcher(); - frappe.theme_switcher.show(); + if (!is_dialog_open) { + frappe.theme_switcher.show(); + is_dialog_open = true; + } + else { + frappe.theme_switcher.hide(); + is_dialog_open = false; + } } }); diff --git a/frappe/public/js/frappe/ui/dialog.js b/frappe/public/js/frappe/ui/dialog.js index 1618db9939..2c0ff0dcf4 100644 --- a/frappe/public/js/frappe/ui/dialog.js +++ b/frappe/public/js/frappe/ui/dialog.js @@ -216,6 +216,7 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { hide() { this.$wrapper.modal("hide"); this.is_visible = false; + frappe.ui.hide_open_dialog(); } get_close_btn() { diff --git a/frappe/public/js/frappe/ui/theme_switcher.js b/frappe/public/js/frappe/ui/theme_switcher.js index 2c1d93a2ec..1725577079 100644 --- a/frappe/public/js/frappe/ui/theme_switcher.js +++ b/frappe/public/js/frappe/ui/theme_switcher.js @@ -25,6 +25,8 @@ frappe.ui.ThemeSwitcher = class ThemeSwitcher { increment_by = 1; } else if (key === "left") { increment_by = -1; + } else if (e.keyCode === 13) { // keycode 13 is for 'enter' + this.hide(); } else { return; } From 16d84c558576005f2565e54c83fdc5746faced8e Mon Sep 17 00:00:00 2001 From: lapardnemihk1099 Date: Fri, 11 Mar 2022 13:41:12 +0530 Subject: [PATCH 02/55] chore: formatting issue fixed --- frappe/public/js/frappe/desk.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 36df5f450e..c0fcfeee92 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -64,8 +64,7 @@ frappe.Application = class Application { if (!is_dialog_open) { frappe.theme_switcher.show(); is_dialog_open = true; - } - else { + } else { frappe.theme_switcher.hide(); is_dialog_open = false; } From b0c813ec214af93fdb59e1477b8544d95d0e6d1f Mon Sep 17 00:00:00 2001 From: lapardnemihk1099 Date: Wed, 23 Mar 2022 17:13:34 +0530 Subject: [PATCH 03/55] chore: fixed bad programming --- frappe/public/js/frappe/desk.js | 15 +++++++-------- frappe/public/js/frappe/ui/dialog.js | 1 - frappe/public/js/frappe/ui/theme_switcher.js | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index c0fcfeee92..4f80218c14 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -54,19 +54,18 @@ frappe.Application = class Application { this.setup_copy_doc_listener(); frappe.ui.keys.setup(); - - let is_dialog_open = false; + frappe.ui.keys.add_shortcut({ shortcut: 'shift+ctrl+g', description: __('Switch Theme'), action: () => { - frappe.theme_switcher = new frappe.ui.ThemeSwitcher(); - if (!is_dialog_open) { - frappe.theme_switcher.show(); - is_dialog_open = true; - } else { + if (cur_dialog) { frappe.theme_switcher.hide(); - is_dialog_open = false; + } else { + if (!frappe.theme_switcher) { + frappe.theme_switcher = new frappe.ui.ThemeSwitcher(); + } + frappe.theme_switcher.show(); } } }); diff --git a/frappe/public/js/frappe/ui/dialog.js b/frappe/public/js/frappe/ui/dialog.js index 2c0ff0dcf4..1618db9939 100644 --- a/frappe/public/js/frappe/ui/dialog.js +++ b/frappe/public/js/frappe/ui/dialog.js @@ -216,7 +216,6 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { hide() { this.$wrapper.modal("hide"); this.is_visible = false; - frappe.ui.hide_open_dialog(); } get_close_btn() { diff --git a/frappe/public/js/frappe/ui/theme_switcher.js b/frappe/public/js/frappe/ui/theme_switcher.js index 1725577079..c2e1094675 100644 --- a/frappe/public/js/frappe/ui/theme_switcher.js +++ b/frappe/public/js/frappe/ui/theme_switcher.js @@ -138,7 +138,7 @@ frappe.ui.ThemeSwitcher = class ThemeSwitcher { } hide() { - this.dialog.hide(); + frappe.ui.hide_open_dialog(); } }; From c2c725c8ce08ab932f7391c6a1deff40a282b1bf Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 25 Mar 2022 19:10:11 +0100 Subject: [PATCH 04/55] fix: store reference to leaflet drawControl --- frappe/public/js/frappe/form/controls/geolocation.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js index 280eac3941..83a85d9317 100644 --- a/frappe/public/js/frappe/form/controls/geolocation.js +++ b/frappe/public/js/frappe/form/controls/geolocation.js @@ -146,9 +146,8 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f }; // create control and add to map - var drawControl = new L.Control.Draw(options); - - this.map.addControl(drawControl); + this.drawControl = new L.Control.Draw(options); + this.map.addControl(this.drawControl); this.map.on('draw:created', (e) => { var type = e.layerType, From b8487cd6016248f0c36e9c512a4399eaca4b8857 Mon Sep 17 00:00:00 2001 From: Khimendrapal Solanki <84378369+lapardnemihk1099@users.noreply.github.com> Date: Mon, 28 Mar 2022 17:17:29 +0530 Subject: [PATCH 05/55] chore: properly closing theme-switcher dialog Co-authored-by: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> --- frappe/public/js/frappe/ui/theme_switcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/theme_switcher.js b/frappe/public/js/frappe/ui/theme_switcher.js index c2e1094675..1725577079 100644 --- a/frappe/public/js/frappe/ui/theme_switcher.js +++ b/frappe/public/js/frappe/ui/theme_switcher.js @@ -138,7 +138,7 @@ frappe.ui.ThemeSwitcher = class ThemeSwitcher { } hide() { - frappe.ui.hide_open_dialog(); + this.dialog.hide(); } }; From 34aa1f97b8e70f6761e124c424ed4f88ba41ac0a Mon Sep 17 00:00:00 2001 From: Khimendrapal Solanki <84378369+lapardnemihk1099@users.noreply.github.com> Date: Mon, 28 Mar 2022 17:20:41 +0530 Subject: [PATCH 06/55] chore: fix add_shortcut conditions Co-authored-by: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> --- frappe/public/js/frappe/desk.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 4f80218c14..c1fc21fe76 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -59,12 +59,10 @@ frappe.Application = class Application { shortcut: 'shift+ctrl+g', description: __('Switch Theme'), action: () => { - if (cur_dialog) { + if (frappe.theme_switcher && frappe.theme_switcher.dialog.is_visible) { frappe.theme_switcher.hide(); } else { - if (!frappe.theme_switcher) { - frappe.theme_switcher = new frappe.ui.ThemeSwitcher(); - } + frappe.theme_switcher = new frappe.ui.ThemeSwitcher(); frappe.theme_switcher.show(); } } From cf0728de725d33786d16a65b43100e339dd9b35c Mon Sep 17 00:00:00 2001 From: lapardnemihk1099 Date: Mon, 28 Mar 2022 17:41:56 +0530 Subject: [PATCH 07/55] chore: fixed linter issue --- frappe/public/js/frappe/desk.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index c1fc21fe76..72e8010605 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -54,7 +54,7 @@ frappe.Application = class Application { this.setup_copy_doc_listener(); frappe.ui.keys.setup(); - + frappe.ui.keys.add_shortcut({ shortcut: 'shift+ctrl+g', description: __('Switch Theme'), From cd7d45757b14e8e41767d0abc3395b858167bdb1 Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Tue, 29 Mar 2022 15:23:03 +0530 Subject: [PATCH 08/55] test: Added test script for control type "Date" --- cypress/integration/control_date.js | 122 ++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 cypress/integration/control_date.js diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js new file mode 100644 index 0000000000..d1ef62dd90 --- /dev/null +++ b/cypress/integration/control_date.js @@ -0,0 +1,122 @@ +context('Date Control', () => { + before(() => { + cy.login(); + cy.visit('/app/doctype'); + return cy.window().its('frappe').then(frappe => { + return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { + name: 'Test Date Control', + fields: [ + { + "label": "Date", + "fieldname": "date", + "fieldtype": "Date", + "in_list_view": 1 + }, + ] + }); + }); + }); + it('Selecting a date from the datepicker', () => { + cy.new_form('Test Date Control'); + cy.get_field('date','Date').click(); + cy.get('.datepicker--nav-title').click(); + cy.get('.datepicker--nav-title').click({force: true}); + + + //Inputing values in the date field + cy.get('.datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]').click(); + cy.get('.datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]').click(); + cy.get('.datepicker--days > .datepicker--cells > .datepicker--cell[data-date=15]').click(); + + //Verifying if the selected date is displayed in the date field + cy.get_field('date','Date').should('have.value', '01-15-2020'); + }); + + it('Checking next and previous button', () => { + cy.get_field('date','Date').click(); + + //Clicking on the next button in the datepicker + cy.get('.datepicker--nav-action[data-action=next]').click(); + + //Selecting a date from the datepicker + cy.get('.datepicker--cell[data-date=15]').click({force: true}); + + //Verifying if the selected date has been displayed in the date field + cy.get_field('date','Date').should('have.value', '02-15-2020'); + cy.wait(500); + cy.get_field('date','Date').click(); + + //Clicking on the previous button in the datepicker + cy.get('.datepicker--nav-action[data-action=prev]').click(); + + //Selecting a date from the datepicker + cy.get('.datepicker--cell[data-date=15]').click({force: true}); + + //Verifying if the selected date has been displayed in the date field + cy.get_field('date','Date').should('have.value', '01-15-2020'); + }); + + it('Clicking on "Today" button gives todays date', () => { + cy.get_field('date','Date').click(); + + //Clicking on "Today" button + cy.get('.datepicker--button').click(); + + //Picking up the todays date + const todaysDate = Cypress.moment().format('MM-DD-YYYY'); + cy.log(todaysDate); + + //Verifying if clicking on "Today" button matches today's date + cy.get_field('date','Date').should('have.value', todaysDate); + }); + + it.only('Configuring first day of the week', () => { + //Visiting "System Settings" page + cy.visit('/app/system-settings/System%20Settings'); + + //Visiting the "Date and Number Format" section + cy.contains('Date and Number Format').click(); + + //Changing the configuration for "First day of the week" field + cy.get('select[data-fieldname="first_day_of_the_week"]').select('Tuesday'); + cy.get('.page-head .page-actions').findByRole('button', {name: 'Save'}).click(); + cy.new_form('Test Date Control'); + cy.get_field('date','Date').click(); + + //Checking if the first day shown in the datepicker is the one which is configured in the System Settings Page + cy.get('.datepicker--days-names').eq(0).should('contain.text', 'Tu'); + cy.visit('/app/doctype'); + + //Adding filter in the doctype list + cy.add_filter(); + cy.get('.fieldname-select-area').type('Created On{enter}'); + cy.get('.filter-field > .form-group > .input-with-feedback').click(); + + //Checking if the first day shown in the datepicker is the one which is configured in the System Settings Page + cy.get('.datepicker--days-names').eq(0).should('contain.text', 'Tu'); + + //Adding event + cy.visit('/app/event'); + cy.click_listview_primary_button('Add Event'); + cy.get('textarea[data-fieldname=subject]').type('Test'); + //cy.fill_field('subject','Test','textarea'); + cy.get('form > .has-error > .form-group > .control-input-wrapper > .control-input > .input-with-feedback[data-fieldtype="Datetime"]').click(); + cy.get('.datepicker.active > .datepicker--content > .datepicker--days > .datepicker--cells > .datepicker--cell[data-date=10]').click({force: true}); + cy.click_listview_primary_button('Save'); + cy.visit('/app/event'); + cy.get('.custom-btn-group > .btn').click(); + + //Opening Calendar view for the event created + cy.get('[data-view="Calendar"] > .grey-link').click(); + + //Checking if the calendar view has the first day as the configured day in the System Settings Page + cy.get('.fc-head-container').eq(0).should('contain.text', 'Tue'); + + //Deleting the created event + cy.visit('/app/event'); + cy.get('.list-row-checkbox').eq(0).click(); + cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); + cy.click_modal_primary_button('Yes'); + }); +}); \ No newline at end of file From 23fa892aca775d53f4d15c7821a935a577a18975 Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Tue, 29 Mar 2022 15:30:02 +0530 Subject: [PATCH 09/55] test: Fixing sider issues --- cypress/integration/control_date.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index d1ef62dd90..d6220c3c51 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -18,7 +18,7 @@ context('Date Control', () => { }); it('Selecting a date from the datepicker', () => { cy.new_form('Test Date Control'); - cy.get_field('date','Date').click(); + cy.get_field('date', 'Date').click(); cy.get('.datepicker--nav-title').click(); cy.get('.datepicker--nav-title').click({force: true}); @@ -29,11 +29,11 @@ context('Date Control', () => { cy.get('.datepicker--days > .datepicker--cells > .datepicker--cell[data-date=15]').click(); //Verifying if the selected date is displayed in the date field - cy.get_field('date','Date').should('have.value', '01-15-2020'); + cy.get_field('date', 'Date').should('have.value', '01-15-2020'); }); it('Checking next and previous button', () => { - cy.get_field('date','Date').click(); + cy.get_field('date', 'Date').click(); //Clicking on the next button in the datepicker cy.get('.datepicker--nav-action[data-action=next]').click(); @@ -42,9 +42,9 @@ context('Date Control', () => { cy.get('.datepicker--cell[data-date=15]').click({force: true}); //Verifying if the selected date has been displayed in the date field - cy.get_field('date','Date').should('have.value', '02-15-2020'); + cy.get_field('date', 'Date').should('have.value', '02-15-2020'); cy.wait(500); - cy.get_field('date','Date').click(); + cy.get_field('date', 'Date').click(); //Clicking on the previous button in the datepicker cy.get('.datepicker--nav-action[data-action=prev]').click(); @@ -53,11 +53,11 @@ context('Date Control', () => { cy.get('.datepicker--cell[data-date=15]').click({force: true}); //Verifying if the selected date has been displayed in the date field - cy.get_field('date','Date').should('have.value', '01-15-2020'); + cy.get_field('date', 'Date').should('have.value', '01-15-2020'); }); it('Clicking on "Today" button gives todays date', () => { - cy.get_field('date','Date').click(); + cy.get_field('date', 'Date').click(); //Clicking on "Today" button cy.get('.datepicker--button').click(); @@ -67,7 +67,7 @@ context('Date Control', () => { cy.log(todaysDate); //Verifying if clicking on "Today" button matches today's date - cy.get_field('date','Date').should('have.value', todaysDate); + cy.get_field('date', 'Date').should('have.value', todaysDate); }); it.only('Configuring first day of the week', () => { @@ -81,7 +81,7 @@ context('Date Control', () => { cy.get('select[data-fieldname="first_day_of_the_week"]').select('Tuesday'); cy.get('.page-head .page-actions').findByRole('button', {name: 'Save'}).click(); cy.new_form('Test Date Control'); - cy.get_field('date','Date').click(); + cy.get_field('date', 'Date').click(); //Checking if the first day shown in the datepicker is the one which is configured in the System Settings Page cy.get('.datepicker--days-names').eq(0).should('contain.text', 'Tu'); From cc3a046f465e84bc9f3528e5bcb61406b7e42c2d Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Tue, 29 Mar 2022 15:40:26 +0530 Subject: [PATCH 10/55] test: Corrected selectors --- cypress/integration/control_date.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index d6220c3c51..27be1c3264 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -98,10 +98,8 @@ context('Date Control', () => { //Adding event cy.visit('/app/event'); cy.click_listview_primary_button('Add Event'); - cy.get('textarea[data-fieldname=subject]').type('Test'); - //cy.fill_field('subject','Test','textarea'); - cy.get('form > .has-error > .form-group > .control-input-wrapper > .control-input > .input-with-feedback[data-fieldtype="Datetime"]').click(); - cy.get('.datepicker.active > .datepicker--content > .datepicker--days > .datepicker--cells > .datepicker--cell[data-date=10]').click({force: true}); + cy.fill_field('subject', 'Test', 'Textarea'); + cy.fill_field('starts_on', '01-01-2022 00:00:00', 'Datetime'); cy.click_listview_primary_button('Save'); cy.visit('/app/event'); cy.get('.custom-btn-group > .btn').click(); From eed29df34dd4681d8df0260b11d27ca931a1c7a1 Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Tue, 29 Mar 2022 16:31:03 +0530 Subject: [PATCH 11/55] test: Corrected Selector --- cypress/integration/control_date.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index 27be1c3264..bdc3a7237f 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -98,7 +98,7 @@ context('Date Control', () => { //Adding event cy.visit('/app/event'); cy.click_listview_primary_button('Add Event'); - cy.fill_field('subject', 'Test', 'Textarea'); + cy.fill_field('subject', 'Test', 'Small Text'); cy.fill_field('starts_on', '01-01-2022 00:00:00', 'Datetime'); cy.click_listview_primary_button('Save'); cy.visit('/app/event'); From 3edf254ef3171b2148e915df1ddcba1583fe7f36 Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Tue, 29 Mar 2022 17:09:40 +0530 Subject: [PATCH 12/55] test: Corrected selector --- cypress/integration/control_date.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index bdc3a7237f..c45602b4d9 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -98,7 +98,7 @@ context('Date Control', () => { //Adding event cy.visit('/app/event'); cy.click_listview_primary_button('Add Event'); - cy.fill_field('subject', 'Test', 'Small Text'); + cy.get('textarea[data-fieldname=subject]').type('Test'); cy.fill_field('starts_on', '01-01-2022 00:00:00', 'Datetime'); cy.click_listview_primary_button('Save'); cy.visit('/app/event'); From 8296d6e84a29b156421414aafb596ca125083b77 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 31 Mar 2022 18:12:09 +0530 Subject: [PATCH 13/55] fix: use backticks for fieldname while preparing filters --- frappe/model/db_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 16056d382a..b573e1d301 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -476,7 +476,7 @@ class DatabaseQuery(object): if 'ifnull(' in f.fieldname: column_name = self.cast_name(f.fieldname, "ifnull(") else: - column_name = self.cast_name(f"{tname}.{f.fieldname}") + column_name = self.cast_name(f"{tname}.`{f.fieldname}`") if f.operator.lower() in additional_filters_config: f.update(get_additional_filter_field(additional_filters_config, f, f.value)) From d032822093615b6e677854fbe5fa4cc84d7eb276 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 31 Mar 2022 21:25:07 +0530 Subject: [PATCH 14/55] fix: use backticks in test_cast_name --- frappe/tests/test_db_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index b4c7c7cce7..62842dc167 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -507,10 +507,10 @@ class TestReportview(unittest.TestCase): if frappe.db.db_type == "postgres": self.assertTrue("strpos( cast( \"tabautoinc_dt_test\".\"name\" as varchar), \'1\')" in query) - self.assertTrue("where cast(\"tabautoinc_dt_test\".name as varchar) = \'1\'" in query) + self.assertTrue("where cast(\"tabautoinc_dt_test\".\"name\" as varchar) = \'1\'" in query) else: self.assertTrue("locate(\'1\', `tabautoinc_dt_test`.`name`)" in query) - self.assertTrue("where `tabautoinc_dt_test`.name = 1" in query) + self.assertTrue("where `tabautoinc_dt_test`.`name` = 1" in query) dt.delete(ignore_permissions=True) From a77381de2b970a0c5cfc8766a3eca787f8e0e369 Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Wed, 30 Mar 2022 16:10:13 +0530 Subject: [PATCH 15/55] feat: Added fuzzy matching to awesome bar --- .../js/frappe/ui/toolbar/fuzzy_match.js | 196 ++++++++++++++++++ .../js/frappe/ui/toolbar/search_utils.js | 125 ++++------- 2 files changed, 231 insertions(+), 90 deletions(-) create mode 100644 frappe/public/js/frappe/ui/toolbar/fuzzy_match.js diff --git a/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js b/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js new file mode 100644 index 0000000000..755882384b --- /dev/null +++ b/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js @@ -0,0 +1,196 @@ +// LICENSE +// +// This software is dual-licensed to the public domain and under the following +// license: you are granted a perpetual, irrevocable license to copy, modify, +// publish, and distribute this file as you see fit. +// +// VERSION +// 0.1.0 (2016-03-28) Initial release +// +// AUTHOR +// Forrest Smith +// +// CONTRIBUTORS +// J�rgen Tjern� - async helper +// Anurag Awasthi - updated to 0.2.0 + +const SEQUENTIAL_BONUS = 15; // bonus for adjacent matches +const SEPARATOR_BONUS = 30; // bonus if match occurs after a separator +const CAMEL_BONUS = 30; // bonus if match is uppercase and prev is lower +const FIRST_LETTER_BONUS = 15; // bonus if the first letter is matched + +const LEADING_LETTER_PENALTY = -5; // penalty applied for every letter in str before the first match +const MAX_LEADING_LETTER_PENALTY = -15; // maximum penalty for leading letters +const UNMATCHED_LETTER_PENALTY = -1; + + +/** + * Does a fuzzy search to find pattern inside a string. + * @param {*} pattern string pattern to search for + * @param {*} str string string which is being searched + * @returns [boolean, number] a boolean which tells if pattern was + * found or not and a search score + */ +function fuzzy_match(pattern, str) { + const recursionCount = 0; + const recursionLimit = 10; + const matches = []; + const maxMatches = 256; + + return fuzzyMatchRecursive( + pattern, + str, + 0 /* patternCurIndex */, + 0 /* strCurrIndex */, + null /* srcMatces */, + matches, + maxMatches, + 0 /* nextMatch */, + recursionCount, + recursionLimit + ); +} + +function fuzzyMatchRecursive( + pattern, + str, + patternCurIndex, + strCurrIndex, + srcMatces, + matches, + maxMatches, + nextMatch, + recursionCount, + recursionLimit +) { + let outScore = 0; + + // Return if recursion limit is reached. + if (++recursionCount >= recursionLimit) { + return [false, outScore]; + } + + // Return if we reached ends of strings. + if (patternCurIndex === pattern.length || strCurrIndex === str.length) { + return [false, outScore]; + } + + // Recursion params + let recursiveMatch = false; + let bestRecursiveMatches = []; + let bestRecursiveScore = 0; + + // Loop through pattern and str looking for a match. + let firstMatch = true; + while (patternCurIndex < pattern.length && strCurrIndex < str.length) { + // Match found. + if ( + pattern[patternCurIndex].toLowerCase() === str[strCurrIndex].toLowerCase() + ) { + if (nextMatch >= maxMatches) { + return [false, outScore]; + } + + if (firstMatch && srcMatces) { + matches = [...srcMatces]; + firstMatch = false; + } + + const recursiveMatches = []; + const [matched, recursiveScore] = fuzzyMatchRecursive( + pattern, + str, + patternCurIndex, + strCurrIndex + 1, + matches, + recursiveMatches, + maxMatches, + nextMatch, + recursionCount, + recursionLimit + ); + + if (matched) { + // Pick best recursive score. + if (!recursiveMatch || recursiveScore > bestRecursiveScore) { + bestRecursiveMatches = [...recursiveMatches]; + bestRecursiveScore = recursiveScore; + } + recursiveMatch = true; + } + + matches[nextMatch++] = strCurrIndex; + ++patternCurIndex; + } + ++strCurrIndex; + } + + const matched = patternCurIndex === pattern.length; + + if (matched) { + outScore = 100; + + // Apply leading letter penalty + let penalty = LEADING_LETTER_PENALTY * matches[0]; + penalty = + penalty < MAX_LEADING_LETTER_PENALTY + ? MAX_LEADING_LETTER_PENALTY + : penalty; + outScore += penalty; + + //Apply unmatched penalty + const unmatched = str.length - nextMatch; + outScore += UNMATCHED_LETTER_PENALTY * unmatched; + + // Apply ordering bonuses + for (let i = 0; i < nextMatch; i++) { + const currIdx = matches[i]; + + if (i > 0) { + const prevIdx = matches[i - 1]; + if (currIdx == prevIdx + 1) { + outScore += SEQUENTIAL_BONUS; + } + } + + // Check for bonuses based on neighbor character value. + if (currIdx > 0) { + // Camel case + const neighbor = str[currIdx - 1]; + const curr = str[currIdx]; + if ( + neighbor !== neighbor.toUpperCase() && + curr !== curr.toLowerCase() + ) { + outScore += CAMEL_BONUS; + } + const isNeighbourSeparator = neighbor == "_" || neighbor == " "; + if (isNeighbourSeparator) { + outScore += SEPARATOR_BONUS; + } + } else { + // First letter + outScore += FIRST_LETTER_BONUS; + } + } + + // Return best result + if (recursiveMatch && (!matched || bestRecursiveScore > outScore)) { + // Recursive score is better than "this" + matches = [...bestRecursiveMatches]; + outScore = bestRecursiveScore; + return [true, outScore]; + } else if (matched) { + // "this" score is better than recursive + return [true, outScore]; + } else { + return [false, outScore]; + } + } + return [false, outScore]; +} + + +module.exports = { + fuzzy_match +}; diff --git a/frappe/public/js/frappe/ui/toolbar/search_utils.js b/frappe/public/js/frappe/ui/toolbar/search_utils.js index 9700276568..8572160737 100644 --- a/frappe/public/js/frappe/ui/toolbar/search_utils.js +++ b/frappe/public/js/frappe/ui/toolbar/search_utils.js @@ -1,4 +1,6 @@ frappe.provide('frappe.search'); +import { fuzzyMatch } from './fuzzy_match.js' + frappe.search.utils = { setup_recent: function() { @@ -533,101 +535,44 @@ frappe.search.utils = { }, fuzzy_search: function(keywords, _item) { - // Returns 10 for case-perfect contain, 0 for not found - // 9 for perfect contain, - // 0 - 6 for fuzzy contain - - // **Specific use-case step** - keywords = keywords || ''; - var item = __(_item || ''); - var item_without_hyphen = item.replace(/-/g, " "); - - var item_length = item.length; - var query_length = keywords.length; - var length_ratio = query_length / item_length; - var max_skips = 3, max_mismatch_len = 2; - - if (query_length > item_length) { - return 0; - } - - // check for perfect string matches or - // matches that start with the keyword - if ([item, item_without_hyphen].includes(keywords) - || [item, item_without_hyphen].some((txt) => txt.toLowerCase().indexOf(keywords) === 0)) { - return 10 + length_ratio; - } - - if (item.indexOf(keywords) !== -1 && keywords !== keywords.toLowerCase()) { - return 9 + length_ratio; - } - - item = item.toLowerCase(); - keywords = keywords.toLowerCase(); - - if (item.indexOf(keywords) !== -1) { - return 8 + length_ratio; - } - - var skips = 0, mismatches = 0; - outer: for (var i = 0, j = 0; i < query_length; i++) { - if (mismatches !== 0) skips++; - if (skips > max_skips) return 0; - var k_ch = keywords.charCodeAt(i); - mismatches = 0; - while (j < item_length) { - if (item.charCodeAt(j++) === k_ch) { - continue outer; - } - if(++mismatches > max_mismatch_len) return 0 ; - } - return 0; - } - - // Since indexOf didn't pass, there will be atleast 1 skip - // hence no divide by zero, but just to be safe - if((skips + mismatches) > 0) { - return (5 + length_ratio)/(skips + mismatches); - } else { - return 0; - } + var match = fuzzyMatch(keywords, _item); + return match[1]; }, bolden_match_part: function(str, subseq) { - var rendered = ""; - if(this.fuzzy_search(subseq, str) === 0) { + if(fuzzyMatch(subseq, str)[0] === false) { return str; - } else if(this.fuzzy_search(subseq, str) > 6) { - var regEx = new RegExp("("+ subseq +")", "ig"); - return str.replace(regEx, '$1'); - } else { - var str_orig = str; - var str = str.toLowerCase(); - var str_len = str.length; - var subseq = subseq.toLowerCase(); - - outer: for(var i = 0, j = 0; i < subseq.length; i++) { - var sub_ch = subseq.charCodeAt(i); - while(j < str_len) { - if(str.charCodeAt(j) === sub_ch) { - var str_char = str_orig.charAt(j); - if(str_char === str_char.toLowerCase()) { - rendered += '' + subseq.charAt(i) + ''; - } else { - rendered += '' + subseq.charAt(i).toUpperCase() + ''; - } - j++; - continue outer; - } - rendered += str_orig.charAt(j); - j++; - } - return str_orig; - } - rendered += str_orig.slice(j); - return rendered; } - + if(str.indexOf(subseq) == 0) { + var tail = str.split(subseq)[1]; + return '' + subseq + '' + tail; + } + var rendered = ""; + var str_orig = str; + var str = str.toLowerCase(); + var str_len = str.length; + var subseq = subseq.toLowerCase(); + + outer: for(var i = 0, j = 0; i < subseq.length; i++) { + var sub_ch = subseq.charCodeAt(i); + while(j < str_len) { + if(str.charCodeAt(j) === sub_ch) { + var str_char = str_orig.charAt(j); + if(str_char === str_char.toLowerCase()) { + rendered += '' + subseq.charAt(i) + ''; + } else { + rendered += '' + subseq.charAt(i).toUpperCase() + ''; + } + j++; + continue outer; + } + rendered += str_orig.charAt(j); + j++; + } + return str_orig; + } + rendered += str_orig.slice(j); + return rendered; }, get_executables(keywords) { From 096d6459f7aa6fb84aff726ce59125a0598b03c9 Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Fri, 1 Apr 2022 10:57:48 +0530 Subject: [PATCH 16/55] fix: Fix case conversions --- .../js/frappe/ui/toolbar/fuzzy_match.js | 150 +++++++++--------- .../js/frappe/ui/toolbar/search_utils.js | 6 +- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js b/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js index 755882384b..0d122faad4 100644 --- a/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js +++ b/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js @@ -32,103 +32,103 @@ const UNMATCHED_LETTER_PENALTY = -1; * found or not and a search score */ function fuzzy_match(pattern, str) { - const recursionCount = 0; - const recursionLimit = 10; + const recursion_count = 0; + const recursion_limit = 10; const matches = []; - const maxMatches = 256; + const max_matches = 256; - return fuzzyMatchRecursive( + return fuzzy_match_recursive( pattern, str, - 0 /* patternCurIndex */, - 0 /* strCurrIndex */, - null /* srcMatces */, + 0 /* pattern_cur_index */, + 0 /* str_curr_index */, + null /* src_matces */, matches, - maxMatches, - 0 /* nextMatch */, - recursionCount, - recursionLimit + max_matches, + 0 /* next_match */, + recursion_count, + recursion_limit ); } -function fuzzyMatchRecursive( +function fuzzy_match_recursive( pattern, str, - patternCurIndex, - strCurrIndex, - srcMatces, + pattern_cur_index, + str_curr_index, + src_matces, matches, - maxMatches, - nextMatch, - recursionCount, - recursionLimit + max_matches, + next_match, + recursion_count, + recursion_limit ) { - let outScore = 0; + let out_score = 0; // Return if recursion limit is reached. - if (++recursionCount >= recursionLimit) { - return [false, outScore]; + if (++recursion_count >= recursion_limit) { + return [false, out_score]; } // Return if we reached ends of strings. - if (patternCurIndex === pattern.length || strCurrIndex === str.length) { - return [false, outScore]; + if (pattern_cur_index === pattern.length || str_curr_index === str.length) { + return [false, out_score]; } // Recursion params - let recursiveMatch = false; - let bestRecursiveMatches = []; - let bestRecursiveScore = 0; + let recursive_match = false; + let best_recursive_matches = []; + let best_recursive_score = 0; // Loop through pattern and str looking for a match. - let firstMatch = true; - while (patternCurIndex < pattern.length && strCurrIndex < str.length) { + let first_match = true; + while (pattern_cur_index < pattern.length && str_curr_index < str.length) { // Match found. if ( - pattern[patternCurIndex].toLowerCase() === str[strCurrIndex].toLowerCase() + pattern[pattern_cur_index].toLowerCase() === str[str_curr_index].toLowerCase() ) { - if (nextMatch >= maxMatches) { - return [false, outScore]; + if (next_match >= max_matches) { + return [false, out_score]; } - if (firstMatch && srcMatces) { - matches = [...srcMatces]; - firstMatch = false; + if (first_match && src_matces) { + matches = [...src_matces]; + first_match = false; } - const recursiveMatches = []; - const [matched, recursiveScore] = fuzzyMatchRecursive( + const recursive_matches = []; + const [matched, recursive_score] = fuzzy_match_recursive( pattern, str, - patternCurIndex, - strCurrIndex + 1, + pattern_cur_index, + str_curr_index + 1, matches, - recursiveMatches, - maxMatches, - nextMatch, - recursionCount, - recursionLimit + recursive_matches, + max_matches, + next_match, + recursion_count, + recursion_limit ); if (matched) { // Pick best recursive score. - if (!recursiveMatch || recursiveScore > bestRecursiveScore) { - bestRecursiveMatches = [...recursiveMatches]; - bestRecursiveScore = recursiveScore; + if (!recursive_match || recursive_score > best_recursive_score) { + best_recursive_matches = [...recursive_matches]; + best_recursive_score = recursive_score; } recursiveMatch = true; } - matches[nextMatch++] = strCurrIndex; - ++patternCurIndex; + matches[next_match++] = str_curr_index; + ++pattern_cur_index; } - ++strCurrIndex; + ++str_curr_index; } - const matched = patternCurIndex === pattern.length; + const matched = pattern_cur_index === pattern.length; if (matched) { - outScore = 100; + out_score = 100; // Apply leading letter penalty let penalty = LEADING_LETTER_PENALTY * matches[0]; @@ -136,58 +136,58 @@ function fuzzyMatchRecursive( penalty < MAX_LEADING_LETTER_PENALTY ? MAX_LEADING_LETTER_PENALTY : penalty; - outScore += penalty; + out_score += penalty; //Apply unmatched penalty - const unmatched = str.length - nextMatch; - outScore += UNMATCHED_LETTER_PENALTY * unmatched; + const unmatched = str.length - next_match; + out_score += UNMATCHED_LETTER_PENALTY * unmatched; // Apply ordering bonuses - for (let i = 0; i < nextMatch; i++) { - const currIdx = matches[i]; + for (let i = 0; i < next_match; i++) { + const curr_idx = matches[i]; if (i > 0) { - const prevIdx = matches[i - 1]; - if (currIdx == prevIdx + 1) { - outScore += SEQUENTIAL_BONUS; + const prev_idx = matches[i - 1]; + if (curr_idx == prev_idx + 1) { + out_score += SEQUENTIAL_BONUS; } } // Check for bonuses based on neighbor character value. - if (currIdx > 0) { + if (curr_idx > 0) { // Camel case - const neighbor = str[currIdx - 1]; - const curr = str[currIdx]; + const neighbor = str[curr_idx - 1]; + const curr = str[curr_idx]; if ( neighbor !== neighbor.toUpperCase() && curr !== curr.toLowerCase() ) { - outScore += CAMEL_BONUS; + out_score += CAMEL_BONUS; } - const isNeighbourSeparator = neighbor == "_" || neighbor == " "; - if (isNeighbourSeparator) { - outScore += SEPARATOR_BONUS; + const is_neighbour_separator = neighbor == "_" || neighbor == " "; + if (is_neighbour_separator) { + out_score += SEPARATOR_BONUS; } } else { // First letter - outScore += FIRST_LETTER_BONUS; + out_score += FIRST_LETTER_BONUS; } } // Return best result - if (recursiveMatch && (!matched || bestRecursiveScore > outScore)) { + if (recursive_match && (!matched || best_recursive_score > out_score)) { // Recursive score is better than "this" - matches = [...bestRecursiveMatches]; - outScore = bestRecursiveScore; - return [true, outScore]; + matches = [...best_recursive_matches]; + out_score = best_recursive_score; + return [true, out_score]; } else if (matched) { // "this" score is better than recursive - return [true, outScore]; + return [true, out_score]; } else { - return [false, outScore]; + return [false, out_score]; } } - return [false, outScore]; + return [false, out_score]; } diff --git a/frappe/public/js/frappe/ui/toolbar/search_utils.js b/frappe/public/js/frappe/ui/toolbar/search_utils.js index 8572160737..23da05e2b6 100644 --- a/frappe/public/js/frappe/ui/toolbar/search_utils.js +++ b/frappe/public/js/frappe/ui/toolbar/search_utils.js @@ -1,5 +1,5 @@ frappe.provide('frappe.search'); -import { fuzzyMatch } from './fuzzy_match.js' +import { fuzzy_match } from './fuzzy_match.js' frappe.search.utils = { @@ -535,12 +535,12 @@ frappe.search.utils = { }, fuzzy_search: function(keywords, _item) { - var match = fuzzyMatch(keywords, _item); + var match = fuzzy_match(keywords, _item); return match[1]; }, bolden_match_part: function(str, subseq) { - if(fuzzyMatch(subseq, str)[0] === false) { + if(fuzzy_match(subseq, str)[0] === false) { return str; } if(str.indexOf(subseq) == 0) { From 49e1e54e4a933144ef230a9bd6f171f186f3b46c Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Fri, 1 Apr 2022 11:21:14 +0530 Subject: [PATCH 17/55] fix: Fix sider issues --- .../js/frappe/ui/toolbar/search_utils.js | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/frappe/public/js/frappe/ui/toolbar/search_utils.js b/frappe/public/js/frappe/ui/toolbar/search_utils.js index 23da05e2b6..c9fe3f85a8 100644 --- a/frappe/public/js/frappe/ui/toolbar/search_utils.js +++ b/frappe/public/js/frappe/ui/toolbar/search_utils.js @@ -1,5 +1,5 @@ frappe.provide('frappe.search'); -import { fuzzy_match } from './fuzzy_match.js' +import { fuzzy_match } from './fuzzy_match.js'; frappe.search.utils = { @@ -535,44 +535,44 @@ frappe.search.utils = { }, fuzzy_search: function(keywords, _item) { - var match = fuzzy_match(keywords, _item); - return match[1]; + var match = fuzzy_match(keywords, _item); + return match[1]; }, bolden_match_part: function(str, subseq) { if(fuzzy_match(subseq, str)[0] === false) { return str; } - if(str.indexOf(subseq) == 0) { - var tail = str.split(subseq)[1]; - return '' + subseq + '' + tail; - } - var rendered = ""; - var str_orig = str; - var str = str.toLowerCase(); - var str_len = str.length; - var subseq = subseq.toLowerCase(); - - outer: for(var i = 0, j = 0; i < subseq.length; i++) { - var sub_ch = subseq.charCodeAt(i); - while(j < str_len) { - if(str.charCodeAt(j) === sub_ch) { - var str_char = str_orig.charAt(j); - if(str_char === str_char.toLowerCase()) { - rendered += '' + subseq.charAt(i) + ''; - } else { - rendered += '' + subseq.charAt(i).toUpperCase() + ''; - } - j++; - continue outer; - } - rendered += str_orig.charAt(j); - j++; - } - return str_orig; - } - rendered += str_orig.slice(j); - return rendered; + if(str.indexOf(subseq) == 0) { + var tail = str.split(subseq)[1]; + return '' + subseq + '' + tail; + } + var rendered = ""; + var str_orig = str; + var str = str.toLowerCase(); + var str_len = str.length; + var subseq = subseq.toLowerCase(); + + outer: for(var i = 0, j = 0; i < subseq.length; i++) { + var sub_ch = subseq.charCodeAt(i); + while(j < str_len) { + if(str.charCodeAt(j) === sub_ch) { + var str_char = str_orig.charAt(j); + if(str_char === str_char.toLowerCase()) { + rendered += '' + subseq.charAt(i) + ''; + } else { + rendered += '' + subseq.charAt(i).toUpperCase() + ''; + } + j++; + continue outer; + } + rendered += str_orig.charAt(j); + j++; + } + return str_orig; + } + rendered += str_orig.slice(j); + return rendered; }, get_executables(keywords) { From c94edc21f41a4a44feb2ef235fd6546fc576da13 Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Fri, 1 Apr 2022 11:24:30 +0530 Subject: [PATCH 18/55] fix: Code clean up --- .../js/frappe/ui/toolbar/fuzzy_match.js | 17 +++---- .../js/frappe/ui/toolbar/search_utils.js | 48 +++++++++---------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js b/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js index 0d122faad4..59fccbccc5 100644 --- a/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js +++ b/frappe/public/js/frappe/ui/toolbar/fuzzy_match.js @@ -31,7 +31,7 @@ const UNMATCHED_LETTER_PENALTY = -1; * @returns [boolean, number] a boolean which tells if pattern was * found or not and a search score */ -function fuzzy_match(pattern, str) { +export function fuzzy_match(pattern, str) { const recursion_count = 0; const recursion_limit = 10; const matches = []; @@ -42,7 +42,7 @@ function fuzzy_match(pattern, str) { str, 0 /* pattern_cur_index */, 0 /* str_curr_index */, - null /* src_matces */, + null /* src_matches */, matches, max_matches, 0 /* next_match */, @@ -56,7 +56,7 @@ function fuzzy_match_recursive( str, pattern_cur_index, str_curr_index, - src_matces, + src_matches, matches, max_matches, next_match, @@ -91,8 +91,8 @@ function fuzzy_match_recursive( return [false, out_score]; } - if (first_match && src_matces) { - matches = [...src_matces]; + if (first_match && src_matches) { + matches = [...src_matches]; first_match = false; } @@ -116,7 +116,7 @@ function fuzzy_match_recursive( best_recursive_matches = [...recursive_matches]; best_recursive_score = recursive_score; } - recursiveMatch = true; + recursive_match = true; } matches[next_match++] = str_curr_index; @@ -189,8 +189,3 @@ function fuzzy_match_recursive( } return [false, out_score]; } - - -module.exports = { - fuzzy_match -}; diff --git a/frappe/public/js/frappe/ui/toolbar/search_utils.js b/frappe/public/js/frappe/ui/toolbar/search_utils.js index c9fe3f85a8..db411feffd 100644 --- a/frappe/public/js/frappe/ui/toolbar/search_utils.js +++ b/frappe/public/js/frappe/ui/toolbar/search_utils.js @@ -540,36 +540,36 @@ frappe.search.utils = { }, bolden_match_part: function(str, subseq) { - if(fuzzy_match(subseq, str)[0] === false) { + if (fuzzy_match(subseq, str)[0] === false) { return str; } - if(str.indexOf(subseq) == 0) { - var tail = str.split(subseq)[1]; - return '' + subseq + '' + tail; + if (str.indexOf(subseq) == 0) { + var tail = str.split(subseq)[1]; + return '' + subseq + '' + tail; } var rendered = ""; var str_orig = str; - var str = str.toLowerCase(); var str_len = str.length; - var subseq = subseq.toLowerCase(); - - outer: for(var i = 0, j = 0; i < subseq.length; i++) { - var sub_ch = subseq.charCodeAt(i); - while(j < str_len) { - if(str.charCodeAt(j) === sub_ch) { - var str_char = str_orig.charAt(j); - if(str_char === str_char.toLowerCase()) { - rendered += '' + subseq.charAt(i) + ''; - } else { - rendered += '' + subseq.charAt(i).toUpperCase() + ''; - } - j++; - continue outer; - } - rendered += str_orig.charAt(j); - j++; - } - return str_orig; + str = str.toLowerCase(); + subseq = subseq.toLowerCase(); + + outer: for (var i = 0, j = 0; i < subseq.length; i++) { + var sub_ch = subseq.charCodeAt(i); + while (j < str_len) { + if (str.charCodeAt(j) === sub_ch) { + var str_char = str_orig.charAt(j); + if (str_char === str_char.toLowerCase()) { + rendered += '' + subseq.charAt(i) + ''; + } else { + rendered += '' + subseq.charAt(i).toUpperCase() + ''; + } + j++; + continue outer; + } + rendered += str_orig.charAt(j); + j++; + } + return str_orig; } rendered += str_orig.slice(j); return rendered; From f406aab62d88e82d298b3a21de0de758d3fda900 Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Fri, 1 Apr 2022 12:30:06 +0530 Subject: [PATCH 19/55] fix: Fix search issue with translated strings --- frappe/public/js/frappe/ui/toolbar/search_utils.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/toolbar/search_utils.js b/frappe/public/js/frappe/ui/toolbar/search_utils.js index db411feffd..1571522489 100644 --- a/frappe/public/js/frappe/ui/toolbar/search_utils.js +++ b/frappe/public/js/frappe/ui/toolbar/search_utils.js @@ -535,7 +535,9 @@ frappe.search.utils = { }, fuzzy_search: function(keywords, _item) { - var match = fuzzy_match(keywords, _item); + keywords = keywords || ''; + var item = __(_item || ''); + var match = fuzzy_match(keywords, item); return match[1]; }, From 7e1a0ed5de92fefa183b0626cec5d9ba62b8a687 Mon Sep 17 00:00:00 2001 From: phot0n Date: Thu, 31 Mar 2022 22:20:39 +0530 Subject: [PATCH 20/55] test: test for fieldname which start with int --- frappe/tests/test_db_query.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 62842dc167..47a5e775ee 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -514,6 +514,33 @@ class TestReportview(unittest.TestCase): dt.delete(ignore_permissions=True) + def test_fieldname_starting_with_int(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype + + dt = new_doctype( + "dt_with_int_named_fieldname", + fields=[{ + "label": "1field", + "fieldname": "1field", + "fieldtype": "Int" + }] + ).insert(ignore_permissions=True) + + frappe.get_doc({ + "doctype": "dt_with_int_named_fieldname", + "1field": 10 + }).insert(ignore_permissions=True) + + + query = DatabaseQuery("dt_with_int_named_fieldname") + self.assertTrue(query.execute(filters={"1field": 10})) + self.assertTrue(query.execute(filters={"1field": ["like", "1%"]})) + self.assertTrue(query.execute(filters={"1field": ["in", "1,2,10"]})) + self.assertTrue(query.execute(filters={"1field": ["is", "set"]})) + self.assertFalse(query.execute(filters={"1field": ["not like", "1%"]})) + + dt.delete() + def add_child_table_to_blog_post(): child_table = frappe.get_doc({ From 3cabfdaa1d9be4cde164208f0fe5b811c946e2f0 Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Tue, 5 Apr 2022 09:18:54 +0530 Subject: [PATCH 21/55] test: Fix cypress tests --- cypress/integration/awesome_bar.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cypress/integration/awesome_bar.js b/cypress/integration/awesome_bar.js index 8e503cce46..053d015366 100644 --- a/cypress/integration/awesome_bar.js +++ b/cypress/integration/awesome_bar.js @@ -13,7 +13,7 @@ context('Awesome Bar', () => { it('navigates to doctype list', () => { cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('todo', { delay: 700 }); cy.get('.awesomplete').findByRole('listbox').should('be.visible'); - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('{downarrow}{enter}', { delay: 700 }); + cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('{enter}', { delay: 700 }); cy.get('.title-text').should('contain', 'To Do'); @@ -22,7 +22,7 @@ context('Awesome Bar', () => { it('find text in doctype list', () => { cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('test in todo{downarrow}{enter}', { delay: 700 }); + .type('test in todo{enter}', { delay: 700 }); cy.get('.title-text').should('contain', 'To Do'); @@ -32,7 +32,7 @@ context('Awesome Bar', () => { it('navigates to new form', () => { cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('new blog post{downarrow}{enter}', { delay: 700 }); + .type('new blog post{enter}', { delay: 700 }); cy.get('.title-text:visible').should('have.text', 'New Blog Post'); }); From b9af0ac37e47cc54bc77aa97794927ba2bb51aeb Mon Sep 17 00:00:00 2001 From: phot0n Date: Mon, 4 Apr 2022 18:30:33 +0530 Subject: [PATCH 22/55] feat(minor): use specific email signature via from field * fix: increase debounce timeout so that events dont clash * fix: signature getting appended multiple time upon dialog open/close --- .../js/frappe/form/footer/form_timeline.js | 4 +- frappe/public/js/frappe/ui/field_group.js | 4 ++ .../public/js/frappe/views/communication.js | 43 ++++++++++++++----- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/form/footer/form_timeline.js b/frappe/public/js/frappe/form/footer/form_timeline.js index 0070d384d7..203859a144 100644 --- a/frappe/public/js/frappe/form/footer/form_timeline.js +++ b/frappe/public/js/frappe/form/footer/form_timeline.js @@ -429,13 +429,13 @@ class FormTimeline extends BaseTimeline { } if (this.frm.doctype === "Communication") { - args.txt = ""; + args.message = ""; args.last_email = this.frm.doc; args.recipients = this.frm.doc.sender; args.subject = __("Re: {0}", [this.frm.doc.subject]); } else { const comment_value = frappe.markdown(this.frm.comment_box.get_value()); - args.txt = strip_html(comment_value) ? comment_value : ''; + args.message = strip_html(comment_value) ? comment_value : ''; } new frappe.views.CommunicationComposer(args); diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index 1936f5115e..178d1a65cb 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -138,6 +138,10 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { }); } + has_field(fieldname) { + return !!this.fields_dict[fieldname]; + } + set_input(key, val) { return this.set_value(key, val); } diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index 1d219a7044..2c420dd1be 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -97,7 +97,7 @@ frappe.views.CommunicationComposer = class { fieldname: "content", onchange: frappe.utils.debounce( this.save_as_draft.bind(this), - 300 + 500 ) }, { fieldtype: "Section Break" }, @@ -153,7 +153,14 @@ frappe.views.CommunicationComposer = class { fieldname: "sender", options: email_accounts.map(function(e) { return e.email_id; - }) + }), + change: async () => { + let sender_email = this.dialog.get_value("sender"); + this.reply_set = !!sender_email; + this.content_set = sender_email && this.sender && this.sender != sender_email; + await this.set_content(sender_email); + this.sender = sender_email; + } }); } @@ -239,7 +246,9 @@ frappe.views.CommunicationComposer = class { // some email clients (outlook) may not send the message id to identify // the thread. So as a backup we use the name of the document as identifier const identifier = `#${this.frm.doc.name}`; - if (!this.subject.includes(identifier)) { + + // converting to str for int names + if (!cstr(this.subject).includes(identifier)) { this.subject = `${this.subject} (${identifier})`; } } @@ -350,7 +359,7 @@ frappe.views.CommunicationComposer = class { } async set_values_from_last_edited_communication() { - if (this.txt || this.message) return; + if (this.message) return; const last_edited = this.get_last_edited_communication(); if (!last_edited.content) return; @@ -709,10 +718,10 @@ frappe.views.CommunicationComposer = class { } } - async set_content() { + async set_content(sender_email) { if (this.content_set) return; - let message = this.txt || this.message || ""; + let message = this.message || ""; if (!message && this.frm) { const { doctype, docname } = this.frm; message = await localforage.getItem(doctype + docname) || ""; @@ -722,22 +731,32 @@ frappe.views.CommunicationComposer = class { this.content_set = true; } - message += await this.get_signature(); + message += await this.get_signature(sender_email || null); - if (this.is_a_reply) { + if (this.is_a_reply && !this.reply_set) { message += this.get_earlier_reply(); } await this.dialog.set_value("content", message); } - async get_signature() { + async get_signature(sender_email) { let signature = frappe.boot.user.email_signature; if (!signature) { + let filters = {}; + if (sender_email) { + filters['email_id'] = sender_email; + } else { + if (this.dialog.has_field("sender")) return ""; + + filters['default_outgoing'] = 1; + } + filters['add_signature'] = 1; + const response = await frappe.db.get_value( 'Email Account', - {'default_outgoing': 1, 'add_signature': 1}, + filters, 'signature' ); @@ -754,6 +773,8 @@ frappe.views.CommunicationComposer = class { } get_earlier_reply() { + this.reply_set = false; + const last_email = ( this.last_email || this.frm && this.frm.timeline.get_last_email(true) @@ -777,6 +798,8 @@ frappe.views.CommunicationComposer = class { last_email.communication_date || last_email.creation ); + this.reply_set = true; + return `

${separator_element || ''} From 4c901a7074cdc0da13e549420f0b0ba636c7fbdd Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 5 Apr 2022 20:38:42 +0200 Subject: [PATCH 23/55] fix: stabilize leaflet map - disable flyToBounds - invalidateSize after loading to avoid gray tiles --- frappe/public/js/frappe/form/controls/geolocation.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/geolocation.js b/frappe/public/js/frappe/form/controls/geolocation.js index 280eac3941..7a1df64f28 100644 --- a/frappe/public/js/frappe/form/controls/geolocation.js +++ b/frappe/public/js/frappe/form/controls/geolocation.js @@ -58,7 +58,7 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f })); this.add_non_group_layers(data_layers, this.editableLayers); try { - this.map.flyToBounds(this.editableLayers.getBounds(), { + this.map.fitBounds(this.editableLayers.getBounds(), { padding: [50,50] }); } @@ -66,10 +66,10 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f // suppress error if layer has a point. } this.editableLayers.addTo(this.map); - this.map._onResize(); - } else if ((value===undefined) || (value == JSON.stringify(new L.FeatureGroup().toGeoJSON()))) { - this.locate_control.start(); + } else { + this.map.setView(frappe.utils.map_defaults.center, frappe.utils.map_defaults.zoom); } + this.map.invalidateSize(); } bind_leaflet_map() { @@ -97,8 +97,7 @@ frappe.ui.form.ControlGeolocation = class ControlGeolocation extends frappe.ui.f }); L.Icon.Default.imagePath = '/assets/frappe/images/leaflet/'; - this.map = L.map(this.map_id).setView(frappe.utils.map_defaults.center, - frappe.utils.map_defaults.zoom); + this.map = L.map(this.map_id); L.tileLayer(frappe.utils.map_defaults.tiles, frappe.utils.map_defaults.options).addTo(this.map); From 253d0cabb43e3ce045929c712ee38ff7d5cf9d70 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 6 Apr 2022 10:04:33 +0530 Subject: [PATCH 24/55] fix: Move custom-actions under page-actions Change in custom-actions was affecting custom-actions in timeline the issue was introduced with https://github.com/frappe/frappe/pull/16470 --- frappe/public/scss/desk/page.scss | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/public/scss/desk/page.scss b/frappe/public/scss/desk/page.scss index 5206e2919c..b0a24eed38 100644 --- a/frappe/public/scss/desk/page.scss +++ b/frappe/public/scss/desk/page.scss @@ -51,11 +51,6 @@ } } -.custom-actions { - display: flex; - align-items: center; -} - .page-actions { align-items: center; .btn { @@ -72,6 +67,11 @@ .custom-btn-group { display: inline-flex; } + + .custom-actions { + display: flex; + align-items: center; + } } .layout-main-section-wrapper { From 7c467366549d89253d9f19392d0958cfd64c1ec8 Mon Sep 17 00:00:00 2001 From: ChillarAnand Date: Wed, 6 Apr 2022 12:26:26 +0530 Subject: [PATCH 25/55] fix: Use app name for parsing app name --- frappe/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/installer.py b/frappe/installer.py index ffb595e9ad..c7dacc4ac1 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -222,7 +222,7 @@ def install_app(name, verbose=False, set_as_patched=True): # install pre-requisites if app_hooks.required_apps: for app in app_hooks.required_apps: - name = parse_app_name(name) + name = parse_app_name(app) install_app(name, verbose=verbose) frappe.flags.in_install = name From 9943423a1ff3897ed1b9e04d6916aedb8d430255 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 6 Apr 2022 16:10:14 +0530 Subject: [PATCH 26/55] fix: strip html from blog comments to prevent spam --- frappe/templates/includes/comments/comment.html | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/frappe/templates/includes/comments/comment.html b/frappe/templates/includes/comments/comment.html index e0fc1c3c54..5be36d65a9 100644 --- a/frappe/templates/includes/comments/comment.html +++ b/frappe/templates/includes/comments/comment.html @@ -1,13 +1,17 @@ {% from "frappe/templates/includes/avatar_macro.html" import avatar %} -
+
- {{ avatar(user_id=(comment.comment_email or comment.sender), size='avatar-medium') }} + {{ avatar(user_id=(frappe.utils.strip_html(comment.comment_email or comment.sender)), size='avatar-medium') }}
-
- {{ comment.sender_full_name or comment.comment_by }} - {{ frappe.utils.pretty_date(comment.creation) }} +
+ + {{ frappe.utils.strip_html(comment.sender_full_name or comment.comment_by) | e }} + + + {{ frappe.utils.pretty_date(comment.creation) }} +
{{ comment.content | markdown }}
From e13c74b53ffba6be8989e2f52d1875c46c87717e Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 6 Apr 2022 16:28:57 +0530 Subject: [PATCH 27/55] fix: strip html from comment content --- frappe/templates/includes/comments/comment.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/includes/comments/comment.html b/frappe/templates/includes/comments/comment.html index 5be36d65a9..4713ee498d 100644 --- a/frappe/templates/includes/comments/comment.html +++ b/frappe/templates/includes/comments/comment.html @@ -13,6 +13,6 @@ {{ frappe.utils.pretty_date(comment.creation) }}
-
{{ comment.content | markdown }}
+
{{ frappe.utils.strip_html(comment.content) | markdown }}
\ No newline at end of file From ae335f5e1c96ac8febb7a42ac6c19b6ae3393a70 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 6 Apr 2022 16:29:22 +0530 Subject: [PATCH 28/55] test: spam links shouldn't render on blog post --- .../doctype/blog_post/test_blog_post.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frappe/website/doctype/blog_post/test_blog_post.py b/frappe/website/doctype/blog_post/test_blog_post.py index d649d25f7e..575b6c0fc0 100644 --- a/frappe/website/doctype/blog_post/test_blog_post.py +++ b/frappe/website/doctype/blog_post/test_blog_post.py @@ -117,6 +117,34 @@ class TestBlogPost(unittest.TestCase): frappe.flags.force_website_cache = True + def test_spam_comments(self): + # Make a temporary Blog Post (and a Blog Category) + blog = make_test_blog('Test Spam Comment') + + # Create a spam comment + frappe.get_doc( + doctype="Comment", + comment_type="Comment", + reference_doctype="Blog Post", + reference_name=blog.name, + comment_email="spam", + comment_by="spam", + published=1, + content="More spam content. spam with link.", + ).insert() + + # Visit the blog post page + set_request(path=blog.route) + blog_page_response = get_response() + blog_page_html = frappe.safe_decode(blog_page_response.get_data()) + + self.assertNotIn('spam', blog_page_html) + self.assertIn("More spam content. spam with link.", blog_page_html) + + # Cleanup + frappe.delete_doc("Blog Post", blog.name) + frappe.delete_doc("Blog Category", blog.blog_category) + def scrub(text): return WebsiteGenerator.scrub(None, text) From 5483caa8f59074b53ef541f0c26d0e1a9246dbd9 Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Wed, 6 Apr 2022 18:56:11 +0530 Subject: [PATCH 29/55] test: Added test script for control type "Date" --- cypress/integration/control_date.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index c45602b4d9..0feffbb05d 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -63,14 +63,13 @@ context('Date Control', () => { cy.get('.datepicker--button').click(); //Picking up the todays date - const todaysDate = Cypress.moment().format('MM-DD-YYYY'); - cy.log(todaysDate); + const todays_date = Cypress.moment().format('MM-DD-YYYY'); //Verifying if clicking on "Today" button matches today's date - cy.get_field('date', 'Date').should('have.value', todaysDate); + cy.get_field('date', 'Date').should('have.value', todays_date); }); - it.only('Configuring first day of the week', () => { + it('Configuring first day of the week', () => { //Visiting "System Settings" page cy.visit('/app/system-settings/System%20Settings'); From 27cbcc3e8dfd0eb2e7a4fd4027cf0d728a1c2ef9 Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 Date: Thu, 7 Apr 2022 11:19:08 +0530 Subject: [PATCH 30/55] test: Removed first_day_of_week test case --- cypress/integration/control_date.js | 48 ----------------------------- 1 file changed, 48 deletions(-) diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index 0feffbb05d..35c585306c 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -68,52 +68,4 @@ context('Date Control', () => { //Verifying if clicking on "Today" button matches today's date cy.get_field('date', 'Date').should('have.value', todays_date); }); - - it('Configuring first day of the week', () => { - //Visiting "System Settings" page - cy.visit('/app/system-settings/System%20Settings'); - - //Visiting the "Date and Number Format" section - cy.contains('Date and Number Format').click(); - - //Changing the configuration for "First day of the week" field - cy.get('select[data-fieldname="first_day_of_the_week"]').select('Tuesday'); - cy.get('.page-head .page-actions').findByRole('button', {name: 'Save'}).click(); - cy.new_form('Test Date Control'); - cy.get_field('date', 'Date').click(); - - //Checking if the first day shown in the datepicker is the one which is configured in the System Settings Page - cy.get('.datepicker--days-names').eq(0).should('contain.text', 'Tu'); - cy.visit('/app/doctype'); - - //Adding filter in the doctype list - cy.add_filter(); - cy.get('.fieldname-select-area').type('Created On{enter}'); - cy.get('.filter-field > .form-group > .input-with-feedback').click(); - - //Checking if the first day shown in the datepicker is the one which is configured in the System Settings Page - cy.get('.datepicker--days-names').eq(0).should('contain.text', 'Tu'); - - //Adding event - cy.visit('/app/event'); - cy.click_listview_primary_button('Add Event'); - cy.get('textarea[data-fieldname=subject]').type('Test'); - cy.fill_field('starts_on', '01-01-2022 00:00:00', 'Datetime'); - cy.click_listview_primary_button('Save'); - cy.visit('/app/event'); - cy.get('.custom-btn-group > .btn').click(); - - //Opening Calendar view for the event created - cy.get('[data-view="Calendar"] > .grey-link').click(); - - //Checking if the calendar view has the first day as the configured day in the System Settings Page - cy.get('.fc-head-container').eq(0).should('contain.text', 'Tue'); - - //Deleting the created event - cy.visit('/app/event'); - cy.get('.list-row-checkbox').eq(0).click(); - cy.get('.actions-btn-group > .btn').contains('Actions').click(); - cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); - }); }); \ No newline at end of file From a096b85b52b84a26e9610c4c561f7d708eb2dd86 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Thu, 7 Apr 2022 12:23:41 +0530 Subject: [PATCH 31/55] fix: Read Only fields visible despite no value --- .../js/frappe/form/controls/base_control.js | 16 ++++++++++++---- frappe/public/js/frappe/form/controls/control.js | 1 - frappe/public/js/frappe/form/controls/data.js | 2 ++ .../public/js/frappe/form/controls/read_only.js | 8 -------- frappe/public/js/frappe/model/perm.js | 7 +++++-- frappe/public/js/frappe/ui/field_group.js | 13 ++++++------- 6 files changed, 25 insertions(+), 22 deletions(-) delete mode 100644 frappe/public/js/frappe/form/controls/read_only.js diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 4ee52d16b8..e22235f60f 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -44,6 +44,8 @@ frappe.ui.form.Control = class BaseControl { } if ((!this.doctype && !this.docname) || this.df.parenttype === 'Web Form' || this.df.is_web_form) { + let status = "Write"; + // like in case of a dialog box if (cint(this.df.hidden)) { // eslint-disable-next-line @@ -55,10 +57,10 @@ frappe.ui.form.Control = class BaseControl { if(explain) console.log("By Hidden Dependency: None"); // eslint-disable-line no-console return "None"; - } else if (cint(this.df.read_only || this.df.is_virtual)) { + } else if (cint(this.df.read_only || this.df.is_virtual || this.df.fieldtype === "Read Only")) { // eslint-disable-next-line if (explain) console.log("By Read Only: Read"); // eslint-disable-line no-console - return "Read"; + status = "Read"; } else if ((this.grid && this.grid.display_status == 'Read') || @@ -67,10 +69,16 @@ frappe.ui.form.Control = class BaseControl { this.layout.grid.display_status == 'Read')) { // parent grid is read if (explain) console.log("By Parent Grid Read-only: Read"); // eslint-disable-line no-console - return "Read"; + status = "Read"; } - return "Write"; + if ( + status === "Read" && + is_null(this.value) && + !in_list(["HTML", "Image", "Button"], this.df.fieldtype) + ) status = "None"; + + return status; } var status = frappe.perm.get_field_display_status(this.df, diff --git a/frappe/public/js/frappe/form/controls/control.js b/frappe/public/js/frappe/form/controls/control.js index bd04938e35..90e8697b1c 100644 --- a/frappe/public/js/frappe/form/controls/control.js +++ b/frappe/public/js/frappe/form/controls/control.js @@ -23,7 +23,6 @@ import './table'; import './color'; import './signature'; import './password'; -import './read_only'; import './button'; import './html'; import './markdown_editor'; diff --git a/frappe/public/js/frappe/form/controls/data.js b/frappe/public/js/frappe/form/controls/data.js index f4c9849528..95abba616a 100644 --- a/frappe/public/js/frappe/form/controls/data.js +++ b/frappe/public/js/frappe/form/controls/data.js @@ -262,3 +262,5 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp return this.grid || this.layout && this.layout.grid; } }; + +frappe.ui.form.ControlReadOnly = frappe.ui.form.ControlData; diff --git a/frappe/public/js/frappe/form/controls/read_only.js b/frappe/public/js/frappe/form/controls/read_only.js deleted file mode 100644 index 2f1d1a2bca..0000000000 --- a/frappe/public/js/frappe/form/controls/read_only.js +++ /dev/null @@ -1,8 +0,0 @@ -frappe.ui.form.ControlReadOnly = class ControlReadOnly extends frappe.ui.form.ControlData { - get_status(explain) { - var status = super.get_status(explain); - if(status==="Write") - status = "Read"; - return; - } -}; diff --git a/frappe/public/js/frappe/model/perm.js b/frappe/public/js/frappe/model/perm.js index 0eabfdd337..3ea9c6bc95 100644 --- a/frappe/public/js/frappe/model/perm.js +++ b/frappe/public/js/frappe/model/perm.js @@ -225,7 +225,10 @@ $.extend(frappe.perm, { if (explain) console.log("By Workflow:" + status); // read only field is checked - if (status === "Write" && cint(df.read_only)) { + if (status === "Write" && ( + cint(df.read_only) || + df.fieldtype === "Read Only" + )) { status = "Read"; } if (explain) console.log("By Read Only:" + status); @@ -276,4 +279,4 @@ $.extend(frappe.perm, { return allowed_docs; } } -}); \ No newline at end of file +}); diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index 1936f5115e..479c020fbb 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -22,17 +22,15 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { super.make(); this.refresh(); // set default - $.each(this.fields_list, function(i, field) { - if (field.df["default"]) { - let def_value = field.df["default"]; + $.each(this.fields_list, (_, field) => { + if (!is_null(field.df.default)) { + let def_value = field.df.default; - if (def_value == 'Today' && field.df["fieldtype"] == 'Date') { + if (def_value === "Today" && field.df.fieldtype === "Date") { def_value = frappe.datetime.get_today(); } - field.set_input(def_value); - // if default and has depends_on, render its fields. - me.refresh_dependency(); + this.set_value(field.df.fieldname, def_value); } }) @@ -129,6 +127,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { if (f) { f.set_value(val).then(() => { f.set_input(val); + f.refresh(); this.refresh_dependency(); resolve(); }); From 0a1595e94e5bd1af7c44bcf4fbd45f3d14438100 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 7 Apr 2022 12:57:11 +0530 Subject: [PATCH 32/55] fix: added add signature button if multiple sender emails available --- .../public/js/frappe/views/communication.js | 57 ++++++++++++------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index 2c420dd1be..6cfebd7ec8 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -100,6 +100,17 @@ frappe.views.CommunicationComposer = class { 500 ) }, + { + fieldtype: "Button", + label: __("Add Signature"), + fieldname: 'add_signature', + hidden: 1, + click: async () => { + let sender_email = this.dialog.get_value('sender') || ""; + this.content_set = false; + await this.set_content(sender_email); + } + }, { fieldtype: "Section Break" }, { label: __("Send me a copy"), @@ -146,21 +157,16 @@ frappe.views.CommunicationComposer = class { }); if (email_accounts.length) { + this.user_email_accounts = email_accounts.map(function(e) { + return e.email_id; + }); + fields.unshift({ label: __("From"), fieldtype: "Select", reqd: 1, fieldname: "sender", - options: email_accounts.map(function(e) { - return e.email_id; - }), - change: async () => { - let sender_email = this.dialog.get_value("sender"); - this.reply_set = !!sender_email; - this.content_set = sender_email && this.sender && this.sender != sender_email; - await this.set_content(sender_email); - this.sender = sender_email; - } + options: this.user_email_accounts }); } @@ -184,9 +190,15 @@ frappe.views.CommunicationComposer = class { this.setup_email(); this.setup_email_template(); this.setup_last_edited_communication(); + this.setup_add_signature_button(); this.set_values(); } + setup_add_signature_button() { + let is_sender = this.dialog.has_field('sender'); + this.dialog.set_df_property('add_signature', 'hidden', !is_sender); + } + setup_multiselect_queries() { ['recipients', 'cc', 'bcc'].forEach(field => { this.dialog.fields_dict[field].get_data = () => { @@ -744,23 +756,28 @@ frappe.views.CommunicationComposer = class { let signature = frappe.boot.user.email_signature; if (!signature) { - let filters = {}; + let filters = { + 'add_signature': 1 + }; + if (sender_email) { filters['email_id'] = sender_email; } else { - if (this.dialog.has_field("sender")) return ""; - filters['default_outgoing'] = 1; } - filters['add_signature'] = 1; - const response = await frappe.db.get_value( - 'Email Account', - filters, - 'signature' - ); + const email = await frappe.db.get_list("Email Account", { + filters: filters, + fields: ['signature', 'email_id'], + limit: 1 + }); - signature = response.message.signature; + signature = email && email[0].signature; + + if (this.user_email_accounts && + this.user_email_accounts.includes(email[0].email_id)) { + this.dialog.set_value('sender', email[0].email_id); + } } if (!signature) return ""; From 7679d5e82cc5e596ba78075649f211f7da81090f Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Thu, 7 Apr 2022 13:07:04 +0530 Subject: [PATCH 33/55] chore: better variable name --- frappe/public/js/frappe/views/communication.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/views/communication.js b/frappe/public/js/frappe/views/communication.js index 6cfebd7ec8..90d81c10e5 100755 --- a/frappe/public/js/frappe/views/communication.js +++ b/frappe/public/js/frappe/views/communication.js @@ -97,7 +97,7 @@ frappe.views.CommunicationComposer = class { fieldname: "content", onchange: frappe.utils.debounce( this.save_as_draft.bind(this), - 500 + 300 ) }, { @@ -195,8 +195,8 @@ frappe.views.CommunicationComposer = class { } setup_add_signature_button() { - let is_sender = this.dialog.has_field('sender'); - this.dialog.set_df_property('add_signature', 'hidden', !is_sender); + let has_sender = this.dialog.has_field('sender'); + this.dialog.set_df_property('add_signature', 'hidden', !has_sender); } setup_multiselect_queries() { From 428b0bc4b130545bda186daddc2575a9b64839ef Mon Sep 17 00:00:00 2001 From: kamaljohnson Date: Thu, 7 Apr 2022 19:46:53 +0530 Subject: [PATCH 34/55] feat: add after insert event to server script this option was not available in the events dropdown. --- frappe/core/doctype/server_script/server_script.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 520c0008c5..548d21bb60 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -49,7 +49,7 @@ "fieldname": "doctype_event", "fieldtype": "Select", "label": "DocType Event", - "options": "Before Insert\nBefore Validate\nBefore Save\nAfter Save\nBefore Submit\nAfter Submit\nBefore Cancel\nAfter Cancel\nBefore Delete\nAfter Delete\nBefore Save (Submitted Document)\nAfter Save (Submitted Document)" + "options": "Before Insert\nBefore Validate\nBefore Save\nAfter Insert\nAfter Save\nBefore Submit\nAfter Submit\nBefore Cancel\nAfter Cancel\nBefore Delete\nAfter Delete\nBefore Save (Submitted Document)\nAfter Save (Submitted Document)" }, { "depends_on": "eval:doc.script_type==='API'", @@ -109,10 +109,11 @@ "link_fieldname": "server_script" } ], - "modified": "2021-09-04 12:02:43.671240", + "modified": "2022-04-07 19:41:23.178772", "modified_by": "Administrator", "module": "Core", "name": "Server Script", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -130,5 +131,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file From 7ceb8fd74799ca17909134244503d67e4c34db81 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 05:35:43 +0530 Subject: [PATCH 35/55] fix(style): minor style fixes --- frappe/public/scss/website/base.scss | 12 ++++++------ frappe/public/scss/website/footer.scss | 2 +- frappe/public/scss/website/page_builder.scss | 2 +- .../section_with_videos/section_with_videos.html | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frappe/public/scss/website/base.scss b/frappe/public/scss/website/base.scss index 7e5d9b5b66..5a6791815f 100644 --- a/frappe/public/scss/website/base.scss +++ b/frappe/public/scss/website/base.scss @@ -1,10 +1,10 @@ $font-size-xs: 0.7rem; $font-size-sm: 0.85rem; $font-size-lg: 1.12rem; -$font-size-xl: 1.25rem; -$font-size-2xl: 1.5rem; -$font-size-3xl: 2rem; -$font-size-4xl: 2.5rem; +$font-size-xl: 1.2rem; +$font-size-2xl: 1.4rem; +$font-size-3xl: 1.9rem; +$font-size-4xl: 2.4rem; $font-size-5xl: 3rem; $font-size-6xl: 4rem; @@ -51,12 +51,12 @@ h1 { h2 { font-size: $font-size-2xl; margin-top: 2rem; - margin-bottom: 0.75rem; + margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { font-size: $font-size-3xl; margin-top: 4rem; - margin-bottom: 1rem; + margin-bottom: 0.75rem; } @include media-breakpoint-up(xl) { font-size: $font-size-4xl; diff --git a/frappe/public/scss/website/footer.scss b/frappe/public/scss/website/footer.scss index e5dae72808..9a36d7ab6d 100644 --- a/frappe/public/scss/website/footer.scss +++ b/frappe/public/scss/website/footer.scss @@ -1,5 +1,5 @@ .web-footer { - margin: 5rem 0; + padding: 3rem 0; min-height: 140px; background-color: var(--fg-color); border-top: 1px solid $border-color; diff --git a/frappe/public/scss/website/page_builder.scss b/frappe/public/scss/website/page_builder.scss index 21058dcf53..cb4cb7ed4b 100644 --- a/frappe/public/scss/website/page_builder.scss +++ b/frappe/public/scss/website/page_builder.scss @@ -549,7 +549,7 @@ font-weight: 600; @include media-breakpoint-up(md) { - font-size: $font-size-2xl; + font-size: $font-size-xl; } } diff --git a/frappe/website/web_template/section_with_videos/section_with_videos.html b/frappe/website/web_template/section_with_videos/section_with_videos.html index 369d469e87..30bea5dead 100644 --- a/frappe/website/web_template/section_with_videos/section_with_videos.html +++ b/frappe/website/web_template/section_with_videos/section_with_videos.html @@ -13,7 +13,7 @@ {%- if video.title -%} -

{{ video.title }}

+

{{ video.title }}

{%- endif -%} {%- if video.content -%}

{{ video.content }}

From ddd807e818e2c00e5f82fe772d91fd2ad72a7cb5 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 05:57:06 +0530 Subject: [PATCH 36/55] fix(style): minor style fixes --- frappe/public/scss/website/footer.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/scss/website/footer.scss b/frappe/public/scss/website/footer.scss index 9a36d7ab6d..ae7c15ab05 100644 --- a/frappe/public/scss/website/footer.scss +++ b/frappe/public/scss/website/footer.scss @@ -1,5 +1,6 @@ .web-footer { padding: 3rem 0; + margin-top: 3rem; min-height: 140px; background-color: var(--fg-color); border-top: 1px solid $border-color; From 63ad1efe3d5c753d5b81bfefa49324203b3a0626 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 06:12:32 +0530 Subject: [PATCH 37/55] fix(style): minor style fixes --- frappe/public/scss/desk/avatar.scss | 2 ++ frappe/public/scss/website/base.scss | 1 - frappe/public/scss/website/blog.scss | 2 +- frappe/public/scss/website/footer.scss | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/public/scss/desk/avatar.scss b/frappe/public/scss/desk/avatar.scss index 638256c21d..073f90e20f 100644 --- a/frappe/public/scss/desk/avatar.scss +++ b/frappe/public/scss/desk/avatar.scss @@ -73,6 +73,7 @@ display: inline-block; width: 100%; height: 100%; + object-fit: cover; background-color: var(--avatar-frame-bg); background-size: cover; background-repeat: no-repeat; @@ -145,6 +146,7 @@ .standard-image { width: 100%; height: 100%; + object-fit: cover; display: flex; justify-content: center; align-items: center; diff --git a/frappe/public/scss/website/base.scss b/frappe/public/scss/website/base.scss index 5a6791815f..abb2cec4dc 100644 --- a/frappe/public/scss/website/base.scss +++ b/frappe/public/scss/website/base.scss @@ -37,7 +37,6 @@ h1 { @include media-breakpoint-up(sm) { font-size: $font-size-5xl; - line-height: 2.5rem; margin-top: 3.5rem; margin-bottom: 1.25rem; } diff --git a/frappe/public/scss/website/blog.scss b/frappe/public/scss/website/blog.scss index 4f289db125..a6f06df112 100644 --- a/frappe/public/scss/website/blog.scss +++ b/frappe/public/scss/website/blog.scss @@ -98,7 +98,7 @@ .blog-header { margin-bottom: 3rem; - margin-top: 3rem; + margin-top: 5rem; } } diff --git a/frappe/public/scss/website/footer.scss b/frappe/public/scss/website/footer.scss index ae7c15ab05..3a15fa2017 100644 --- a/frappe/public/scss/website/footer.scss +++ b/frappe/public/scss/website/footer.scss @@ -1,6 +1,6 @@ .web-footer { padding: 3rem 0; - margin-top: 3rem; + margin-top: 4rem; min-height: 140px; background-color: var(--fg-color); border-top: 1px solid $border-color; From 4c350f41a2e7d8ce125850bb1d32bc564ba3cfdf Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 06:37:44 +0530 Subject: [PATCH 38/55] fix(style): minor style fixes --- frappe/public/scss/website/base.scss | 36 +++++++------------ frappe/public/scss/website/blog.scss | 5 +++ frappe/public/scss/website/error-state.scss | 1 + frappe/public/scss/website/footer.scss | 1 - .../blog_post/templates/blog_post.html | 2 +- .../blog_post/templates/blog_post_list.html | 4 +-- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/frappe/public/scss/website/base.scss b/frappe/public/scss/website/base.scss index abb2cec4dc..90453fc9db 100644 --- a/frappe/public/scss/website/base.scss +++ b/frappe/public/scss/website/base.scss @@ -1,13 +1,3 @@ -$font-size-xs: 0.7rem; -$font-size-sm: 0.85rem; -$font-size-lg: 1.12rem; -$font-size-xl: 1.2rem; -$font-size-2xl: 1.4rem; -$font-size-3xl: 1.9rem; -$font-size-4xl: 2.4rem; -$font-size-5xl: 3rem; -$font-size-6xl: 4rem; - html { height: 100%; } @@ -29,66 +19,66 @@ h1, h2, h3, h4 { } h1 { - font-size: $font-size-3xl; + font-size: 2rem; line-height: 1.25; letter-spacing: -0.025em; margin-top: 3rem; margin-bottom: 0.75rem; @include media-breakpoint-up(sm) { - font-size: $font-size-5xl; + font-size: 2.5rem; margin-top: 3.5rem; margin-bottom: 1.25rem; } @include media-breakpoint-up(xl) { - font-size: $font-size-6xl; + font-size: 3.5rem; line-height: 1; margin-top: 4rem; } } h2 { - font-size: $font-size-2xl; + font-size: 1.4rem; margin-top: 2rem; margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { - font-size: $font-size-3xl; + font-size: 2rem; margin-top: 4rem; margin-bottom: 0.75rem; } @include media-breakpoint-up(xl) { - font-size: $font-size-4xl; + font-size: 2.5rem; margin-top: 4rem; } } h3 { - font-size: $font-size-xl; + font-size: 1.2rem; margin-top: 1.5rem; margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { - font-size: $font-size-2xl; + font-size: 1.4rem; margin-top: 2.5rem; } @include media-breakpoint-up(xl) { - font-size: $font-size-3xl; + font-size: 1.9rem; margin-top: 3.5rem; } } h4 { - font-size: $font-size-lg; + font-size: 1.1rem; margin-top: 1rem; margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { - font-size: $font-size-xl; + font-size: 1.3rem; margin-top: 1.25rem; } @include media-breakpoint-up(xl) { - font-size: $font-size-2xl; + font-size: 1.5rem; margin-top: 1.75rem; } @@ -98,5 +88,5 @@ h4 { } .btn.btn-lg { - font-size: $font-size-lg; + font-size: 1.1rem; } diff --git a/frappe/public/scss/website/blog.scss b/frappe/public/scss/website/blog.scss index a6f06df112..b16e088f69 100644 --- a/frappe/public/scss/website/blog.scss +++ b/frappe/public/scss/website/blog.scss @@ -102,6 +102,11 @@ } } + .blog-comments { + margin-top: 1rem; + margin-bottom: 5rem; + } + .feedback-item svg { vertical-align: sub; diff --git a/frappe/public/scss/website/error-state.scss b/frappe/public/scss/website/error-state.scss index c869e9e1df..7a83fc0084 100644 --- a/frappe/public/scss/website/error-state.scss +++ b/frappe/public/scss/website/error-state.scss @@ -1,4 +1,5 @@ .error-page { + margin: 3rem 0; text-align: center; .img-404 { diff --git a/frappe/public/scss/website/footer.scss b/frappe/public/scss/website/footer.scss index 3a15fa2017..9a36d7ab6d 100644 --- a/frappe/public/scss/website/footer.scss +++ b/frappe/public/scss/website/footer.scss @@ -1,6 +1,5 @@ .web-footer { padding: 3rem 0; - margin-top: 4rem; min-height: 140px; background-color: var(--fg-color); border-top: 1px solid $border-color; diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index d8ece09f46..4bab50c33e 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -66,7 +66,7 @@ {% endif %} {% if not disable_comments %} -
+
{% include 'templates/includes/comments/comments.html' %}
{% endif %} diff --git a/frappe/website/doctype/blog_post/templates/blog_post_list.html b/frappe/website/doctype/blog_post/templates/blog_post_list.html index 1aa30316fe..4cb53d065c 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post_list.html +++ b/frappe/website/doctype/blog_post/templates/blog_post_list.html @@ -8,8 +8,8 @@
-

{{ blog_title or _('Blog') }}

-

{{ blog_introduction or '' }}

+

{{ blog_title or _('Blog') }}

+

{{ blog_introduction or '' }}

From 96a62339533f2c8cffca3e8db9250b88817af48f Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 07:19:44 +0530 Subject: [PATCH 39/55] fix(style): minor style fixes --- frappe/public/scss/website/base.scss | 4 ++++ frappe/public/scss/website/markdown.scss | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/public/scss/website/base.scss b/frappe/public/scss/website/base.scss index 90453fc9db..90c98cfb45 100644 --- a/frappe/public/scss/website/base.scss +++ b/frappe/public/scss/website/base.scss @@ -87,6 +87,10 @@ h4 { } } +p { + line-height: 1.7; +} + .btn.btn-lg { font-size: 1.1rem; } diff --git a/frappe/public/scss/website/markdown.scss b/frappe/public/scss/website/markdown.scss index c2592b61e9..178b29e505 100644 --- a/frappe/public/scss/website/markdown.scss +++ b/frappe/public/scss/website/markdown.scss @@ -30,7 +30,8 @@ } p, li { - font-size: $font-size-lg; + font-size: 1.1rem; + line-height: 1.7; } li { From c0d8dfed157e2eca11b1d1aa73d691f4bdf6e29c Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 07:50:07 +0530 Subject: [PATCH 40/55] fix(style): minor style fixes --- frappe/public/scss/website/markdown.scss | 5 ++++- frappe/public/scss/website/page_builder.scss | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/public/scss/website/markdown.scss b/frappe/public/scss/website/markdown.scss index 178b29e505..6b5f7cbb6e 100644 --- a/frappe/public/scss/website/markdown.scss +++ b/frappe/public/scss/website/markdown.scss @@ -5,7 +5,6 @@ } .from-markdown { - color: $gray-700; line-height: 1.7; > :first-child { @@ -34,6 +33,10 @@ line-height: 1.7; } + p.lead { + @extend .lead; + } + li { padding-top: 1px; padding-bottom: 1px; diff --git a/frappe/public/scss/website/page_builder.scss b/frappe/public/scss/website/page_builder.scss index cb4cb7ed4b..da6c6b8e58 100644 --- a/frappe/public/scss/website/page_builder.scss +++ b/frappe/public/scss/website/page_builder.scss @@ -23,9 +23,11 @@ } .lead { + color: var(--text-muted); font-weight: normal; font-size: 1.25rem; - margin-bottom: 1.5rem; + margin-top: -1rem; + margin-bottom: 2.5rem; } .hero-subtitle { From c54f694e7c57aa781c03ce6e5a1f61fc3308a1ed Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 07:56:23 +0530 Subject: [PATCH 41/55] fix(style): minor style fixes --- frappe/public/scss/website/base.scss | 6 +++--- frappe/public/scss/website/page_builder.scss | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frappe/public/scss/website/base.scss b/frappe/public/scss/website/base.scss index 90c98cfb45..30f7b81008 100644 --- a/frappe/public/scss/website/base.scss +++ b/frappe/public/scss/website/base.scss @@ -70,16 +70,16 @@ h3 { h4 { font-size: 1.1rem; - margin-top: 1rem; + margin-top: 1.7rem; margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { font-size: 1.3rem; - margin-top: 1.25rem; + margin-top: 2rem; } @include media-breakpoint-up(xl) { font-size: 1.5rem; - margin-top: 1.75rem; + margin-top: 2.5rem; } a { diff --git a/frappe/public/scss/website/page_builder.scss b/frappe/public/scss/website/page_builder.scss index da6c6b8e58..d16ea4e27d 100644 --- a/frappe/public/scss/website/page_builder.scss +++ b/frappe/public/scss/website/page_builder.scss @@ -16,12 +16,6 @@ } } -.hero-title, .hero-subtitle { - max-width: 42rem; - margin-top: 0rem; - margin-bottom: 0.5rem; -} - .lead { color: var(--text-muted); font-weight: normal; @@ -40,6 +34,12 @@ } } +.hero-title, .hero-subtitle { + max-width: 42rem; + margin-top: 0rem; + margin-bottom: 0.5rem; +} + .hero.align-center { h1, .hero-title, .hero-subtitle, .hero-buttons { text-align: center; From a682516f712504fd92c0a71933ad94dd72735cb7 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 8 Apr 2022 10:38:03 +0530 Subject: [PATCH 42/55] fix(style): minor style fixes --- frappe/public/scss/website/base.scss | 8 ++++---- frappe/public/scss/website/blog.scss | 4 ++++ frappe/public/scss/website/index.scss | 4 ++-- frappe/public/scss/website/markdown.scss | 5 ++++- frappe/public/scss/website/page_builder.scss | 11 +++++++++-- .../section_with_cta/section_with_cta.html | 1 + .../section_with_features/section_with_features.html | 3 ++- .../section_with_small_cta.html | 1 + 8 files changed, 27 insertions(+), 10 deletions(-) diff --git a/frappe/public/scss/website/base.scss b/frappe/public/scss/website/base.scss index 30f7b81008..d666bcd410 100644 --- a/frappe/public/scss/website/base.scss +++ b/frappe/public/scss/website/base.scss @@ -55,7 +55,7 @@ h2 { h3 { font-size: 1.2rem; - margin-top: 1.5rem; + margin-top: 2rem; margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { @@ -70,16 +70,16 @@ h3 { h4 { font-size: 1.1rem; - margin-top: 1.7rem; + margin-top: 2rem; margin-bottom: 0.5rem; @include media-breakpoint-up(sm) { font-size: 1.3rem; - margin-top: 2rem; + margin-top: 2.5rem; } @include media-breakpoint-up(xl) { font-size: 1.5rem; - margin-top: 2.5rem; + margin-top: 3rem; } a { diff --git a/frappe/public/scss/website/blog.scss b/frappe/public/scss/website/blog.scss index b16e088f69..ebc147b238 100644 --- a/frappe/public/scss/website/blog.scss +++ b/frappe/public/scss/website/blog.scss @@ -14,6 +14,10 @@ } } +.blog-list-content { + margin-bottom: 3rem; +} + .blog-card { margin-bottom: 2rem; position: relative; diff --git a/frappe/public/scss/website/index.scss b/frappe/public/scss/website/index.scss index 0c96c62c17..933ac7ae22 100644 --- a/frappe/public/scss/website/index.scss +++ b/frappe/public/scss/website/index.scss @@ -114,8 +114,8 @@ @media (max-width: map-get($grid-breakpoints, "lg")) { .page-content-wrapper .container { - padding-left: 1rem; - padding-right: 1rem; + padding-left: 1.5rem; + padding-right: 1.5rem; } } diff --git a/frappe/public/scss/website/markdown.scss b/frappe/public/scss/website/markdown.scss index 6b5f7cbb6e..2b17c209cd 100644 --- a/frappe/public/scss/website/markdown.scss +++ b/frappe/public/scss/website/markdown.scss @@ -29,8 +29,11 @@ } p, li { - font-size: 1.1rem; line-height: 1.7; + + @include media-breakpoint-up(sm) { + font-size: 1.05rem; + } } p.lead { diff --git a/frappe/public/scss/website/page_builder.scss b/frappe/public/scss/website/page_builder.scss index d16ea4e27d..2ca51067b7 100644 --- a/frappe/public/scss/website/page_builder.scss +++ b/frappe/public/scss/website/page_builder.scss @@ -20,8 +20,14 @@ color: var(--text-muted); font-weight: normal; font-size: 1.25rem; - margin-top: -1rem; - margin-bottom: 2.5rem; + + margin-top: -0.5rem; + margin-bottom: 1.5rem; + + @include media-breakpoint-up(sm) { + margin-top: -1rem; + margin-bottom: 2.5rem; + } } .hero-subtitle { @@ -53,6 +59,7 @@ .section-description { max-width: 56rem; + color: var(--text-muted); margin-top: 0.5rem; font-size: $font-size-lg; diff --git a/frappe/website/web_template/section_with_cta/section_with_cta.html b/frappe/website/web_template/section_with_cta/section_with_cta.html index 4494dccecb..4015c05517 100644 --- a/frappe/website/web_template/section_with_cta/section_with_cta.html +++ b/frappe/website/web_template/section_with_cta/section_with_cta.html @@ -22,4 +22,5 @@
{%- endif -%} + {% if cta_url %}{% endif %}
diff --git a/frappe/website/web_template/section_with_features/section_with_features.html b/frappe/website/web_template/section_with_features/section_with_features.html index e9a1966f3d..b8cc3c2b90 100644 --- a/frappe/website/web_template/section_with_features/section_with_features.html +++ b/frappe/website/web_template/section_with_features/section_with_features.html @@ -6,7 +6,8 @@

{{ subtitle }}

{%- endif -%} -
+
{%- for feature in features -%}
diff --git a/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html index 7bd8c905ad..b898ebdcbe 100644 --- a/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html +++ b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html @@ -16,4 +16,5 @@ {%- endif -%}
+ {% if cta_url %}{% endif %}
From 02286e4e6f4178d7137a0f8757a24e130ecd96dc Mon Sep 17 00:00:00 2001 From: Komal-Saraf0609 <81952590+Komal-Saraf0609@users.noreply.github.com> Date: Fri, 8 Apr 2022 19:12:58 +0530 Subject: [PATCH 43/55] test: Added test script for control type "Attach" (#16355) Adding automation test script for control/fieldtype "Attach". The above test script does the following testing: 1. Creating a new doctype with attach fieldtype. 2. Attaching a new image using the "Link" option from the options which the "Attach" button offers. 3. Checking if the URL of the attached image is getting displayed in the field of the newly created doctype. 4. Checking if the clicking on the "Clear" button clears the text in the field and again displays the "Attach" button. 5. Doing all the above testing by using the "Library" option from the options which the "Attach" button offers. --- cypress/integration/control_attach.js | 90 +++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 cypress/integration/control_attach.js diff --git a/cypress/integration/control_attach.js b/cypress/integration/control_attach.js new file mode 100644 index 0000000000..0552780737 --- /dev/null +++ b/cypress/integration/control_attach.js @@ -0,0 +1,90 @@ +context('Attach Control', () => { + before(() => { + cy.login(); + cy.visit('/app/doctype'); + return cy.window().its('frappe').then(frappe => { + return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { + name: 'Test Attach Control', + fields: [ + { + "label": "Attach File or Image", + "fieldname": "attach", + "fieldtype": "Attach", + "in_list_view": 1, + }, + ] + }); + }); + }); + it('Checking functionality for "Link" button in the "Attach" fieldtype', () => { + //Navigating to the new form for the newly created doctype + cy.new_form('Test Attach Control'); + + //Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype + cy.findByRole('button', {name: 'Attach'}).click(); + + //Clicking on "Link" button to attach a file using the "Link" button + cy.findByRole('button', {name: 'Link'}).click(); + cy.findByPlaceholderText('Attach a web link').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + + //Clicking on the Upload button to upload the file + cy.intercept("POST", "/api/method/upload_file").as("upload_image"); + cy.get('.modal-footer').findByRole("button", {name: "Upload"}).click({delay: 500}); + cy.wait("@upload_image"); + cy.findByRole('button', {name: 'Save'}).click(); + + //Checking if the URL of the attached image is getting displayed in the field of the newly created doctype + cy.get('.attached-file > .ellipsis > .attached-file-link') + .should('have.attr', 'href') + .and('equal', 'https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + + //Clicking on the "Clear" button + cy.get('[data-action="clear_attachment"]').click(); + + //Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button + cy.get('.control-input > .btn-sm').should('contain', 'Attach'); + + //Deleting the doc + cy.go_to_list('Test Attach Control'); + cy.get('.list-row-checkbox').eq(0).click(); + cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); + cy.click_modal_primary_button('Yes'); + }); + + it('Checking functionality for "Library" button in the "Attach" fieldtype', () => { + //Navigating to the new form for the newly created doctype + cy.new_form('Test Attach Control'); + + //Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype + cy.findByRole('button', {name: 'Attach'}).click(); + + //Clicking on "Library" button to attach a file using the "Library" button + cy.findByRole('button', {name: 'Library'}).click(); + cy.contains('72402.jpg').click(); + + //Clicking on the Upload button to upload the file + cy.intercept("POST", "/api/method/upload_file").as("upload_image"); + cy.get('.modal-footer').findByRole("button", {name: "Upload"}).click({delay: 500}); + cy.wait("@upload_image"); + cy.findByRole('button', {name: 'Save'}).click(); + + //Checking if the URL of the attached image is getting displayed in the field of the newly created doctype + cy.get('.attached-file > .ellipsis > .attached-file-link') + .should('have.attr', 'href') + .and('equal', 'https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + + //Clicking on the "Clear" button + cy.get('[data-action="clear_attachment"]').click(); + + //Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button + cy.get('.control-input > .btn-sm').should('contain', 'Attach'); + + //Deleting the doc + cy.go_to_list('Test Attach Control'); + cy.get('.list-row-checkbox').eq(0).click(); + cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); + cy.click_modal_primary_button('Yes'); + }); +}); \ No newline at end of file From 42cfcdadf94ff262515cdfd44af3b838b10b31cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Apr 2022 01:08:28 +0000 Subject: [PATCH 44/55] chore(deps): bump moment from 2.24.0 to 2.29.2 Bumps [moment](https://github.com/moment/moment) from 2.24.0 to 2.29.2. - [Release notes](https://github.com/moment/moment/releases) - [Changelog](https://github.com/moment/moment/blob/develop/CHANGELOG.md) - [Commits](https://github.com/moment/moment/compare/2.24.0...2.29.2) --- updated-dependencies: - dependency-name: moment dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d2c1de7be5..fd27bc223b 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "js-sha256": "^0.9.0", "jsbarcode": "^3.9.0", "localforage": "^1.9.0", - "moment": "^2.20.1", + "moment": "^2.29.2", "moment-timezone": "^0.5.28", "node-sass": "^7.0.0", "plyr": "^3.6.2", diff --git a/yarn.lock b/yarn.lock index 339e4c5669..12021982ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2998,10 +2998,10 @@ moment-timezone@^0.5.28: dependencies: moment ">= 2.9.0" -"moment@>= 2.9.0", moment@^2.20.1: - version "2.24.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" - integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== +"moment@>= 2.9.0", moment@^2.29.2: + version "2.29.2" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4" + integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg== ms@2.0.0: version "2.0.0" From 9e4182d91f7130c9ce4c7957df59730ac3fb91ea Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 9 Apr 2022 16:37:59 +0530 Subject: [PATCH 45/55] fix: new route syntax for Logs in System Console --- frappe/desk/doctype/system_console/system_console.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json index 657e9df89d..c92b2005ed 100644 --- a/frappe/desk/doctype/system_console/system_console.json +++ b/frappe/desk/doctype/system_console/system_console.json @@ -1,7 +1,7 @@ { "actions": [ { - "action": "#List/Console Log/List", + "action": "app/console-log", "action_type": "Route", "label": "Logs" }, @@ -86,7 +86,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-09-15 17:17:44.844767", + "modified": "2022-04-09 16:35:32.345542", "modified_by": "Administrator", "module": "Desk", "name": "System Console", @@ -104,5 +104,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file From 60814c4e3fb8f3eb6be0e54cad7ba2a97c2b2104 Mon Sep 17 00:00:00 2001 From: Mohammed Redah Date: Sun, 10 Apr 2022 05:16:43 +0300 Subject: [PATCH 46/55] fix: Export Links in Customize Form (#16333) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/modules/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index 4768faff48..0383327b68 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -45,7 +45,7 @@ def export_customizations(module, doctype, sync_on_migrate=0, with_permissions=0 if not frappe.get_conf().developer_mode: raise Exception('Not developer mode') - custom = {'custom_fields': [], 'property_setters': [], 'custom_perms': [], + custom = {'custom_fields': [], 'property_setters': [], 'custom_perms': [],'links':[], 'doctype': doctype, 'sync_on_migrate': sync_on_migrate} def add(_doctype): @@ -53,6 +53,8 @@ def export_customizations(module, doctype, sync_on_migrate=0, with_permissions=0 fields='*', filters={'dt': _doctype}) custom['property_setters'] += frappe.get_all('Property Setter', fields='*', filters={'doc_type': _doctype}) + custom['links'] += frappe.get_all('DocType Link', + fields='*', filters={'parent': _doctype}) add(doctype) From cc9613f5c238f1d515691d9547d564775e5e3e8c Mon Sep 17 00:00:00 2001 From: HENRY Florian Date: Sun, 10 Apr 2022 04:18:55 +0200 Subject: [PATCH 47/55] fix: update french translation (#16550) --- frappe/translations/fr.csv | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/frappe/translations/fr.csv b/frappe/translations/fr.csv index 112130e945..67aca1443c 100644 --- a/frappe/translations/fr.csv +++ b/frappe/translations/fr.csv @@ -293,7 +293,7 @@ old_parent,grand_parent, (Ctrl + G),(Ctrl + G), ** Failed: {0} to {1}: {2},** Échec: {0} à {1}: {2}, **Currency** Master,Données de Base **Devise**, -0 - Draft; 1 - Submitted; 2 - Cancelled,0 - Brouillon; 1 - Soumis; 2 - Annulé, +0 - Draft; 1 - Submitted; 2 - Cancelled,0 - Brouillon; 1 - Validé; 2 - Annulé, 0 is highest,0 est le plus élevé, 1 Currency = [?] Fraction\nFor e.g. 1 USD = 100 Cent,1 Devise = [?] Fraction \nE.g. 1 USD = 100 centimes, 1 comment,1 commentaire, @@ -377,7 +377,7 @@ Align Labels to the Right,Alignez les Étiquettes à Droite, Align Value,Aligner la Valeur, All Images attached to Website Slideshow should be public,Toutes les images jointes au diaporama du site Web doivent être publiques, All customizations will be removed. Please confirm.,Toutes les personnalisations seront supprimées. Veuillez confirmer., -"All possible Workflow States and roles of the workflow. Docstatus Options: 0 is""Saved"", 1 is ""Submitted"" and 2 is ""Cancelled""","Tous les États et Rôles possibles du Flux de Travail. Options de Statut du Document : 0 est ""Enregistré"", 1 est ""Soumis"" et 2 est ""Annulé""", +"All possible Workflow States and roles of the workflow. Docstatus Options: 0 is""Saved"", 1 is ""Submitted"" and 2 is ""Cancelled""","Tous les États et Rôles possibles du Flux de Travail. Options de Statut du Document : 0 est ""Enregistré"", 1 est ""Validé"" et 2 est ""Annulé""", All-uppercase is almost as easy to guess as all-lowercase.,Tout en majuscules est presque aussi facile à deviner que tout en minuscules., Allocated To,Attribué à, Allow,Autoriser, @@ -404,7 +404,7 @@ Allow Self Approval,Autoriser l'auto-approbation, Allow approval for creator of the document,Autoriser l'approbation par le créateur du document, Allow events in timeline,Autoriser les événements dans la chronologie, Allow in Quick Entry,Autoriser dans les entrées rapides, -Allow on Submit,Autoriser à la Soumission, +Allow on Submit,Autoriser à la Validation, Allow only one session per user,Autoriser une seule session par utilisateur, Allow page break inside tables,Autoriser les sauts de page dans les tables, Allow saving if mandatory fields are not filled,Autoriser l'enregistrement si les champs obligatoires ne sont pas remplis, @@ -594,7 +594,7 @@ Cancelled Document restored as Draft,Le document annulé a été restauré en ta Cancelling,Annulation, Cancelling {0},Annulation de {0}, Cannot Remove,Ne peut être retiré, -Cannot cancel before submitting. See Transition {0},Impossible d'annuler avant de soumettre. Voir Transition {0}, +Cannot cancel before submitting. See Transition {0},Impossible d'annuler avant de valider. Voir Transition {0}, Cannot change docstatus from 0 to 2,Impossible de changer le statut du document de 0 à 2, Cannot change docstatus from 1 to 0,Impossible de changer le statut du document de 1 à 0, Cannot change header content,Impossible de changer le contenu de l'en-tête, @@ -627,7 +627,7 @@ Card Details,Détails de la carte, Categorize blog posts.,Catégoriser les posts de blog., Category Description,Description de la Catégorie, Cent,Centime, -"Certain documents, like an Invoice, should not be changed once final. The final state for such documents is called Submitted. You can restrict which roles can Submit.","Certains documents, comme une Facture, ne devraient pas être modifiés une fois finalisés. L'état final de ces documents est appelée Soumis. Vous pouvez limiter les rôles pouvant Soumettre.", +"Certain documents, like an Invoice, should not be changed once final. The final state for such documents is called Submitted. You can restrict which roles can Submit.","Certains documents, comme une Facture, ne devraient pas être modifiés une fois finalisés. L'état final de ces documents est appelée Validé. Vous pouvez limiter les rôles pouvant Valider.", Chain Integrity,Intégrité de la chaîne, Chaining Hash,Hachage de chaînage, Change Label (via Custom Translation),Modifier le libellé (via Traduction Personnalisée ), @@ -896,7 +896,7 @@ DocType {0} provided for the field {1} must have atleast one Link DocType can not be merged,DocType ne peut pas être fusionné, DocType can only be renamed by Administrator,DocType ne peut être renommé que par l'Administrateur, DocType is a Table / Form in the application.,DocType est un Tableau / Formulaire dans l'application., -DocType must be Submittable for the selected Doc Event,Le DocType doit être soumissible pour l'événement Doc sélectionné, +DocType must be Submittable for the selected Doc Event,Le DocType doit être validable pour l'événement Doc sélectionné, DocType on which this Workflow is applicable.,DocType pour lequel ce Flux de Travail est applicable., "DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores","Le nom du DocType doit commencer par une lettre et il peut uniquement se composer de lettres, des chiffres, d’espaces et du tiret bas (underscore)", Doctype required,Doctype requis, @@ -908,7 +908,7 @@ Document Restored,Document Restauré, Document Share Report,Rapport de Partage de Document, Document States,États du Document, Document Type is not importable,Le type de document n'est pas importable, -Document Type is not submittable,Le type de document n'est pas soumis, +Document Type is not submittable,Le type de document n'est pas valider, Document Type to Track,Type de document à suivre, Document Types,Types de documents, Document can't saved.,Le document ne peut pas être enregistré., @@ -1392,7 +1392,7 @@ Is Published Field must be a valid fieldname,Le Champ Publié doit-il être un n Is Single,Est Seul, Is Spam,Est Spam, Is Standard,Est Standard, -Is Submittable,Est Soumissible, +Is Submittable,Est Validable, Is Table,Est Table, Is Your Company Address,Est l'Adresse de votre Entreprise, It is risky to delete this file: {0}. Please contact your System Manager.,Il est risqué de supprimer ce fichier : {0}. Veuillez contactez votre Administrateur Système., @@ -1541,7 +1541,7 @@ Max Value,Valeur Max, Max width for type Currency is 100px in row {0},Largeur max pour le type Devise est 100px dans la ligne {0}, Maximum Attachment Limit for this record reached.,Taille maximale des Pièces Jointes pour cet enregistrement est atteint., Maximum {0} rows allowed,Maximum {0} lignes autorisés, -"Meaning of Submit, Cancel, Amend","Signification de Soumettre, Annuler, Modifier", +"Meaning of Submit, Cancel, Amend","Signification de Valider, Annuler, Modifier", Mention transaction completion page URL,Mentionnez la page URL de fin de transaction, Mentions,Mentions, Menu,Menu, @@ -1737,7 +1737,7 @@ Old Password,Ancien Mot De Passe, Old Password Required.,Ancien Mot de Passe Requis., Older backups will be automatically deleted,Les anciennes sauvegardes seront automatiquement supprimées, "On {0}, {1} wrote:","Sur {0}, {1} a écrit :", -"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended.","Une fois soumis, les documents à soumettre ne peuvent plus être modifiés. Ils ne peuvent être annulés et amendés.", +"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended.","Une fois validé, les documents à valider ne peuvent plus être modifiés. Ils ne peuvent être annulés et amendés.", "Once you have set this, the users will only be able access documents (eg. Blog Post) where the link exists (eg. Blogger).","Une fois que vous avez défini ceci, les utilisateurs ne pourront accèder qu'aux documents (e.g. Article de Blog) où le lien existe (e.g. Blogger) .", One Last Step,Une Dernière Étape, One Time Password (OTP) Registration Code from {},Code de Mot de Passe Unique (OTP) à partir de {}, @@ -1829,7 +1829,7 @@ Percent Complete,Pourcentage d'Avancement, Perm Level,Niveau d'Autorisation, Permanent,Permanent, Permanently Cancel {0}?,Annuler de Manière Permanente {0} ?, -Permanently Submit {0}?,Soumettre de Manière Permanente {0} ?, +Permanently Submit {0}?,Valider de Manière Permanente {0} ?, Permanently delete {0}?,Supprimer de Manière Permanente {0} ?, Permission Error,Erreur d'autorisation, Permission Level,Niveau d'Autorisation, @@ -1837,7 +1837,7 @@ Permission Levels,Niveaux d'Autorisation, Permission Rules,Règles d'Autorisation, Permissions,Autorisations, Permissions are automatically applied to Standard Reports and searches.,Les autorisations sont automatiquement appliquées aux rapports standard et aux recherches., -"Permissions are set on Roles and Document Types (called DocTypes) by setting rights like Read, Write, Create, Delete, Submit, Cancel, Amend, Report, Import, Export, Print, Email and Set User Permissions.","Les Autorisations sont définies sur les Rôles et les Types de Documents (appelés DocTypes) en définissant des droits , tels que Lire, Écrire, Créer, Supprimer, Soumettre, Annuler, Modifier, Rapporter, Importer, Exporter, Imprimer, Envoyer un Email et Définir les Autorisations de l'Utilisateur .", +"Permissions are set on Roles and Document Types (called DocTypes) by setting rights like Read, Write, Create, Delete, Submit, Cancel, Amend, Report, Import, Export, Print, Email and Set User Permissions.","Les Autorisations sont définies sur les Rôles et les Types de Documents (appelés DocTypes) en définissant des droits , tels que Lire, Écrire, Créer, Supprimer, Valider, Annuler, Modifier, Rapporter, Importer, Exporter, Imprimer, Envoyer un Email et Définir les Autorisations de l'Utilisateur .", Permissions at higher levels are Field Level permissions. All Fields have a Permission Level set against them and the rules defined at that permissions apply to the field. This is useful in case you want to hide or make certain field read-only for certain Roles.,Les Autorisations aux niveaux supérieurs sont des permissions de Niveau Champ. Un Niveau d'Autorisation est défini pour chaque Champ et les règles définies pour ces Autorisations s’appliquent au Champ. Ceci est utile si vous voulez cacher ou mettre certains champs en lecture seule pour certains Rôles., "Permissions at level 0 are Document Level permissions, i.e. they are primary for access to the document.","Les Autorisations au niveau 0 sont les autorisations de Niveau Document, c’est à dire qu'elles sont nécessaires pour accéder au document.", Permissions get applied on Users based on what Roles they are assigned.,Autorisations sont appliqués aux utilisateurs en fonction des Rôles qui leurs sont affectés., @@ -2123,7 +2123,7 @@ Row No,Rangée No, Row Status,État de la ligne, Row Values Changed,Valeurs de Lignes Modifiées, Row {0}: Not allowed to disable Mandatory for standard fields,Ligne {0}: impossible de désactiver Obligatoire pour les champs standard, -Row {0}: Not allowed to enable Allow on Submit for standard fields,Ligne {0} : Il n’est pas autorisé d’activer Autoriser à la Soumission pour les champs standards, +Row {0}: Not allowed to enable Allow on Submit for standard fields,Ligne {0} : Il n’est pas autorisé d’activer Autoriser à la Validation pour les champs standards, Rows Added,Lignes Ajoutées, Rows Removed,Lignes Supprimées, Rule,Règle, @@ -2395,13 +2395,13 @@ Stylesheets for Print Formats,Feuilles de style pour les Formats d'Impression, Sub-domain provided by erpnext.com,Sous-domaine fourni par erpnext.com, Subdomain,Sous-domaine, Subject Field,Champ de sujet, -Submit after importing,Soumettre après l'import, -Submit an Issue,Soumettre un ticket, -Submit this document to confirm,Soumettre ce document pour confirmer, -Submit {0} documents?,Soumettre {0} documents ?, -Submiting {0},Soumission de {0}, -Submitted Document cannot be converted back to draft. Transition row {0},Document Soumis ne peut pas être reconvertis en Brouillon. Ligne de transition {0}, -Submitting,Soumission, +Submit after importing,Valider après l'import, +Submit an Issue,Valider un ticket, +Submit this document to confirm,Valider ce document pour confirmer, +Submit {0} documents?,Valider {0} documents ?, +Submiting {0},Validation de {0}, +Submitted Document cannot be converted back to draft. Transition row {0},Document Valider ne peut pas être reconvertis en Brouillon. Ligne de transition {0}, +Submitting,Validation, Subscription Notification,Notification d'abonnement, Subsidiary,Filiale, Success Action,Action de succès, @@ -2784,7 +2784,7 @@ You are not permitted to view the newsletter.,Vous n'êtes pas autorisé à You are now following this document. You will receive daily updates via email. You can change this in User Settings.,Vous suivez maintenant ce document. Vous recevrez des mises à jour quotidiennes par courrier électronique. Vous pouvez modifier cela dans les paramètres de l'utilisateur., You can add dynamic properties from the document by using Jinja templating.,Vous pouvez ajouter des propriétés dynamiques au document à l'aide des modèles Jinja., You can also copy-paste this ,Vous pouvez également copier-coller cette, -"You can change Submitted documents by cancelling them and then, amending them.","Vous pouvez modifier les documents Soumis en les annulant et ensuite, en les modifiant.", +"You can change Submitted documents by cancelling them and then, amending them.","Vous pouvez modifier les documents Validés en les annulant et ensuite, en les modifiant.", You can find things by asking 'find orange in customers',Vous pouvez trouver des choses en demandant 'trouver orange dans clients', You can only upload upto 5000 records in one go. (may be less in some cases),Vous pouvez seulement charger jusqu'à 5000 enregistrement en une seule fois. (peut-être moins dans certains cas), You can use Customize Form to set levels on fields.,Vous pouvez utiliser Personaliser le Formulaire pour définir les niveaux de champs., @@ -2807,7 +2807,7 @@ You gained {0} points,Vous avez gagné {0} points, You have a new message from: ,Vous avez un nouveau message de:, You have been successfully logged out,Vous avez été déconnecté avec succès, You have unsaved changes in this form. Please save before you continue.,Vous avez des modifications non enregistrées dans ce formulaire. Veuillez enregistrer avant de continuer., -You must login to submit this form,Vous devez vous connecter pour soumettre ce formulaire, +You must login to submit this form,Vous devez vous connecter pour valider ce formulaire, You need to be in developer mode to edit a Standard Web Form,Vous devez être en Mode Développeur pour modifier un Formulaire Web Standard, You need to be logged in and have System Manager Role to be able to access backups.,Vous devez être connecté et avoir le Role Responsable Système pour pouvoir accéder aux sauvegardes., You need to be logged in to access this {0}.,Vous devez être connecté pour accéder à ce(tte) {0}., @@ -2820,7 +2820,7 @@ Your Language,Votre Langue, Your Name,Votre Nom, Your account has been locked and will resume after {0} seconds,Votre compte a été verrouillé et reprendra après {0} secondes, Your connection request to Google Calendar was successfully accepted,Votre demande de connexion à Google Agenda a été acceptée avec succès, -Your information has been submitted,Vos informations ont été soumises, +Your information has been submitted,Vos informations ont été validées, Your login id is,Votre id de connexion est, Your organization name and address for the email footer.,Le nom de votre société et l'adresse pour le pied de l'email., Your payment has been successfully registered.,Votre paiement a été enregistré avec succès., @@ -2982,7 +2982,7 @@ star,étoile, star-empty,étoile-vide, step-backward,vers-larrière, step-forward,vers-l'avant, -submitted this document,a soumis ce document, +submitted this document,a validé ce document, text in document type,Texte dans le type de document, text-height,Hauteur-texte, text-width,largeur-text, @@ -3094,11 +3094,11 @@ zoom-out,Réduire, "{0}, Row {1}","{0}, Ligne {1}", "{0}: '{1}' ({3}) will get truncated, as max characters allowed is {2}",{0} : {1} '({3}) sera tronqué car le nombre de caractères max est {2}, {0}: Cannot set Amend without Cancel,{0} : Impossible de choisir Modifier sans Annuler, -{0}: Cannot set Assign Amend if not Submittable,{0} : Impossible de définir ‘Assigner Modifier’ si non Soumissible, -{0}: Cannot set Assign Submit if not Submittable,{0} : Impossible de définir ‘Assigner Soumettre’ si non Soumissible, -{0}: Cannot set Cancel without Submit,{0} : Impossible de choisir Annuler sans Soumettre, +{0}: Cannot set Assign Amend if not Submittable,{0} : Impossible de définir ‘Assigner Modifier’ si non Validable, +{0}: Cannot set Assign Submit if not Submittable,{0} : Impossible de définir ‘Assigner Valider’ si non Validable, +{0}: Cannot set Cancel without Submit,{0} : Impossible de choisir Annuler sans Valider, {0}: Cannot set Import without Create,{0} : Impossible de choisir Import sans Créer, -"{0}: Cannot set Submit, Cancel, Amend without Write","{0} : Vous ne pouvez pas choisir Envoyer, Annuler, Modifier sans Écrire", +"{0}: Cannot set Submit, Cancel, Amend without Write","{0} : Vous ne pouvez pas choisir Valider, Annuler, Modifier sans Écrire", {0}: Cannot set import as {1} is not importable,{0} : Impossible de choisir import car {1} n'est pas importable, {0}: No basic permissions set,{0} : Aucune autorisation de base définie, "{0}: Only one rule allowed with the same Role, Level and {1}","{0} : Une seule règle est permise avec le même Rôle, Niveau et {1}", @@ -3153,8 +3153,8 @@ Administration,Administration, After Cancel,Après annuler, After Delete,Après la suppression, After Save,Après l'enregistrement, -After Save (Submitted Document),Après l'enregistrement (document soumis), -After Submit,Après soumettre, +After Save (Submitted Document),Après l'enregistrement (document valider), +After Submit,Après validation, Aggregate Function Based On,Fonction d'agrégation basée sur, Aggregate Function field is required to create a dashboard chart,Le champ Fonction d'agrégation est requis pour créer un graphique de tableau de bord, All Records,Tous les enregistrements, @@ -3199,8 +3199,8 @@ Before Cancel,Avant d'annuler, Before Delete,Avant de supprimer, Before Insert,Avant l'insertion, Before Save,Avant de sauvegarder, -Before Save (Submitted Document),Avant de sauvegarder (document soumis), -Before Submit,Avant de soumettre, +Before Save (Submitted Document),Avant de sauvegarder (document valider), +Before Submit,Avant de valider, Blank Template,Modèle vierge, Callback URL,URL de rappel, Cancel All Documents,Annuler tous les documents, @@ -3556,11 +3556,11 @@ Skipping column {0},Colonne ignorée {0}, Social Home,Maison sociale, Some columns might get cut off when printing to PDF. Try to keep number of columns under 10.,Certaines colonnes peuvent être coupées lors de l'impression au format PDF. Essayez de garder le nombre de colonnes sous 10., Something went wrong during the token generation. Click on {0} to generate a new one.,Quelque chose s'est mal passé pendant la génération de jetons. Cliquez sur {0} pour en générer un nouveau., -Submit After Import,Soumettre après importation, -Submitting...,Soumission..., +Submit After Import,Validation après importation, +Submitting...,Validation..., Success! You are good to go 👍,Succès! Vous êtes bon pour aller, Successful Transactions,Transactions réussies, -Successfully Submitted!,Soumis avec succès!, +Successfully Submitted!,Validation avec succès!, Successfully imported {0} record.,{0} enregistrement importé avec succès., Successfully imported {0} records.,{0} enregistrements importés avec succès., Successfully updated {0} record.,{0} enregistrement mis à jour avec succès., @@ -3659,7 +3659,7 @@ choose an,choisir un, empty,vide, of,de, or attach a,ou attacher un, -submitted this document {0},a soumis ce document {0}, +submitted this document {0},a validé ce document {0}, "tag name..., e.g. #tag","nom de tag ..., par exemple #tag", uploaded file,fichier téléchargé, via Data Import,via importation de données, @@ -3678,7 +3678,7 @@ via Data Import,via importation de données, {0} shared a document {1} {2} with you,{0} a partagé un document {1} {2} avec vous, {0} should not be same as {1},{0} ne doit pas être identique à {1}, {0} translations pending,{0} traductions en attente, -{0} {1} is linked with the following submitted documents: {2},{0} {1} est lié aux documents soumis suivants: {2}, +{0} {1} is linked with the following submitted documents: {2},{0} {1} est lié aux documents validés suivants: {2}, "{0}: Failed to attach new recurring document. To enable attaching document in the auto repeat notification email, enable {1} in Print Settings","{0}: Impossible de joindre un nouveau document récurrent. Pour activer la pièce jointe dans l'e-mail de notification de répétition automatique, activez {1} dans Paramètres d'impression", {0}: Fieldname cannot be one of {1},{0}: le nom de champ ne peut pas être l'un des {1}, {} Complete,{} Achevée, @@ -3793,7 +3793,7 @@ Sr,Sr, Start,Démarrer, Start Time,Heure de Début, Status,Statut, -Submitted,Soumis, +Submitted,Validé, Tag,Étiquette, Template,Modèle, Thursday,Jeudi, @@ -4146,7 +4146,7 @@ Collapse,Réduire, "Invalid token, please provide a valid token with prefix 'Basic' or 'Token'.","Jeton non valide, veuillez fournir un jeton valide avec le préfixe «Basic» ou «Token».", {0} is not a valid Name,{0} n'est pas un nom valide, Your system is being updated. Please refresh again after a few moments.,Votre système est en cours de mise à jour. Veuillez actualiser à nouveau après quelques instants., -{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.,{0} {1}: l'enregistrement soumis ne peut pas être supprimé. Vous devez d'abord {2} l'annuler {3}., +{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.,{0} {1}: l'enregistrement validé ne peut pas être supprimé. Vous devez d'abord {2} l'annuler {3}., Invalid naming series (. missing) for {0},Série de noms non valide (. Manquante) pour {0}, Error has occurred in {0},Une erreur s'est produite dans {0}, Status Updated,Statut mis à jour, @@ -4510,7 +4510,7 @@ Oops,Oups, Skip Step,Passer l'étape, "You're doing great, let's take you back to the onboarding page.","Vous vous débrouillez très bien, revenons à la page d'intégration.", Good Work 🎉,Bon travail 🎉, -Submit this document to complete this step.,Soumettez ce document pour terminer cette étape., +Submit this document to complete this step.,Validez ce document pour terminer cette étape., Great,Génial, You may continue with onboarding,Vous pouvez continuer avec l'intégration, You seem good to go!,Vous semblez prêt à partir!, From 1e63475b2ae6bfced2a473f7554f8963b2b43681 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Mon, 11 Apr 2022 13:07:52 +0530 Subject: [PATCH 48/55] style: Added gap between from and to field --- frappe/public/scss/common/modal.scss | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/frappe/public/scss/common/modal.scss b/frappe/public/scss/common/modal.scss index c9217a075e..0de34f4ae4 100644 --- a/frappe/public/scss/common/modal.scss +++ b/frappe/public/scss/common/modal.scss @@ -210,16 +210,22 @@ body.modal-open[style^="padding-right"] { form { display: flex; align-items: center; - .frappe-control:first-child { - flex: 1; - margin-bottom: 0px; - } - .frappe-control:last-child { - margin-left: 10px; - margin-bottom: -24px; - button { - // same as form-control input - height: calc(1.5em + .75rem + 2px); + + .frappe-control { + &[data-fieldname="sender"] { + flex: 1; + margin-bottom: 0px; + } + &[data-fieldname="recipients"] { + margin-left: 10px; + } + &[data-fieldname="option_toggle_button"] { + margin-left: 10px; + margin-bottom: -24px; + button { + // same as form-control input + height: calc(1.5em + .75rem + 2px); + } } } } From 27d226f29546cd91ac653c62ecb6d15860f3d713 Mon Sep 17 00:00:00 2001 From: lapardnemihk1099 Date: Mon, 11 Apr 2022 14:32:14 +0530 Subject: [PATCH 49/55] test: theme switcher dialog shortcut test --- cypress/integration/theme_switcher_dialog.js | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 cypress/integration/theme_switcher_dialog.js diff --git a/cypress/integration/theme_switcher_dialog.js b/cypress/integration/theme_switcher_dialog.js new file mode 100644 index 0000000000..7d3c1305ba --- /dev/null +++ b/cypress/integration/theme_switcher_dialog.js @@ -0,0 +1,30 @@ +context('Theme Switcher Shortcut', () => { + before(() => { + cy.login(); + cy.visit('/app'); + }); + beforeEach(() => { + cy.reload(); + }); + it('Check Toggle', () => { + cy.open_theme_dialog('{ctrl+shift+g}'); + cy.get('.modal-backdrop').should('exist'); + cy.get('.theme-grid > div').first().click(); + cy.close_theme('{ctrl+shift+g}'); + cy.get('.modal-backdrop').should('not.exist'); + }); + it('Check Enter', () => { + cy.open_theme_dialog('{ctrl+shift+g}'); + cy.get('.theme-grid > div').first().click(); + cy.close_theme('{enter}'); + cy.get('.modal-backdrop').should('not.exist'); + }); + +}); + +Cypress.Commands.add('open_theme_dialog', (shortcut_keys) => { + cy.get('body').type(shortcut_keys); +}); +Cypress.Commands.add('close_theme', (shortcut_keys) => { + cy.get('.modal-header').type(shortcut_keys); +}); \ No newline at end of file From 5993a8ba57b6098dbb0d1e0c9dbad6d0bdaf3ee8 Mon Sep 17 00:00:00 2001 From: lapardnemihk1099 Date: Mon, 11 Apr 2022 14:45:13 +0530 Subject: [PATCH 50/55] chore: sider issues fixed --- cypress/integration/theme_switcher_dialog.js | 44 ++++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cypress/integration/theme_switcher_dialog.js b/cypress/integration/theme_switcher_dialog.js index 7d3c1305ba..b4297e5674 100644 --- a/cypress/integration/theme_switcher_dialog.js +++ b/cypress/integration/theme_switcher_dialog.js @@ -1,30 +1,30 @@ context('Theme Switcher Shortcut', () => { - before(() => { - cy.login(); - cy.visit('/app'); - }); - beforeEach(() => { - cy.reload(); - }); - it('Check Toggle', () => { - cy.open_theme_dialog('{ctrl+shift+g}'); - cy.get('.modal-backdrop').should('exist'); - cy.get('.theme-grid > div').first().click(); - cy.close_theme('{ctrl+shift+g}'); - cy.get('.modal-backdrop').should('not.exist'); - }); - it('Check Enter', () => { - cy.open_theme_dialog('{ctrl+shift+g}'); - cy.get('.theme-grid > div').first().click(); - cy.close_theme('{enter}'); - cy.get('.modal-backdrop').should('not.exist'); - }); + before(() => { + cy.login(); + cy.visit('/app'); + }); + beforeEach(() => { + cy.reload(); + }); + it('Check Toggle', () => { + cy.open_theme_dialog('{ctrl+shift+g}'); + cy.get('.modal-backdrop').should('exist'); + cy.get('.theme-grid > div').first().click(); + cy.close_theme('{ctrl+shift+g}'); + cy.get('.modal-backdrop').should('not.exist'); + }); + it('Check Enter', () => { + cy.open_theme_dialog('{ctrl+shift+g}'); + cy.get('.theme-grid > div').first().click(); + cy.close_theme('{enter}'); + cy.get('.modal-backdrop').should('not.exist'); + }); }); Cypress.Commands.add('open_theme_dialog', (shortcut_keys) => { - cy.get('body').type(shortcut_keys); + cy.get('body').type(shortcut_keys); }); Cypress.Commands.add('close_theme', (shortcut_keys) => { - cy.get('.modal-header').type(shortcut_keys); + cy.get('.modal-header').type(shortcut_keys); }); \ No newline at end of file From b51d8379b5e0fc675a5414e1f42100f75d97bb9c Mon Sep 17 00:00:00 2001 From: Shariq Ansari <30859809+shariquerik@users.noreply.github.com> Date: Mon, 11 Apr 2022 16:15:35 +0530 Subject: [PATCH 51/55] fix: Card link indicator hover color same as link background color (#16568) --- frappe/public/scss/desk/desktop.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss index e2684706f0..52157c888a 100644 --- a/frappe/public/scss/desk/desktop.scss +++ b/frappe/public/scss/desk/desktop.scss @@ -578,7 +578,7 @@ body { background-color: var(--fg-hover-color); .indicator-pill { - background-color: var(--fg-color); + background-color: var(--fg-hover-color); } } From 5c8856d66ea02f6d35e026fd8131d92d111f64e7 Mon Sep 17 00:00:00 2001 From: Abhishek Saxena <33656173+saxenabhishek@users.noreply.github.com> Date: Tue, 12 Apr 2022 10:37:25 +0530 Subject: [PATCH 52/55] refactor: db.sql calls to frappe.qb (#16107) # Changes - Introduces `subqry` class to use in where clause when there is a non-column condition. eg. > .where(subqry(no_of_roles) == 0) - Convert SQL queries to frappe.qb # Testing Functions with query refactors - frappe.boot.get_user_pages_or_reports() -> Same output of `get_bootinfo()` as develop - frappe.boot.get_unseen_notes() -> Forms the same query as develop ```sql SELECT `name`,`title`,`content`,`notify_on_every_login` FROM `tabNote` WHERE `notify_on_every_login`=1 AND `expire_notification_on`>'2022-03-30 01:10:53.393874' AND (SELECT `nsb`.`user` FROM `tabNote Seen By` `nsb` WHERE `nsb`.`parent`=`tabNote`.`name`) NOT IN ('Administrator') ``` - frappe.installer._delete_doctypes() -> installed and uninsalled a dummy app to drop tables ### Not tested - frappe.make_property_setter() - frappe.realtime.get_pending_tasks_for_doc() [whitelist method] - frappe.sessions.Session.start() - frappe.twofactor.cache_2fa_data() --- frappe/__init__.py | 7 ++- frappe/boot.py | 113 +++++++++++++++++++--------------- frappe/query_builder/terms.py | 1 + frappe/realtime.py | 5 -- frappe/sessions.py | 18 +++--- frappe/twofactor.py | 17 +++-- 6 files changed, 86 insertions(+), 75 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index f92f76c98c..08dcb9bf3f 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1290,7 +1290,12 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp {'parent': 'DocField', 'fieldname': args.property}, 'fieldtype') or 'Data' if not args.doctype: - doctype_list = db.sql_list('select distinct parent from tabDocField where fieldname=%s', args.fieldname) + DocField_doctype = qb.DocType("DocField") + doctype_list = ( + qb.from_(DocField_doctype).select(DocField_doctype.parent) + .where(DocField_doctype.fieldname == args.fieldname).distinct() + ).run(as_list=True) + else: doctype_list = [args.doctype] diff --git a/frappe/boot.py b/frappe/boot.py index b5008f778a..ae5bdfa8c0 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -18,6 +18,10 @@ from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_p from frappe.model.base_document import get_controller from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo from frappe.utils import get_time_zone, add_user_info +from frappe.query_builder import DocType +from frappe.query_builder.functions import Count +from frappe.query_builder.terms import subqry + def get_bootinfo(): """build and return boot info""" @@ -129,43 +133,53 @@ def get_user_pages_or_reports(parent, cache=False): roles = frappe.get_roles() has_role = {} - column = get_column(parent) + + page = DocType("Page") + report = DocType("Report") + + if parent == "Report": + columns = (report.name.as_("title"), report.ref_doctype, report.report_type) + else: + columns = (page.title.as_("title"), ) + + + customRole = DocType("Custom Role") + hasRole = DocType("Has Role") + parentTable = DocType(parent) # get pages or reports set on custom role - pages_with_custom_roles = frappe.db.sql(""" - select - `tabCustom Role`.{field} as name, - `tabCustom Role`.modified, - `tabCustom Role`.ref_doctype, - {column} - from `tabCustom Role`, `tabHas Role`, `tab{parent}` - where - `tabHas Role`.parent = `tabCustom Role`.name - and `tab{parent}`.name = `tabCustom Role`.{field} - and `tabCustom Role`.{field} is not null - and `tabHas Role`.role in ({roles}) - """.format(field=parent.lower(), parent=parent, column=column, - roles = ', '.join(['%s']*len(roles))), roles, as_dict=1) + pages_with_custom_roles = ( + frappe.qb.from_(customRole).from_(hasRole).from_(parentTable) + .select(customRole[parent.lower()].as_("name"), customRole.modified, customRole.ref_doctype, *columns) + .where( + (hasRole.parent == customRole.name) + & (parentTable.name == customRole[parent.lower()]) + & (customRole[parent.lower()].isnotnull()) + & (hasRole.role.isin(roles))) + ).run(as_dict=True) for p in pages_with_custom_roles: has_role[p.name] = {"modified":p.modified, "title": p.title, "ref_doctype": p.ref_doctype} - pages_with_standard_roles = frappe.db.sql(""" - select distinct - `tab{parent}`.name as name, - `tab{parent}`.modified, - {column} - from `tabHas Role`, `tab{parent}` - where - `tabHas Role`.role in ({roles}) - and `tabHas Role`.parent = `tab{parent}`.name - and `tab{parent}`.`name` not in ( - select `tabCustom Role`.{field} from `tabCustom Role` - where `tabCustom Role`.{field} is not null) - {condition} - """.format(parent=parent, column=column, roles = ', '.join(['%s']*len(roles)), - field=parent.lower(), condition="and `tabReport`.disabled=0" if parent == "Report" else ""), - roles, as_dict=True) + subq = ( + frappe.qb.from_(customRole).select(customRole[parent.lower()]) + .where(customRole[parent.lower()].isnotnull()) + ) + + pages_with_standard_roles = ( + frappe.qb.from_(hasRole).from_(parentTable) + .select(parentTable.name.as_("name"), parentTable.modified, *columns) + .where( + (hasRole.role.isin(roles)) + & (hasRole.parent == parentTable.name) + & (parentTable.name.notin(subq)) + ).distinct() + ) + + if parent == "Report": + pages_with_standard_roles = pages_with_standard_roles.where(report.disabled == 0) + + pages_with_standard_roles = pages_with_standard_roles.run(as_dict=True) for p in pages_with_standard_roles: if p.name not in has_role: @@ -173,16 +187,16 @@ def get_user_pages_or_reports(parent, cache=False): if parent == "Report": has_role[p.name].update({'ref_doctype': p.ref_doctype}) + no_of_roles = (frappe.qb.from_(hasRole).select(Count("*")) + .where(hasRole.parent == parentTable.name) + ) + # pages with no role are allowed if parent =="Page": - pages_with_no_roles = frappe.db.sql(""" - select - `tab{parent}`.name, `tab{parent}`.modified, {column} - from `tab{parent}` - where - (select count(*) from `tabHas Role` - where `tabHas Role`.parent=`tab{parent}`.`name`) = 0 - """.format(parent=parent, column=column), as_dict=1) + + pages_with_no_roles = (frappe.qb.from_(parentTable).select(parentTable.name, parentTable.modified, *columns) + .where(subqry(no_of_roles) == 0) + ).run(as_dict=True) for p in pages_with_no_roles: if p.name not in has_role: @@ -201,13 +215,6 @@ def get_user_pages_or_reports(parent, cache=False): _cache.set_value('has_role:' + parent, has_role, frappe.session.user, 21600) return has_role -def get_column(doctype): - column = "`tabPage`.title as title" - if doctype == "Report": - column = "`tabReport`.`name` as title, `tabReport`.ref_doctype, `tabReport`.report_type" - - return column - def load_translations(bootinfo): messages = frappe.get_lang_dict("boot") @@ -271,10 +278,16 @@ def load_print_css(bootinfo, print_settings): bootinfo.print_css = frappe.www.printview.get_print_style(print_settings.print_style or "Redesign", for_legacy=True) def get_unseen_notes(): - return frappe.db.sql('''select `name`, title, content, notify_on_every_login from `tabNote` where notify_on_login=1 - and expire_notification_on > %s and %s not in - (select user from `tabNote Seen By` nsb - where nsb.parent=`tabNote`.name)''', (frappe.utils.now(), frappe.session.user), as_dict=True) + note = DocType("Note") + nsb = DocType("Note Seen By").as_("nsb") + + return ( + frappe.qb.from_(note).select(note.name, note.title, note.content, note.notify_on_every_login) + .where( + (note.notify_on_every_login == 1) + & (note.expire_notification_on > frappe.utils.now()) + & (subqry(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin([frappe.session.user]))) + ).run(as_dict=1) def get_success_action(): return frappe.get_all("Success Action", fields=["*"]) diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index d3785e049a..aee6bf029e 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -103,6 +103,7 @@ class ParameterizedFunction(Function): return function_sql + class subqry(Criterion): def __init__(self, subq: QueryBuilder, alias: Optional[str] = None,) -> None: super().__init__(alias) diff --git a/frappe/realtime.py b/frappe/realtime.py index 940a3220a4..dc47599923 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -9,11 +9,6 @@ import redis redis_server = None -@frappe.whitelist() -def get_pending_tasks_for_doc(doctype, docname): - return frappe.db.sql_list("select name from `tabAsync Task` where status in ('Queued', 'Running') and reference_doctype=%s and reference_name=%s", (doctype, docname)) - - def publish_progress(percent, title=None, doctype=None, docname=None, description=None): publish_realtime('progress', {'percent': percent, 'title': title, 'description': description}, user=frappe.session.user, doctype=doctype, docname=docname) diff --git a/frappe/sessions.py b/frappe/sessions.py index 6a5771b617..4bbcaaa2ae 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -244,16 +244,14 @@ class Session: # update user user = frappe.get_doc("User", self.data['user']) - frappe.db.sql("""UPDATE `tabUser` - SET - last_login = %(now)s, - last_ip = %(ip)s, - last_active = %(now)s - WHERE name=%(name)s""", { - 'now': frappe.utils.now(), - 'ip': frappe.local.request_ip, - 'name': self.data['user'] - }) + user_doctype=frappe.qb.DocType("User") + (frappe.qb.update(user_doctype) + .set(user_doctype.last_login, frappe.utils.now()) + .set(user_doctype.last_ip, frappe.local.request_ip) + .set(user_doctype.last_active, frappe.utils.now()) + .where(user_doctype.name == self.data['user']) + ).run() + user.run_notifications("before_change") user.run_notifications("on_update") frappe.db.commit() diff --git a/frappe/twofactor.py b/frappe/twofactor.py index bd49d588b0..bb063faf5a 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -84,22 +84,21 @@ def two_factor_is_enabled_for_(user): return False if isinstance(user, str): - user = frappe.get_doc('User', user) + user = frappe.get_doc("User", user) - roles = [frappe.db.escape(d.role) for d in user.roles or []] - roles.append("'All'") + roles = [d.role for d in user.roles or []] + ["All"] - query = """SELECT `name` - FROM `tabRole` - WHERE `two_factor_auth`= 1 - AND `name` IN ({0}) - LIMIT 1""".format(", ".join(roles)) + role_doctype = frappe.qb.DocType("Role") + no_of_users = frappe.db.count(role_doctype, filters= + ((role_doctype.two_factor_auth == 1) & (role_doctype.name.isin(roles))), + ) - if len(frappe.db.sql(query)) > 0: + if int(no_of_users) > 0: return True return False + def get_otpsecret_for_(user): '''Set OTP Secret for user even if not set.''' otp_secret = frappe.db.get_default(user + '_otpsecret') From c0c5b2ebdddbe8898ce2d5e5365f4931ff73b6bf Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 12 Apr 2022 10:59:25 +0530 Subject: [PATCH 53/55] style: format all python files using black (#16453) Co-authored-by: Frappe Bot --- .git-blame-ignore-revs | 3 + .pre-commit-config.yaml | 11 + frappe/__init__.py | 945 ++++++--- frappe/api.py | 87 +- frappe/app.py | 192 +- frappe/auth.py | 191 +- .../assignment_rule/assignment_rule.py | 187 +- .../assignment_rule/test_assignment_rule.py | 326 +-- .../assignment_rule_day.py | 1 + .../assignment_rule_user.py | 1 + .../doctype/auto_repeat/auto_repeat.py | 290 +-- .../doctype/auto_repeat/test_auto_repeat.py | 249 ++- .../auto_repeat_day/auto_repeat_day.py | 1 + .../automation/doctype/milestone/milestone.py | 2 + .../doctype/milestone/test_milestone.py | 3 +- .../milestone_tracker/milestone_tracker.py | 43 +- .../test_milestone_tracker.py | 50 +- frappe/boot.py | 174 +- frappe/build.py | 48 +- frappe/cache_manager.py | 147 +- frappe/client.py | 226 ++- frappe/commands/__init__.py | 54 +- frappe/commands/redis_utils.py | 80 +- frappe/commands/scheduler.py | 113 +- frappe/commands/site.py | 662 ++++--- frappe/commands/translate.py | 59 +- frappe/commands/utils.py | 626 +++--- frappe/config/__init__.py | 30 +- frappe/contacts/address_and_contact.py | 112 +- frappe/contacts/doctype/address/address.py | 139 +- .../contacts/doctype/address/test_address.py | 41 +- .../address_template/address_template.py | 28 +- .../address_template/test_address_template.py | 28 +- frappe/contacts/doctype/contact/contact.py | 176 +- .../contacts/doctype/contact/test_contact.py | 17 +- .../doctype/contact_email/contact_email.py | 1 + .../doctype/contact_phone/contact_phone.py | 1 + frappe/contacts/doctype/gender/gender.py | 1 + frappe/contacts/doctype/gender/test_gender.py | 1 + .../contacts/doctype/salutation/salutation.py | 1 + .../doctype/salutation/test_salutation.py | 1 + .../addresses_and_contacts.py | 50 +- .../test_addresses_and_contacts.py | 124 +- frappe/core/doctype/__init__.py | 1 - frappe/core/doctype/access_log/access_log.py | 44 +- .../doctype/access_log/test_access_log.py | 62 +- .../core/doctype/activity_log/activity_log.py | 23 +- frappe/core/doctype/activity_log/feed.py | 81 +- .../doctype/activity_log/test_activity_log.py | 70 +- .../core/doctype/block_module/block_module.py | 1 + frappe/core/doctype/comment/comment.py | 134 +- frappe/core/doctype/comment/test_comment.py | 86 +- frappe/core/doctype/communication/__init__.py | 1 - .../doctype/communication/communication.py | 298 +-- frappe/core/doctype/communication/email.py | 150 +- frappe/core/doctype/communication/mixins.py | 150 +- .../communication/test_communication.py | 371 ++-- .../communication_link/communication_link.py | 4 +- .../doctype/custom_docperm/custom_docperm.py | 3 +- .../custom_docperm/test_custom_docperm.py | 4 +- .../core/doctype/custom_role/custom_role.py | 10 +- .../doctype/custom_role/test_custom_role.py | 4 +- .../core/doctype/data_export/data_export.py | 1 + frappe/core/doctype/data_export/exporter.py | 268 ++- .../doctype/data_export/test_data_exporter.py | 108 +- .../core/doctype/data_import/data_import.py | 55 +- frappe/core/doctype/data_import/exporter.py | 27 +- frappe/core/doctype/data_import/importer.py | 251 +-- .../doctype/data_import/test_data_import.py | 1 + .../core/doctype/data_import/test_exporter.py | 16 +- .../core/doctype/data_import/test_importer.py | 249 ++- .../data_import_log/data_import_log.py | 1 + .../data_import_log/test_data_import_log.py | 1 + frappe/core/doctype/defaultvalue/__init__.py | 1 - .../core/doctype/defaultvalue/defaultvalue.py | 19 +- .../deleted_document/deleted_document.py | 19 +- .../deleted_document/test_deleted_document.py | 4 +- frappe/core/doctype/docfield/__init__.py | 1 - frappe/core/doctype/docfield/docfield.py | 26 +- frappe/core/doctype/docperm/__init__.py | 1 - frappe/core/doctype/docperm/docperm.py | 2 +- frappe/core/doctype/docshare/docshare.py | 34 +- frappe/core/doctype/docshare/test_docshare.py | 30 +- frappe/core/doctype/doctype/__init__.py | 1 - frappe/core/doctype/doctype/doctype.py | 896 ++++++--- .../core/doctype/doctype/patches/set_route.py | 7 +- frappe/core/doctype/doctype/test_doctype.py | 417 ++-- .../doctype/doctype_action/doctype_action.py | 1 + .../core/doctype/doctype_link/doctype_link.py | 1 + .../doctype/doctype_state/doctype_state.py | 1 + .../document_naming_rule.py | 30 +- .../test_document_naming_rule.py | 81 +- .../document_naming_rule_condition.py | 1 + .../test_document_naming_rule_condition.py | 1 + frappe/core/doctype/domain/domain.py | 72 +- frappe/core/doctype/domain/test_domain.py | 4 +- .../domain_settings/domain_settings.py | 52 +- .../core/doctype/dynamic_link/dynamic_link.py | 5 +- frappe/core/doctype/error_log/error_log.py | 15 +- .../core/doctype/error_log/test_error_log.py | 4 +- .../doctype/error_snapshot/error_snapshot.py | 14 +- .../error_snapshot/test_error_snapshot.py | 4 +- frappe/core/doctype/feedback/feedback.py | 1 + frappe/core/doctype/feedback/test_feedback.py | 11 +- frappe/core/doctype/file/file.py | 408 ++-- frappe/core/doctype/file/test_file.py | 648 +++--- frappe/core/doctype/has_domain/has_domain.py | 1 + frappe/core/doctype/has_role/has_role.py | 1 + .../installed_application.py | 1 + .../installed_applications.py | 16 +- .../test_installed_applications.py | 1 + frappe/core/doctype/language/language.py | 54 +- frappe/core/doctype/language/test_language.py | 4 +- .../log_setting_user/log_setting_user.py | 1 + .../log_setting_user/test_log_setting_user.py | 1 + .../core/doctype/log_settings/log_settings.py | 21 +- .../doctype/log_settings/test_log_settings.py | 26 +- frappe/core/doctype/module_def/__init__.py | 1 - frappe/core/doctype/module_def/module_def.py | 12 +- .../doctype/module_def/test_module_def.py | 4 +- .../doctype/module_profile/module_profile.py | 5 +- .../module_profile/test_module_profile.py | 36 +- .../core/doctype/navbar_item/navbar_item.py | 1 + .../doctype/navbar_item/test_navbar_item.py | 1 + .../navbar_settings/navbar_settings.py | 23 +- .../navbar_settings/test_navbar_settings.py | 1 + frappe/core/doctype/package/package.py | 13 +- frappe/core/doctype/package/test_package.py | 145 +- .../doctype/package_import/package_import.py | 41 +- .../package_import/test_package_import.py | 1 + .../package_release/package_release.py | 85 +- .../package_release/test_package_release.py | 1 + frappe/core/doctype/page/__init__.py | 1 - frappe/core/doctype/page/page.py | 107 +- .../doctype/page/patches/drop_unused_pages.py | 5 +- frappe/core/doctype/page/test_page.py | 16 +- frappe/core/doctype/patch_log/__init__.py | 1 - frappe/core/doctype/patch_log/patch_log.py | 4 +- .../core/doctype/patch_log/test_patch_log.py | 4 +- .../payment_gateway/payment_gateway.py | 3 +- .../payment_gateway/test_payment_gateway.py | 4 +- .../prepared_report/prepared_report.py | 80 +- .../prepared_report/test_prepared_report.py | 31 +- frappe/core/doctype/report/__init__.py | 1 - .../doctype/report/boilerplate/controller.py | 1 + frappe/core/doctype/report/report.py | 230 ++- frappe/core/doctype/report/test_report.py | 416 ++-- .../doctype/report_column/report_column.py | 1 + .../doctype/report_filter/report_filter.py | 1 + frappe/core/doctype/role/__init__.py | 1 - .../v13_set_default_desk_properties.py | 10 +- frappe/core/doctype/role/role.py | 65 +- frappe/core/doctype/role/test_role.py | 31 +- .../role_permission_for_page_and_report.py | 44 +- .../core/doctype/role_profile/role_profile.py | 11 +- .../doctype/role_profile/test_role_profile.py | 42 +- .../scheduled_job_log/scheduled_job_log.py | 1 + .../test_scheduled_job_log.py | 1 + .../scheduled_job_type/scheduled_job_type.py | 50 +- .../test_scheduled_job_type.py | 78 +- .../doctype/server_script/server_script.py | 59 +- .../server_script/server_script_utils.py | 62 +- .../server_script/test_server_script.py | 148 +- .../session_default/session_default.py | 1 + .../session_default_settings.py | 29 +- .../test_session_default_settings.py | 21 +- .../doctype/sms_parameter/sms_parameter.py | 4 +- .../core/doctype/sms_settings/sms_settings.py | 55 +- .../doctype/sms_settings/test_sms_settings.py | 4 +- .../doctype/success_action/success_action.py | 1 + .../system_settings/system_settings.py | 42 +- .../system_settings/test_system_settings.py | 4 +- frappe/core/doctype/test/test.py | 5 +- frappe/core/doctype/test/test_test.py | 1 + .../transaction_log/test_transaction_log.py | 57 +- .../transaction_log/transaction_log.py | 19 +- .../doctype/translation/test_translation.py | 51 +- .../core/doctype/translation/translation.py | 69 +- frappe/core/doctype/user/test_user.py | 263 ++- frappe/core/doctype/user/user.py | 609 +++--- .../user_document_type/user_document_type.py | 1 + frappe/core/doctype/user_email/user_email.py | 1 + .../doctype/user_group/test_user_group.py | 1 + frappe/core/doctype/user_group/user_group.py | 8 +- .../test_user_group_member.py | 1 + .../user_group_member/user_group_member.py | 1 + .../user_permission/test_user_permission.py | 189 +- .../user_permission/user_permission.py | 248 ++- .../user_select_document_type.py | 1 + .../user_social_login/user_social_login.py | 1 + .../core/doctype/user_type/test_user_type.py | 54 +- frappe/core/doctype/user_type/user_type.py | 240 ++- .../doctype/user_type/user_type_dashboard.py | 12 +- .../user_type_module/user_type_module.py | 1 + frappe/core/doctype/version/test_version.py | 30 +- frappe/core/doctype/version/version.py | 49 +- frappe/core/doctype/view_log/test_view_log.py | 35 +- frappe/core/doctype/view_log/view_log.py | 1 + frappe/core/notifications.py | 17 +- frappe/core/page/__init__.py | 1 - .../page/background_jobs/background_jobs.py | 46 +- .../core/page/permission_manager/__init__.py | 1 - .../permission_manager/permission_manager.py | 108 +- frappe/core/report/__init__.py | 1 - .../permitted_documents_for_user.py | 29 +- .../transaction_log_report.py | 65 +- frappe/core/utils.py | 38 +- .../web_form/edit_profile/edit_profile.py | 1 + frappe/coverage.py | 37 +- .../custom/doctype/client_script/__init__.py | 1 - .../doctype/client_script/client_script.py | 5 +- .../client_script/test_client_script.py | 4 +- .../custom/doctype/custom_field/__init__.py | 1 - .../doctype/custom_field/custom_field.py | 94 +- .../doctype/custom_field/test_custom_field.py | 15 +- .../custom/doctype/customize_form/__init__.py | 1 - .../doctype/customize_form/customize_form.py | 487 ++--- .../customize_form/test_customize_form.py | 211 +- .../doctype/customize_form_field/__init__.py | 1 - .../customize_form_field.py | 4 +- .../doctype/doctype_layout/doctype_layout.py | 2 +- .../convert_web_forms_to_doctype_layout.py | 25 +- .../doctype_layout/test_doctype_layout.py | 1 + .../doctype_layout_field.py | 1 + .../doctype/property_setter/__init__.py | 1 - .../property_setter/property_setter.py | 57 +- .../property_setter/test_property_setter.py | 4 +- .../test_rename_new/test_rename_new.py | 1 + .../test_rename_new/test_test_rename_new.py | 1 + .../connectors/base.py | 4 +- .../connectors/frappe_connection.py | 15 +- .../data_migration_connector.py | 42 +- .../test_data_migration_connector.py | 1 + .../data_migration_mapping.py | 29 +- .../test_data_migration_mapping.py | 1 + .../data_migration_mapping_detail.py | 1 + .../data_migration_plan.py | 40 +- .../test_data_migration_plan.py | 1 + .../data_migration_plan_mapping.py | 1 + .../data_migration_run/data_migration_run.py | 328 ++-- .../test_data_migration_run.py | 181 +- frappe/database/__init__.py | 36 +- frappe/database/database.py | 407 ++-- frappe/database/db_manager.py | 37 +- frappe/database/mariadb/database.py | 191 +- frappe/database/mariadb/schema.py | 65 +- frappe/database/mariadb/setup_db.py | 79 +- frappe/database/postgres/database.py | 246 ++- frappe/database/postgres/schema.py | 65 +- frappe/database/postgres/setup_db.py | 40 +- frappe/database/query.py | 90 +- frappe/database/schema.py | 167 +- frappe/database/sequence.py | 10 +- frappe/defaults.py | 81 +- frappe/deferred_insert.py | 17 +- frappe/desk/__init__.py | 1 - frappe/desk/calendar.py | 19 +- frappe/desk/desk_page.py | 18 +- frappe/desk/desktop.py | 230 ++- .../desk/doctype/bulk_update/bulk_update.py | 39 +- .../doctype/calendar_view/calendar_view.py | 1 + .../desk/doctype/console_log/console_log.py | 1 + .../doctype/console_log/test_console_log.py | 1 + frappe/desk/doctype/dashboard/dashboard.py | 63 +- .../desk/doctype/dashboard/test_dashboard.py | 1 + .../dashboard_chart/dashboard_chart.py | 261 +-- .../dashboard_chart/test_dashboard_chart.py | 348 ++-- .../dashboard_chart_field.py | 1 + .../dashboard_chart_link.py | 1 + .../dashboard_chart_source.py | 22 +- .../test_dashboard_chart_source.py | 1 + .../dashboard_settings/dashboard_settings.py | 22 +- .../desk/doctype/desktop_icon/desktop_icon.py | 416 ++-- frappe/desk/doctype/event/__init__.py | 1 - frappe/desk/doctype/event/event.py | 177 +- frappe/desk/doctype/event/test_event.py | 83 +- .../event_participants/event_participants.py | 3 +- .../desk/doctype/form_tour/test_form_tour.py | 1 + .../doctype/form_tour_step/form_tour_step.py | 1 + .../global_search_doctype.py | 1 + .../global_search_settings.py | 21 +- .../desk/doctype/kanban_board/kanban_board.py | 106 +- .../doctype/kanban_board/test_kanban_board.py | 4 +- .../kanban_board_column.py | 1 + .../desk/doctype/list_filter/list_filter.py | 5 +- .../list_view_settings/list_view_settings.py | 43 +- .../test_list_view_settings.py | 1 + .../module_onboarding/module_onboarding.py | 4 +- .../test_module_onboarding.py | 1 + frappe/desk/doctype/note/note.py | 12 +- frappe/desk/doctype/note/test_note.py | 59 +- .../desk/doctype/note_seen_by/note_seen_by.py | 1 + .../notification_log/notification_log.py | 97 +- .../notification_log/test_notification_log.py | 53 +- .../notification_settings.py | 35 +- .../test_notification_settings.py | 1 + .../notification_subscribed_document.py | 1 + .../desk/doctype/number_card/number_card.py | 125 +- .../doctype/number_card/test_number_card.py | 1 + .../number_card_link/number_card_link.py | 1 + .../test_onboarding_permission.py | 1 + .../onboarding_step/onboarding_step.py | 10 +- .../onboarding_step/test_onboarding_step.py | 1 + .../doctype/route_history/route_history.py | 38 +- .../doctype/system_console/system_console.py | 33 +- .../system_console/test_system_console.py | 10 +- frappe/desk/doctype/tag/tag.py | 105 +- frappe/desk/doctype/tag/test_tag.py | 33 +- frappe/desk/doctype/tag_link/tag_link.py | 1 + frappe/desk/doctype/tag_link/test_tag_link.py | 1 + frappe/desk/doctype/todo/__init__.py | 1 - frappe/desk/doctype/todo/test_todo.py | 129 +- frappe/desk/doctype/todo/todo.py | 98 +- .../desk/doctype/workspace/test_workspace.py | 95 +- frappe/desk/doctype/workspace/workspace.py | 202 +- .../workspace_chart/workspace_chart.py | 1 + .../doctype/workspace_link/workspace_link.py | 1 + .../workspace_shortcut/workspace_shortcut.py | 1 + frappe/desk/form/__init__.py | 1 - frappe/desk/form/assign_to.py | 183 +- frappe/desk/form/document_follow.py | 263 +-- frappe/desk/form/linked_with.py | 354 ++-- frappe/desk/form/load.py | 291 +-- frappe/desk/form/meta.py | 100 +- frappe/desk/form/save.py | 21 +- frappe/desk/form/test_form.py | 7 +- frappe/desk/form/utils.py | 71 +- frappe/desk/gantt.py | 7 +- frappe/desk/leaderboard.py | 62 +- frappe/desk/like.py | 56 +- frappe/desk/link_preview.py | 39 +- frappe/desk/listview.py | 27 +- frappe/desk/moduleview.py | 278 +-- frappe/desk/notifications.py | 88 +- frappe/desk/page/activity/activity.py | 33 +- frappe/desk/page/backups/backups.py | 55 +- frappe/desk/page/leaderboard/leaderboard.py | 5 +- .../page/setup_wizard/install_fixtures.py | 37 +- frappe/desk/page/setup_wizard/setup_wizard.py | 268 +-- frappe/desk/page/user_profile/user_profile.py | 101 +- frappe/desk/query_report.py | 99 +- frappe/desk/report/todo/todo.py | 64 +- frappe/desk/report_dump.py | 35 +- frappe/desk/reportview.py | 295 +-- frappe/desk/search.py | 219 ++- frappe/desk/treeview.py | 57 +- frappe/desk/utils.py | 18 +- frappe/email/__init__.py | 53 +- .../auto_email_report/auto_email_report.py | 193 +- .../test_auto_email_report.py | 48 +- .../document_follow/document_follow.py | 2 +- .../document_follow/test_document_follow.py | 118 +- .../doctype/email_account/email_account.py | 430 ++-- .../email_account/test_email_account.py | 297 +-- .../doctype/email_domain/email_domain.py | 84 +- .../doctype/email_domain/test_email_domain.py | 9 +- .../email_flag_queue/email_flag_queue.py | 1 + .../email_flag_queue/test_email_flag_queue.py | 4 +- .../email/doctype/email_group/email_group.py | 63 +- .../doctype/email_group/test_email_group.py | 4 +- .../email_group_member/email_group_member.py | 6 +- .../test_email_group_member.py | 4 +- .../email/doctype/email_queue/email_queue.py | 302 +-- .../doctype/email_queue/test_email_queue.py | 4 +- .../email_queue_recipient.py | 8 +- frappe/email/doctype/email_rule/email_rule.py | 1 + .../doctype/email_rule/test_email_rule.py | 4 +- .../doctype/email_template/email_template.py | 14 +- .../email_template/test_email_template.py | 1 + .../email_unsubscribe/email_unsubscribe.py | 31 +- .../test_email_unsubscribe.py | 4 +- .../email/doctype/imap_folder/imap_folder.py | 1 + frappe/email/doctype/newsletter/exceptions.py | 3 + frappe/email/doctype/newsletter/newsletter.py | 88 +- .../doctype/newsletter/test_newsletter.py | 83 +- .../newsletter_attachment.py | 1 + .../newsletter_email_group.py | 1 + .../doctype/notification/notification.py | 258 +-- .../doctype/notification/test_notification.py | 331 ++-- .../notification_recipient.py | 1 + .../unhandled_email/test_unhandled_email.py | 4 +- .../unhandled_email/unhandled_email.py | 7 +- frappe/email/email_body.py | 374 ++-- frappe/email/inbox.py | 110 +- frappe/email/queue.py | 113 +- frappe/email/receive.py | 365 ++-- frappe/email/smtp.py | 29 +- frappe/email/test_email_body.py | 160 +- frappe/email/test_smtp.py | 69 +- frappe/email/utils.py | 10 +- .../document_type_mapping.py | 107 +- .../test_document_type_mapping.py | 1 + .../doctype/event_consumer/event_consumer.py | 136 +- .../event_consumer/test_event_consumer.py | 1 + .../doctype/event_producer/event_producer.py | 220 ++- .../event_producer/test_event_producer.py | 384 ++-- .../event_producer_last_update.py | 1 + .../test_event_producer_last_update.py | 1 + .../event_sync_log/test_event_sync_log.py | 1 + .../event_update_log/event_update_log.py | 171 +- .../event_update_log/test_event_update_log.py | 1 + .../event_update_log_consumer.py | 1 + frappe/exceptions.py | 240 ++- frappe/frappeclient.py | 271 +-- frappe/geo/country_info.py | 31 +- frappe/geo/doctype/country/country.py | 4 +- frappe/geo/doctype/country/test_country.py | 3 +- frappe/geo/doctype/currency/currency.py | 4 +- frappe/geo/doctype/currency/test_currency.py | 3 +- frappe/geo/utils.py | 61 +- frappe/handler.py | 137 +- frappe/hooks.py | 76 +- frappe/installer.py | 171 +- .../braintree_settings/braintree_settings.py | 256 ++- .../test_braintree_settings.py | 1 + .../doctype/connected_app/connected_app.py | 64 +- .../connected_app/test_connected_app.py | 98 +- .../dropbox_settings/dropbox_settings.py | 208 +- .../dropbox_settings/test_dropbox_settings.py | 1 + .../google_calendar/google_calendar.py | 304 ++- .../google_contacts/google_contacts.py | 203 +- .../doctype/google_drive/google_drive.py | 104 +- .../doctype/google_drive/test_google_drive.py | 1 + .../google_settings/google_settings.py | 4 +- .../google_settings/test_google_settings.py | 5 +- .../integration_request.py | 8 +- .../test_integration_request.py | 4 +- .../ldap_group_mapping/ldap_group_mapping.py | 1 + .../doctype/ldap_settings/ldap_settings.py | 175 +- .../ldap_settings/test_ldap_settings.py | 647 +++--- .../oauth_authorization_code.py | 1 + .../test_oauth_authorization_code.py | 4 +- .../oauth_bearer_token/oauth_bearer_token.py | 6 +- .../test_oauth_bearer_token.py | 4 +- .../doctype/oauth_client/oauth_client.py | 16 +- .../doctype/oauth_client/test_oauth_client.py | 4 +- .../oauth_provider_settings.py | 14 +- .../doctype/oauth_scope/oauth_scope.py | 1 + .../paypal_settings/paypal_settings.py | 323 +-- .../doctype/paytm_settings/paytm_settings.py | 164 +- .../paytm_settings/test_paytm_settings.py | 1 + .../query_parameters/query_parameters.py | 1 + .../razorpay_settings/razorpay_settings.py | 309 +-- .../s3_backup_settings/s3_backup_settings.py | 87 +- .../test_s3_backup_settings.py | 1 + .../slack_webhook_url/slack_webhook_url.py | 17 +- .../test_slack_webhook_url.py | 1 + .../social_login_key/social_login_key.py | 136 +- .../social_login_key/test_social_login_key.py | 60 +- .../social_login_keys/social_login_keys.py | 1 + .../stripe_settings/stripe_settings.py | 234 ++- .../stripe_settings/test_stripe_settings.py | 1 + .../doctype/token_cache/test_token_cache.py | 25 +- .../doctype/token_cache/token_cache.py | 34 +- .../integrations/doctype/webhook/__init__.py | 34 +- .../doctype/webhook/test_webhook.py | 59 +- .../integrations/doctype/webhook/webhook.py | 44 +- .../doctype/webhook_data/webhook_data.py | 1 + .../doctype/webhook_header/webhook_header.py | 1 + .../test_webhook_request_log.py | 1 + .../webhook_request_log.py | 1 + .../frappe_providers/frappecloud.py | 13 +- frappe/integrations/oauth2.py | 31 +- frappe/integrations/oauth2_logins.py | 12 +- frappe/integrations/offsite_backup_utils.py | 26 +- frappe/integrations/utils.py | 76 +- frappe/middlewares.py | 10 +- frappe/migrate.py | 17 +- frappe/model/__init__.py | 262 ++- frappe/model/base_document.py | 371 ++-- frappe/model/create_new.py | 71 +- frappe/model/db_query.py | 517 +++-- frappe/model/delete_doc.py | 317 +-- frappe/model/docfield.py | 36 +- frappe/model/document.py | 506 +++-- frappe/model/dynamic_links.py | 21 +- frappe/model/mapper.py | 91 +- frappe/model/meta.py | 312 +-- frappe/model/naming.py | 174 +- frappe/model/rename_doc.py | 345 ++-- frappe/model/sync.py | 36 +- frappe/model/utils/__init__.py | 78 +- frappe/model/utils/link_count.py | 33 +- frappe/model/utils/rename_doc.py | 20 +- frappe/model/utils/rename_field.py | 96 +- frappe/model/utils/user_settings.py | 78 +- frappe/model/workflow.py | 142 +- frappe/modules/__init__.py | 3 +- frappe/modules/export_file.py | 55 +- frappe/modules/import_file.py | 85 +- frappe/modules/patch_handler.py | 49 +- frappe/modules/utils.py | 197 +- frappe/monitor.py | 7 +- frappe/oauth.py | 77 +- frappe/parallel_test_runner.py | 113 +- ..._chat_by_default_within_system_settings.py | 10 +- frappe/patches/v10_0/enhance_security.py | 4 +- .../increase_single_table_column_length.py | 2 +- .../v10_0/migrate_passwords_passlib.py | 15 +- .../v10_0/modify_naming_series_table.py | 8 +- .../modify_smallest_currency_fraction.py | 3 +- .../v10_0/refactor_social_login_keys.py | 67 +- .../v10_0/reload_countries_and_currencies.py | 1 - ...remove_custom_field_for_disabled_domain.py | 2 +- .../patches/v10_0/set_default_locking_time.py | 3 +- .../v10_0/set_no_copy_to_workflow_state.py | 9 +- .../apply_customization_to_custom_doctype.py | 31 +- .../v11_0/change_email_signature_fieldtype.py | 15 +- .../v11_0/copy_fetch_data_from_options.py | 20 +- .../patches/v11_0/create_contact_for_user.py | 25 +- .../v11_0/delete_all_prepared_reports.py | 4 +- .../delete_duplicate_user_permissions.py | 19 +- .../drop_column_apply_user_permissions.py | 11 +- .../v11_0/fix_order_by_in_reports_json.py | 33 +- ...all_prepared_report_attachments_private.py | 13 +- ...igrate_report_settings_for_new_listview.py | 48 +- .../v11_0/multiple_references_in_events.py | 19 +- .../v11_0/reload_and_rename_view_log.py | 10 +- ...pe_user_permissions_for_page_and_report.py | 3 +- .../patches/v11_0/remove_skip_for_doctype.py | 46 +- .../rename_email_alert_to_notification.py | 14 +- .../v11_0/rename_google_maps_doctype.py | 8 +- ...rename_standard_reply_to_email_template.py | 6 +- ...rkflow_action_to_workflow_action_master.py | 9 +- .../v11_0/replicate_old_user_permissions.py | 55 +- .../set_allow_self_approval_in_workflow.py | 4 +- .../v11_0/set_default_letter_head_source.py | 7 +- .../patches/v11_0/set_dropbox_file_backup.py | 8 +- ...and_modified_value_for_user_permissions.py | 7 +- .../sync_stripe_settings_before_migrate.py | 17 +- .../v11_0/update_list_user_settings.py | 33 +- ...change_existing_dashboard_chart_filters.py | 17 +- .../patches/v12_0/copy_to_parent_for_tags.py | 3 +- .../create_notification_settings_for_user.py | 14 +- .../patches/v12_0/delete_duplicate_indexes.py | 14 +- .../delete_feedback_request_if_exists.py | 2 +- .../patches/v12_0/delete_gsuite_if_exists.py | 9 +- .../patches/v12_0/fix_email_id_formatting.py | 55 +- .../patches/v12_0/fix_public_private_files.py | 16 +- frappe/patches/v12_0/init_desk_settings.py | 2 + .../move_email_and_phone_to_child_table.py | 114 +- ..._form_attachments_to_attachments_folder.py | 7 +- .../move_timeline_links_to_dynamic_links.py | 48 +- .../remove_deprecated_fields_from_doctype.py | 17 +- .../remove_example_email_thread_notify.py | 6 +- .../patches/v12_0/remove_feedback_rating.py | 13 +- .../patches/v12_0/remove_gcalendar_gmaps.py | 15 +- .../patches/v12_0/rename_events_repeat_on.py | 31 +- .../rename_uploaded_files_with_proper_name.py | 26 +- .../v12_0/replace_null_values_in_tables.py | 25 +- frappe/patches/v12_0/reset_home_settings.py | 9 +- .../v12_0/set_correct_assign_value_in_docs.py | 7 +- .../patches/v12_0/set_correct_url_in_files.py | 42 +- .../v12_0/set_default_incoming_email_port.py | 33 +- .../v12_0/set_default_password_reset_limit.py | 2 +- .../v12_0/set_primary_key_in_series.py | 18 +- .../setup_comments_from_communications.py | 14 +- frappe/patches/v12_0/setup_email_linking.py | 3 +- frappe/patches/v12_0/setup_tags.py | 28 +- ..._auto_repeat_status_and_not_submittable.py | 22 +- frappe/patches/v12_0/update_global_search.py | 3 +- .../patches/v12_0/update_print_format_type.py | 13 +- ...webpage_migrate_description_to_meta_tag.py | 10 +- .../patches/v12_0/website_meta_tag_parent.py | 7 +- .../v13_0/add_standard_navbar_items.py | 8 +- .../add_switch_theme_to_navbar_settings.py | 23 +- .../add_toggle_width_in_navbar_settings.py | 23 +- frappe/patches/v13_0/cleanup_desk_cards.py | 48 +- ...eate_custom_dashboards_cards_and_charts.py | 43 +- ...delete_event_producer_and_consumer_keys.py | 1 + frappe/patches/v13_0/email_unsubscribe.py | 5 +- frappe/patches/v13_0/enable_custom_script.py | 7 +- .../generate_theme_files_in_public_folder.py | 4 +- .../patches/v13_0/increase_password_length.py | 1 + frappe/patches/v13_0/jinja_hook.py | 13 +- frappe/patches/v13_0/make_user_type.py | 12 +- .../v13_0/migrate_translation_column_data.py | 7 +- frappe/patches/v13_0/queryreport_columns.py | 13 +- frappe/patches/v13_0/remove_chat.py | 6 +- frappe/patches/v13_0/remove_custom_link.py | 19 +- .../v13_0/remove_duplicate_navbar_items.py | 4 +- .../remove_invalid_options_for_data_fields.py | 10 +- .../remove_tailwind_from_page_builder.py | 1 - .../patches/v13_0/remove_twilio_settings.py | 14 +- frappe/patches/v13_0/remove_web_view.py | 3 +- .../v13_0/rename_desk_page_to_workspace.py | 21 +- ...name_is_custom_field_in_dashboard_chart.py | 9 +- ...list_view_setting_to_list_view_settings.py | 15 +- .../v13_0/rename_notification_fields.py | 5 +- frappe/patches/v13_0/rename_onboarding.py | 2 +- ...place_field_target_with_open_in_new_tab.py | 5 +- .../patches/v13_0/replace_old_data_import.py | 3 +- ...set_existing_dashboard_charts_as_public.py | 12 +- .../v13_0/set_first_day_of_the_week.py | 3 +- .../set_path_for_homepage_in_web_page_view.py | 3 +- frappe/patches/v13_0/set_read_times.py | 13 +- .../v13_0/set_route_for_blog_category.py | 1 + frappe/patches/v13_0/set_social_icons.py | 3 +- .../patches/v13_0/set_unique_for_page_view.py | 7 +- frappe/patches/v13_0/site_wise_logging.py | 5 +- .../update_date_filters_in_user_settings.py | 46 +- .../patches/v13_0/update_duration_options.py | 25 +- .../update_icons_in_customized_desk_pages.py | 11 +- .../v13_0/update_newsletter_content_type.py | 9 +- .../update_notification_channel_if_empty.py | 7 +- .../patches/v13_0/web_template_set_module.py | 11 +- .../v13_0/website_theme_custom_scss.py | 26 +- frappe/patches/v14_0/copy_mail_data.py | 20 +- .../patches/v14_0/drop_data_import_legacy.py | 3 +- frappe/patches/v14_0/remove_db_aggregation.py | 27 +- .../v14_0/remove_post_and_post_comment.py | 1 + .../patches/v14_0/reset_creation_datetime.py | 13 +- .../patches/v14_0/save_ratings_in_fraction.py | 4 +- .../update_auto_account_deletion_duration.py | 1 + ...date_color_names_in_kanban_board_column.py | 26 +- .../v14_0/update_is_system_generated_flag.py | 7 +- frappe/patches/v14_0/update_workspace2.py | 48 +- frappe/permissions.py | 464 +++-- .../doctype/letter_head/letter_head.py | 33 +- .../doctype/letter_head/test_letter_head.py | 14 +- .../network_printer_settings.py | 23 +- .../test_network_printer_settings.py | 1 + .../doctype/print_format/print_format.py | 81 +- .../doctype/print_format/test_print_format.py | 16 +- .../print_format_field_template.py | 2 +- .../test_print_format_field_template.py | 1 + .../doctype/print_heading/print_heading.py | 1 + .../print_heading/test_print_heading.py | 4 +- .../doctype/print_settings/print_settings.py | 7 +- .../print_settings/test_print_settings.py | 1 + .../doctype/print_style/print_style.py | 10 +- .../doctype/print_style/test_print_style.py | 4 +- frappe/printing/page/print/print.py | 5 +- .../print_format_builder.py | 12 +- .../print_format_builder_beta.py | 4 +- frappe/query_builder/__init__.py | 10 +- frappe/query_builder/custom.py | 30 +- frappe/query_builder/functions.py | 54 +- frappe/query_builder/terms.py | 20 +- frappe/query_builder/utils.py | 24 +- frappe/rate_limiter.py | 29 +- frappe/realtime.py | 64 +- frappe/recorder.py | 10 +- frappe/search/__init__.py | 7 +- frappe/search/full_text_search.py | 53 +- frappe/search/test_full_text_search.py | 95 +- frappe/search/website_search.py | 38 +- frappe/sessions.py | 196 +- frappe/share.py | 128 +- .../energy_point_log/energy_point_log.py | 321 +-- .../energy_point_log/test_energy_point_log.py | 309 +-- .../energy_point_rule/energy_point_rule.py | 92 +- .../energy_point_settings.py | 31 +- .../test_energy_point_settings.py | 1 + .../doctype/review_level/review_level.py | 1 + .../templates/includes/comments/comments.py | 44 +- .../templates/includes/feedback/feedback.py | 23 +- .../pages/integrations/braintree_checkout.py | 41 +- .../pages/integrations/payment_cancel.py | 1 + .../pages/integrations/payment_success.py | 11 +- .../pages/integrations/paytm_checkout.py | 22 +- .../pages/integrations/razorpay_checkout.py | 56 +- .../pages/integrations/stripe_checkout.py | 51 +- frappe/test_runner.py | 224 ++- frappe/tests/__init__.py | 7 +- frappe/tests/test_api.py | 36 +- frappe/tests/test_assign.py | 64 +- frappe/tests/test_auth.py | 65 +- frappe/tests/test_background_jobs.py | 4 +- frappe/tests/test_boilerplate.py | 17 +- frappe/tests/test_bot.py | 1 + frappe/tests/test_child_table.py | 37 +- frappe/tests/test_client.py | 128 +- frappe/tests/test_commands.py | 189 +- frappe/tests/test_cors.py | 82 +- frappe/tests/test_db.py | 358 ++-- frappe/tests/test_db_query.py | 673 ++++--- frappe/tests/test_db_update.py | 91 +- frappe/tests/test_defaults.py | 22 +- frappe/tests/test_document.py | 162 +- frappe/tests/test_document_locks.py | 9 +- frappe/tests/test_domainification.py | 79 +- frappe/tests/test_dynamic_links.py | 102 +- frappe/tests/test_email.py | 311 ++- frappe/tests/test_exporter_fixtures.py | 68 +- frappe/tests/test_fixture_import.py | 4 +- frappe/tests/test_fmt_datetime.py | 70 +- frappe/tests/test_fmt_money.py | 12 +- frappe/tests/test_form_load.py | 115 +- frappe/tests/test_formatter.py | 22 +- frappe/tests/test_frappe_client.py | 144 +- frappe/tests/test_geo_ip.py | 4 +- frappe/tests/test_global_search.py | 159 +- frappe/tests/test_goal.py | 6 +- frappe/tests/test_hooks.py | 38 +- frappe/tests/test_linked_with.py | 107 +- frappe/tests/test_listview.py | 20 +- frappe/tests/test_monitor.py | 3 +- frappe/tests/test_naming.py | 197 +- frappe/tests/test_oauth20.py | 168 +- frappe/tests/test_password.py | 75 +- frappe/tests/test_patches.py | 37 +- frappe/tests/test_pdf.py | 6 +- frappe/tests/test_permissions.py | 356 ++-- frappe/tests/test_printview.py | 5 +- frappe/tests/test_query_builder.py | 175 +- frappe/tests/test_query_report.py | 2 +- frappe/tests/test_rate_limiter.py | 3 +- frappe/tests/test_recorder.py | 65 +- frappe/tests/test_redis.py | 45 +- frappe/tests/test_rename_doc.py | 70 +- frappe/tests/test_safe_exec.py | 49 +- frappe/tests/test_scheduler.py | 91 +- frappe/tests/test_search.py | 244 ++- frappe/tests/test_seen.py | 46 +- frappe/tests/test_sitemap.py | 16 +- frappe/tests/test_translate.py | 44 +- frappe/tests/test_twofactor.py | 174 +- frappe/tests/test_utils.py | 291 ++- frappe/tests/test_website.py | 239 +-- frappe/tests/tests_geo_utils.py | 48 +- frappe/tests/ui_test_helpers.py | 375 ++-- frappe/tests/utils.py | 14 +- frappe/translate.py | 449 +++-- frappe/twofactor.py | 398 ++-- frappe/utils/__init__.py | 385 ++-- frappe/utils/background_jobs.py | 204 +- frappe/utils/backups.py | 77 +- frappe/utils/bench_helper.py | 85 +- frappe/utils/boilerplate.py | 66 +- frappe/utils/change_log.py | 147 +- frappe/utils/commands.py | 25 +- frappe/utils/connections.py | 4 +- frappe/utils/csvutils.py | 96 +- frappe/utils/dashboard.py | 65 +- frappe/utils/data.py | 844 +++++--- frappe/utils/dateutils.py | 113 +- frappe/utils/diff.py | 4 +- frappe/utils/doctor.py | 44 +- frappe/utils/error.py | 117 +- frappe/utils/file_lock.py | 21 +- frappe/utils/file_manager.py | 226 ++- frappe/utils/fixtures.py | 22 +- frappe/utils/formatters.py | 58 +- frappe/utils/global_search.py | 276 +-- frappe/utils/goal.py | 21 +- frappe/utils/html_utils.py | 769 ++++++-- frappe/utils/identicon.py | 80 +- frappe/utils/image.py | 31 +- frappe/utils/install.py | 290 +-- frappe/utils/jinja.py | 74 +- frappe/utils/jinja_globals.py | 11 +- frappe/utils/lazy_loader.py | 3 +- frappe/utils/logger.py | 30 +- frappe/utils/make_random.py | 31 +- frappe/utils/momentjs.py | 1739 ++++++----------- frappe/utils/nestedset.py | 187 +- frappe/utils/oauth.py | 126 +- frappe/utils/password.py | 68 +- frappe/utils/password_strength.py | 66 +- frappe/utils/pdf.py | 93 +- frappe/utils/print_format.py | 86 +- frappe/utils/redis_queue.py | 48 +- frappe/utils/redis_wrapper.py | 18 +- frappe/utils/response.py | 187 +- frappe/utils/safe_exec.py | 385 ++-- frappe/utils/scheduler.py | 54 +- frappe/utils/testutils.py | 20 +- frappe/utils/user.py | 98 +- frappe/utils/verified_command.py | 35 +- frappe/utils/weasyprint.py | 74 +- frappe/utils/xlsxutils.py | 22 +- frappe/website/dashboard_fixtures.py | 65 +- .../about_us_settings/about_us_settings.py | 9 +- .../test_about_us_settings.py | 1 + .../about_us_team_member.py | 4 +- .../doctype/blog_category/blog_category.py | 5 +- .../blog_category/test_blog_category.py | 4 +- frappe/website/doctype/blog_post/blog_post.py | 167 +- .../doctype/blog_post/test_blog_post.py | 101 +- .../doctype/blog_settings/blog_settings.py | 8 +- .../blog_settings/test_blog_settings.py | 1 + frappe/website/doctype/blogger/blogger.py | 18 +- .../website/doctype/blogger/test_blogger.py | 3 +- frappe/website/doctype/color/color.py | 1 + frappe/website/doctype/color/test_color.py | 1 + .../company_history/company_history.py | 4 +- .../contact_us_settings.py | 6 +- .../discussion_reply/discussion_reply.py | 62 +- .../discussion_reply/test_discussion_reply.py | 1 + .../discussion_topic/discussion_topic.py | 27 +- .../discussion_topic/test_discussion_topic.py | 1 + .../doctype/help_article/help_article.py | 68 +- .../doctype/help_article/test_help_article.py | 4 +- .../doctype/help_category/help_category.py | 12 +- .../help_category/test_help_category.py | 4 +- .../personal_data_deletion_request.py | 60 +- .../test_personal_data_deletion_request.py | 16 +- .../personal_data_deletion_step.py | 1 + .../personal_data_download_request.py | 60 +- .../test_personal_data_download_request.py | 70 +- .../portal_menu_item/portal_menu_item.py | 1 + .../portal_settings/portal_settings.py | 29 +- .../portal_settings/test_portal_settings.py | 1 + .../social_link_settings.py | 1 + .../doctype/top_bar_item/top_bar_item.py | 4 +- .../website/doctype/web_form/test_web_form.py | 55 +- frappe/website/doctype/web_form/web_form.py | 279 +-- .../doctype/web_form_field/web_form_field.py | 1 + .../website/doctype/web_page/test_web_page.py | 66 +- frappe/website/doctype/web_page/web_page.py | 64 +- .../web_page_view/test_web_page_view.py | 1 + .../doctype/web_page_view/web_page_view.py | 13 +- .../doctype/web_template/test_web_template.py | 5 +- .../doctype/web_template/web_template.py | 8 +- .../test_web_template_field.py | 1 + .../web_template_field/web_template_field.py | 1 + .../website_meta_tag/website_meta_tag.py | 9 +- .../test_website_route_meta.py | 28 +- .../website_route_meta/website_route_meta.py | 3 +- .../website_route_redirect.py | 1 + .../doctype/website_script/website_script.py | 8 +- .../website_settings/google_indexing.py | 35 +- .../website_settings/test_website_settings.py | 1 + .../website_settings/website_settings.py | 123 +- .../website_sidebar/test_website_sidebar.py | 4 +- .../website_sidebar/website_sidebar.py | 2 +- .../website_sidebar_item.py | 1 + .../test_website_slideshow.py | 4 +- .../website_slideshow/website_slideshow.py | 12 +- .../website_slideshow_item.py | 4 +- .../website_theme/test_website_theme.py | 42 +- .../doctype/website_theme/website_theme.py | 90 +- .../website_theme_ignore_app.py | 1 + .../website/page_renderers/base_renderer.py | 16 +- .../page_renderers/base_template_page.py | 28 +- .../website/page_renderers/document_page.py | 14 +- frappe/website/page_renderers/error_page.py | 5 +- frappe/website/page_renderers/list_page.py | 5 +- .../website/page_renderers/not_found_page.py | 9 +- .../page_renderers/not_permitted_page.py | 14 +- frappe/website/page_renderers/print_page.py | 17 +- .../website/page_renderers/redirect_page.py | 14 +- frappe/website/page_renderers/static_page.py | 13 +- .../website/page_renderers/template_page.py | 133 +- frappe/website/page_renderers/web_form.py | 7 +- frappe/website/path_resolver.py | 97 +- .../website_analytics/website_analytics.py | 100 +- frappe/website/router.py | 155 +- frappe/website/serve.py | 3 +- frappe/website/utils.py | 274 +-- .../request_to_delete_data.py | 2 +- frappe/website/website_components/metatags.py | 37 +- frappe/website/website_generator.py | 81 +- frappe/workflow/doctype/workflow/__init__.py | 1 - .../doctype/workflow/test_workflow.py | 204 +- frappe/workflow/doctype/workflow/workflow.py | 110 +- .../doctype/workflow_action/__init__.py | 1 - .../workflow_action/test_workflow_action.py | 1 + .../workflow_action/workflow_action.py | 374 ++-- .../workflow_action_master.py | 1 + .../workflow_action_permitted_role.py | 1 + .../workflow_document_state/__init__.py | 1 - .../workflow_document_state.py | 4 +- .../doctype/workflow_state/__init__.py | 1 - .../workflow_state/test_workflow_state.py | 2 +- .../doctype/workflow_state/workflow_state.py | 4 +- .../doctype/workflow_transition/__init__.py | 1 - .../workflow_transition.py | 4 +- frappe/www/404.py | 1 + frappe/www/_test/_test_folder/_test_page.py | 2 +- frappe/www/_test/_test_home_page.py | 2 +- frappe/www/_test/_test_no_context.py | 3 +- frappe/www/about.py | 1 + frappe/www/app.py | 60 +- frappe/www/complete_signup.py | 1 - frappe/www/contact.py | 49 +- frappe/www/error.py | 6 +- frappe/www/list.py | 95 +- frappe/www/login.py | 70 +- frappe/www/me.py | 5 +- frappe/www/message.py | 8 +- frappe/www/printview.py | 245 ++- frappe/www/profile.py | 3 +- frappe/www/qrcode.py | 30 +- frappe/www/robots.py | 11 +- frappe/www/rss.py | 27 +- frappe/www/search.py | 31 +- frappe/www/sitemap.py | 33 +- frappe/www/third_party_apps.py | 34 +- frappe/www/unsubscribe.py | 19 +- frappe/www/update_password.py | 3 +- frappe/www/website_script.py | 11 +- pyproject.toml | 11 + setup.py | 40 +- 895 files changed, 35868 insertions(+), 25531 deletions(-) create mode 100644 pyproject.toml diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 96e9be8b3c..44f8a593d2 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -19,3 +19,6 @@ fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85 # Clean up whitespace b2fc959307c7c79f5584625569d5aed04133ba13 + +# Format codebase and sort imports +cb6f68e8c106ee2d037dd4b39dbb6d7c68caf1c8 \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3c3447cb3..b39f1ca85d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,17 @@ repos: - id: check-merge-conflict - id: check-ast + - repo: https://github.com/adityahase/black + rev: 9cb0a69f4d0030cdf687eddf314468b39ed54119 + hooks: + - id: black + additional_dependencies: ['click==8.0.4'] + + - repo: https://github.com/timothycrosley/isort + rev: 5.9.1 + hooks: + - id: isort + exclude: ".*setup.py$" ci: autoupdate_schedule: weekly diff --git a/frappe/__init__.py b/frappe/__init__.py index 08dcb9bf3f..32c8a75a2d 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -10,72 +10,87 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ -import os, warnings +import os +import warnings -STANDARD_USERS = ('Guest', 'Administrator') +STANDARD_USERS = ("Guest", "Administrator") -_dev_server = os.environ.get('DEV_SERVER', False) +_dev_server = os.environ.get("DEV_SERVER", False) if _dev_server: - warnings.simplefilter('always', DeprecationWarning) - warnings.simplefilter('always', PendingDeprecationWarning) + warnings.simplefilter("always", DeprecationWarning) + warnings.simplefilter("always", PendingDeprecationWarning) + +import importlib +import inspect +import json +import sys +from typing import TYPE_CHECKING, Dict, List, Union -import sys, importlib, inspect, json import click from werkzeug.local import Local, release_local -from typing import TYPE_CHECKING, Dict, List, Union + +from frappe.query_builder import get_query_builder, patch_query_aggregation, patch_query_execute +from frappe.utils.data import cstr # Local application imports from .exceptions import * -from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader) -from .utils.lazy_loader import lazy_import - -from frappe.query_builder import ( - get_query_builder, - patch_query_execute, - patch_query_aggregation, +from .utils.jinja import ( + get_email_from_template, + get_jenv, + get_jloader, + get_template, + render_template, ) -from frappe.utils.data import cstr +from .utils.lazy_loader import lazy_import -__version__ = '14.0.0-dev' +__version__ = "14.0.0-dev" __title__ = "Frappe Framework" local = Local() controllers = {} + class _dict(dict): """dict like object that exposes keys as attributes""" + def __getattr__(self, key): ret = self.get(key) # "__deepcopy__" exception added to fix frappe#14833 via DFP if not ret and key.startswith("__") and key != "__deepcopy__": raise AttributeError() return ret + def __setattr__(self, key, value): self[key] = value + def __getstate__(self): return self + def __setstate__(self, d): self.update(d) + def update(self, d): """update and return self -- the missing dict feature in python""" super(_dict, self).update(d) return self + def copy(self): return _dict(dict(self).copy()) + def _(msg, lang=None, context=None): """Returns translated string in current lang, if exists. - Usage: - _('Change') - _('Change', context='Coins') + Usage: + _('Change') + _('Change', context='Coins') """ from frappe.translate import get_full_dict - from frappe.utils import strip_html_tags, is_html + from frappe.utils import is_html, strip_html_tags - if not hasattr(local, 'lang'): - local.lang = lang or 'en' + if not hasattr(local, "lang"): + local.lang = lang or "en" if not lang: lang = local.lang @@ -88,9 +103,9 @@ def _(msg, lang=None, context=None): # msg should always be unicode msg = as_unicode(msg).strip() - translated_string = '' + translated_string = "" if context: - string_key = '{msg}:{context}'.format(msg=msg, context=context) + string_key = "{msg}:{context}".format(msg=msg, context=context) translated_string = get_full_dict(lang).get(string_key) if not translated_string: @@ -99,30 +114,36 @@ def _(msg, lang=None, context=None): # return lang_full_dict according to lang passed parameter return translated_string or non_translated_string -def as_unicode(text, encoding='utf-8'): - '''Convert to unicode if required''' + +def as_unicode(text, encoding="utf-8"): + """Convert to unicode if required""" if isinstance(text, str): return text elif text is None: - return '' + return "" elif isinstance(text, bytes): return str(text, encoding) else: return str(text) + def get_lang_dict(fortype, name=None): """Returns the translated language dict for the given type and name. - :param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot` - :param name: name of the document for which assets are to be returned.""" + :param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot` + :param name: name of the document for which assets are to be returned.""" from frappe.translate import get_dict + return get_dict(fortype, name) + def set_user_lang(user, user_language=None): """Guess and set user language for the session. `frappe.local.lang`""" from frappe.translate import get_user_lang + local.lang = get_user_lang(user) + # local-globals db = local("db") @@ -155,31 +176,34 @@ if TYPE_CHECKING: # end: static analysis hack + def init(site, sites_path=None, new_site=False): """Initialize frappe for the current site. Reset thread locals `frappe.local`""" if getattr(local, "initialised", None): return if not sites_path: - sites_path = '.' + sites_path = "." local.error_log = [] local.message_log = [] local.debug_log = [] local.realtime_log = [] - local.flags = _dict({ - "currently_saving": [], - "redirect_location": "", - "in_install_db": False, - "in_install_app": False, - "in_import": False, - "in_test": False, - "mute_messages": False, - "ignore_links": False, - "mute_emails": False, - "has_dataurl": False, - "new_site": new_site - }) + local.flags = _dict( + { + "currently_saving": [], + "redirect_location": "", + "in_install_db": False, + "in_install_app": False, + "in_import": False, + "in_test": False, + "mute_messages": False, + "ignore_links": False, + "mute_emails": False, + "has_dataurl": False, + "new_site": new_site, + } + ) local.rollback_observers = [] local.before_commit = [] local.test_objects = {} @@ -190,7 +214,7 @@ def init(site, sites_path=None, new_site=False): local.all_apps = None local.request_ip = None - local.response = _dict({"docs":[]}) + local.response = _dict({"docs": []}) local.task_id = None local.conf = _dict(get_site_config()) @@ -210,7 +234,7 @@ def init(site, sites_path=None, new_site=False): local.link_count = {} local.jenv = None - local.jloader =None + local.jloader = None local.cache = {} local.document_cache = {} local.meta_cache = {} @@ -226,6 +250,7 @@ def init(site, sites_path=None, new_site=False): local.initialised = True + def connect(site=None, db_name=None, set_admin_as_user=True): """Connect to site database instance. @@ -234,6 +259,7 @@ def connect(site=None, db_name=None, set_admin_as_user=True): :param set_admin_as_user: Set Administrator as current user. """ from frappe.database import get_db + if site: init(site) @@ -241,8 +267,10 @@ def connect(site=None, db_name=None, set_admin_as_user=True): if set_admin_as_user: set_user("Administrator") + def connect_replica(): from frappe.database import get_db + user = local.conf.db_name password = local.conf.db_password port = local.conf.replica_db_port @@ -257,6 +285,7 @@ def connect_replica(): local.primary_db = local.db local.db = local.replica_db + def get_site_config(sites_path=None, site_path=None): """Returns `site_config.json` combined with `sites/common_site_config.json`. `site_config` is a set of site wide settings like database name, password, email etc.""" @@ -287,8 +316,9 @@ def get_site_config(sites_path=None, site_path=None): return _dict(config) + def get_conf(site=None): - if hasattr(local, 'conf'): + if hasattr(local, "conf"): return local.conf else: @@ -296,10 +326,11 @@ def get_conf(site=None): with init_site(site): return local.conf + class init_site: def __init__(self, site=None): - '''If site is None, initialize it for empty site ('') to load common_site_config.json''' - self.site = site or '' + """If site is None, initialize it for empty site ('') to load common_site_config.json""" + self.site = site or "" def __enter__(self): init(self.site) @@ -308,6 +339,7 @@ class init_site: def __exit__(self, type, value, traceback): destroy() + def destroy(): """Closes connection and releases werkzeug local.""" if db: @@ -315,21 +347,27 @@ def destroy(): release_local(local) + redis_server = None + + def cache() -> "RedisWrapper": """Returns redis connection.""" global redis_server if not redis_server: from frappe.utils.redis_wrapper import RedisWrapper - redis_server = RedisWrapper.from_url(conf.get('redis_cache') - or "redis://localhost:11311") + + redis_server = RedisWrapper.from_url(conf.get("redis_cache") or "redis://localhost:11311") return redis_server + def get_traceback(): """Returns error traceback.""" from frappe.utils import get_traceback + return get_traceback() + def errprint(msg): """Log error. This is sent back as `exc` in response. @@ -340,8 +378,10 @@ def errprint(msg): error_log.append({"exc": msg}) + def print_sql(enable=True): - return cache().set_value('flag_print_sql', enable) + return cache().set_value("flag_print_sql", enable) + def log(msg): """Add to `debug_log`. @@ -353,7 +393,19 @@ def log(msg): debug_log.append(as_unicode(msg)) -def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False, indicator=None, alert=False, primary_action=None, is_minimizable=None, wide=None): + +def msgprint( + msg, + title=None, + raise_exception=0, + as_table=False, + as_list=False, + indicator=None, + alert=False, + primary_action=None, + is_minimizable=None, + wide=None, +): """Print a message to the user (via HTTP response). Messages are sent in the `__server_messages` property in the response JSON and shown in a pop-up / modal. @@ -399,7 +451,7 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False, out.title = title or _("Message", context="Default title of the message dialog") if not indicator and raise_exception: - indicator = 'red' + indicator = "red" if indicator: out.indicator = indicator @@ -421,14 +473,16 @@ def msgprint(msg, title=None, raise_exception=0, as_table=False, as_list=False, message_log.append(json.dumps(out)) - if raise_exception and hasattr(raise_exception, '__name__'): - local.response['exc_type'] = raise_exception.__name__ + if raise_exception and hasattr(raise_exception, "__name__"): + local.response["exc_type"] = raise_exception.__name__ _raise_exception() + def clear_messages(): local.message_log = [] + def get_message_log(): log = [] for msg_out in local.message_log: @@ -436,21 +490,33 @@ def get_message_log(): return log + def clear_last_message(): if len(local.message_log) > 0: local.message_log = local.message_log[:-1] + def throw(msg, exc=ValidationError, title=None, is_minimizable=None, wide=None, as_list=False): """Throw execption and show message (`msgprint`). :param msg: Message. :param exc: Exception class. Default `frappe.ValidationError`""" - msgprint(msg, raise_exception=exc, title=title, indicator='red', is_minimizable=is_minimizable, wide=wide, as_list=as_list) + msgprint( + msg, + raise_exception=exc, + title=title, + indicator="red", + is_minimizable=is_minimizable, + wide=wide, + as_list=as_list, + ) + def emit_js(js, user=False, **kwargs): if user is False: user = session.user - publish_realtime('eval_js', js, user=user, **kwargs) + publish_realtime("eval_js", js, user=user, **kwargs) + def create_folder(path, with_init=False): """Create a folder in the given path and add an `__init__.py` file (optional). @@ -458,12 +524,14 @@ def create_folder(path, with_init=False): :param path: Folder path. :param with_init: Create `__init__.py` in the new folder.""" from frappe.utils import touch_file + if not os.path.exists(path): os.makedirs(path) if with_init: touch_file(os.path.join(path, "__init__.py")) + def set_user(username): """Set current user. @@ -478,19 +546,24 @@ def set_user(username): local.new_doc_templates = {} local.user_perms = None + def get_user(): from frappe.utils.user import UserPermissions + if not local.user_perms: local.user_perms = UserPermissions(local.session.user) return local.user_perms + def get_roles(username=None): """Returns roles of current user.""" if not local.session: return ["Guest"] import frappe.permissions + return frappe.permissions.get_roles(username or local.session.user) + def get_request_header(key, default=None): """Return HTTP request header. @@ -498,13 +571,45 @@ def get_request_header(key, default=None): :param default: Default value.""" return request.headers.get(key, default) -def sendmail(recipients=None, sender="", subject="No Subject", message="No Message", - as_markdown=False, delayed=True, reference_doctype=None, reference_name=None, - unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, add_unsubscribe_link=1, - attachments=None, content=None, doctype=None, name=None, reply_to=None, queue_separately=False, - cc=None, bcc=None, message_id=None, in_reply_to=None, send_after=None, expose_recipients=None, - send_priority=1, communication=None, retry=1, now=None, read_receipt=None, is_notification=False, - inline_images=None, template=None, args=None, header=None, print_letterhead=False, with_container=False): + +def sendmail( + recipients=None, + sender="", + subject="No Subject", + message="No Message", + as_markdown=False, + delayed=True, + reference_doctype=None, + reference_name=None, + unsubscribe_method=None, + unsubscribe_params=None, + unsubscribe_message=None, + add_unsubscribe_link=1, + attachments=None, + content=None, + doctype=None, + name=None, + reply_to=None, + queue_separately=False, + cc=None, + bcc=None, + message_id=None, + in_reply_to=None, + send_after=None, + expose_recipients=None, + send_priority=1, + communication=None, + retry=1, + now=None, + read_receipt=None, + is_notification=False, + inline_images=None, + template=None, + args=None, + header=None, + print_letterhead=False, + with_container=False, +): """Send email using user's default **Email Account** or global default **Email Account**. @@ -548,20 +653,44 @@ def sendmail(recipients=None, sender="", subject="No Subject", message="No Messa if as_markdown: from frappe.utils import md_to_html + message = md_to_html(message) if not delayed: now = True from frappe.email.doctype.email_queue.email_queue import QueueBuilder - builder = QueueBuilder(recipients=recipients, sender=sender, - subject=subject, message=message, text_content=text_content, - reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link, - unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message, - attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to, - send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately, - communication=communication, read_receipt=read_receipt, is_notification=is_notification, - inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container) + + builder = QueueBuilder( + recipients=recipients, + sender=sender, + subject=subject, + message=message, + text_content=text_content, + reference_doctype=doctype or reference_doctype, + reference_name=name or reference_name, + add_unsubscribe_link=add_unsubscribe_link, + unsubscribe_method=unsubscribe_method, + unsubscribe_params=unsubscribe_params, + unsubscribe_message=unsubscribe_message, + attachments=attachments, + reply_to=reply_to, + cc=cc, + bcc=bcc, + message_id=message_id, + in_reply_to=in_reply_to, + send_after=send_after, + expose_recipients=expose_recipients, + send_priority=send_priority, + queue_separately=queue_separately, + communication=communication, + read_receipt=read_receipt, + is_notification=is_notification, + inline_images=inline_images, + header=header, + print_letterhead=print_letterhead, + with_container=with_container, + ) # build email queue and send the email if send_now is True. builder.process(send_now=now) @@ -572,6 +701,7 @@ guest_methods = [] xss_safe_methods = [] allowed_http_methods_for_whitelisted_func = {} + def whitelist(allow_guest=False, xss_safe=False, methods=None): """ Decorator for whitelisting a function and making it accessible via HTTP. @@ -582,13 +712,13 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None): Use as: - @frappe.whitelist() - def myfunc(param1, param2): - pass + @frappe.whitelist() + def myfunc(param1, param2): + pass """ if not methods: - methods = ['GET', 'POST', 'PUT', 'DELETE'] + methods = ["GET", "POST", "PUT", "DELETE"] def innerfn(fn): global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func @@ -596,7 +726,7 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None): # get function from the unbound / bound method # this is needed because functions can be compared, but not methods method = None - if hasattr(fn, '__func__'): + if hasattr(fn, "__func__"): method = fn fn = method.__func__ @@ -613,10 +743,11 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None): return innerfn + def is_whitelisted(method): from frappe.utils import sanitize_html - is_guest = session['user'] == 'Guest' + is_guest = session["user"] == "Guest" if method not in whitelisted or is_guest and method not in guest_methods: throw(_("Not permitted"), PermissionError) @@ -627,6 +758,7 @@ def is_whitelisted(method): if isinstance(value, str): form_dict[key] = sanitize_html(value) + def read_only(): def innfn(fn): def wrapper_fn(*args, **kwargs): @@ -636,14 +768,17 @@ def read_only(): try: retval = fn(*args, **get_newargs(fn, kwargs)) finally: - if local and hasattr(local, 'primary_db'): + if local and hasattr(local, "primary_db"): local.db.close() local.db = local.primary_db return retval + return wrapper_fn + return innfn + def write_only(): # if replica connection exists, we have to replace it momentarily with the primary connection def innfn(fn): @@ -664,9 +799,12 @@ def write_only(): local.db = replica_db return retval + return wrapper_fn + return innfn + def only_for(roles, message=False): """Raise `frappe.PermissionError` if the user does not have any of the given **Roles**. @@ -680,14 +818,17 @@ def only_for(roles, message=False): myroles = set(get_roles()) if not roles.intersection(myroles): if message: - msgprint(_('This action is only allowed for {}').format(bold(', '.join(roles))), _('Not Permitted')) + msgprint( + _("This action is only allowed for {}").format(bold(", ".join(roles))), _("Not Permitted") + ) raise PermissionError + def get_domain_data(module): try: - domain_data = get_hooks('domains') + domain_data = get_hooks("domains") if module in domain_data: - return _dict(get_attr(get_hooks('domains')[module][0] + '.data')) + return _dict(get_attr(get_hooks("domains")[module][0] + ".data")) else: return _dict() except ImportError: @@ -703,13 +844,15 @@ def clear_cache(user=None, doctype=None): :param user: If user is given, only user cache is cleared. :param doctype: If doctype is given, only DocType cache is cleared.""" import frappe.cache_manager + if doctype: frappe.cache_manager.clear_doctype_cache(doctype) reset_metadata_version() elif user: frappe.cache_manager.clear_user_cache(user) - else: # everything + else: # everything from frappe import translate + frappe.cache_manager.clear_user_cache() frappe.cache_manager.clear_domain_cache() translate.clear_cache() @@ -722,6 +865,7 @@ def clear_cache(user=None, doctype=None): local.role_permissions = {} + def only_has_select_perm(doctype, user=None, ignore_permissions=False): if ignore_permissions: return False @@ -730,14 +874,18 @@ def only_has_select_perm(doctype, user=None, ignore_permissions=False): user = local.session.user import frappe.permissions + permissions = frappe.permissions.get_role_permissions(doctype, user=user) - if permissions.get('select') and not permissions.get('read'): + if permissions.get("select") and not permissions.get("read"): return True else: return False -def has_permission(doctype=None, ptype="read", doc=None, user=None, verbose=False, throw=False, parent_doctype=None): + +def has_permission( + doctype=None, ptype="read", doc=None, user=None, verbose=False, throw=False, parent_doctype=None +): """Raises `frappe.PermissionError` if not permitted. :param doctype: DocType for which permission is to be check. @@ -750,8 +898,15 @@ def has_permission(doctype=None, ptype="read", doc=None, user=None, verbose=Fals if not doctype and doc: doctype = doc.doctype - out = frappe.permissions.has_permission(doctype, ptype, doc=doc, verbose=verbose, user=user, - raise_exception=throw, parent_doctype=parent_doctype) + out = frappe.permissions.has_permission( + doctype, + ptype, + doc=doc, + verbose=verbose, + user=user, + raise_exception=throw, + parent_doctype=parent_doctype, + ) if throw and not out: # mimics frappe.throw @@ -760,15 +915,16 @@ def has_permission(doctype=None, ptype="read", doc=None, user=None, verbose=Fals _("No permission for {0}").format(document_label), raise_exception=ValidationError, title=None, - indicator='red', + indicator="red", is_minimizable=None, wide=None, - as_list=False + as_list=False, ) return out -def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doctype=None): + +def has_website_permission(doc=None, ptype="read", user=None, verbose=False, doctype=None): """Raises `frappe.PermissionError` if not permitted. :param doctype: DocType for which permission is to be check. @@ -789,7 +945,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc return True # check permission in controller - if hasattr(doc, 'has_website_permission'): + if hasattr(doc, "has_website_permission"): return doc.has_website_permission(ptype, user, verbose=verbose) hooks = (get_hooks("has_website_permission") or {}).get(doctype, []) @@ -806,36 +962,46 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc else: return False + def is_table(doctype): """Returns True if `istable` property (indicating child Table) is set for given DocType.""" + def get_tables(): - return db.get_values( - "DocType", filters={"istable": 1}, order_by=None, pluck=True - ) + return db.get_values("DocType", filters={"istable": 1}, order_by=None, pluck=True) tables = cache().get_value("is_table", get_tables) return doctype in tables + def get_precision(doctype, fieldname, currency=None, doc=None): """Get precision for a given field""" from frappe.model.meta import get_field_precision + return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency) + def generate_hash(txt=None, length=None): """Generates random hash for given text + current timestamp + random string.""" - import hashlib, time + import hashlib + import time + from .utils import random_string - digest = hashlib.sha224(((txt or "") + repr(time.time()) + repr(random_string(8))).encode()).hexdigest() + + digest = hashlib.sha224( + ((txt or "") + repr(time.time()) + repr(random_string(8))).encode() + ).hexdigest() if length: digest = digest[:length] return digest + def reset_metadata_version(): """Reset `metadata_version` (Client (Javascript) build ID) hash.""" v = generate_hash() cache().set_value("metadata_version", v) return v + def new_doc(doctype, parent_doc=None, parentfield=None, as_dict=False): """Returns a new document of the given DocType with defaults set. @@ -843,13 +1009,17 @@ def new_doc(doctype, parent_doc=None, parentfield=None, as_dict=False): :param parent_doc: [optional] add to parent document. :param parentfield: [optional] add against this `parentfield`.""" from frappe.model.create_new import get_new_doc + return get_new_doc(doctype, parent_doc, parentfield, as_dict=as_dict) + def set_value(doctype, docname, fieldname, value=None): """Set document value. Calls `frappe.client.set_value`""" import frappe.client + return frappe.client.set_value(doctype, docname, fieldname, value) + def get_cached_doc(*args, **kwargs): allow_dict = kwargs.pop("_allow_dict", False) @@ -868,7 +1038,7 @@ def get_cached_doc(*args, **kwargs): return _respond(doc) # redis cache - if doc := cache().hget('document_cache', key): + if doc := cache().hget("document_cache", key): return _respond(doc, True) # database @@ -876,6 +1046,7 @@ def get_cached_doc(*args, **kwargs): return doc + def can_cache_doc(args): """ Determine if document should be cached based on get_doc params. @@ -892,15 +1063,18 @@ def can_cache_doc(args): if isinstance(doctype, str) and isinstance(name, str): return get_document_cache_key(doctype, name) + def get_document_cache_key(doctype, name): - return f'{doctype}::{name}' + return f"{doctype}::{name}" + def clear_document_cache(doctype, name): cache().hdel("last_modified", doctype) key = get_document_cache_key(doctype, name) if key in local.document_cache: del local.document_cache[key] - cache().hdel('document_cache', key) + cache().hdel("document_cache", key) + def get_cached_value(doctype, name, fieldname="name", as_dict=False): try: @@ -911,7 +1085,7 @@ def get_cached_value(doctype, name, fieldname="name", as_dict=False): if isinstance(fieldname, str): if as_dict: - throw('Cannot make dict for single fieldname') + throw("Cannot make dict for single fieldname") return doc.get(fieldname) values = [doc.get(f) for f in fieldname] @@ -919,6 +1093,7 @@ def get_cached_value(doctype, name, fieldname="name", as_dict=False): return _dict(zip(fieldname, values)) return values + def get_doc(*args, **kwargs): """Return a `frappe.model.document.Document` object of the given type and name. @@ -927,53 +1102,65 @@ def get_doc(*args, **kwargs): Examples: - # insert a new document - todo = frappe.get_doc({"doctype":"ToDo", "description": "test"}) - todo.insert() + # insert a new document + todo = frappe.get_doc({"doctype":"ToDo", "description": "test"}) + todo.insert() - # open an existing document - todo = frappe.get_doc("ToDo", "TD0001") + # open an existing document + todo = frappe.get_doc("ToDo", "TD0001") """ import frappe.model.document + doc = frappe.model.document.get_doc(*args, **kwargs) # set in cache if key := can_cache_doc(args): local.document_cache[key] = doc - cache().hset('document_cache', key, doc.as_dict()) + cache().hset("document_cache", key, doc.as_dict()) return doc + def get_last_doc(doctype, filters=None, order_by="creation desc"): """Get last created document of this type.""" - d = get_all( - doctype, - filters=filters, - limit_page_length=1, - order_by=order_by, - pluck="name" - ) + d = get_all(doctype, filters=filters, limit_page_length=1, order_by=order_by, pluck="name") if d: return get_doc(doctype, d[0]) else: raise DoesNotExistError + def get_single(doctype): """Return a `frappe.model.document.Document` object of the given Single doctype.""" return get_doc(doctype, doctype) + def get_meta(doctype, cached=True): """Get `frappe.model.meta.Meta` instance of given doctype name.""" import frappe.model.meta + return frappe.model.meta.get_meta(doctype, cached=cached) + def get_meta_module(doctype): import frappe.modules + return frappe.modules.load_doctype_module(doctype) -def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reload=False, - ignore_permissions=False, flags=None, ignore_on_trash=False, ignore_missing=True, delete_permanently=False): + +def delete_doc( + doctype=None, + name=None, + force=0, + ignore_doctypes=None, + for_reload=False, + ignore_permissions=False, + flags=None, + ignore_on_trash=False, + ignore_missing=True, + delete_permanently=False, +): """Delete a document. Calls `frappe.model.delete_doc.delete_doc`. :param doctype: DocType of document to be delete. @@ -984,17 +1171,36 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa :param ignore_permissions: Ignore user permissions. :param delete_permanently: Do not create a Deleted Document for the document.""" import frappe.model.delete_doc - frappe.model.delete_doc.delete_doc(doctype, name, force, ignore_doctypes, for_reload, - ignore_permissions, flags, ignore_on_trash, ignore_missing, delete_permanently) + + frappe.model.delete_doc.delete_doc( + doctype, + name, + force, + ignore_doctypes, + for_reload, + ignore_permissions, + flags, + ignore_on_trash, + ignore_missing, + delete_permanently, + ) + def delete_doc_if_exists(doctype, name, force=0): """Delete document if exists.""" delete_doc(doctype, name, force=force, ignore_missing=True) + def reload_doctype(doctype, force=False, reset_permissions=False): """Reload DocType from model (`[module]/[doctype]/[name]/[name].json`) files.""" - reload_doc(scrub(db.get_value("DocType", doctype, "module")), "doctype", scrub(doctype), - force=force, reset_permissions=reset_permissions) + reload_doc( + scrub(db.get_value("DocType", doctype, "module")), + "doctype", + scrub(doctype), + force=force, + reset_permissions=reset_permissions, + ) + def reload_doc(module, dt=None, dn=None, force=False, reset_permissions=False): """Reload Document from model (`[module]/[doctype]/[name]/[name].json`) files. @@ -1006,32 +1212,39 @@ def reload_doc(module, dt=None, dn=None, force=False, reset_permissions=False): """ import frappe.modules + return frappe.modules.reload_doc(module, dt, dn, force=force, reset_permissions=reset_permissions) + @whitelist() def rename_doc(*args, **kwargs): """ - Renames a doc(dt, old) to doc(dt, new) and updates all linked fields of type "Link" + Renames a doc(dt, old) to doc(dt, new) and updates all linked fields of type "Link" - Calls `frappe.model.rename_doc.rename_doc` + Calls `frappe.model.rename_doc.rename_doc` """ - kwargs.pop('ignore_permissions', None) - kwargs.pop('cmd', None) + kwargs.pop("ignore_permissions", None) + kwargs.pop("cmd", None) from frappe.model.rename_doc import rename_doc + return rename_doc(*args, **kwargs) + def get_module(modulename): """Returns a module object for given Python module name using `importlib.import_module`.""" return importlib.import_module(modulename) + def scrub(txt): """Returns sluggified string. e.g. `Sales Order` becomes `sales_order`.""" - return cstr(txt).replace(' ', '_').replace('-', '_').lower() + return cstr(txt).replace(" ", "_").replace("-", "_").lower() + def unscrub(txt): """Returns titlified string. e.g. `sales_order` becomes `Sales Order`.""" - return txt.replace('_', ' ').replace('-', ' ').title() + return txt.replace("_", " ").replace("-", " ").title() + def get_module_path(module, *joins): """Get the path of the given module name. @@ -1041,6 +1254,7 @@ def get_module_path(module, *joins): module = scrub(module) return get_pymodule_path(local.module_app[module] + "." + module, *joins) + def get_app_path(app_name, *joins): """Return path of given app. @@ -1048,12 +1262,14 @@ def get_app_path(app_name, *joins): :param *joins: Join additional path elements using `os.path.join`.""" return get_pymodule_path(app_name, *joins) + def get_site_path(*joins): """Return path of current site. :param *joins: Join additional path elements using `os.path.join`.""" return os.path.join(local.site_path, *joins) + def get_pymodule_path(modulename, *joins): """Return path of given Python module name. @@ -1061,12 +1277,14 @@ def get_pymodule_path(modulename, *joins): :param *joins: Join additional path elements using `os.path.join`.""" if not "public" in joins: joins = [scrub(part) for part in joins] - return os.path.join(os.path.dirname(get_module(scrub(modulename)).__file__ or ''), *joins) + return os.path.join(os.path.dirname(get_module(scrub(modulename)).__file__ or ""), *joins) + def get_module_list(app_name): """Get list of modules for given all via `app/modules.txt`.""" return get_file_items(os.path.join(os.path.dirname(get_module(app_name).__file__), "modules.txt")) + def get_all_apps(with_internal_apps=True, sites_path=None): """Get list of all apps via `sites/apps.txt`.""" if not sites_path: @@ -1081,10 +1299,11 @@ def get_all_apps(with_internal_apps=True, sites_path=None): if "frappe" in apps: apps.remove("frappe") - apps.insert(0, 'frappe') + apps.insert(0, "frappe") return apps + def get_installed_apps(sort=False, frappe_last=False): """Get list of installed apps in current site.""" if getattr(flags, "in_install_db", True): @@ -1094,7 +1313,7 @@ def get_installed_apps(sort=False, frappe_last=False): connect() if not local.all_apps: - local.all_apps = cache().get_value('all_apps', get_all_apps) + local.all_apps = cache().get_value("all_apps", get_all_apps) installed = json.loads(db.get_global("installed_apps") or "[]") @@ -1102,16 +1321,17 @@ def get_installed_apps(sort=False, frappe_last=False): installed = [app for app in local.all_apps if app in installed] if frappe_last: - if 'frappe' in installed: - installed.remove('frappe') - installed.append('frappe') + if "frappe" in installed: + installed.remove("frappe") + installed.append("frappe") return installed + def get_doc_hooks(): - '''Returns hooked methods for given doc. It will expand the dict tuple if required.''' - if not hasattr(local, 'doc_events_hooks'): - hooks = get_hooks('doc_events', {}) + """Returns hooked methods for given doc. It will expand the dict tuple if required.""" + if not hasattr(local, "doc_events_hooks"): + hooks = get_hooks("doc_events", {}) out = {} for key, value in hooks.items(): if isinstance(key, tuple): @@ -1124,16 +1344,18 @@ def get_doc_hooks(): return local.doc_events_hooks + def get_hooks(hook=None, default=None, app_name=None): """Get hooks via `app/hooks.py` :param hook: Name of the hook. Will gather all hooks for this name and return as a list. :param default: Default if no hook found. :param app_name: Filter by app.""" + def load_app_hooks(app_name=None): hooks = {} for app in [app_name] if app_name else get_installed_apps(sort=True): - app = "frappe" if app=="webnotes" else app + app = "frappe" if app == "webnotes" else app try: app_hooks = get_module(app + ".hooks") except ImportError: @@ -1165,14 +1387,15 @@ def get_hooks(hook=None, default=None, app_name=None): else: return hooks + def append_hook(target, key, value): - '''appends a hook to the the target dict. + """appends a hook to the the target dict. If the hook key, exists, it will make it a key. If the hook value is a dict, like doc_events, it will listify the values against the key. - ''' + """ if isinstance(value, dict): # dict? make a list of values against each key target.setdefault(key, {}) @@ -1185,6 +1408,7 @@ def append_hook(target, key, value): value = [value] target[key].extend(value) + def setup_module_map(): """Rebuild map of all modules (internal).""" _cache = cache() @@ -1206,6 +1430,7 @@ def setup_module_map(): _cache.set_value("app_modules", local.app_modules) _cache.set_value("module_app", local.module_app) + def get_file_items(path, raise_not_found=False, ignore_empty_lines=True): """Returns items from text file as a list. Ignores empty lines.""" import frappe.utils @@ -1215,17 +1440,20 @@ def get_file_items(path, raise_not_found=False, ignore_empty_lines=True): content = frappe.utils.strip(content) return [ - p.strip() for p in content.splitlines() + p.strip() + for p in content.splitlines() if (not ignore_empty_lines) or (p.strip() and not p.startswith("#")) ] else: return [] + def get_file_json(path): """Read a file and return parsed JSON object.""" - with open(path, 'r') as f: + with open(path, "r") as f: return json.load(f) + def read_file(path, raise_not_found=False): """Open a file and return its content as Unicode.""" if isinstance(path, str): @@ -1239,16 +1467,22 @@ def read_file(path, raise_not_found=False): else: return None + def get_attr(method_string): """Get python method object from its name.""" app_name = method_string.split(".")[0] - if not local.flags.in_uninstall and not local.flags.in_install and app_name not in get_installed_apps(): + if ( + not local.flags.in_uninstall + and not local.flags.in_install + and app_name not in get_installed_apps() + ): throw(_("App {0} is not installed").format(app_name), AppNotInstalledError) - modulename = '.'.join(method_string.split('.')[:-1]) - methodname = method_string.split('.')[-1] + modulename = ".".join(method_string.split(".")[:-1]) + methodname = method_string.split(".")[-1] return getattr(get_module(modulename), methodname) + def call(fn, *args, **kwargs): """Call a function and match arguments.""" if isinstance(fn, str): @@ -1258,8 +1492,9 @@ def call(fn, *args, **kwargs): return fn(*args, **newargs) + def get_newargs(fn, kwargs): - if hasattr(fn, 'fnargs'): + if hasattr(fn, "fnargs"): fnargs = fn.fnargs else: fullargspec = inspect.getfullargspec(fn) @@ -1277,23 +1512,30 @@ def get_newargs(fn, kwargs): return newargs -def make_property_setter(args, ignore_validate=False, validate_fields_for_doctype=True, is_system_generated=True): + +def make_property_setter( + args, ignore_validate=False, validate_fields_for_doctype=True, is_system_generated=True +): """Create a new **Property Setter** (for overriding DocType and DocField properties). If doctype is not specified, it will create a property setter for all fields with the given fieldname""" args = _dict(args) if not args.doctype_or_field: - args.doctype_or_field = 'DocField' + args.doctype_or_field = "DocField" if not args.property_type: - args.property_type = db.get_value('DocField', - {'parent': 'DocField', 'fieldname': args.property}, 'fieldtype') or 'Data' + args.property_type = ( + db.get_value("DocField", {"parent": "DocField", "fieldname": args.property}, "fieldtype") + or "Data" + ) if not args.doctype: DocField_doctype = qb.DocType("DocField") doctype_list = ( - qb.from_(DocField_doctype).select(DocField_doctype.parent) - .where(DocField_doctype.fieldname == args.fieldname).distinct() + qb.from_(DocField_doctype) + .select(DocField_doctype.parent) + .where(DocField_doctype.fieldname == args.fieldname) + .distinct() ).run(as_list=True) else: @@ -1301,33 +1543,40 @@ def make_property_setter(args, ignore_validate=False, validate_fields_for_doctyp for doctype in doctype_list: if not args.property_type: - args.property_type = db.get_value('DocField', - {'parent': doctype, 'fieldname': args.fieldname}, 'fieldtype') or 'Data' - - ps = get_doc({ - 'doctype': "Property Setter", - 'doctype_or_field': args.doctype_or_field, - 'doc_type': doctype, - 'field_name': args.fieldname, - 'row_name': args.row_name, - 'property': args.property, - 'value': args.value, - 'property_type': args.property_type or "Data", - 'is_system_generated': is_system_generated, - '__islocal': 1 - }) + args.property_type = ( + db.get_value("DocField", {"parent": doctype, "fieldname": args.fieldname}, "fieldtype") + or "Data" + ) + + ps = get_doc( + { + "doctype": "Property Setter", + "doctype_or_field": args.doctype_or_field, + "doc_type": doctype, + "field_name": args.fieldname, + "row_name": args.row_name, + "property": args.property, + "value": args.value, + "property_type": args.property_type or "Data", + "is_system_generated": is_system_generated, + "__islocal": 1, + } + ) ps.flags.ignore_validate = ignore_validate ps.flags.validate_fields_for_doctype = validate_fields_for_doctype ps.validate_fieldtype_change() ps.insert() + def import_doc(path): """Import a file using Data Import.""" from frappe.core.doctype.data_import.data_import import import_doc + import_doc(path) + def copy_doc(doc, ignore_no_copy=True): - """ No_copy fields also get copied.""" + """No_copy fields also get copied.""" import copy def remove_no_copy_fields(d): @@ -1335,7 +1584,7 @@ def copy_doc(doc, ignore_no_copy=True): if hasattr(d, df.fieldname): d.set(df.fieldname, None) - fields_to_clear = ['name', 'owner', 'creation', 'modified', 'modified_by'] + fields_to_clear = ["name", "owner", "creation", "modified", "modified_by"] if not local.flags.in_test: fields_to_clear.append("docstatus") @@ -1347,7 +1596,7 @@ def copy_doc(doc, ignore_no_copy=True): newdoc = get_doc(copy.deepcopy(d)) newdoc.set("__islocal", 1) - for fieldname in (fields_to_clear + ['amended_from', 'amendment_date']): + for fieldname in fields_to_clear + ["amended_from", "amendment_date"]: newdoc.set(fieldname, None) if not ignore_no_copy: @@ -1364,6 +1613,7 @@ def copy_doc(doc, ignore_no_copy=True): return newdoc + def compare(val1, condition, val2): """Compare two values using `frappe.utils.compare` @@ -1381,11 +1631,23 @@ def compare(val1, condition, val2): - "None" """ import frappe.utils + return frappe.utils.compare(val1, condition, val2) -def respond_as_web_page(title, html, success=None, http_status_code=None, context=None, - indicator_color=None, primary_action='/', primary_label = None, fullpage=False, - width=None, template='message'): + +def respond_as_web_page( + title, + html, + success=None, + http_status_code=None, + context=None, + indicator_color=None, + primary_action="/", + primary_label=None, + fullpage=False, + width=None, + template="message", +): """Send response as a web page with a message rather than JSON. Used to show permission errors etc. :param title: Page title and heading. @@ -1402,33 +1664,34 @@ def respond_as_web_page(title, html, success=None, http_status_code=None, contex """ local.message_title = title local.message = html - local.response['type'] = 'page' - local.response['route'] = template + local.response["type"] = "page" + local.response["route"] = template local.no_cache = 1 if http_status_code: - local.response['http_status_code'] = http_status_code + local.response["http_status_code"] = http_status_code if not context: context = {} if not indicator_color: if success: - indicator_color = 'green' + indicator_color = "green" elif http_status_code and http_status_code > 300: - indicator_color = 'red' + indicator_color = "red" else: - indicator_color = 'blue' + indicator_color = "blue" - context['indicator_color'] = indicator_color - context['primary_label'] = primary_label - context['primary_action'] = primary_action - context['error_code'] = http_status_code - context['fullpage'] = fullpage + context["indicator_color"] = indicator_color + context["primary_label"] = primary_label + context["primary_action"] = primary_action + context["error_code"] = http_status_code + context["fullpage"] = fullpage if width: - context['card_width'] = width + context["card_width"] = width + + local.response["context"] = context - local.response['context'] = context def redirect_to_message(title, html, http_status_code=None, context=None, indicator_color=None): """Redirects to /message?id=random @@ -1439,41 +1702,35 @@ def redirect_to_message(title, html, http_status_code=None, context=None, indica :param http_status_code: HTTP status code. Example Usage: - frappe.redirect_to_message(_('Thank you'), "

You will receive an email at test@example.com

") + frappe.redirect_to_message(_('Thank you'), "

You will receive an email at test@example.com

") """ message_id = generate_hash(length=8) - message = { - 'context': context or {}, - 'http_status_code': http_status_code or 200 - } - message['context'].update({ - 'header': title, - 'title': title, - 'message': html - }) + message = {"context": context or {}, "http_status_code": http_status_code or 200} + message["context"].update({"header": title, "title": title, "message": html}) if indicator_color: - message['context'].update({ - "indicator_color": indicator_color - }) + message["context"].update({"indicator_color": indicator_color}) cache().set_value("message_id:{0}".format(message_id), message, expires_in_sec=60) - location = '/message?id={0}'.format(message_id) + location = "/message?id={0}".format(message_id) - if not getattr(local, 'is_ajax', False): + if not getattr(local, "is_ajax", False): local.response["type"] = "redirect" local.response["location"] = location else: return location + def build_match_conditions(doctype, as_condition=True): """Return match (User permissions) for given doctype as list or SQL.""" import frappe.desk.reportview + return frappe.desk.reportview.build_match_conditions(doctype, as_condition=as_condition) + def get_list(doctype, *args, **kwargs): """List database query via `frappe.model.db_query`. Will also check for permissions. @@ -1486,18 +1743,20 @@ def get_list(doctype, *args, **kwargs): Example usage: - # simple dict filter - frappe.get_list("ToDo", fields=["name", "description"], filters = {"owner":"test@example.com"}) + # simple dict filter + frappe.get_list("ToDo", fields=["name", "description"], filters = {"owner":"test@example.com"}) - # filter as a list of lists - frappe.get_list("ToDo", fields="*", filters = [["modified", ">", "2014-01-01"]]) + # filter as a list of lists + frappe.get_list("ToDo", fields="*", filters = [["modified", ">", "2014-01-01"]]) - # filter as a list of dicts - frappe.get_list("ToDo", fields="*", filters = {"description": ("like", "test%")}) + # filter as a list of dicts + frappe.get_list("ToDo", fields="*", filters = {"description": ("like", "test%")}) """ import frappe.model.db_query + return frappe.model.db_query.DatabaseQuery(doctype).execute(*args, **kwargs) + def get_all(doctype, *args, **kwargs): """List database query via `frappe.model.db_query`. Will **not** check for permissions. Parameters are same as `frappe.get_list` @@ -1511,20 +1770,21 @@ def get_all(doctype, *args, **kwargs): Example usage: - # simple dict filter - frappe.get_all("ToDo", fields=["name", "description"], filters = {"owner":"test@example.com"}) + # simple dict filter + frappe.get_all("ToDo", fields=["name", "description"], filters = {"owner":"test@example.com"}) - # filter as a list of lists - frappe.get_all("ToDo", fields=["*"], filters = [["modified", ">", "2014-01-01"]]) + # filter as a list of lists + frappe.get_all("ToDo", fields=["*"], filters = [["modified", ">", "2014-01-01"]]) - # filter as a list of dicts - frappe.get_all("ToDo", fields=["*"], filters = {"description": ("like", "test%")}) + # filter as a list of dicts + frappe.get_all("ToDo", fields=["*"], filters = {"description": ("like", "test%")}) """ kwargs["ignore_permissions"] = True if not "limit_page_length" in kwargs: kwargs["limit_page_length"] = 0 return get_list(doctype, *args, **kwargs) + def get_value(*args, **kwargs): """Returns a document property or list of properties. @@ -1539,49 +1799,74 @@ def get_value(*args, **kwargs): """ return db.get_value(*args, **kwargs) + def as_json(obj: Union[Dict, List], indent=1) -> str: from frappe.utils.response import json_handler try: - return json.dumps(obj, indent=indent, sort_keys=True, default=json_handler, separators=(',', ': ')) + return json.dumps( + obj, indent=indent, sort_keys=True, default=json_handler, separators=(",", ": ") + ) except TypeError: # this would break in case the keys are not all os "str" type - as defined in the JSON # adding this to ensure keys are sorted (expected behaviour) sorted_obj = dict(sorted(obj.items(), key=lambda kv: str(kv[0]))) - return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=(',', ': ')) + return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=(",", ": ")) + def are_emails_muted(): from frappe.utils import cint + return flags.mute_emails or cint(conf.get("mute_emails") or 0) or False + def get_test_records(doctype): """Returns list of objects from `test_records.json` in the given doctype's folder.""" from frappe.modules import get_doctype_module, get_module_path - path = os.path.join(get_module_path(get_doctype_module(doctype)), "doctype", scrub(doctype), "test_records.json") + + path = os.path.join( + get_module_path(get_doctype_module(doctype)), "doctype", scrub(doctype), "test_records.json" + ) if os.path.exists(path): with open(path, "r") as f: return json.loads(f.read()) else: return [] + def format_value(*args, **kwargs): """Format value with given field properties. :param value: Value to be formatted. :param df: (Optional) DocField object with properties `fieldtype`, `options` etc.""" import frappe.utils.formatters + return frappe.utils.formatters.format_value(*args, **kwargs) + def format(*args, **kwargs): """Format value with given field properties. :param value: Value to be formatted. :param df: (Optional) DocField object with properties `fieldtype`, `options` etc.""" import frappe.utils.formatters + return frappe.utils.formatters.format_value(*args, **kwargs) -def get_print(doctype=None, name=None, print_format=None, style=None, html=None, - as_pdf=False, doc=None, output=None, no_letterhead=0, password=None, pdf_options=None): + +def get_print( + doctype=None, + name=None, + print_format=None, + style=None, + html=None, + as_pdf=False, + doc=None, + output=None, + no_letterhead=0, + password=None, + pdf_options=None, +): """Get Print Format for given document. :param doctype: DocType of document. @@ -1590,8 +1875,8 @@ def get_print(doctype=None, name=None, print_format=None, style=None, html=None, :param style: Print Format style. :param as_pdf: Return as PDF. Default False. :param password: Password to encrypt the pdf with. Default None""" - from frappe.website.serve import get_response_content from frappe.utils.pdf import get_pdf + from frappe.website.serve import get_response_content local.form_dict.doctype = doctype local.form_dict.name = name @@ -1602,7 +1887,7 @@ def get_print(doctype=None, name=None, print_format=None, style=None, html=None, pdf_options = pdf_options or {} if password: - pdf_options['password'] = password + pdf_options["password"] = password if not html: html = get_response_content("printview") @@ -1612,19 +1897,32 @@ def get_print(doctype=None, name=None, print_format=None, style=None, html=None, else: return html -def attach_print(doctype, name, file_name=None, print_format=None, - style=None, html=None, doc=None, lang=None, print_letterhead=True, password=None): + +def attach_print( + doctype, + name, + file_name=None, + print_format=None, + style=None, + html=None, + doc=None, + lang=None, + print_letterhead=True, + password=None, +): from frappe.utils import scrub_urls - if not file_name: file_name = name - file_name = file_name.replace(' ','').replace('/','-') + if not file_name: + file_name = name + file_name = file_name.replace(" ", "").replace("/", "-") print_settings = db.get_singles_dict("Print Settings") _lang = local.lang - #set lang as specified in print format attachment - if lang: local.lang = lang + # set lang as specified in print format attachment + if lang: + local.lang = lang local.flags.ignore_print_permissions = True no_letterhead = not print_letterhead @@ -1635,29 +1933,27 @@ def attach_print(doctype, name, file_name=None, print_format=None, html=html, doc=doc, no_letterhead=no_letterhead, - password=password + password=password, ) - content = '' + content = "" if int(print_settings.send_print_as_pdf or 0): ext = ".pdf" kwargs["as_pdf"] = True content = get_print(doctype, name, **kwargs) else: ext = ".html" - content = scrub_urls(get_print(doctype, name, **kwargs)).encode('utf-8') + content = scrub_urls(get_print(doctype, name, **kwargs)).encode("utf-8") - out = { - "fname": file_name + ext, - "fcontent": content - } + out = {"fname": file_name + ext, "fcontent": content} local.flags.ignore_print_permissions = False - #reset lang to original local lang + # reset lang to original local lang local.lang = _lang return out + def publish_progress(*args, **kwargs): """Show the user progress for a long request @@ -1668,8 +1964,10 @@ def publish_progress(*args, **kwargs): :param description: Optional description """ import frappe.realtime + return frappe.realtime.publish_progress(*args, **kwargs) + def publish_realtime(*args, **kwargs): """Publish real-time updates @@ -1685,6 +1983,7 @@ def publish_realtime(*args, **kwargs): return frappe.realtime.publish_realtime(*args, **kwargs) + def local_cache(namespace, key, generator, regenerate_if_none=False): """A key value store for caching within a request @@ -1705,41 +2004,48 @@ def local_cache(namespace, key, generator, regenerate_if_none=False): return local.cache[namespace][key] + def enqueue(*args, **kwargs): - ''' - Enqueue method to be executed using a background worker - - :param method: method string or method object - :param queue: (optional) should be either long, default or short - :param timeout: (optional) should be set according to the functions - :param event: this is passed to enable clearing of jobs from queues - :param is_async: (optional) if is_async=False, the method is executed immediately, else via a worker - :param job_name: (optional) can be used to name an enqueue call, which can be used to prevent duplicate calls - :param kwargs: keyword arguments to be passed to the method - ''' + """ + Enqueue method to be executed using a background worker + + :param method: method string or method object + :param queue: (optional) should be either long, default or short + :param timeout: (optional) should be set according to the functions + :param event: this is passed to enable clearing of jobs from queues + :param is_async: (optional) if is_async=False, the method is executed immediately, else via a worker + :param job_name: (optional) can be used to name an enqueue call, which can be used to prevent duplicate calls + :param kwargs: keyword arguments to be passed to the method + """ import frappe.utils.background_jobs + return frappe.utils.background_jobs.enqueue(*args, **kwargs) + def task(**task_kwargs): def decorator_task(f): f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs) return f + return decorator_task + def enqueue_doc(*args, **kwargs): - ''' - Enqueue method to be executed using a background worker - - :param doctype: DocType of the document on which you want to run the event - :param name: Name of the document on which you want to run the event - :param method: method string or method object - :param queue: (optional) should be either long, default or short - :param timeout: (optional) should be set according to the functions - :param kwargs: keyword arguments to be passed to the method - ''' + """ + Enqueue method to be executed using a background worker + + :param doctype: DocType of the document on which you want to run the event + :param name: Name of the document on which you want to run the event + :param method: method string or method object + :param queue: (optional) should be either long, default or short + :param timeout: (optional) should be set according to the functions + :param kwargs: keyword arguments to be passed to the method + """ import frappe.utils.background_jobs + return frappe.utils.background_jobs.enqueue_doc(*args, **kwargs) + def get_doctype_app(doctype): def _get_doctype_app(): doctype_module = local.db.get_value("DocType", doctype, "module") @@ -1747,15 +2053,29 @@ def get_doctype_app(doctype): return local_cache("doctype_app", doctype, generator=_get_doctype_app) + loggers = {} log_level = None -def logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): - '''Returns a python logger that uses StreamHandler''' + + +def logger( + module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20 +): + """Returns a python logger that uses StreamHandler""" from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter, max_size=max_size, file_count=file_count) + + return get_logger( + module=module, + with_more_info=with_more_info, + allow_site=allow_site, + filter=filter, + max_size=max_size, + file_count=file_count, + ) + def log_error(message=None, title=_("Error")): - '''Log error to Error Log''' + """Log error to Error Log""" # AI ALERT: # the title and message may be swapped @@ -1763,74 +2083,81 @@ def log_error(message=None, title=_("Error")): # this hack tries to be smart about whats a title (single line ;-)) and fixes it if message: - if '\n' in title: + if "\n" in title: error, title = title, message else: error = message else: error = get_traceback() - return get_doc(dict(doctype='Error Log', error=as_unicode(error), - method=title)).insert(ignore_permissions=True) + return get_doc(dict(doctype="Error Log", error=as_unicode(error), method=title)).insert( + ignore_permissions=True + ) + def get_desk_link(doctype, name): - html = '{doctype_local} {name}' - return html.format( - doctype=doctype, - name=name, - doctype_local=_(doctype) + html = ( + '{doctype_local} {name}' ) + return html.format(doctype=doctype, name=name, doctype_local=_(doctype)) + def bold(text): - return '{0}'.format(text) + return "{0}".format(text) + def safe_eval(code, eval_globals=None, eval_locals=None): - '''A safer `eval`''' - whitelisted_globals = { - "int": int, - "float": float, - "long": int, - "round": round - } + """A safer `eval`""" + whitelisted_globals = {"int": int, "float": float, "long": int, "round": round} UNSAFE_ATTRIBUTES = { # Generator Attributes - "gi_frame", "gi_code", + "gi_frame", + "gi_code", # Coroutine Attributes - "cr_frame", "cr_code", "cr_origin", + "cr_frame", + "cr_code", + "cr_origin", # Async Generator Attributes - "ag_code", "ag_frame", + "ag_code", + "ag_frame", # Traceback Attributes - "tb_frame", "tb_next", + "tb_frame", + "tb_next", # Format Attributes - "format", "format_map", + "format", + "format_map", } for attribute in UNSAFE_ATTRIBUTES: if attribute in code: throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute)) - if '__' in code: + if "__" in code: throw('Illegal rule {0}. Cannot use "__"'.format(bold(code))) if not eval_globals: eval_globals = {} - eval_globals['__builtins__'] = {} + eval_globals["__builtins__"] = {} eval_globals.update(whitelisted_globals) return eval(code, eval_globals, eval_locals) + def get_system_settings(key): if key not in local.system_settings: - local.system_settings.update({key: db.get_single_value('System Settings', key)}) + local.system_settings.update({key: db.get_single_value("System Settings", key)}) return local.system_settings.get(key) + def get_active_domains(): from frappe.core.doctype.domain_settings.domain_settings import get_active_domains + return get_active_domains() + def get_version(doctype, name, limit=None, head=False, raise_err=True): - ''' + """ Returns a list of version information of a given DocType. Note: Applicable only if DocType has changes tracked. @@ -1839,51 +2166,52 @@ def get_version(doctype, name, limit=None, head=False, raise_err=True): >>> frappe.get_version('User', 'foobar@gmail.com') >>> [ - { - "version": [version.data], # Refer Version DocType get_diff method and data attribute - "user": "admin@gmail.com", # User that created this version - "creation": # Creation timestamp of that object. - } + { + "version": [version.data], # Refer Version DocType get_diff method and data attribute + "user": "admin@gmail.com", # User that created this version + "creation": # Creation timestamp of that object. + } ] - ''' + """ meta = get_meta(doctype) if meta.track_changes: - names = db.get_all('Version', filters={ - 'ref_doctype': doctype, - 'docname': name, - 'order_by': 'creation' if head else None, - 'limit': limit - }, as_list=1) + names = db.get_all( + "Version", + filters={ + "ref_doctype": doctype, + "docname": name, + "order_by": "creation" if head else None, + "limit": limit, + }, + as_list=1, + ) - from frappe.utils import squashify, dictify, safe_json_loads + from frappe.utils import dictify, safe_json_loads, squashify versions = [] for name in names: name = squashify(name) - doc = get_doc('Version', name) + doc = get_doc("Version", name) data = doc.data data = safe_json_loads(data) - data = dictify(dict( - version=data, - user=doc.owner, - creation=doc.creation - )) + data = dictify(dict(version=data, user=doc.owner, creation=doc.creation)) versions.append(data) return versions else: if raise_err: - raise ValueError(_('{0} has no versions tracked.').format(doctype)) + raise ValueError(_("{0} has no versions tracked.").format(doctype)) + @whitelist(allow_guest=True) def ping(): return "pong" -def safe_encode(param, encoding='utf-8'): +def safe_encode(param, encoding="utf-8"): try: param = param.encode(encoding) except Exception: @@ -1891,31 +2219,38 @@ def safe_encode(param, encoding='utf-8'): return param -def safe_decode(param, encoding='utf-8'): +def safe_decode(param, encoding="utf-8"): try: param = param.decode(encoding) except Exception: pass return param + def parse_json(val): from frappe.utils import parse_json + return parse_json(val) -def mock(type, size=1, locale='en'): + +def mock(type, size=1, locale="en"): import faker + results = [] fake = faker.Faker(locale) if type not in dir(fake): - raise ValueError('Not a valid mock type.') + raise ValueError("Not a valid mock type.") else: for i in range(size): data = getattr(fake, type)() results.append(data) from frappe.utils import squashify + return squashify(results) + def validate_and_sanitize_search_inputs(fn): from frappe.desk.search import validate_and_sanitize_search_inputs as func + return func(fn) diff --git a/frappe/api.py b/frappe/api.py index 226853c47b..32e19a1b43 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -9,8 +9,8 @@ import frappe import frappe.client import frappe.handler from frappe import _ -from frappe.utils.response import build_response from frappe.utils.data import sbool +from frappe.utils.response import build_response def handle(): @@ -22,22 +22,22 @@ def handle(): `/api/method/{methodname}` will call a whitelisted method `/api/resource/{doctype}` will query a table - examples: - - `?fields=["name", "owner"]` - - `?filters=[["Task", "name", "like", "%005"]]` - - `?limit_start=0` - - `?limit_page_length=20` + examples: + - `?fields=["name", "owner"]` + - `?filters=[["Task", "name", "like", "%005"]]` + - `?limit_start=0` + - `?limit_page_length=20` `/api/resource/{doctype}/{name}` will point to a resource - `GET` will return doclist - `POST` will insert - `PUT` will update - `DELETE` will delete + `GET` will return doclist + `POST` will insert + `PUT` will update + `DELETE` will delete `/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method """ - parts = frappe.request.path[1:].split("/",3) + parts = frappe.request.path[1:].split("/", 3) call = doctype = name = None if len(parts) > 1: @@ -49,22 +49,22 @@ def handle(): if len(parts) > 3: name = parts[3] - if call=="method": + if call == "method": frappe.local.form_dict.cmd = doctype return frappe.handler.handle() - elif call=="resource": + elif call == "resource": if "run_method" in frappe.local.form_dict: method = frappe.local.form_dict.pop("run_method") doc = frappe.get_doc(doctype, name) doc.is_whitelisted(method) - if frappe.local.request.method=="GET": + if frappe.local.request.method == "GET": if not doc.has_permission("read"): frappe.throw(_("Not permitted"), frappe.PermissionError) frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)}) - if frappe.local.request.method=="POST": + if frappe.local.request.method == "POST": if not doc.has_permission("write"): frappe.throw(_("Not permitted"), frappe.PermissionError) @@ -73,13 +73,13 @@ def handle(): else: if name: - if frappe.local.request.method=="GET": + if frappe.local.request.method == "GET": doc = frappe.get_doc(doctype, name) if not doc.has_permission("read"): raise frappe.PermissionError frappe.local.response.update({"data": doc}) - if frappe.local.request.method=="PUT": + if frappe.local.request.method == "PUT": data = get_request_form_data() doc = frappe.get_doc(doctype, name, for_update=True) @@ -90,9 +90,7 @@ def handle(): # Not checking permissions here because it's checked in doc.save doc.update(data) - frappe.local.response.update({ - "data": doc.save().as_dict() - }) + frappe.local.response.update({"data": doc.save().as_dict()}) # check for child table doctype if doc.get("parenttype"): @@ -183,7 +181,7 @@ def validate_oauth(authorization_header): Authenticate request using OAuth and set session user Args: - authorization_header (list of str): The 'Authorization' header containing the prefix and token + authorization_header (list of str): The 'Authorization' header containing the prefix and token """ from frappe.integrations.oauth2 import get_oauth_server @@ -194,7 +192,9 @@ def validate_oauth(authorization_header): req = frappe.request parsed_url = urlparse(req.url) access_token = {"access_token": token} - uri = parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token) + uri = ( + parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token) + ) http_method = req.method headers = req.headers body = req.get_data() @@ -202,8 +202,12 @@ def validate_oauth(authorization_header): body = None try: - required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter()) - valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) + required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split( + get_url_delimiter() + ) + valid, oauthlib_request = get_oauth_server().verify_request( + uri, http_method, body, headers, required_scopes + ) if valid: frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) frappe.local.form_dict = form_dict @@ -216,48 +220,43 @@ def validate_auth_via_api_keys(authorization_header): Authenticate request using API keys and set session user Args: - authorization_header (list of str): The 'Authorization' header containing the prefix and token + authorization_header (list of str): The 'Authorization' header containing the prefix and token """ try: auth_type, auth_token = authorization_header authorization_source = frappe.get_request_header("Frappe-Authorization-Source") - if auth_type.lower() == 'basic': + if auth_type.lower() == "basic": api_key, api_secret = frappe.safe_decode(base64.b64decode(auth_token)).split(":") validate_api_key_secret(api_key, api_secret, authorization_source) - elif auth_type.lower() == 'token': + elif auth_type.lower() == "token": api_key, api_secret = auth_token.split(":") validate_api_key_secret(api_key, api_secret, authorization_source) except binascii.Error: - frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken) + frappe.throw( + _("Failed to decode token, please provide a valid base64-encoded token."), + frappe.InvalidAuthorizationToken, + ) except (AttributeError, TypeError, ValueError): pass def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None): """frappe_authorization_source to provide api key and secret for a doctype apart from User""" - doctype = frappe_authorization_source or 'User' - doc = frappe.db.get_value( - doctype=doctype, - filters={"api_key": api_key}, - fieldname=["name"] - ) + doctype = frappe_authorization_source or "User" + doc = frappe.db.get_value(doctype=doctype, filters={"api_key": api_key}, fieldname=["name"]) form_dict = frappe.local.form_dict - doc_secret = frappe.utils.password.get_decrypted_password(doctype, doc, fieldname='api_secret') + doc_secret = frappe.utils.password.get_decrypted_password(doctype, doc, fieldname="api_secret") if api_secret == doc_secret: - if doctype == 'User': - user = frappe.db.get_value( - doctype="User", - filters={"api_key": api_key}, - fieldname=["name"] - ) + if doctype == "User": + user = frappe.db.get_value(doctype="User", filters={"api_key": api_key}, fieldname=["name"]) else: - user = frappe.db.get_value(doctype, doc, 'user') - if frappe.local.login_manager.user in ('', 'Guest'): + user = frappe.db.get_value(doctype, doc, "user") + if frappe.local.login_manager.user in ("", "Guest"): frappe.set_user(user) frappe.local.form_dict = form_dict def validate_auth_via_hooks(): - for auth_hook in frappe.get_hooks('auth_hooks', []): + for auth_hook in frappe.get_hooks("auth_hooks", []): frappe.get_attr(auth_hook)() diff --git a/frappe/app.py b/frappe/app.py index 975a2e2002..e6df29fbd9 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -2,37 +2,37 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import os import logging +import os -from werkzeug.local import LocalManager -from werkzeug.wrappers import Request, Response from werkzeug.exceptions import HTTPException, NotFound +from werkzeug.local import LocalManager from werkzeug.middleware.profiler import ProfilerMiddleware from werkzeug.middleware.shared_data import SharedDataMiddleware +from werkzeug.wrappers import Request, Response import frappe -import frappe.handler -import frappe.auth import frappe.api +import frappe.auth +import frappe.handler +import frappe.monitor +import frappe.rate_limiter +import frappe.recorder import frappe.utils.response -from frappe.utils import get_site_name, sanitize_html +from frappe import _ +from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request from frappe.middlewares import StaticDataMiddleware -from frappe.website.serve import get_response +from frappe.utils import get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot -from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request -from frappe import _ -import frappe.recorder -import frappe.monitor -import frappe.rate_limiter +from frappe.website.serve import get_response local_manager = LocalManager([frappe.local]) _site = None _sites_path = os.environ.get("SITES_PATH", ".") -class RequestContext(object): +class RequestContext(object): def __init__(self, environ): self.request = Request(environ) @@ -42,6 +42,7 @@ class RequestContext(object): def __exit__(self, type, value, traceback): frappe.destroy() + @Request.application def application(request): response = None @@ -65,13 +66,13 @@ def application(request): elif request.path.startswith("/api/"): response = frappe.api.handle() - elif request.path.startswith('/backups'): + elif request.path.startswith("/backups"): response = frappe.utils.response.download_backup(request.path) - elif request.path.startswith('/private/files/'): + elif request.path.startswith("/private/files/"): response = frappe.utils.response.download_private_file(request.path) - elif request.method in ('GET', 'HEAD', 'POST'): + elif request.method in ("GET", "HEAD", "POST"): response = get_response() else: @@ -103,41 +104,45 @@ def application(request): return response + def init_request(request): frappe.local.request = request - frappe.local.is_ajax = frappe.get_request_header("X-Requested-With")=="XMLHttpRequest" + frappe.local.is_ajax = frappe.get_request_header("X-Requested-With") == "XMLHttpRequest" - site = _site or request.headers.get('X-Frappe-Site-Name') or get_site_name(request.host) + site = _site or request.headers.get("X-Frappe-Site-Name") or get_site_name(request.host) frappe.init(site=site, sites_path=_sites_path) if not (frappe.local.conf and frappe.local.conf.db_name): # site does not exist raise NotFound - if frappe.local.conf.get('maintenance_mode'): + if frappe.local.conf.get("maintenance_mode"): frappe.connect() - raise frappe.SessionStopped('Session Stopped') + raise frappe.SessionStopped("Session Stopped") else: frappe.connect(set_admin_as_user=False) - request.max_content_length = frappe.local.conf.get('max_file_size') or 10 * 1024 * 1024 + request.max_content_length = frappe.local.conf.get("max_file_size") or 10 * 1024 * 1024 make_form_dict(request) if request.method != "OPTIONS": frappe.local.http_request = frappe.auth.HTTPRequest() + def log_request(request, response): - if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: - frappe.logger("frappe.web", allow_site=frappe.local.site).info({ - "site": get_site_name(request.host), - "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), - "base_url": getattr(request, "base_url", "NOTFOUND"), - "full_path": getattr(request, "full_path", "NOTFOUND"), - "method": getattr(request, "method", "NOTFOUND"), - "scheme": getattr(request, "scheme", "NOTFOUND"), - "http_status_code": getattr(response, "status_code", "NOTFOUND") - }) + if hasattr(frappe.local, "conf") and frappe.local.conf.enable_frappe_logger: + frappe.logger("frappe.web", allow_site=frappe.local.site).info( + { + "site": get_site_name(request.host), + "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), + "base_url": getattr(request, "base_url", "NOTFOUND"), + "full_path": getattr(request, "full_path", "NOTFOUND"), + "method": getattr(request, "method", "NOTFOUND"), + "scheme": getattr(request, "scheme", "NOTFOUND"), + "http_status_code": getattr(response, "status_code", "NOTFOUND"), + } + ) def process_response(response): @@ -145,19 +150,20 @@ def process_response(response): return # set cookies - if hasattr(frappe.local, 'cookie_manager'): + if hasattr(frappe.local, "cookie_manager"): frappe.local.cookie_manager.flush_cookies(response=response) # rate limiter headers - if hasattr(frappe.local, 'rate_limiter'): + if hasattr(frappe.local, "rate_limiter"): response.headers.extend(frappe.local.rate_limiter.headers()) # CORS headers - if hasattr(frappe.local, 'conf') and frappe.conf.allow_cors: + if hasattr(frappe.local, "conf") and frappe.conf.allow_cors: set_cors_headers(response) + def set_cors_headers(response): - origin = frappe.request.headers.get('Origin') + origin = frappe.request.headers.get("Origin") allow_cors = frappe.conf.allow_cors if not (origin and allow_cors): return @@ -169,20 +175,25 @@ def set_cors_headers(response): if origin not in allow_cors: return - response.headers.extend({ - 'Access-Control-Allow-Origin': origin, - 'Access-Control-Allow-Credentials': 'true', - 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', - 'Access-Control-Allow-Headers': ('Authorization,DNT,X-Mx-ReqToken,' - 'Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,' - 'Cache-Control,Content-Type') - }) + response.headers.extend( + { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Credentials": "true", + "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", + "Access-Control-Allow-Headers": ( + "Authorization,DNT,X-Mx-ReqToken," + "Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since," + "Cache-Control,Content-Type" + ), + } + ) + def make_form_dict(request): import json request_data = request.get_data(as_text=True) - if 'application/json' in (request.content_type or '') and request_data: + if "application/json" in (request.content_type or "") and request_data: args = json.loads(request_data) else: args = {} @@ -198,20 +209,19 @@ def make_form_dict(request): # _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict frappe.local.form_dict.pop("_") + def handle_exception(e): response = None http_status_code = getattr(e, "http_status_code", 500) return_as_message = False accept_header = frappe.get_request_header("Accept") or "" respond_as_json = ( - frappe.get_request_header('Accept') - and (frappe.local.is_ajax or 'application/json' in accept_header) - or ( - frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text") - ) + frappe.get_request_header("Accept") + and (frappe.local.is_ajax or "application/json" in accept_header) + or (frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text")) ) - if frappe.conf.get('developer_mode'): + if frappe.conf.get("developer_mode"): # don't fail silently print(frappe.get_traceback()) @@ -220,27 +230,38 @@ def handle_exception(e): # if the request is ajax, send back the trace or error message response = frappe.utils.response.report_error(http_status_code) - elif (http_status_code==500 + elif ( + http_status_code == 500 and (frappe.db and isinstance(e, frappe.db.InternalError)) - and (frappe.db and (frappe.db.is_deadlocked(e) or frappe.db.is_timedout(e)))): - http_status_code = 508 + and (frappe.db and (frappe.db.is_deadlocked(e) or frappe.db.is_timedout(e))) + ): + http_status_code = 508 - elif http_status_code==401: - frappe.respond_as_web_page(_("Session Expired"), + elif http_status_code == 401: + frappe.respond_as_web_page( + _("Session Expired"), _("Your session has expired, please login again to continue."), - http_status_code=http_status_code, indicator_color='red') + http_status_code=http_status_code, + indicator_color="red", + ) return_as_message = True - elif http_status_code==403: - frappe.respond_as_web_page(_("Not Permitted"), + elif http_status_code == 403: + frappe.respond_as_web_page( + _("Not Permitted"), _("You do not have enough permissions to complete the action"), - http_status_code=http_status_code, indicator_color='red') + http_status_code=http_status_code, + indicator_color="red", + ) return_as_message = True - elif http_status_code==404: - frappe.respond_as_web_page(_("Not Found"), + elif http_status_code == 404: + frappe.respond_as_web_page( + _("Not Found"), _("The resource you are looking for is not available"), - http_status_code=http_status_code, indicator_color='red') + http_status_code=http_status_code, + indicator_color="red", + ) return_as_message = True elif http_status_code == 429: @@ -252,9 +273,9 @@ def handle_exception(e): if frappe.local.flags.disable_traceback and not frappe.local.dev_server: traceback = "" - frappe.respond_as_web_page("Server Error", - traceback, http_status_code=http_status_code, - indicator_color='red', width=640) + frappe.respond_as_web_page( + "Server Error", traceback, http_status_code=http_status_code, indicator_color="red", width=640 + ) return_as_message = True if e.__class__ == frappe.AuthenticationError: @@ -269,6 +290,7 @@ def handle_exception(e): return response + def after_request(rollback): if (frappe.local.request.method in ("POST", "PUT") or frappe.local.flags.commit) and frappe.db: if frappe.db.transaction_writes: @@ -286,41 +308,47 @@ def after_request(rollback): return rollback + application = local_manager.make_middleware(application) -def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=None, sites_path='.'): + +def serve( + port=8000, profile=False, no_reload=False, no_threading=False, site=None, sites_path="." +): global application, _site, _sites_path _site = site _sites_path = sites_path from werkzeug.serving import run_simple - if profile or os.environ.get('USE_PROFILER'): - application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls')) + if profile or os.environ.get("USE_PROFILER"): + application = ProfilerMiddleware(application, sort_by=("cumtime", "calls")) - if not os.environ.get('NO_STATICS'): - application = SharedDataMiddleware(application, { - str('/assets'): str(os.path.join(sites_path, 'assets')) - }) + if not os.environ.get("NO_STATICS"): + application = SharedDataMiddleware( + application, {str("/assets"): str(os.path.join(sites_path, "assets"))} + ) - application = StaticDataMiddleware(application, { - str('/files'): str(os.path.abspath(sites_path)) - }) + application = StaticDataMiddleware( + application, {str("/files"): str(os.path.abspath(sites_path))} + ) application.debug = True - application.config = { - 'SERVER_NAME': 'localhost:8000' - } + application.config = {"SERVER_NAME": "localhost:8000"} - log = logging.getLogger('werkzeug') + log = logging.getLogger("werkzeug") log.propagate = False - in_test_env = os.environ.get('CI') + in_test_env = os.environ.get("CI") if in_test_env: log.setLevel(logging.ERROR) - run_simple('0.0.0.0', int(port), application, + run_simple( + "0.0.0.0", + int(port), + application, use_reloader=False if in_test_env else not no_reload, use_debugger=not in_test_env, use_evalex=not in_test_env, - threaded=not no_threading) + threaded=not no_threading, + ) diff --git a/frappe/auth.py b/frappe/auth.py index d4778eb0c1..dc53c20f28 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -11,7 +11,12 @@ from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.modules.patch_handler import check_session_stopped from frappe.sessions import Session, clear_sessions, delete_session from frappe.translate import get_language -from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, get_cached_user_pass, should_run_2fa +from frappe.twofactor import ( + authenticate_for_2factor, + confirm_otp_token, + get_cached_user_pass, + should_run_2fa, +) from frappe.utils import cint, date_diff, datetime, get_datetime, today from frappe.utils.password import check_password from frappe.website.utils import get_home_page @@ -47,20 +52,20 @@ class HTTPRequest: def domain(self): if not getattr(self, "_domain", None): self._domain = frappe.request.host - if self._domain and self._domain.startswith('www.'): + if self._domain and self._domain.startswith("www."): self._domain = self._domain[4:] return self._domain def set_request_ip(self): - if frappe.get_request_header('X-Forwarded-For'): - frappe.local.request_ip = (frappe.get_request_header('X-Forwarded-For').split(",")[0]).strip() + if frappe.get_request_header("X-Forwarded-For"): + frappe.local.request_ip = (frappe.get_request_header("X-Forwarded-For").split(",")[0]).strip() - elif frappe.get_request_header('REMOTE_ADDR'): - frappe.local.request_ip = frappe.get_request_header('REMOTE_ADDR') + elif frappe.get_request_header("REMOTE_ADDR"): + frappe.local.request_ip = frappe.get_request_header("REMOTE_ADDR") else: - frappe.local.request_ip = '127.0.0.1' + frappe.local.request_ip = "127.0.0.1" def set_cookies(self): frappe.local.cookie_manager = CookieManager() @@ -75,7 +80,7 @@ class HTTPRequest: if ( not frappe.local.session.data.csrf_token or frappe.local.session.data.device == "mobile" - or frappe.conf.get('ignore_csrf', None) + or frappe.conf.get("ignore_csrf", None) ): # not via boot return @@ -99,10 +104,10 @@ class HTTPRequest: def connect(self): """connect to db, from ac_name or db_name""" frappe.local.db = frappe.database.get_db( - user=self.get_db_name(), - password=getattr(conf, 'db_password', '') + user=self.get_db_name(), password=getattr(conf, "db_password", "") ) + class LoginManager: def __init__(self): self.user = None @@ -110,13 +115,15 @@ class LoginManager: self.full_name = None self.user_type = None - if frappe.local.form_dict.get('cmd')=='login' or frappe.local.request.path=="/api/method/login": + if ( + frappe.local.form_dict.get("cmd") == "login" or frappe.local.request.path == "/api/method/login" + ): if self.login() is False: return self.resume = False # run login triggers - self.run_trigger('on_session_creation') + self.run_trigger("on_session_creation") else: try: self.resume = True @@ -131,12 +138,14 @@ class LoginManager: def login(self): # clear cache - frappe.clear_cache(user = frappe.form_dict.get('usr')) + frappe.clear_cache(user=frappe.form_dict.get("usr")) user, pwd = get_cached_user_pass() self.authenticate(user=user, pwd=pwd) if self.force_user_to_reset_password(): doc = frappe.get_doc("User", self.user) - frappe.local.response["redirect_to"] = doc.reset_password(send_email=False, password_expired=True) + frappe.local.response["redirect_to"] = doc.reset_password( + send_email=False, password_expired=True + ) frappe.local.response["message"] = "Password Reset" return False @@ -147,7 +156,7 @@ class LoginManager: self.post_login() def post_login(self): - self.run_trigger('on_login') + self.run_trigger("on_login") validate_ip_address(self.user) self.validate_hour() self.get_user_info() @@ -156,8 +165,9 @@ class LoginManager: self.set_user_info() def get_user_info(self): - self.info = frappe.db.get_value("User", self.user, - ["user_type", "first_name", "last_name", "user_image"], as_dict=1) + self.info = frappe.db.get_value( + "User", self.user, ["user_type", "first_name", "last_name", "user_image"], as_dict=1 + ) self.user_type = self.info.user_type @@ -170,28 +180,27 @@ class LoginManager: # set sid again frappe.local.cookie_manager.init_cookies() - self.full_name = " ".join(filter(None, [self.info.first_name, - self.info.last_name])) + self.full_name = " ".join(filter(None, [self.info.first_name, self.info.last_name])) - if self.info.user_type=="Website User": + if self.info.user_type == "Website User": frappe.local.cookie_manager.set_cookie("system_user", "no") if not resume: frappe.local.response["message"] = "No App" - frappe.local.response["home_page"] = '/' + get_home_page() + frappe.local.response["home_page"] = "/" + get_home_page() else: frappe.local.cookie_manager.set_cookie("system_user", "yes") if not resume: - frappe.local.response['message'] = 'Logged In' + frappe.local.response["message"] = "Logged In" frappe.local.response["home_page"] = "/app" if not resume: frappe.response["full_name"] = self.full_name # redirect information - redirect_to = frappe.cache().hget('redirect_after_login', self.user) + redirect_to = frappe.cache().hget("redirect_after_login", self.user) if redirect_to: frappe.local.response["redirect_to"] = redirect_to - frappe.cache().hdel('redirect_after_login', self.user) + frappe.cache().hdel("redirect_after_login", self.user) frappe.local.cookie_manager.set_cookie("full_name", self.full_name) frappe.local.cookie_manager.set_cookie("user_id", self.user) @@ -202,8 +211,9 @@ class LoginManager: def make_session(self, resume=False): # start session - frappe.local.session_obj = Session(user=self.user, resume=resume, - full_name=self.full_name, user_type=self.user_type) + frappe.local.session_obj = Session( + user=self.user, resume=resume, full_name=self.full_name, user_type=self.user_type + ) # reset user if changed to Guest self.user = frappe.local.session_obj.user @@ -212,7 +222,10 @@ class LoginManager: def clear_active_sessions(self): """Clear other sessions of the current user if `deny_multiple_sessions` is not set""" - if not (cint(frappe.conf.get("deny_multiple_sessions")) or cint(frappe.db.get_system_setting('deny_multiple_sessions'))): + if not ( + cint(frappe.conf.get("deny_multiple_sessions")) + or cint(frappe.db.get_system_setting("deny_multiple_sessions")) + ): return if frappe.session.user != "Guest": @@ -222,27 +235,27 @@ class LoginManager: from frappe.core.doctype.user.user import User if not (user and pwd): - user, pwd = frappe.form_dict.get('usr'), frappe.form_dict.get('pwd') + user, pwd = frappe.form_dict.get("usr"), frappe.form_dict.get("pwd") if not (user and pwd): - self.fail(_('Incomplete login details'), user=user) + self.fail(_("Incomplete login details"), user=user) user = User.find_by_credentials(user, pwd) if not user: - self.fail('Invalid login credentials') + self.fail("Invalid login credentials") # Current login flow uses cached credentials for authentication while checking OTP. # Incase of OTP check, tracker for auth needs to be disabled(If not, it can remove tracker history as it is going to succeed anyway) # Tracker is activated for 2FA incase of OTP. - ignore_tracker = should_run_2fa(user.name) and ('otp' in frappe.form_dict) + ignore_tracker = should_run_2fa(user.name) and ("otp" in frappe.form_dict) tracker = None if ignore_tracker else get_login_attempt_tracker(user.name) if not user.is_authenticated: tracker and tracker.add_failure_attempt() - self.fail('Invalid login credentials', user=user.name) - elif not (user.name == 'Administrator' or user.enabled): + self.fail("Invalid login credentials", user=user.name) + elif not (user.name == "Administrator" or user.enabled): tracker and tracker.add_failure_attempt() - self.fail('User disabled or missing', user=user.name) + self.fail("User disabled or missing", user=user.name) else: tracker and tracker.add_success_attempt() self.user = user.name @@ -254,12 +267,14 @@ class LoginManager: if self.user in frappe.STANDARD_USERS: return False - reset_pwd_after_days = cint(frappe.db.get_single_value("System Settings", - "force_user_to_reset_password")) + reset_pwd_after_days = cint( + frappe.db.get_single_value("System Settings", "force_user_to_reset_password") + ) if reset_pwd_after_days: - last_password_reset_date = frappe.db.get_value("User", - self.user, "last_password_reset_date") or today() + last_password_reset_date = ( + frappe.db.get_value("User", self.user, "last_password_reset_date") or today() + ) last_pwd_reset_days = date_diff(today(), last_password_reset_date) @@ -272,30 +287,31 @@ class LoginManager: # returns user in correct case return check_password(user, pwd) except frappe.AuthenticationError: - self.fail('Incorrect password', user=user) + self.fail("Incorrect password", user=user) def fail(self, message, user=None): if not user: - user = _('Unknown User') - frappe.local.response['message'] = message + user = _("Unknown User") + frappe.local.response["message"] = message add_authentication_log(message, user, status="Failed") frappe.db.commit() raise frappe.AuthenticationError - def run_trigger(self, event='on_login'): + def run_trigger(self, event="on_login"): for method in frappe.get_hooks().get(event, []): frappe.call(frappe.get_attr(method), login_manager=self) def validate_hour(self): """check if user is logging in during restricted hours""" - login_before = int(frappe.db.get_value('User', self.user, 'login_before', ignore=True) or 0) - login_after = int(frappe.db.get_value('User', self.user, 'login_after', ignore=True) or 0) + login_before = int(frappe.db.get_value("User", self.user, "login_before", ignore=True) or 0) + login_after = int(frappe.db.get_value("User", self.user, "login_after", ignore=True) or 0) if not (login_before or login_after): return from frappe.utils import now_datetime - current_hour = int(now_datetime().strftime('%H')) + + current_hour = int(now_datetime().strftime("%H")) if login_before and current_hour > login_before: frappe.throw(_("Login not allowed at this time"), frappe.AuthenticationError) @@ -311,9 +327,10 @@ class LoginManager: self.user = user self.post_login() - def logout(self, arg='', user=None): - if not user: user = frappe.session.user - self.run_trigger('on_logout') + def logout(self, arg="", user=None): + if not user: + user = frappe.session.user + self.run_trigger("on_logout") if user == frappe.session.user: delete_session(frappe.session.sid, user=user, reason="User Manually Logged Out") @@ -324,13 +341,15 @@ class LoginManager: def clear_cookies(self): clear_cookies() + class CookieManager: def __init__(self): self.cookies = {} self.to_delete = [] def init_cookies(self): - if not frappe.local.session.get('sid'): return + if not frappe.local.session.get("sid"): + return # sid expires in 3 days expires = datetime.datetime.now() + datetime.timedelta(days=3) @@ -340,7 +359,7 @@ class CookieManager: self.set_cookie("country", frappe.session.session_country) def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): - if not secure and hasattr(frappe.local, 'request'): + if not secure and hasattr(frappe.local, "request"): secure = frappe.local.request.scheme == "https" # Cordova does not work with Lax @@ -352,7 +371,7 @@ class CookieManager: "expires": expires, "secure": secure, "httponly": httponly, - "samesite": samesite + "samesite": samesite, } def delete_cookie(self, to_delete): @@ -363,11 +382,14 @@ class CookieManager: def flush_cookies(self, response): for key, opts in self.cookies.items(): - response.set_cookie(key, quote((opts.get("value") or "").encode('utf-8')), + response.set_cookie( + key, + quote((opts.get("value") or "").encode("utf-8")), expires=opts.get("expires"), secure=opts.get("secure"), httponly=opts.get("httponly"), - samesite=opts.get("samesite")) + samesite=opts.get("samesite"), + ) # expires yesterday! expires = datetime.datetime.now() + datetime.timedelta(days=-1) @@ -379,19 +401,29 @@ class CookieManager: def get_logged_user(): return frappe.session.user + def clear_cookies(): if hasattr(frappe.local, "session"): frappe.session.sid = "" - frappe.local.cookie_manager.delete_cookie(["full_name", "user_id", "sid", "user_image", "system_user"]) + frappe.local.cookie_manager.delete_cookie( + ["full_name", "user_id", "sid", "user_image", "system_user"] + ) + def validate_ip_address(user): """check if IP Address is valid""" - user = frappe.get_cached_doc("User", user) if not frappe.flags.in_test else frappe.get_doc("User", user) + user = ( + frappe.get_cached_doc("User", user) if not frappe.flags.in_test else frappe.get_doc("User", user) + ) ip_list = user.get_restricted_ip_list() if not ip_list: return - system_settings = frappe.get_cached_doc("System Settings") if not frappe.flags.in_test else frappe.get_single("System Settings") + system_settings = ( + frappe.get_cached_doc("System Settings") + if not frappe.flags.in_test + else frappe.get_single("System Settings") + ) # check if bypass restrict ip is enabled for all users bypass_restrict_ip_check = system_settings.bypass_restrict_ip_check_if_2fa_enabled @@ -406,6 +438,7 @@ def validate_ip_address(user): frappe.throw(_("Access not allowed from this IP Address"), frappe.AuthenticationError) + def get_login_attempt_tracker(user_name: str, raise_locked_exception: bool = True): """Get login attempt tracker instance. @@ -413,18 +446,22 @@ def get_login_attempt_tracker(user_name: str, raise_locked_exception: bool = Tru :param raise_locked_exception: If set, raises an exception incase of user not allowed to login """ sys_settings = frappe.get_doc("System Settings") - track_login_attempts = (sys_settings.allow_consecutive_login_attempts >0) + track_login_attempts = sys_settings.allow_consecutive_login_attempts > 0 tracker_kwargs = {} if track_login_attempts: - tracker_kwargs['lock_interval'] = sys_settings.allow_login_after_fail - tracker_kwargs['max_consecutive_login_attempts'] = sys_settings.allow_consecutive_login_attempts + tracker_kwargs["lock_interval"] = sys_settings.allow_login_after_fail + tracker_kwargs["max_consecutive_login_attempts"] = sys_settings.allow_consecutive_login_attempts tracker = LoginAttemptTracker(user_name, **tracker_kwargs) if raise_locked_exception and track_login_attempts and not tracker.is_user_allowed(): - frappe.throw(_("Your account has been locked and will resume after {0} seconds") - .format(sys_settings.allow_login_after_fail), frappe.SecurityException) + frappe.throw( + _("Your account has been locked and will resume after {0} seconds").format( + sys_settings.allow_login_after_fail + ), + frappe.SecurityException, + ) return tracker @@ -433,8 +470,11 @@ class LoginAttemptTracker(object): Lock the account for s number of seconds if there have been n consecutive unsuccessful attempts to log in. """ - def __init__(self, user_name: str, max_consecutive_login_attempts: int=3, lock_interval:int = 5*60): - """ Initialize the tracker. + + def __init__( + self, user_name: str, max_consecutive_login_attempts: int = 3, lock_interval: int = 5 * 60 + ): + """Initialize the tracker. :param user_name: Name of the loggedin user :param max_consecutive_login_attempts: Maximum allowed consecutive failed login attempts @@ -446,15 +486,15 @@ class LoginAttemptTracker(object): @property def login_failed_count(self): - return frappe.cache().hget('login_failed_count', self.user_name) + return frappe.cache().hget("login_failed_count", self.user_name) @login_failed_count.setter def login_failed_count(self, count): - frappe.cache().hset('login_failed_count', self.user_name, count) + frappe.cache().hset("login_failed_count", self.user_name, count) @login_failed_count.deleter def login_failed_count(self): - frappe.cache().hdel('login_failed_count', self.user_name) + frappe.cache().hdel("login_failed_count", self.user_name) @property def login_failed_time(self): @@ -462,23 +502,23 @@ class LoginAttemptTracker(object): For every user we track only First failed login attempt time within lock interval of time. """ - return frappe.cache().hget('login_failed_time', self.user_name) + return frappe.cache().hget("login_failed_time", self.user_name) @login_failed_time.setter def login_failed_time(self, timestamp): - frappe.cache().hset('login_failed_time', self.user_name, timestamp) + frappe.cache().hset("login_failed_time", self.user_name, timestamp) @login_failed_time.deleter def login_failed_time(self): - frappe.cache().hdel('login_failed_time', self.user_name) + frappe.cache().hdel("login_failed_time", self.user_name) def add_failure_attempt(self): - """ Log user failure attempts into the system. + """Log user failure attempts into the system. Increase the failure count if new failure is with in current lock interval time period, if not reset the login failure count. """ login_failed_time = self.login_failed_time - login_failed_count = self.login_failed_count # Consecutive login failure count + login_failed_count = self.login_failed_count # Consecutive login failure count current_time = get_datetime() if not (login_failed_time and login_failed_count): @@ -493,8 +533,7 @@ class LoginAttemptTracker(object): self.login_failed_count = login_failed_count def add_success_attempt(self): - """Reset login failures. - """ + """Reset login failures.""" del self.login_failed_count del self.login_failed_time @@ -507,6 +546,10 @@ class LoginAttemptTracker(object): login_failed_count = self.login_failed_count or 0 current_time = get_datetime() - if login_failed_time and login_failed_time + self.lock_interval > current_time and login_failed_count > self.max_failed_logins: + if ( + login_failed_time + and login_failed_time + self.lock_interval > current_time + and login_failed_count > self.max_failed_logins + ): return False return True diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index 90099eebb6..f3dfa4cf0a 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -24,9 +24,7 @@ class AssignmentRule(Document): def validate_document_types(self): if self.document_type == "ToDo": frappe.throw( - _('Assignment Rule is not allowed on {0} document type').format( - frappe.bold("ToDo") - ) + _("Assignment Rule is not allowed on {0} document type").format(frappe.bold("ToDo")) ) def validate_assignment_days(self): @@ -38,70 +36,70 @@ class AssignmentRule(Document): frappe.throw( _("Assignment Day{0} {1} has been repeated.").format( - plural, - frappe.bold(", ".join(repeated_days)) + plural, frappe.bold(", ".join(repeated_days)) ) ) def apply_unassign(self, doc, assignments): - if (self.unassign_condition and - self.name in [d.assignment_rule for d in assignments]): + if self.unassign_condition and self.name in [d.assignment_rule for d in assignments]: return self.clear_assignment(doc) return False def apply_assign(self, doc): - if self.safe_eval('assign_condition', doc): + if self.safe_eval("assign_condition", doc): return self.do_assignment(doc) def do_assignment(self, doc): # clear existing assignment, to reassign - assign_to.clear(doc.get('doctype'), doc.get('name')) + assign_to.clear(doc.get("doctype"), doc.get("name")) user = self.get_user(doc) if user: - assign_to.add(dict( - assign_to = [user], - doctype = doc.get('doctype'), - name = doc.get('name'), - description = frappe.render_template(self.description, doc), - assignment_rule = self.name, - notify = True, - date = doc.get(self.due_date_based_on) if self.due_date_based_on else None - )) + assign_to.add( + dict( + assign_to=[user], + doctype=doc.get("doctype"), + name=doc.get("name"), + description=frappe.render_template(self.description, doc), + assignment_rule=self.name, + notify=True, + date=doc.get(self.due_date_based_on) if self.due_date_based_on else None, + ) + ) # set for reference in round robin - self.db_set('last_user', user) + self.db_set("last_user", user) return True return False def clear_assignment(self, doc): - '''Clear assignments''' - if self.safe_eval('unassign_condition', doc): - return assign_to.clear(doc.get('doctype'), doc.get('name')) + """Clear assignments""" + if self.safe_eval("unassign_condition", doc): + return assign_to.clear(doc.get("doctype"), doc.get("name")) def close_assignments(self, doc): - '''Close assignments''' - if self.safe_eval('close_condition', doc): - return assign_to.close_all_assignments(doc.get('doctype'), doc.get('name')) + """Close assignments""" + if self.safe_eval("close_condition", doc): + return assign_to.close_all_assignments(doc.get("doctype"), doc.get("name")) def get_user(self, doc): - ''' + """ Get the next user for assignment - ''' - if self.rule == 'Round Robin': + """ + if self.rule == "Round Robin": return self.get_user_round_robin() - elif self.rule == 'Load Balancing': + elif self.rule == "Load Balancing": return self.get_user_load_balancing() - elif self.rule == 'Based on Field': + elif self.rule == "Based on Field": return self.get_user_based_on_field(doc) def get_user_round_robin(self): - ''' + """ Get next user based on round robin - ''' + """ # first time, or last in list, pick the first if not self.last_user or self.last_user == self.users[-1].user: @@ -110,32 +108,33 @@ class AssignmentRule(Document): # find out the next user in the list for i, d in enumerate(self.users): if self.last_user == d.user: - return self.users[i+1].user + return self.users[i + 1].user # bad last user, assign to the first one return self.users[0].user def get_user_load_balancing(self): - '''Assign to the user with least number of open assignments''' + """Assign to the user with least number of open assignments""" counts = [] for d in self.users: - counts.append(dict( - user = d.user, - count = frappe.db.count('ToDo', dict( - reference_type = self.document_type, - allocated_to = d.user, - status = "Open")) - )) + counts.append( + dict( + user=d.user, + count=frappe.db.count( + "ToDo", dict(reference_type=self.document_type, allocated_to=d.user, status="Open") + ), + ) + ) # sort by dict value - sorted_counts = sorted(counts, key = lambda k: k['count']) + sorted_counts = sorted(counts, key=lambda k: k["count"]) # pick the first user - return sorted_counts[0].get('user') + return sorted_counts[0].get("user") def get_user_based_on_field(self, doc): val = doc.get(self.field) - if frappe.db.exists('User', val): + if frappe.db.exists("User", val): return val def safe_eval(self, fieldname, doc): @@ -145,12 +144,12 @@ class AssignmentRule(Document): except Exception as e: # when assignment fails, don't block the document as it may be # a part of the email pulling - frappe.msgprint(frappe._('Auto assignment failed: {0}').format(str(e)), indicator = 'orange') + frappe.msgprint(frappe._("Auto assignment failed: {0}").format(str(e)), indicator="orange") return False def get_assignment_days(self): - return [d.day for d in self.get('assignment_days', [])] + return [d.day for d in self.get("assignment_days", [])] def is_rule_not_applicable_today(self): today = frappe.flags.assignment_day or frappe.utils.get_weekday() @@ -159,11 +158,14 @@ class AssignmentRule(Document): def get_assignments(doc) -> List[Dict]: - return frappe.get_all('ToDo', fields = ['name', 'assignment_rule'], filters = dict( - reference_type = doc.get('doctype'), - reference_name = doc.get('name'), - status = ('!=', 'Cancelled') - ), limit=5) + return frappe.get_all( + "ToDo", + fields=["name", "assignment_rule"], + filters=dict( + reference_type=doc.get("doctype"), reference_name=doc.get("name"), status=("!=", "Cancelled") + ), + limit=5, + ) @frappe.whitelist() @@ -173,21 +175,30 @@ def bulk_apply(doctype, docnames): for name in docnames: if background: - frappe.enqueue('frappe.automation.doctype.assignment_rule.assignment_rule.apply', doc=None, doctype=doctype, name=name) + frappe.enqueue( + "frappe.automation.doctype.assignment_rule.assignment_rule.apply", + doc=None, + doctype=doctype, + name=name, + ) else: apply(doctype=doctype, name=name) def reopen_closed_assignment(doc): - todo_list = frappe.get_all("ToDo", filters={ - "reference_type": doc.doctype, - "reference_name": doc.name, - "status": "Closed", - }, pluck="name") + todo_list = frappe.get_all( + "ToDo", + filters={ + "reference_type": doc.doctype, + "reference_name": doc.name, + "status": "Closed", + }, + pluck="name", + ) for todo in todo_list: - todo_doc = frappe.get_doc('ToDo', todo) - todo_doc.status = 'Open' + todo_doc = frappe.get_doc("ToDo", todo) + todo_doc.status = "Open" todo_doc.save(ignore_permissions=True) return bool(todo_list) @@ -209,13 +220,16 @@ def apply(doc=None, method=None, doctype=None, name=None): if not doc and doctype and name: doc = frappe.get_doc(doctype, name) - assignment_rules = get_doctype_map("Assignment Rule", doc.doctype, filters={ - "document_type": doc.doctype, "disabled": 0 - }, order_by="priority desc") + assignment_rules = get_doctype_map( + "Assignment Rule", + doc.doctype, + filters={"document_type": doc.doctype, "disabled": 0}, + order_by="priority desc", + ) # multiple auto assigns assignment_rule_docs: List[AssignmentRule] = [ - frappe.get_cached_doc("Assignment Rule", d.get('name')) for d in assignment_rules + frappe.get_cached_doc("Assignment Rule", d.get("name")) for d in assignment_rules ] if not assignment_rule_docs: @@ -224,8 +238,8 @@ def apply(doc=None, method=None, doctype=None, name=None): doc = doc.as_dict() assignments = get_assignments(doc) - clear = True # are all assignments cleared - new_apply = False # are new assignments applied + clear = True # are all assignments cleared + new_apply = False # are new assignments applied if assignments: # first unassign @@ -260,14 +274,18 @@ def apply(doc=None, method=None, doctype=None, name=None): if not new_apply: # only reopen if close condition is not satisfied - to_close_todos = assignment_rule.safe_eval('close_condition', doc) + to_close_todos = assignment_rule.safe_eval("close_condition", doc) if to_close_todos: # close todo status - todos_to_close = frappe.get_all("ToDo", filters={ - "reference_type": doc.doctype, - "reference_name": doc.name, - }, pluck="name") + todos_to_close = frappe.get_all( + "ToDo", + filters={ + "reference_type": doc.doctype, + "reference_name": doc.name, + }, + pluck="name", + ) for todo in todos_to_close: _todo = frappe.get_doc("ToDo", todo) @@ -286,8 +304,7 @@ def apply(doc=None, method=None, doctype=None, name=None): def update_due_date(doc, state=None): - """Run on_update on every Document (via hooks.py) - """ + """Run on_update on every Document (via hooks.py)""" skip_document_update = ( frappe.flags.in_migrate or frappe.flags.in_patch @@ -306,7 +323,7 @@ def update_due_date(doc, state=None): "due_date_based_on": ["is", "set"], "document_type": doc.doctype, "disabled": 0, - } + }, ) for rule in assignment_rules: @@ -319,20 +336,24 @@ def update_due_date(doc, state=None): ) if field_updated: - assignment_todos = frappe.get_all("ToDo", filters={ - "assignment_rule": rule.get("name"), - "reference_type": doc.doctype, - "reference_name": doc.name, - "status": "Open", - }, pluck="name") + assignment_todos = frappe.get_all( + "ToDo", + filters={ + "assignment_rule": rule.get("name"), + "reference_type": doc.doctype, + "reference_name": doc.name, + "status": "Open", + }, + pluck="name", + ) for todo in assignment_todos: - todo_doc = frappe.get_doc('ToDo', todo) + todo_doc = frappe.get_doc("ToDo", todo) todo_doc.date = doc.get(due_date_field) todo_doc.flags.updater_reference = { - 'doctype': 'Assignment Rule', - 'docname': rule.get('name'), - 'label': _('via Assignment Rule') + "doctype": "Assignment Rule", + "docname": rule.get("name"), + "label": _("via Assignment Rule"), } todo_doc.save(ignore_permissions=True) diff --git a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py index 63dbf69d3b..2ab2e4d263 100644 --- a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py @@ -20,13 +20,13 @@ class TestAutoAssign(unittest.TestCase): def setUp(self): make_test_records("User") days = [ - dict(day = 'Sunday'), - dict(day = 'Monday'), - dict(day = 'Tuesday'), - dict(day = 'Wednesday'), - dict(day = 'Thursday'), - dict(day = 'Friday'), - dict(day = 'Saturday'), + dict(day="Sunday"), + dict(day="Monday"), + dict(day="Tuesday"), + dict(day="Wednesday"), + dict(day="Thursday"), + dict(day="Friday"), + dict(day="Saturday"), ] self.days = days self.assignment_rule = get_assignment_rule([days, days]) @@ -36,20 +36,22 @@ class TestAutoAssign(unittest.TestCase): note = make_note(dict(public=1)) # check if auto assigned to first user - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), 'test@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + "test@example.com", + ) note = make_note(dict(public=1)) # check if auto assigned to second user - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), 'test1@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + "test1@example.com", + ) clear_assignments() @@ -57,35 +59,41 @@ class TestAutoAssign(unittest.TestCase): # check if auto assigned to third user, even if # previous assignments where closed - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), 'test2@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + "test2@example.com", + ) # check loop back to first user note = make_note(dict(public=1)) - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), 'test@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + "test@example.com", + ) def test_load_balancing(self): - self.assignment_rule.rule = 'Load Balancing' + self.assignment_rule.rule = "Load Balancing" self.assignment_rule.save() for _ in range(30): note = make_note(dict(public=1)) # check if each user has 10 assignments (?) - for user in ('test@example.com', 'test1@example.com', 'test2@example.com'): - self.assertEqual(len(frappe.get_all('ToDo', dict(allocated_to = user, reference_type = 'Note'))), 10) + for user in ("test@example.com", "test1@example.com", "test2@example.com"): + self.assertEqual( + len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type="Note"))), 10 + ) # clear 5 assignments for first user # can't do a limit in "delete" since postgres does not support it - for d in frappe.get_all('ToDo', dict(reference_type = 'Note', allocated_to = 'test@example.com'), limit=5): + for d in frappe.get_all( + "ToDo", dict(reference_type="Note", allocated_to="test@example.com"), limit=5 + ): frappe.db.delete("ToDo", {"name": d.name}) # add 5 more assignments @@ -93,56 +101,59 @@ class TestAutoAssign(unittest.TestCase): make_note(dict(public=1)) # check if each user still has 10 assignments - for user in ('test@example.com', 'test1@example.com', 'test2@example.com'): - self.assertEqual(len(frappe.get_all('ToDo', dict(allocated_to = user, reference_type = 'Note'))), 10) + for user in ("test@example.com", "test1@example.com", "test2@example.com"): + self.assertEqual( + len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type="Note"))), 10 + ) def test_based_on_field(self): - self.assignment_rule.rule = 'Based on Field' - self.assignment_rule.field = 'owner' + self.assignment_rule.rule = "Based on Field" + self.assignment_rule.field = "owner" self.assignment_rule.save() - frappe.set_user('test1@example.com') + frappe.set_user("test1@example.com") note = make_note(dict(public=1)) # check if auto assigned to doc owner, test1@example.com - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'owner'), 'test1@example.com') - - frappe.set_user('test2@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "owner" + ), + "test1@example.com", + ) + + frappe.set_user("test2@example.com") note = make_note(dict(public=1)) # check if auto assigned to doc owner, test2@example.com - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'owner'), 'test2@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "owner" + ), + "test2@example.com", + ) - frappe.set_user('Administrator') + frappe.set_user("Administrator") def test_assign_condition(self): # check condition note = make_note(dict(public=0)) - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), None) + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + None, + ) def test_clear_assignment(self): note = make_note(dict(public=1)) # check if auto assigned to first user - todo = frappe.get_list('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), limit=1)[0] + todo = frappe.get_list( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), limit=1 + )[0] - todo = frappe.get_doc('ToDo', todo['name']) - self.assertEqual(todo.allocated_to, 'test@example.com') + todo = frappe.get_doc("ToDo", todo["name"]) + self.assertEqual(todo.allocated_to, "test@example.com") # test auto unassign note.public = 0 @@ -151,99 +162,101 @@ class TestAutoAssign(unittest.TestCase): todo.load_from_db() # check if todo is cancelled - self.assertEqual(todo.status, 'Cancelled') + self.assertEqual(todo.status, "Cancelled") def test_close_assignment(self): note = make_note(dict(public=1, content="valid")) # check if auto assigned - todo = frappe.get_list('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), limit=1)[0] + todo = frappe.get_list( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), limit=1 + )[0] - todo = frappe.get_doc('ToDo', todo['name']) - self.assertEqual(todo.allocated_to, 'test@example.com') + todo = frappe.get_doc("ToDo", todo["name"]) + self.assertEqual(todo.allocated_to, "test@example.com") - note.content="Closed" + note.content = "Closed" note.save() todo.load_from_db() # check if todo is closed - self.assertEqual(todo.status, 'Closed') + self.assertEqual(todo.status, "Closed") # check if closed todo retained assignment - self.assertEqual(todo.allocated_to, 'test@example.com') + self.assertEqual(todo.allocated_to, "test@example.com") def check_multiple_rules(self): note = make_note(dict(public=1, notify_on_login=1)) # check if auto assigned to test3 (2nd rule is applied, as it has higher priority) - self.assertEqual(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), 'test@example.com') + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + "test@example.com", + ) def check_assignment_rule_scheduling(self): frappe.db.delete("Assignment Rule") - days_1 = [dict(day = 'Sunday'), dict(day = 'Monday'), dict(day = 'Tuesday')] + days_1 = [dict(day="Sunday"), dict(day="Monday"), dict(day="Tuesday")] - days_2 = [dict(day = 'Wednesday'), dict(day = 'Thursday'), dict(day = 'Friday'), dict(day = 'Saturday')] + days_2 = [dict(day="Wednesday"), dict(day="Thursday"), dict(day="Friday"), dict(day="Saturday")] - get_assignment_rule([days_1, days_2], ['public == 1', 'public == 1']) + get_assignment_rule([days_1, days_2], ["public == 1", "public == 1"]) frappe.flags.assignment_day = "Monday" note = make_note(dict(public=1)) - self.assertIn(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), ['test@example.com', 'test1@example.com', 'test2@example.com']) + self.assertIn( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + ["test@example.com", "test1@example.com", "test2@example.com"], + ) frappe.flags.assignment_day = "Friday" note = make_note(dict(public=1)) - self.assertIn(frappe.db.get_value('ToDo', dict( - reference_type = 'Note', - reference_name = note.name, - status = 'Open' - ), 'allocated_to'), ['test3@example.com']) + self.assertIn( + frappe.db.get_value( + "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + ), + ["test3@example.com"], + ) def test_assignment_rule_condition(self): frappe.db.delete("Assignment Rule") # Add expiry_date custom field from frappe.custom.doctype.custom_field.custom_field import create_custom_field - df = dict(fieldname='expiry_date', label='Expiry Date', fieldtype='Date') - create_custom_field('Note', df) - - assignment_rule = frappe.get_doc(dict( - name = 'Assignment with Due Date', - doctype = 'Assignment Rule', - document_type = 'Note', - assign_condition = 'public == 0', - due_date_based_on = 'expiry_date', - assignment_days = self.days, - users = [ - dict(user = 'test@example.com'), - ] - )).insert() + + df = dict(fieldname="expiry_date", label="Expiry Date", fieldtype="Date") + create_custom_field("Note", df) + + assignment_rule = frappe.get_doc( + dict( + name="Assignment with Due Date", + doctype="Assignment Rule", + document_type="Note", + assign_condition="public == 0", + due_date_based_on="expiry_date", + assignment_days=self.days, + users=[ + dict(user="test@example.com"), + ], + ) + ).insert() expiry_date = frappe.utils.add_days(frappe.utils.nowdate(), 2) - note1 = make_note({'expiry_date': expiry_date}) - note2 = make_note({'expiry_date': expiry_date}) + note1 = make_note({"expiry_date": expiry_date}) + note2 = make_note({"expiry_date": expiry_date}) - note1_todo = frappe.get_all('ToDo', filters=dict( - reference_type = 'Note', - reference_name = note1.name, - status = 'Open' - ))[0] + note1_todo = frappe.get_all( + "ToDo", filters=dict(reference_type="Note", reference_name=note1.name, status="Open") + )[0] - note1_todo_doc = frappe.get_doc('ToDo', note1_todo.name) + note1_todo_doc = frappe.get_doc("ToDo", note1_todo.name) self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), expiry_date) # due date should be updated if the reference doc's date is updated. @@ -253,66 +266,67 @@ class TestAutoAssign(unittest.TestCase): self.assertEqual(frappe.utils.get_date_str(note1_todo_doc.date), note1.expiry_date) # saving one note's expiry should not update other note todo's due date - note2_todo = frappe.get_all('ToDo', filters=dict( - reference_type = 'Note', - reference_name = note2.name, - status = 'Open' - ), fields=['name', 'date'])[0] + note2_todo = frappe.get_all( + "ToDo", + filters=dict(reference_type="Note", reference_name=note2.name, status="Open"), + fields=["name", "date"], + )[0] self.assertNotEqual(frappe.utils.get_date_str(note2_todo.date), note1.expiry_date) self.assertEqual(frappe.utils.get_date_str(note2_todo.date), expiry_date) assignment_rule.delete() + def clear_assignments(): frappe.db.delete("ToDo", {"reference_type": "Note"}) + def get_assignment_rule(days, assign=None): - frappe.delete_doc_if_exists('Assignment Rule', 'For Note 1') + frappe.delete_doc_if_exists("Assignment Rule", "For Note 1") if not assign: - assign = ['public == 1', 'notify_on_login == 1'] - - assignment_rule = frappe.get_doc(dict( - name = 'For Note 1', - doctype = 'Assignment Rule', - priority = 0, - document_type = 'Note', - assign_condition = assign[0], - unassign_condition = 'public == 0 or notify_on_login == 1', - close_condition = '"Closed" in content', - rule = 'Round Robin', - assignment_days = days[0], - users = [ - dict(user = 'test@example.com'), - dict(user = 'test1@example.com'), - dict(user = 'test2@example.com'), - ] - )).insert() - - frappe.delete_doc_if_exists('Assignment Rule', 'For Note 2') + assign = ["public == 1", "notify_on_login == 1"] + + assignment_rule = frappe.get_doc( + dict( + name="For Note 1", + doctype="Assignment Rule", + priority=0, + document_type="Note", + assign_condition=assign[0], + unassign_condition="public == 0 or notify_on_login == 1", + close_condition='"Closed" in content', + rule="Round Robin", + assignment_days=days[0], + users=[ + dict(user="test@example.com"), + dict(user="test1@example.com"), + dict(user="test2@example.com"), + ], + ) + ).insert() + + frappe.delete_doc_if_exists("Assignment Rule", "For Note 2") # 2nd rule - frappe.get_doc(dict( - name = 'For Note 2', - doctype = 'Assignment Rule', - priority = 1, - document_type = 'Note', - assign_condition = assign[1], - unassign_condition = 'notify_on_login == 0', - rule = 'Round Robin', - assignment_days = days[1], - users = [ - dict(user = 'test3@example.com') - ] - )).insert() + frappe.get_doc( + dict( + name="For Note 2", + doctype="Assignment Rule", + priority=1, + document_type="Note", + assign_condition=assign[1], + unassign_condition="notify_on_login == 0", + rule="Round Robin", + assignment_days=days[1], + users=[dict(user="test3@example.com")], + ) + ).insert() return assignment_rule + def make_note(values=None): - note = frappe.get_doc(dict( - doctype = 'Note', - title = random_string(10), - content = random_string(20) - )) + note = frappe.get_doc(dict(doctype="Note", title=random_string(10), content=random_string(20))) if values: note.update(values) diff --git a/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py b/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py index 836ae3d453..2a7e6dd66f 100644 --- a/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py +++ b/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class AssignmentRuleDay(Document): pass diff --git a/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py b/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py index 1bb8953a7a..3f47f3c866 100644 --- a/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py +++ b/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class AssignmentRuleUser(Document): pass diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index 0277b8e402..5ff9e9f3ab 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -2,23 +2,45 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE +from datetime import timedelta + +from dateutil.relativedelta import relativedelta + import frappe from frappe import _ -from datetime import timedelta +from frappe.automation.doctype.assignment_rule.assignment_rule import get_repeated +from frappe.contacts.doctype.contact.contact import ( + get_contacts_linked_from, + get_contacts_linking_to, +) +from frappe.core.doctype.communication.email import make from frappe.desk.form import assign_to -from frappe.utils.jinja import validate_template -from dateutil.relativedelta import relativedelta -from frappe.utils.user import get_system_managers -from frappe.utils import cstr, getdate, split_emails, add_days, today, get_last_day, get_first_day, month_diff from frappe.model.document import Document -from frappe.core.doctype.communication.email import make +from frappe.utils import ( + add_days, + cstr, + get_first_day, + get_last_day, + getdate, + month_diff, + split_emails, + today, +) from frappe.utils.background_jobs import get_jobs -from frappe.automation.doctype.assignment_rule.assignment_rule import get_repeated -from frappe.contacts.doctype.contact.contact import get_contacts_linked_from -from frappe.contacts.doctype.contact.contact import get_contacts_linking_to +from frappe.utils.jinja import validate_template +from frappe.utils.user import get_system_managers + +month_map = {"Monthly": 1, "Quarterly": 3, "Half-yearly": 6, "Yearly": 12} +week_map = { + "Monday": 0, + "Tuesday": 1, + "Wednesday": 2, + "Thursday": 3, + "Friday": 4, + "Saturday": 5, + "Sunday": 6, +} -month_map = {'Monthly': 1, 'Quarterly': 3, 'Half-yearly': 6, 'Yearly': 12} -week_map = {'Monday': 0, 'Tuesday': 1, 'Wednesday': 2, 'Thursday': 3, 'Friday': 4, 'Saturday': 5, 'Sunday': 6} class AutoRepeat(Document): def validate(self): @@ -46,7 +68,7 @@ class AutoRepeat(Document): frappe.get_doc(self.reference_doctype, self.reference_document).notify_update() def on_trash(self): - frappe.db.set_value(self.reference_doctype, self.reference_document, 'auto_repeat', '') + frappe.db.set_value(self.reference_doctype, self.reference_document, "auto_repeat", "") frappe.get_doc(self.reference_doctype, self.reference_document).notify_update() def set_dates(self): @@ -56,29 +78,36 @@ class AutoRepeat(Document): self.next_schedule_date = self.get_next_schedule_date(schedule_date=self.start_date) def unlink_if_applicable(self): - if self.status == 'Completed' or self.disabled: - frappe.db.set_value(self.reference_doctype, self.reference_document, 'auto_repeat', '') + if self.status == "Completed" or self.disabled: + frappe.db.set_value(self.reference_doctype, self.reference_document, "auto_repeat", "") def validate_reference_doctype(self): if frappe.flags.in_test or frappe.flags.in_patch: return if not frappe.get_meta(self.reference_doctype).allow_auto_repeat: - frappe.throw(_("Enable Allow Auto Repeat for the doctype {0} in Customize Form").format(self.reference_doctype)) + frappe.throw( + _("Enable Allow Auto Repeat for the doctype {0} in Customize Form").format( + self.reference_doctype + ) + ) def validate_submit_on_creation(self): if self.submit_on_creation and not frappe.get_meta(self.reference_doctype).is_submittable: - frappe.throw(_('Cannot enable {0} for a non-submittable doctype').format( - frappe.bold('Submit on Creation'))) + frappe.throw( + _("Cannot enable {0} for a non-submittable doctype").format(frappe.bold("Submit on Creation")) + ) def validate_dates(self): if frappe.flags.in_patch: return if self.end_date: - self.validate_from_to_dates('start_date', 'end_date') + self.validate_from_to_dates("start_date", "end_date") if self.end_date == self.start_date: - frappe.throw(_('{0} should not be same as {1}').format(frappe.bold('End Date'), frappe.bold('Start Date'))) + frappe.throw( + _("{0} should not be same as {1}").format(frappe.bold("End Date"), frappe.bold("Start Date")) + ) def validate_email_id(self): if self.notify_by_email: @@ -100,17 +129,17 @@ class AutoRepeat(Document): frappe.throw( _("Auto Repeat Day{0} {1} has been repeated.").format( - plural, - frappe.bold(", ".join(repeated_days)) + plural, frappe.bold(", ".join(repeated_days)) ) ) - def update_auto_repeat_id(self): - #check if document is already on auto repeat + # check if document is already on auto repeat auto_repeat = frappe.db.get_value(self.reference_doctype, self.reference_document, "auto_repeat") if auto_repeat and auto_repeat != self.name and not frappe.flags.in_patch: - frappe.throw(_("The {0} is already on auto repeat {1}").format(self.reference_document, auto_repeat)) + frappe.throw( + _("The {0} is already on auto repeat {1}").format(self.reference_document, auto_repeat) + ) else: frappe.db.set_value(self.reference_doctype, self.reference_document, "auto_repeat", self.name) @@ -136,18 +165,18 @@ class AutoRepeat(Document): row = { "reference_document": self.reference_document, "frequency": self.frequency, - "next_scheduled_date": next_date + "next_scheduled_date": next_date, } schedule_details.append(row) if self.end_date: next_date = self.get_next_schedule_date(schedule_date=start_date, for_full_schedule=True) - while (getdate(next_date) < getdate(end_date)): + while getdate(next_date) < getdate(end_date): row = { - "reference_document" : self.reference_document, - "frequency" : self.frequency, - "next_scheduled_date" : next_date + "reference_document": self.reference_document, + "frequency": self.frequency, + "next_scheduled_date": next_date, } schedule_details.append(row) next_date = self.get_next_schedule_date(schedule_date=next_date, for_full_schedule=True) @@ -169,9 +198,9 @@ class AutoRepeat(Document): def make_new_document(self): reference_doc = frappe.get_doc(self.reference_doctype, self.reference_document) - new_doc = frappe.copy_doc(reference_doc, ignore_no_copy = False) + new_doc = frappe.copy_doc(reference_doc, ignore_no_copy=False) self.update_doc(new_doc, reference_doc) - new_doc.insert(ignore_permissions = True) + new_doc.insert(ignore_permissions=True) if self.submit_on_creation: new_doc.submit() @@ -180,61 +209,72 @@ class AutoRepeat(Document): def update_doc(self, new_doc, reference_doc): new_doc.docstatus = 0 - if new_doc.meta.get_field('set_posting_time'): - new_doc.set('set_posting_time', 1) - - if new_doc.meta.get_field('auto_repeat'): - new_doc.set('auto_repeat', self.name) - - for fieldname in ['naming_series', 'ignore_pricing_rule', 'posting_time', 'select_print_heading', 'user_remark', 'remarks', 'owner']: + if new_doc.meta.get_field("set_posting_time"): + new_doc.set("set_posting_time", 1) + + if new_doc.meta.get_field("auto_repeat"): + new_doc.set("auto_repeat", self.name) + + for fieldname in [ + "naming_series", + "ignore_pricing_rule", + "posting_time", + "select_print_heading", + "user_remark", + "remarks", + "owner", + ]: if new_doc.meta.get_field(fieldname): new_doc.set(fieldname, reference_doc.get(fieldname)) for data in new_doc.meta.fields: - if data.fieldtype == 'Date' and data.reqd: + if data.fieldtype == "Date" and data.reqd: new_doc.set(data.fieldname, self.next_schedule_date) self.set_auto_repeat_period(new_doc) - auto_repeat_doc = frappe.get_doc('Auto Repeat', self.name) + auto_repeat_doc = frappe.get_doc("Auto Repeat", self.name) - #for any action that needs to take place after the recurring document creation - #on recurring method of that doctype is triggered - new_doc.run_method('on_recurring', reference_doc = reference_doc, auto_repeat_doc = auto_repeat_doc) + # for any action that needs to take place after the recurring document creation + # on recurring method of that doctype is triggered + new_doc.run_method("on_recurring", reference_doc=reference_doc, auto_repeat_doc=auto_repeat_doc) def set_auto_repeat_period(self, new_doc): mcount = month_map.get(self.frequency) - if mcount and new_doc.meta.get_field('from_date') and new_doc.meta.get_field('to_date'): - last_ref_doc = frappe.db.get_all(doctype = self.reference_doctype, - fields = ['name', 'from_date', 'to_date'], - filters = [ - ['auto_repeat', '=', self.name], - ['docstatus', '<', 2], + if mcount and new_doc.meta.get_field("from_date") and new_doc.meta.get_field("to_date"): + last_ref_doc = frappe.db.get_all( + doctype=self.reference_doctype, + fields=["name", "from_date", "to_date"], + filters=[ + ["auto_repeat", "=", self.name], + ["docstatus", "<", 2], ], - order_by = 'creation desc', - limit = 1) + order_by="creation desc", + limit=1, + ) if not last_ref_doc: return from_date = get_next_date(last_ref_doc[0].from_date, mcount) - if (cstr(get_first_day(last_ref_doc[0].from_date)) == cstr(last_ref_doc[0].from_date)) and \ - (cstr(get_last_day(last_ref_doc[0].to_date)) == cstr(last_ref_doc[0].to_date)): + if (cstr(get_first_day(last_ref_doc[0].from_date)) == cstr(last_ref_doc[0].from_date)) and ( + cstr(get_last_day(last_ref_doc[0].to_date)) == cstr(last_ref_doc[0].to_date) + ): to_date = get_last_day(get_next_date(last_ref_doc[0].to_date, mcount)) else: to_date = get_next_date(last_ref_doc[0].to_date, mcount) - new_doc.set('from_date', from_date) - new_doc.set('to_date', to_date) + new_doc.set("from_date", from_date) + new_doc.set("to_date", to_date) def get_next_schedule_date(self, schedule_date, for_full_schedule=False): """ - Returns the next schedule date for auto repeat after a recurring document has been created. - Adds required offset to the schedule_date param and returns the next schedule date. + Returns the next schedule date for auto repeat after a recurring document has been created. + Adds required offset to the schedule_date param and returns the next schedule date. - :param schedule_date: The date when the last recurring document was created. - :param for_full_schedule: If True, returns the immediate next schedule date, else the full schedule. + :param schedule_date: The date when the last recurring document was created. + :param for_full_schedule: If True, returns the immediate next schedule date, else the full schedule. """ if month_map.get(self.frequency): month_count = month_map.get(self.frequency) + month_diff(schedule_date, self.start_date) - 1 @@ -295,60 +335,75 @@ class AutoRepeat(Document): return 7 def get_auto_repeat_days(self): - return [d.day for d in self.get('repeat_on_days', [])] + return [d.day for d in self.get("repeat_on_days", [])] def send_notification(self, new_doc): """Notify concerned people about recurring document generation""" - subject = self.subject or '' - message = self.message or '' + subject = self.subject or "" + message = self.message or "" if not self.subject: subject = _("New {0}: {1}").format(new_doc.doctype, new_doc.name) elif "{" in self.subject: - subject = frappe.render_template(self.subject, {'doc': new_doc}) + subject = frappe.render_template(self.subject, {"doc": new_doc}) - print_format = self.print_format or 'Standard' + print_format = self.print_format or "Standard" error_string = None try: - attachments = [frappe.attach_print(new_doc.doctype, new_doc.name, - file_name=new_doc.name, print_format=print_format)] + attachments = [ + frappe.attach_print( + new_doc.doctype, new_doc.name, file_name=new_doc.name, print_format=print_format + ) + ] except frappe.PermissionError: - error_string = _("A recurring {0} {1} has been created for you via Auto Repeat {2}.").format(new_doc.doctype, new_doc.name, self.name) + error_string = _("A recurring {0} {1} has been created for you via Auto Repeat {2}.").format( + new_doc.doctype, new_doc.name, self.name + ) error_string += "

" - error_string += _("{0}: Failed to attach new recurring document. To enable attaching document in the auto repeat notification email, enable {1} in Print Settings").format( - frappe.bold(_('Note')), - frappe.bold(_('Allow Print for Draft')) - ) - attachments = '[]' + error_string += _( + "{0}: Failed to attach new recurring document. To enable attaching document in the auto repeat notification email, enable {1} in Print Settings" + ).format(frappe.bold(_("Note")), frappe.bold(_("Allow Print for Draft"))) + attachments = "[]" if error_string: message = error_string elif not self.message: message = _("Please find attached {0}: {1}").format(new_doc.doctype, new_doc.name) elif "{" in self.message: - message = frappe.render_template(self.message, {'doc': new_doc}) + message = frappe.render_template(self.message, {"doc": new_doc}) - recipients = self.recipients.split('\n') + recipients = self.recipients.split("\n") - make(doctype=new_doc.doctype, name=new_doc.name, recipients=recipients, - subject=subject, content=message, attachments=attachments, send_email=1) + make( + doctype=new_doc.doctype, + name=new_doc.name, + recipients=recipients, + subject=subject, + content=message, + attachments=attachments, + send_email=1, + ) @frappe.whitelist() def fetch_linked_contacts(self): if self.reference_doctype and self.reference_document: - res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id']) - res += get_contacts_linked_from(self.reference_doctype, self.reference_document, fields=['email_id']) + res = get_contacts_linking_to( + self.reference_doctype, self.reference_document, fields=["email_id"] + ) + res += get_contacts_linked_from( + self.reference_doctype, self.reference_document, fields=["email_id"] + ) email_ids = {d.email_id for d in res} if not email_ids: - frappe.msgprint(_('No contacts linked to document'), alert=True) + frappe.msgprint(_("No contacts linked to document"), alert=True) else: - self.recipients = ', '.join(email_ids) + self.recipients = ", ".join(email_ids) def disable_auto_repeat(self): - frappe.db.set_value('Auto Repeat', self.name, 'disabled', 1) + frappe.db.set_value("Auto Repeat", self.name, "disabled", 1) def notify_error_to_user(self, error_log): recipients = list(get_system_managers(only_name=True)) @@ -356,20 +411,17 @@ class AutoRepeat(Document): subject = _("Auto Repeat Document Creation Failed") form_link = frappe.utils.get_link_to_form(self.reference_doctype, self.reference_document) - auto_repeat_failed_for = _('Auto Repeat failed for {0}').format(form_link) + auto_repeat_failed_for = _("Auto Repeat failed for {0}").format(form_link) - error_log_link = frappe.utils.get_link_to_form('Error Log', error_log.name) - error_log_message = _('Check the Error Log for more information: {0}').format(error_log_link) + error_log_link = frappe.utils.get_link_to_form("Error Log", error_log.name) + error_log_message = _("Check the Error Log for more information: {0}").format(error_log_link) frappe.sendmail( recipients=recipients, subject=subject, template="auto_repeat_fail", - args={ - 'auto_repeat_failed_for': auto_repeat_failed_for, - 'error_log_message': error_log_message - }, - header=[subject, 'red'] + args={"auto_repeat_failed_for": auto_repeat_failed_for, "error_log_message": error_log_message}, + header=[subject, "red"], ) @@ -382,18 +434,18 @@ def get_next_date(dt, mcount, day=None): def get_next_weekday(current_schedule_day, weekdays): days = list(week_map.keys()) if current_schedule_day > 0: - days = days[(current_schedule_day + 1):] + days[:current_schedule_day] + days = days[(current_schedule_day + 1) :] + days[:current_schedule_day] else: - days = days[(current_schedule_day + 1):] + days = days[(current_schedule_day + 1) :] for entry in days: if entry in weekdays: return entry -#called through hooks +# called through hooks def make_auto_repeat_entry(): - enqueued_method = 'frappe.automation.doctype.auto_repeat.auto_repeat.create_repeated_entries' + enqueued_method = "frappe.automation.doctype.auto_repeat.auto_repeat.create_repeated_entries" jobs = get_jobs() if not jobs or enqueued_method not in jobs[frappe.local.site]: @@ -404,7 +456,7 @@ def make_auto_repeat_entry(): def create_repeated_entries(data): for d in data: - doc = frappe.get_doc('Auto Repeat', d.name) + doc = frappe.get_doc("Auto Repeat", d.name) current_date = getdate(today()) schedule_date = getdate(doc.next_schedule_date) @@ -413,33 +465,32 @@ def create_repeated_entries(data): doc.create_documents() schedule_date = doc.get_next_schedule_date(schedule_date=schedule_date) if schedule_date and not doc.disabled: - frappe.db.set_value('Auto Repeat', doc.name, 'next_schedule_date', schedule_date) + frappe.db.set_value("Auto Repeat", doc.name, "next_schedule_date", schedule_date) def get_auto_repeat_entries(date=None): if not date: date = getdate(today()) - return frappe.db.get_all('Auto Repeat', filters=[ - ['next_schedule_date', '<=', date], - ['status', '=', 'Active'] - ]) + return frappe.db.get_all( + "Auto Repeat", filters=[["next_schedule_date", "<=", date], ["status", "=", "Active"]] + ) -#called through hooks +# called through hooks def set_auto_repeat_as_completed(): - auto_repeat = frappe.get_all("Auto Repeat", filters = {'status': ['!=', 'Disabled']}) + auto_repeat = frappe.get_all("Auto Repeat", filters={"status": ["!=", "Disabled"]}) for entry in auto_repeat: doc = frappe.get_doc("Auto Repeat", entry.name) if doc.is_completed(): - doc.status = 'Completed' + doc.status = "Completed" doc.save() @frappe.whitelist() -def make_auto_repeat(doctype, docname, frequency = 'Daily', start_date = None, end_date = None): +def make_auto_repeat(doctype, docname, frequency="Daily", start_date=None, end_date=None): if not start_date: start_date = getdate(today()) - doc = frappe.new_doc('Auto Repeat') + doc = frappe.new_doc("Auto Repeat") doc.reference_doctype = doctype doc.reference_document = docname doc.frequency = frequency @@ -449,24 +500,34 @@ def make_auto_repeat(doctype, docname, frequency = 'Daily', start_date = None, e doc.save() return doc + # method for reference_doctype filter @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters): - res = frappe.db.get_all('Property Setter', { - 'property': 'allow_auto_repeat', - 'value': '1', - }, ['doc_type']) + res = frappe.db.get_all( + "Property Setter", + { + "property": "allow_auto_repeat", + "value": "1", + }, + ["doc_type"], + ) docs = [r.doc_type for r in res] - res = frappe.db.get_all('DocType', { - 'allow_auto_repeat': 1, - }, ['name']) + res = frappe.db.get_all( + "DocType", + { + "allow_auto_repeat": 1, + }, + ["name"], + ) docs += [r.name for r in res] docs = set(list(docs)) return [[d] for d in docs] + @frappe.whitelist() def update_reference(docname, reference): result = "" @@ -478,13 +539,14 @@ def update_reference(docname, reference): raise e return result + @frappe.whitelist() def generate_message_preview(reference_dt, reference_doc, message=None, subject=None): frappe.has_permission("Auto Repeat", "write", throw=True) doc = frappe.get_doc(reference_dt, reference_doc) subject_preview = _("Please add a subject to your email") - msg_preview = frappe.render_template(message, {'doc': doc}) + msg_preview = frappe.render_template(message, {"doc": doc}) if subject: - subject_preview = frappe.render_template(subject, {'doc': doc}) + subject_preview = frappe.render_template(subject, {"doc": doc}) - return {'message': msg_preview, 'subject': subject_preview} + return {"message": msg_preview, "subject": subject_preview} diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py index 30a0310a92..e1db7ca6d1 100644 --- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py @@ -4,24 +4,40 @@ import unittest import frappe +from frappe.automation.doctype.auto_repeat.auto_repeat import ( + create_repeated_entries, + get_auto_repeat_entries, + week_map, +) from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.automation.doctype.auto_repeat.auto_repeat import get_auto_repeat_entries, create_repeated_entries, week_map -from frappe.utils import today, add_days, getdate, add_months +from frappe.utils import add_days, add_months, getdate, today + def add_custom_fields(): df = dict( - fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', insert_after='sender', - options='Auto Repeat', hidden=1, print_hide=1, read_only=1) - create_custom_field('ToDo', df) + fieldname="auto_repeat", + label="Auto Repeat", + fieldtype="Link", + insert_after="sender", + options="Auto Repeat", + hidden=1, + print_hide=1, + read_only=1, + ) + create_custom_field("ToDo", df) + class TestAutoRepeat(unittest.TestCase): def setUp(self): - if not frappe.db.sql("SELECT `fieldname` FROM `tabCustom Field` WHERE `fieldname`='auto_repeat' and `dt`=%s", "Todo"): + if not frappe.db.sql( + "SELECT `fieldname` FROM `tabCustom Field` WHERE `fieldname`='auto_repeat' and `dt`=%s", "Todo" + ): add_custom_fields() def test_daily_auto_repeat(self): todo = frappe.get_doc( - dict(doctype='ToDo', description='test recurring todo', assigned_by='Administrator')).insert() + dict(doctype="ToDo", description="test recurring todo", assigned_by="Administrator") + ).insert() doc = make_auto_repeat(reference_document=todo.name) self.assertEqual(doc.next_schedule_date, today()) @@ -32,19 +48,25 @@ class TestAutoRepeat(unittest.TestCase): todo = frappe.get_doc(doc.reference_doctype, doc.reference_document) self.assertEqual(todo.auto_repeat, doc.name) - new_todo = frappe.db.get_value('ToDo', - {'auto_repeat': doc.name, 'name': ('!=', todo.name)}, 'name') + new_todo = frappe.db.get_value( + "ToDo", {"auto_repeat": doc.name, "name": ("!=", todo.name)}, "name" + ) - new_todo = frappe.get_doc('ToDo', new_todo) + new_todo = frappe.get_doc("ToDo", new_todo) - self.assertEqual(todo.get('description'), new_todo.get('description')) + self.assertEqual(todo.get("description"), new_todo.get("description")) def test_weekly_auto_repeat(self): todo = frappe.get_doc( - dict(doctype='ToDo', description='test weekly todo', assigned_by='Administrator')).insert() + dict(doctype="ToDo", description="test weekly todo", assigned_by="Administrator") + ).insert() - doc = make_auto_repeat(reference_doctype='ToDo', - frequency='Weekly', reference_document=todo.name, start_date=add_days(today(), -7)) + doc = make_auto_repeat( + reference_doctype="ToDo", + frequency="Weekly", + reference_document=todo.name, + start_date=add_days(today(), -7), + ) self.assertEqual(doc.next_schedule_date, today()) data = get_auto_repeat_entries(getdate(today())) @@ -54,25 +76,29 @@ class TestAutoRepeat(unittest.TestCase): todo = frappe.get_doc(doc.reference_doctype, doc.reference_document) self.assertEqual(todo.auto_repeat, doc.name) - new_todo = frappe.db.get_value('ToDo', - {'auto_repeat': doc.name, 'name': ('!=', todo.name)}, 'name') + new_todo = frappe.db.get_value( + "ToDo", {"auto_repeat": doc.name, "name": ("!=", todo.name)}, "name" + ) - new_todo = frappe.get_doc('ToDo', new_todo) + new_todo = frappe.get_doc("ToDo", new_todo) - self.assertEqual(todo.get('description'), new_todo.get('description')) + self.assertEqual(todo.get("description"), new_todo.get("description")) def test_weekly_auto_repeat_with_weekdays(self): todo = frappe.get_doc( - dict(doctype='ToDo', description='test auto repeat with weekdays', assigned_by='Administrator')).insert() + dict(doctype="ToDo", description="test auto repeat with weekdays", assigned_by="Administrator") + ).insert() weekdays = list(week_map.keys()) current_weekday = getdate().weekday() - days = [ - {'day': weekdays[current_weekday]}, - {'day': weekdays[(current_weekday + 2) % 7]} - ] - doc = make_auto_repeat(reference_doctype='ToDo', - frequency='Weekly', reference_document=todo.name, start_date=add_days(today(), -7), days=days) + days = [{"day": weekdays[current_weekday]}, {"day": weekdays[(current_weekday + 2) % 7]}] + doc = make_auto_repeat( + reference_doctype="ToDo", + frequency="Weekly", + reference_document=todo.name, + start_date=add_days(today(), -7), + days=days, + ) self.assertEqual(doc.next_schedule_date, today()) data = get_auto_repeat_entries(getdate(today())) @@ -90,136 +116,173 @@ class TestAutoRepeat(unittest.TestCase): end_date = add_months(start_date, 12) todo = frappe.get_doc( - dict(doctype='ToDo', description='test recurring todo', assigned_by='Administrator')).insert() + dict(doctype="ToDo", description="test recurring todo", assigned_by="Administrator") + ).insert() - self.monthly_auto_repeat('ToDo', todo.name, start_date, end_date) - #test without end_date - todo = frappe.get_doc(dict(doctype='ToDo', description='test recurring todo without end_date', assigned_by='Administrator')).insert() - self.monthly_auto_repeat('ToDo', todo.name, start_date) + self.monthly_auto_repeat("ToDo", todo.name, start_date, end_date) + # test without end_date + todo = frappe.get_doc( + dict( + doctype="ToDo", description="test recurring todo without end_date", assigned_by="Administrator" + ) + ).insert() + self.monthly_auto_repeat("ToDo", todo.name, start_date) - def monthly_auto_repeat(self, doctype, docname, start_date, end_date = None): + def monthly_auto_repeat(self, doctype, docname, start_date, end_date=None): def get_months(start, end): diff = (12 * end.year + end.month) - (12 * start.year + start.month) return diff + 1 doc = make_auto_repeat( - reference_doctype=doctype, frequency='Monthly', reference_document=docname, start_date=start_date, - end_date=end_date) + reference_doctype=doctype, + frequency="Monthly", + reference_document=docname, + start_date=start_date, + end_date=end_date, + ) doc.disable_auto_repeat() data = get_auto_repeat_entries(getdate(today())) create_repeated_entries(data) - docnames = frappe.get_all(doc.reference_doctype, {'auto_repeat': doc.name}) + docnames = frappe.get_all(doc.reference_doctype, {"auto_repeat": doc.name}) self.assertEqual(len(docnames), 1) - doc = frappe.get_doc('Auto Repeat', doc.name) - doc.db_set('disabled', 0) + doc = frappe.get_doc("Auto Repeat", doc.name) + doc.db_set("disabled", 0) months = get_months(getdate(start_date), getdate(today())) data = get_auto_repeat_entries(getdate(today())) create_repeated_entries(data) - docnames = frappe.get_all(doc.reference_doctype, {'auto_repeat': doc.name}) + docnames = frappe.get_all(doc.reference_doctype, {"auto_repeat": doc.name}) self.assertEqual(len(docnames), months) def test_notification_is_attached(self): todo = frappe.get_doc( - dict(doctype='ToDo', description='Test recurring notification attachment', assigned_by='Administrator')).insert() + dict( + doctype="ToDo", + description="Test recurring notification attachment", + assigned_by="Administrator", + ) + ).insert() - doc = make_auto_repeat(reference_document=todo.name, notify=1, recipients="test@domain.com", subject="New ToDo", - message="A new ToDo has just been created for you") + doc = make_auto_repeat( + reference_document=todo.name, + notify=1, + recipients="test@domain.com", + subject="New ToDo", + message="A new ToDo has just been created for you", + ) data = get_auto_repeat_entries(getdate(today())) create_repeated_entries(data) frappe.db.commit() - new_todo = frappe.db.get_value('ToDo', - {'auto_repeat': doc.name, 'name': ('!=', todo.name)}, 'name') + new_todo = frappe.db.get_value( + "ToDo", {"auto_repeat": doc.name, "name": ("!=", todo.name)}, "name" + ) - linked_comm = frappe.db.exists("Communication", dict(reference_doctype="ToDo", reference_name=new_todo)) + linked_comm = frappe.db.exists( + "Communication", dict(reference_doctype="ToDo", reference_name=new_todo) + ) self.assertTrue(linked_comm) def test_next_schedule_date(self): current_date = getdate(today()) todo = frappe.get_doc( - dict(doctype='ToDo', description='test next schedule date for monthly', assigned_by='Administrator')).insert() - doc = make_auto_repeat(frequency='Monthly', reference_document=todo.name, start_date=add_months(today(), -2)) + dict( + doctype="ToDo", description="test next schedule date for monthly", assigned_by="Administrator" + ) + ).insert() + doc = make_auto_repeat( + frequency="Monthly", reference_document=todo.name, start_date=add_months(today(), -2) + ) # next_schedule_date is set as on or after current date # it should not be a previous month's date self.assertTrue((doc.next_schedule_date >= current_date)) todo = frappe.get_doc( - dict(doctype='ToDo', description='test next schedule date for daily', assigned_by='Administrator')).insert() - doc = make_auto_repeat(frequency='Daily', reference_document=todo.name, start_date=add_days(today(), -2)) + dict( + doctype="ToDo", description="test next schedule date for daily", assigned_by="Administrator" + ) + ).insert() + doc = make_auto_repeat( + frequency="Daily", reference_document=todo.name, start_date=add_days(today(), -2) + ) self.assertEqual(getdate(doc.next_schedule_date), current_date) def test_submit_on_creation(self): - doctype = 'Test Submittable DocType' + doctype = "Test Submittable DocType" create_submittable_doctype(doctype) current_date = getdate() - submittable_doc = frappe.get_doc(dict(doctype=doctype, test='test submit on creation')).insert() + submittable_doc = frappe.get_doc(dict(doctype=doctype, test="test submit on creation")).insert() submittable_doc.submit() - doc = make_auto_repeat(frequency='Daily', reference_doctype=doctype, reference_document=submittable_doc.name, - start_date=add_days(current_date, -1), submit_on_creation=1) + doc = make_auto_repeat( + frequency="Daily", + reference_doctype=doctype, + reference_document=submittable_doc.name, + start_date=add_days(current_date, -1), + submit_on_creation=1, + ) data = get_auto_repeat_entries(current_date) create_repeated_entries(data) - docnames = frappe.db.get_all(doc.reference_doctype, - filters={'auto_repeat': doc.name}, - fields=['docstatus'], - limit=1 + docnames = frappe.db.get_all( + doc.reference_doctype, filters={"auto_repeat": doc.name}, fields=["docstatus"], limit=1 ) self.assertEqual(docnames[0].docstatus, 1) def make_auto_repeat(**args): args = frappe._dict(args) - doc = frappe.get_doc({ - 'doctype': 'Auto Repeat', - 'reference_doctype': args.reference_doctype or 'ToDo', - 'reference_document': args.reference_document or frappe.db.get_value('ToDo', 'name'), - 'submit_on_creation': args.submit_on_creation or 0, - 'frequency': args.frequency or 'Daily', - 'start_date': args.start_date or add_days(today(), -1), - 'end_date': args.end_date or "", - 'notify_by_email': args.notify or 0, - 'recipients': args.recipients or "", - 'subject': args.subject or "", - 'message': args.message or "", - 'repeat_on_days': args.days or [] - }).insert(ignore_permissions=True) + doc = frappe.get_doc( + { + "doctype": "Auto Repeat", + "reference_doctype": args.reference_doctype or "ToDo", + "reference_document": args.reference_document or frappe.db.get_value("ToDo", "name"), + "submit_on_creation": args.submit_on_creation or 0, + "frequency": args.frequency or "Daily", + "start_date": args.start_date or add_days(today(), -1), + "end_date": args.end_date or "", + "notify_by_email": args.notify or 0, + "recipients": args.recipients or "", + "subject": args.subject or "", + "message": args.message or "", + "repeat_on_days": args.days or [], + } + ).insert(ignore_permissions=True) return doc def create_submittable_doctype(doctype, submit_perms=1): - if frappe.db.exists('DocType', doctype): + if frappe.db.exists("DocType", doctype): return else: - doc = frappe.get_doc({ - 'doctype': 'DocType', - '__newname': doctype, - 'module': 'Custom', - 'custom': 1, - 'is_submittable': 1, - 'fields': [{ - 'fieldname': 'test', - 'label': 'Test', - 'fieldtype': 'Data' - }], - 'permissions': [{ - 'role': 'System Manager', - 'read': 1, - 'write': 1, - 'create': 1, - 'delete': 1, - 'submit': submit_perms, - 'cancel': submit_perms, - 'amend': submit_perms - }] - }).insert() + doc = frappe.get_doc( + { + "doctype": "DocType", + "__newname": doctype, + "module": "Custom", + "custom": 1, + "is_submittable": 1, + "fields": [{"fieldname": "test", "label": "Test", "fieldtype": "Data"}], + "permissions": [ + { + "role": "System Manager", + "read": 1, + "write": 1, + "create": 1, + "delete": 1, + "submit": submit_perms, + "cancel": submit_perms, + "amend": submit_perms, + } + ], + } + ).insert() doc.allow_auto_repeat = 1 - doc.save() \ No newline at end of file + doc.save() diff --git a/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py b/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py index 54fc0d14e9..95d75bf9da 100644 --- a/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py +++ b/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class AutoRepeatDay(Document): pass diff --git a/frappe/automation/doctype/milestone/milestone.py b/frappe/automation/doctype/milestone/milestone.py index eff65571fd..4059a2eb73 100644 --- a/frappe/automation/doctype/milestone/milestone.py +++ b/frappe/automation/doctype/milestone/milestone.py @@ -5,8 +5,10 @@ import frappe from frappe.model.document import Document + class Milestone(Document): pass + def on_doctype_update(): frappe.db.add_index("Milestone", ["reference_type", "reference_name"]) diff --git a/frappe/automation/doctype/milestone/test_milestone.py b/frappe/automation/doctype/milestone/test_milestone.py index f8fb910072..1824220497 100644 --- a/frappe/automation/doctype/milestone/test_milestone.py +++ b/frappe/automation/doctype/milestone/test_milestone.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -#import frappe +# import frappe import unittest + class TestMilestone(unittest.TestCase): pass diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py index 042e7b0391..16b2fe9204 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py @@ -3,43 +3,50 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document import frappe.cache_manager from frappe.model import log_types +from frappe.model.document import Document + class MilestoneTracker(Document): def on_update(self): - frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.document_type) + frappe.cache_manager.clear_doctype_map("Milestone Tracker", self.document_type) def on_trash(self): - frappe.cache_manager.clear_doctype_map('Milestone Tracker', self.document_type) + frappe.cache_manager.clear_doctype_map("Milestone Tracker", self.document_type) def apply(self, doc): before_save = doc.get_doc_before_save() from_value = before_save and before_save.get(self.track_field) or None if from_value != doc.get(self.track_field): - frappe.get_doc(dict( - doctype = 'Milestone', - reference_type = doc.doctype, - reference_name = doc.name, - track_field = self.track_field, - from_value = from_value, - value = doc.get(self.track_field), - milestone_tracker = self.name, - )).insert(ignore_permissions=True) + frappe.get_doc( + dict( + doctype="Milestone", + reference_type=doc.doctype, + reference_name=doc.name, + track_field=self.track_field, + from_value=from_value, + value=doc.get(self.track_field), + milestone_tracker=self.name, + ) + ).insert(ignore_permissions=True) + def evaluate_milestone(doc, event): - if (frappe.flags.in_install + if ( + frappe.flags.in_install or frappe.flags.in_migrate or frappe.flags.in_setup_wizard - or doc.doctype in log_types): + or doc.doctype in log_types + ): return # track milestones related to this doctype for d in get_milestone_trackers(doc.doctype): - frappe.get_doc('Milestone Tracker', d.get('name')).apply(doc) + frappe.get_doc("Milestone Tracker", d.get("name")).apply(doc) -def get_milestone_trackers(doctype): - return frappe.cache_manager.get_doctype_map('Milestone Tracker', doctype, - dict(document_type = doctype, disabled=0)) +def get_milestone_trackers(doctype): + return frappe.cache_manager.get_doctype_map( + "Milestone Tracker", doctype, dict(document_type=doctype, disabled=0) + ) diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py index f4d5f00d83..4e53072348 100644 --- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py @@ -1,48 +1,48 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE +import unittest + import frappe import frappe.cache_manager -import unittest + class TestMilestoneTracker(unittest.TestCase): def test_milestone(self): frappe.db.delete("Milestone Tracker") - frappe.cache().delete_key('milestone_tracker_map') + frappe.cache().delete_key("milestone_tracker_map") - milestone_tracker = frappe.get_doc(dict( - doctype = 'Milestone Tracker', - document_type = 'ToDo', - track_field = 'status' - )).insert() + milestone_tracker = frappe.get_doc( + dict(doctype="Milestone Tracker", document_type="ToDo", track_field="status") + ).insert() - todo = frappe.get_doc(dict( - doctype = 'ToDo', - description = 'test milestone', - status = 'Open' - )).insert() + todo = frappe.get_doc(dict(doctype="ToDo", description="test milestone", status="Open")).insert() - milestones = frappe.get_all('Milestone', - fields = ['track_field', 'value', 'milestone_tracker'], - filters = dict(reference_type = todo.doctype, reference_name=todo.name)) + milestones = frappe.get_all( + "Milestone", + fields=["track_field", "value", "milestone_tracker"], + filters=dict(reference_type=todo.doctype, reference_name=todo.name), + ) self.assertEqual(len(milestones), 1) - self.assertEqual(milestones[0].track_field, 'status') - self.assertEqual(milestones[0].value, 'Open') + self.assertEqual(milestones[0].track_field, "status") + self.assertEqual(milestones[0].value, "Open") - todo.status = 'Closed' + todo.status = "Closed" todo.save() - milestones = frappe.get_all('Milestone', - fields = ['track_field', 'value', 'milestone_tracker'], - filters = dict(reference_type = todo.doctype, reference_name=todo.name), - order_by = 'modified desc') + milestones = frappe.get_all( + "Milestone", + fields=["track_field", "value", "milestone_tracker"], + filters=dict(reference_type=todo.doctype, reference_name=todo.name), + order_by="modified desc", + ) self.assertEqual(len(milestones), 2) - self.assertEqual(milestones[0].track_field, 'status') - self.assertEqual(milestones[0].value, 'Closed') + self.assertEqual(milestones[0].track_field, "status") + self.assertEqual(milestones[0].value, "Closed") # cleanup frappe.db.delete("Milestone") - milestone_tracker.delete() \ No newline at end of file + milestone_tracker.delete() diff --git a/frappe/boot.py b/frappe/boot.py index ae5bdfa8c0..1b8e471e00 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -7,20 +7,22 @@ bootstrap client session import frappe import frappe.defaults import frappe.desk.desk_page +from frappe.core.doctype.navbar_settings.navbar_settings import get_app_logo, get_navbar_settings from frappe.desk.doctype.route_history.route_history import frequently_visited_links from frappe.desk.form.load import get_meta_bundle -from frappe.utils.change_log import get_versions -from frappe.translate import get_lang_dict from frappe.email.inbox import get_email_accounts -from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled -from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled -from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.model.base_document import get_controller -from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo -from frappe.utils import get_time_zone, add_user_info from frappe.query_builder import DocType from frappe.query_builder.functions import Count from frappe.query_builder.terms import subqry +from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points +from frappe.social.doctype.energy_point_settings.energy_point_settings import ( + is_energy_point_enabled, +) +from frappe.translate import get_lang_dict +from frappe.utils import add_user_info, get_time_zone +from frappe.utils.change_log import get_versions +from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled def get_bootinfo(): @@ -38,9 +40,9 @@ def get_bootinfo(): bootinfo.sysdefaults = frappe.defaults.get_defaults() bootinfo.server_date = frappe.utils.nowdate() - if frappe.session['user'] != 'Guest': + if frappe.session["user"] != "Guest": bootinfo.user_info = get_user_info() - bootinfo.sid = frappe.session['sid'] + bootinfo.sid = frappe.session["sid"] bootinfo.modules = {} bootinfo.module_list = [] @@ -51,8 +53,10 @@ def get_bootinfo(): add_layouts(bootinfo) bootinfo.module_app = frappe.local.module_app - bootinfo.single_types = [d.name for d in frappe.get_all('DocType', {'issingle': 1})] - bootinfo.nested_set_doctypes = [d.parent for d in frappe.get_all('DocField', {'fieldname': 'lft'}, ['parent'])] + bootinfo.single_types = [d.name for d in frappe.get_all("DocType", {"issingle": 1})] + bootinfo.nested_set_doctypes = [ + d.parent for d in frappe.get_all("DocField", {"fieldname": "lft"}, ["parent"]) + ] add_home_page(bootinfo, doclist) bootinfo.page_info = get_allowed_pages() load_translations(bootinfo) @@ -66,8 +70,8 @@ def get_bootinfo(): set_time_zone(bootinfo) # ipinfo - if frappe.session.data.get('ipinfo'): - bootinfo.ipinfo = frappe.session['data']['ipinfo'] + if frappe.session.data.get("ipinfo"): + bootinfo.ipinfo = frappe.session["data"]["ipinfo"] # add docs bootinfo.docs = doclist @@ -77,7 +81,7 @@ def get_bootinfo(): if bootinfo.lang: bootinfo.lang = str(bootinfo.lang) - bootinfo.versions = {k: v['version'] for k, v in get_versions().items()} + bootinfo.versions = {k: v["version"] for k, v in get_versions().items()} bootinfo.error_report_email = frappe.conf.error_report_email bootinfo.calendars = sorted(frappe.get_hooks("calendars")) @@ -97,37 +101,47 @@ def get_bootinfo(): return bootinfo + def get_letter_heads(): letter_heads = {} - for letter_head in frappe.get_all("Letter Head", fields = ["name", "content", "footer"]): - letter_heads.setdefault(letter_head.name, - {'header': letter_head.content, 'footer': letter_head.footer}) + for letter_head in frappe.get_all("Letter Head", fields=["name", "content", "footer"]): + letter_heads.setdefault( + letter_head.name, {"header": letter_head.content, "footer": letter_head.footer} + ) return letter_heads + def load_conf_settings(bootinfo): from frappe import conf - bootinfo.max_file_size = conf.get('max_file_size') or 10485760 - for key in ('developer_mode', 'socketio_port', 'file_watcher_port'): - if key in conf: bootinfo[key] = conf.get(key) + + bootinfo.max_file_size = conf.get("max_file_size") or 10485760 + for key in ("developer_mode", "socketio_port", "file_watcher_port"): + if key in conf: + bootinfo[key] = conf.get(key) + def load_desktop_data(bootinfo): from frappe.desk.desktop import get_workspace_sidebar_items - bootinfo.allowed_workspaces = get_workspace_sidebar_items().get('pages') + + bootinfo.allowed_workspaces = get_workspace_sidebar_items().get("pages") bootinfo.module_page_map = get_controller("Workspace").get_module_page_map() bootinfo.dashboards = frappe.get_all("Dashboard") + def get_allowed_pages(cache=False): - return get_user_pages_or_reports('Page', cache=cache) + return get_user_pages_or_reports("Page", cache=cache) + def get_allowed_reports(cache=False): - return get_user_pages_or_reports('Report', cache=cache) + return get_user_pages_or_reports("Report", cache=cache) + def get_user_pages_or_reports(parent, cache=False): _cache = frappe.cache() if cache: - has_role = _cache.get_value('has_role:' + parent, user=frappe.session.user) + has_role = _cache.get_value("has_role:" + parent, user=frappe.session.user) if has_role: return has_role @@ -140,8 +154,7 @@ def get_user_pages_or_reports(parent, cache=False): if parent == "Report": columns = (report.name.as_("title"), report.ref_doctype, report.report_type) else: - columns = (page.title.as_("title"), ) - + columns = (page.title.as_("title"),) customRole = DocType("Custom Role") hasRole = DocType("Has Role") @@ -149,31 +162,39 @@ def get_user_pages_or_reports(parent, cache=False): # get pages or reports set on custom role pages_with_custom_roles = ( - frappe.qb.from_(customRole).from_(hasRole).from_(parentTable) - .select(customRole[parent.lower()].as_("name"), customRole.modified, customRole.ref_doctype, *columns) + frappe.qb.from_(customRole) + .from_(hasRole) + .from_(parentTable) + .select( + customRole[parent.lower()].as_("name"), customRole.modified, customRole.ref_doctype, *columns + ) .where( (hasRole.parent == customRole.name) & (parentTable.name == customRole[parent.lower()]) & (customRole[parent.lower()].isnotnull()) - & (hasRole.role.isin(roles))) + & (hasRole.role.isin(roles)) + ) ).run(as_dict=True) for p in pages_with_custom_roles: - has_role[p.name] = {"modified":p.modified, "title": p.title, "ref_doctype": p.ref_doctype} + has_role[p.name] = {"modified": p.modified, "title": p.title, "ref_doctype": p.ref_doctype} subq = ( - frappe.qb.from_(customRole).select(customRole[parent.lower()]) + frappe.qb.from_(customRole) + .select(customRole[parent.lower()]) .where(customRole[parent.lower()].isnotnull()) ) pages_with_standard_roles = ( - frappe.qb.from_(hasRole).from_(parentTable) + frappe.qb.from_(hasRole) + .from_(parentTable) .select(parentTable.name.as_("name"), parentTable.modified, *columns) .where( (hasRole.role.isin(roles)) & (hasRole.parent == parentTable.name) & (parentTable.name.notin(subq)) - ).distinct() + ) + .distinct() ) if parent == "Report": @@ -183,18 +204,20 @@ def get_user_pages_or_reports(parent, cache=False): for p in pages_with_standard_roles: if p.name not in has_role: - has_role[p.name] = {"modified":p.modified, "title": p.title} + has_role[p.name] = {"modified": p.modified, "title": p.title} if parent == "Report": - has_role[p.name].update({'ref_doctype': p.ref_doctype}) + has_role[p.name].update({"ref_doctype": p.ref_doctype}) - no_of_roles = (frappe.qb.from_(hasRole).select(Count("*")) - .where(hasRole.parent == parentTable.name) + no_of_roles = ( + frappe.qb.from_(hasRole).select(Count("*")).where(hasRole.parent == parentTable.name) ) # pages with no role are allowed - if parent =="Page": + if parent == "Page": - pages_with_no_roles = (frappe.qb.from_(parentTable).select(parentTable.name, parentTable.modified, *columns) + pages_with_no_roles = ( + frappe.qb.from_(parentTable) + .select(parentTable.name, parentTable.modified, *columns) .where(subqry(no_of_roles) == 0) ).run(as_dict=True) @@ -203,18 +226,20 @@ def get_user_pages_or_reports(parent, cache=False): has_role[p.name] = {"modified": p.modified, "title": p.title} elif parent == "Report": - reports = frappe.get_all("Report", + reports = frappe.get_all( + "Report", fields=["name", "report_type"], filters={"name": ("in", has_role.keys())}, - ignore_ifnull=True + ignore_ifnull=True, ) for report in reports: has_role[report.name]["report_type"] = report.report_type # Expire every six hours - _cache.set_value('has_role:' + parent, has_role, frappe.session.user, 21600) + _cache.set_value("has_role:" + parent, has_role, frappe.session.user, 21600) return has_role + def load_translations(bootinfo): messages = frappe.get_lang_dict("boot") @@ -225,27 +250,30 @@ def load_translations(bootinfo): messages[name] = frappe._(name) # only untranslated - messages = {k: v for k, v in messages.items() if k!=v} + messages = {k: v for k, v in messages.items() if k != v} bootinfo["__messages"] = messages + def get_user_info(): # get info for current user user_info = frappe._dict() add_user_info(frappe.session.user, user_info) - if frappe.session.user == 'Administrator' and user_info.Administrator.email: + if frappe.session.user == "Administrator" and user_info.Administrator.email: user_info[user_info.Administrator.email] = user_info.Administrator return user_info + def get_user(bootinfo): """get user info""" bootinfo.user = frappe.get_user().load_user() + def add_home_page(bootinfo, docs): """load home page""" - if frappe.session.user=="Guest": + if frappe.session.user == "Guest": return home_page = frappe.db.get_default("desktop:home_page") @@ -255,50 +283,65 @@ def add_home_page(bootinfo, docs): try: page = frappe.desk.desk_page.get(home_page) docs.append(page) - bootinfo['home_page'] = page.name + bootinfo["home_page"] = page.name except (frappe.DoesNotExistError, frappe.PermissionError): if frappe.message_log: frappe.message_log.pop() - bootinfo['home_page'] = 'Workspaces' + bootinfo["home_page"] = "Workspaces" + def add_timezone_info(bootinfo): system = bootinfo.sysdefaults.get("time_zone") import frappe.utils.momentjs - bootinfo.timezone_info = {"zones":{}, "rules":{}, "links":{}} + + bootinfo.timezone_info = {"zones": {}, "rules": {}, "links": {}} frappe.utils.momentjs.update(system, bootinfo.timezone_info) + def load_print(bootinfo, doclist): print_settings = frappe.db.get_singles_dict("Print Settings") print_settings.doctype = ":Print Settings" doclist.append(print_settings) load_print_css(bootinfo, print_settings) + def load_print_css(bootinfo, print_settings): import frappe.www.printview - bootinfo.print_css = frappe.www.printview.get_print_style(print_settings.print_style or "Redesign", for_legacy=True) + + bootinfo.print_css = frappe.www.printview.get_print_style( + print_settings.print_style or "Redesign", for_legacy=True + ) + def get_unseen_notes(): note = DocType("Note") nsb = DocType("Note Seen By").as_("nsb") return ( - frappe.qb.from_(note).select(note.name, note.title, note.content, note.notify_on_every_login) + frappe.qb.from_(note) + .select(note.name, note.title, note.content, note.notify_on_every_login) .where( (note.notify_on_every_login == 1) & (note.expire_notification_on > frappe.utils.now()) - & (subqry(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin([frappe.session.user]))) - ).run(as_dict=1) + & ( + subqry(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin( + [frappe.session.user] + ) + ) + ) + ).run(as_dict=1) + def get_success_action(): return frappe.get_all("Success Action", fields=["*"]) + def get_link_preview_doctypes(): from frappe.utils import cint - link_preview_doctypes = [d.name for d in frappe.db.get_all('DocType', {'show_preview_popup': 1})] - customizations = frappe.get_all("Property Setter", - fields=['doc_type', 'value'], - filters={'property': 'show_preview_popup'} + link_preview_doctypes = [d.name for d in frappe.db.get_all("DocType", {"show_preview_popup": 1})] + customizations = frappe.get_all( + "Property Setter", fields=["doc_type", "value"], filters={"property": "show_preview_popup"} ) for custom in customizations: @@ -309,22 +352,23 @@ def get_link_preview_doctypes(): return link_preview_doctypes + def get_additional_filters_from_hooks(): filter_config = frappe._dict() - filter_hooks = frappe.get_hooks('filters_config') + filter_hooks = frappe.get_hooks("filters_config") for hook in filter_hooks: filter_config.update(frappe.get_attr(hook)()) return filter_config + def add_layouts(bootinfo): # add routes for readable doctypes - bootinfo.doctype_layouts = frappe.get_all('DocType Layout', ['name', 'route', 'document_type']) + bootinfo.doctype_layouts = frappe.get_all("DocType Layout", ["name", "route", "document_type"]) + def get_desk_settings(): - role_list = frappe.get_all('Role', fields=['*'], filters=dict( - name=['in', frappe.get_roles()] - )) + role_list = frappe.get_all("Role", fields=["*"], filters=dict(name=["in", frappe.get_roles()])) desk_settings = {} from frappe.core.doctype.role.role import desk_properties @@ -335,8 +379,10 @@ def get_desk_settings(): return desk_settings + def get_notification_settings(): - return frappe.get_cached_doc('Notification Settings', frappe.session.user) + return frappe.get_cached_doc("Notification Settings", frappe.session.user) + @frappe.whitelist() def get_link_title_doctypes(): @@ -348,8 +394,10 @@ def get_link_title_doctypes(): ) return [d.name for d in dts + custom_dts if d] + def set_time_zone(bootinfo): bootinfo.time_zone = { "system": get_time_zone(), - "user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) or get_time_zone() + "user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) + or get_time_zone(), } diff --git a/frappe/build.py b/frappe/build.py index 7a06ee3a22..e20ee0d698 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -1,8 +1,8 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import os -import shutil import re +import shutil import subprocess from distutils.spawn import find_executable from subprocess import getoutput @@ -25,6 +25,7 @@ sites_path = os.path.abspath(os.getcwd()) class AssetsNotDownloadedError(Exception): pass + class AssetsDontExistError(HTTPError): pass @@ -43,7 +44,7 @@ def download_file(url, prefix): def build_missing_files(): - '''Check which files dont exist yet from the assets.json and run build for those files''' + """Check which files dont exist yet from the assets.json and run build for those files""" missing_assets = [] current_asset_files = [] @@ -60,7 +61,7 @@ def build_missing_files(): assets_json = frappe.parse_json(assets_json) for bundle_file, output_file in assets_json.items(): - if not output_file.startswith('/assets/frappe'): + if not output_file.startswith("/assets/frappe"): continue if os.path.basename(output_file) not in current_asset_files: @@ -78,8 +79,7 @@ def build_missing_files(): def get_assets_link(frappe_head) -> str: tag = getoutput( r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" - r" refs/tags/,,' -e 's/\^{}//'" - % frappe_head + r" refs/tags/,,' -e 's/\^{}//'" % frappe_head ) if tag: @@ -111,6 +111,7 @@ def fetch_assets(url, frappe_head): def setup_assets(assets_archive): import tarfile + directories_created = set() click.secho("\nExtracting assets...\n", fg="yellow") @@ -221,7 +222,16 @@ def setup(): assets_path = os.path.join(frappe.local.sites_path, "assets") -def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None): +def bundle( + mode, + apps=None, + hard_link=False, + make_copy=False, + restore=False, + verbose=False, + skip_frappe=False, + files=None, +): """concat / minify js files""" setup() make_asset_dirs(hard_link=hard_link) @@ -236,7 +246,7 @@ def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, ver command += " --skip_frappe" if files: - command += " --files {files}".format(files=','.join(files)) + command += " --files {files}".format(files=",".join(files)) command += " --run-build-command" @@ -253,9 +263,7 @@ def watch(apps=None): if apps: command += " --apps {apps}".format(apps=apps) - live_reload = frappe.utils.cint( - os.environ.get("LIVE_RELOAD", frappe.conf.live_reload) - ) + live_reload = frappe.utils.cint(os.environ.get("LIVE_RELOAD", frappe.conf.live_reload)) if live_reload: command += " --live-reload" @@ -266,8 +274,8 @@ def watch(apps=None): def check_node_executable(): - node_version = Version(subprocess.getoutput('node -v')[1:]) - warn = '⚠️ ' + node_version = Version(subprocess.getoutput("node -v")[1:]) + warn = "⚠️ " if node_version.major < 14: click.echo(f"{warn} Please update your node version to 14") if not find_executable("yarn"): @@ -276,9 +284,7 @@ def check_node_executable(): def get_node_env(): - node_env = { - "NODE_OPTIONS": f"--max_old_space_size={get_safe_max_old_space_size()}" - } + node_env = {"NODE_OPTIONS": f"--max_old_space_size={get_safe_max_old_space_size()}"} return node_env @@ -345,8 +351,7 @@ def clear_broken_symlinks(): def unstrip(message: str) -> str: - """Pads input string on the right side until the last available column in the terminal - """ + """Pads input string on the right side until the last available column in the terminal""" _len = len(message) try: max_str = os.get_terminal_size().columns @@ -367,7 +372,9 @@ def make_asset_dirs(hard_link=False): symlinks = generate_assets_map() for source, target in symlinks.items(): - start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}") + start_message = unstrip( + f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}" + ) fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}") # Used '\r' instead of '\x1b[1K\r' to print entire lines in smaller terminal sizes @@ -404,10 +411,11 @@ def scrub_html_template(content): # strip comments content = re.sub(r"()", "", content) - return content.replace("'", "\'") + return content.replace("'", "'") def html_to_js_template(path, content): """returns HTML template content as Javascript code, adding it to `frappe.templates`""" return """frappe.templates["{key}"] = '{content}';\n""".format( - key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content)) + key=path.rsplit("/", 1)[-1][:-5], content=scrub_html_template(content) + ) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 94a845639b..b15f8f2234 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -1,33 +1,75 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe +from frappe.desk.notifications import clear_notifications, delete_notification_count_for from frappe.model.document import Document -from frappe.desk.notifications import (delete_notification_count_for, - clear_notifications) common_default_keys = ["__default", "__global"] -doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map', - 'milestone_tracker_map', 'event_consumer_document_type_map') - -bench_cache_keys = ('assets_json',) - -global_cache_keys = ("app_hooks", "installed_apps", 'all_apps', - "app_modules", "module_app", "system_settings", - 'scheduler_events', 'time_zone', 'webhooks', 'active_domains', - 'active_modules', 'assignment_rule', 'server_script_map', 'wkhtmltopdf_version', - 'domain_restricted_doctypes', 'domain_restricted_pages', 'information_schema:counts', - 'sitemap_routes', 'db_tables', 'server_script_autocompletion_items') + doctype_map_keys - -user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang", - "defaults", "user_permissions", "home_page", "linked_with", - "desktop_icons", 'portal_menu_items', 'user_perm_can_read', - "has_role:Page", "has_role:Report", "desk_sidebar_items") +doctype_map_keys = ( + "energy_point_rule_map", + "assignment_rule_map", + "milestone_tracker_map", + "event_consumer_document_type_map", +) + +bench_cache_keys = ("assets_json",) + +global_cache_keys = ( + "app_hooks", + "installed_apps", + "all_apps", + "app_modules", + "module_app", + "system_settings", + "scheduler_events", + "time_zone", + "webhooks", + "active_domains", + "active_modules", + "assignment_rule", + "server_script_map", + "wkhtmltopdf_version", + "domain_restricted_doctypes", + "domain_restricted_pages", + "information_schema:counts", + "sitemap_routes", + "db_tables", + "server_script_autocompletion_items", +) + doctype_map_keys + +user_cache_keys = ( + "bootinfo", + "user_recent", + "roles", + "user_doc", + "lang", + "defaults", + "user_permissions", + "home_page", + "linked_with", + "desktop_icons", + "portal_menu_items", + "user_perm_can_read", + "has_role:Page", + "has_role:Report", + "desk_sidebar_items", +) + +doctype_cache_keys = ( + "meta", + "form_meta", + "table_columns", + "last_modified", + "linked_doctypes", + "notifications", + "workflow", + "data_import_column_header_map", +) + doctype_map_keys -doctype_cache_keys = ("meta", "form_meta", "table_columns", "last_modified", - "linked_doctypes", 'notifications', 'workflow' , - 'data_import_column_header_map') + doctype_map_keys def clear_user_cache(user=None): cache = frappe.cache() @@ -47,11 +89,13 @@ def clear_user_cache(user=None): clear_defaults_cache() clear_global_cache() + def clear_domain_cache(user=None): cache = frappe.cache() - domain_cache_keys = ('domain_restricted_doctypes', 'domain_restricted_pages') + domain_cache_keys = ("domain_restricted_doctypes", "domain_restricted_pages") cache.delete_value(domain_cache_keys) + def clear_global_cache(): from frappe.website.utils import clear_website_cache @@ -61,21 +105,23 @@ def clear_global_cache(): frappe.cache().delete_value(bench_cache_keys) frappe.setup_module_map() + def clear_defaults_cache(user=None): if user: - for p in ([user] + common_default_keys): + for p in [user] + common_default_keys: frappe.cache().hdel("defaults", p) - elif frappe.flags.in_install!="frappe": + elif frappe.flags.in_install != "frappe": frappe.cache().delete_key("defaults") + def clear_doctype_cache(doctype=None): clear_controller_cache(doctype) cache = frappe.cache() - if getattr(frappe.local, 'meta_cache') and (doctype in frappe.local.meta_cache): + if getattr(frappe.local, "meta_cache") and (doctype in frappe.local.meta_cache): del frappe.local.meta_cache[doctype] - for key in ('is_table', 'doctype_modules', 'document_cache'): + for key in ("is_table", "doctype_modules", "document_cache"): cache.delete_value(key) frappe.local.document_cache = {} @@ -89,8 +135,9 @@ def clear_doctype_cache(doctype=None): # clear all parent doctypes - for dt in frappe.db.get_all('DocField', 'parent', - dict(fieldtype=['in', frappe.model.table_fields], options=doctype)): + for dt in frappe.db.get_all( + "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=doctype) + ): clear_single(dt.parent) # clear all notifications @@ -101,6 +148,7 @@ def clear_doctype_cache(doctype=None): for name in doctype_cache_keys: cache.delete_value(name) + def clear_controller_cache(doctype=None): if not doctype: del frappe.controllers @@ -110,9 +158,10 @@ def clear_controller_cache(doctype=None): for site_controllers in frappe.controllers.values(): site_controllers.pop(doctype, None) + def get_doctype_map(doctype, name, filters=None, order_by=None): cache = frappe.cache() - cache_key = frappe.scrub(doctype) + '_map' + cache_key = frappe.scrub(doctype) + "_map" doctype_map = cache.hget(cache_key, name) if doctype_map is not None: @@ -121,7 +170,7 @@ def get_doctype_map(doctype, name, filters=None, order_by=None): else: # non cached, build cache try: - items = frappe.get_all(doctype, filters=filters, order_by = order_by) + items = frappe.get_all(doctype, filters=filters, order_by=order_by) cache.hset(cache_key, name, json.dumps(items)) except frappe.db.TableMissingError: # executed from inside patch, ignore @@ -129,15 +178,19 @@ def get_doctype_map(doctype, name, filters=None, order_by=None): return items + def clear_doctype_map(doctype, name): - frappe.cache().hdel(frappe.scrub(doctype) + '_map', name) + frappe.cache().hdel(frappe.scrub(doctype) + "_map", name) + def build_table_count_cache(): - if (frappe.flags.in_patch + if ( + frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_migrate or frappe.flags.in_import - or frappe.flags.in_setup_wizard): + or frappe.flags.in_setup_wizard + ): return _cache = frappe.cache() @@ -145,39 +198,45 @@ def build_table_count_cache(): table_rows = frappe.qb.Field("table_rows").as_("count") information_schema = frappe.qb.Schema("information_schema") - data = ( - frappe.qb.from_(information_schema.tables).select(table_name, table_rows) - ).run(as_dict=True) - counts = {d.get('name').replace('tab', '', 1): d.get('count', None) for d in data} + data = (frappe.qb.from_(information_schema.tables).select(table_name, table_rows)).run( + as_dict=True + ) + counts = {d.get("name").replace("tab", "", 1): d.get("count", None) for d in data} _cache.set_value("information_schema:counts", counts) return counts + def build_domain_restriced_doctype_cache(*args, **kwargs): - if (frappe.flags.in_patch + if ( + frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_migrate or frappe.flags.in_import - or frappe.flags.in_setup_wizard): + or frappe.flags.in_setup_wizard + ): return _cache = frappe.cache() active_domains = frappe.get_active_domains() - doctypes = frappe.get_all("DocType", filters={'restrict_to_domain': ('IN', active_domains)}) + doctypes = frappe.get_all("DocType", filters={"restrict_to_domain": ("IN", active_domains)}) doctypes = [doc.name for doc in doctypes] _cache.set_value("domain_restricted_doctypes", doctypes) return doctypes + def build_domain_restriced_page_cache(*args, **kwargs): - if (frappe.flags.in_patch + if ( + frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_migrate or frappe.flags.in_import - or frappe.flags.in_setup_wizard): + or frappe.flags.in_setup_wizard + ): return _cache = frappe.cache() active_domains = frappe.get_active_domains() - pages = frappe.get_all("Page", filters={'restrict_to_domain': ('IN', active_domains)}) + pages = frappe.get_all("Page", filters={"restrict_to_domain": ("IN", active_domains)}) pages = [page.name for page in pages] _cache.set_value("domain_restricted_pages", pages) diff --git a/frappe/client.py b/frappe/client.py index 1898994afe..e970a64802 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -1,32 +1,44 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json +import os + import frappe -from frappe import _ import frappe.model import frappe.utils -import json, os -from frappe.utils import get_safe_filters +from frappe import _ from frappe.desk.reportview import validate_args from frappe.model.db_query import check_parent_permission +from frappe.utils import get_safe_filters - -''' +""" Handle RESTful requests that are mapped to the `/api/resource` route. Requests via FrappeClient are also handled here. -''' +""" + @frappe.whitelist() -def get_list(doctype, fields=None, filters=None, order_by=None, - limit_start=None, limit_page_length=20, parent=None, debug=False, as_dict=True, or_filters=None): - '''Returns a list of records by filters, fields, ordering and limit +def get_list( + doctype, + fields=None, + filters=None, + order_by=None, + limit_start=None, + limit_page_length=20, + parent=None, + debug=False, + as_dict=True, + or_filters=None, +): + """Returns a list of records by filters, fields, ordering and limit :param doctype: DocType of the data to be queried :param fields: fields to be returned. Default is `name` :param filters: filter list by this dict :param order_by: Order by this fieldname :param limit_start: Start at this index - :param limit_page_length: Number of records to be returned (default 20)''' + :param limit_page_length: Number of records to be returned (default 20)""" if frappe.is_table(doctype): check_parent_permission(parent, doctype) @@ -40,23 +52,25 @@ def get_list(doctype, fields=None, filters=None, order_by=None, limit_start=limit_start, limit_page_length=limit_page_length, debug=debug, - as_list=not as_dict + as_list=not as_dict, ) validate_args(args) return frappe.get_list(**args) + @frappe.whitelist() def get_count(doctype, filters=None, debug=False, cache=False): return frappe.db.count(doctype, get_safe_filters(filters), debug, cache) + @frappe.whitelist() def get(doctype, name=None, filters=None, parent=None): - '''Returns a document by name or filters + """Returns a document by name or filters :param doctype: DocType of the document to be returned :param name: return document of this `name` - :param filters: If name is not set, filter by these values and return the first match''' + :param filters: If name is not set, filter by these values and return the first match""" if frappe.is_table(doctype): check_parent_permission(parent, doctype) @@ -71,13 +85,14 @@ def get(doctype, name=None, filters=None, parent=None): return frappe.get_doc(doctype, name).as_dict() + @frappe.whitelist() def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, parent=None): - '''Returns a value form a document + """Returns a value form a document :param doctype: DocType to be queried :param fieldname: Field to be returned (default `name`) - :param filters: dict or string for identifying the record''' + :param filters: dict or string for identifying the record""" if frappe.is_table(doctype): check_parent_permission(parent, doctype) @@ -102,7 +117,15 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if frappe.get_meta(doctype).issingle: value = frappe.db.get_values_from_single(fields, filters, doctype, as_dict=as_dict, debug=debug) else: - value = get_list(doctype, filters=filters, fields=fields, debug=debug, limit_page_length=1, parent=parent, as_dict=as_dict) + value = get_list( + doctype, + filters=filters, + fields=fields, + debug=debug, + limit_page_length=1, + parent=parent, + as_dict=as_dict, + ) if as_dict: return value[0] if value else {} @@ -112,6 +135,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren return value[0] if len(fields) > 1 else value[0][0] + @frappe.whitelist() def get_single_value(doctype, field): if not frappe.has_permission(doctype): @@ -119,14 +143,15 @@ def get_single_value(doctype, field): value = frappe.db.get_single_value(doctype, field) return value -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def set_value(doctype, name, fieldname, value=None): - '''Set a value using get_doc, group of values + """Set a value using get_doc, group of values :param doctype: DocType of the document :param name: name of the document :param fieldname: fieldname string or JSON / dict with key value pair - :param value: value if fieldname is JSON / dict''' + :param value: value if fieldname is JSON / dict""" if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): frappe.throw(_("Cannot edit standard fields")) @@ -137,7 +162,7 @@ def set_value(doctype, name, fieldname, value=None): try: values = json.loads(fieldname) except ValueError: - values = {fieldname: ''} + values = {fieldname: ""} else: values = {fieldname: value} @@ -155,11 +180,12 @@ def set_value(doctype, name, fieldname, value=None): return doc.as_dict() -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def insert(doc=None): - '''Insert a document + """Insert a document - :param doc: JSON or dict object to be inserted''' + :param doc: JSON or dict object to be inserted""" if isinstance(doc, str): doc = json.loads(doc) @@ -173,18 +199,19 @@ def insert(doc=None): doc = frappe.get_doc(doc).insert() return doc.as_dict() -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def insert_many(docs=None): - '''Insert multiple documents + """Insert multiple documents - :param docs: JSON or list of dict objects to be inserted in one request''' + :param docs: JSON or list of dict objects to be inserted in one request""" if isinstance(docs, str): docs = json.loads(docs) out = [] if len(docs) > 200: - frappe.throw(_('Only 200 inserts allowed in one request')) + frappe.throw(_("Only 200 inserts allowed in one request")) for doc in docs: if doc.get("parenttype"): @@ -199,11 +226,12 @@ def insert_many(docs=None): return out -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def save(doc): - '''Update (save) an existing document + """Update (save) an existing document - :param doc: JSON or dict object with the properties of the document to be updated''' + :param doc: JSON or dict object with the properties of the document to be updated""" if isinstance(doc, str): doc = json.loads(doc) @@ -212,21 +240,23 @@ def save(doc): return doc.as_dict() -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def rename_doc(doctype, old_name, new_name, merge=False): - '''Rename document + """Rename document :param doctype: DocType of the document to be renamed :param old_name: Current `name` of the document to be renamed - :param new_name: New `name` to be set''' + :param new_name: New `name` to be set""" new_name = frappe.rename_doc(doctype, old_name, new_name, merge=merge) return new_name -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def submit(doc): - '''Submit a document + """Submit a document - :param doc: JSON or dict object to be submitted remotely''' + :param doc: JSON or dict object to be submitted remotely""" if isinstance(doc, str): doc = json.loads(doc) @@ -235,52 +265,57 @@ def submit(doc): return doc.as_dict() -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def cancel(doctype, name): - '''Cancel a document + """Cancel a document :param doctype: DocType of the document to be cancelled - :param name: name of the document to be cancelled''' + :param name: name of the document to be cancelled""" wrapper = frappe.get_doc(doctype, name) wrapper.cancel() return wrapper.as_dict() -@frappe.whitelist(methods=['DELETE', 'POST']) + +@frappe.whitelist(methods=["DELETE", "POST"]) def delete(doctype, name): - '''Delete a remote document + """Delete a remote document :param doctype: DocType of the document to be deleted - :param name: name of the document to be deleted''' + :param name: name of the document to be deleted""" frappe.delete_doc(doctype, name, ignore_missing=False) -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def set_default(key, value, parent=None): """set a user default value""" frappe.db.set_default(key, value, parent or frappe.session.user) frappe.clear_cache(user=frappe.session.user) + @frappe.whitelist() def get_default(key, parent=None): """set a user default value""" return frappe.db.get_default(key, parent) -@frappe.whitelist(methods=['POST', 'PUT']) +@frappe.whitelist(methods=["POST", "PUT"]) def make_width_property_setter(doc): - '''Set width Property Setter + """Set width Property Setter - :param doc: Property Setter document with `width` property''' + :param doc: Property Setter document with `width` property""" if isinstance(doc, str): doc = json.loads(doc) - if doc["doctype"]=="Property Setter" and doc["property"]=="width": - frappe.get_doc(doc).insert(ignore_permissions = True) + if doc["doctype"] == "Property Setter" and doc["property"] == "width": + frappe.get_doc(doc).insert(ignore_permissions=True) -@frappe.whitelist(methods=['POST', 'PUT']) + +@frappe.whitelist(methods=["POST", "PUT"]) def bulk_update(docs): - '''Bulk update documents + """Bulk update documents - :param docs: JSON list of documents to be updated remotely. Each document must have `docname` property''' + :param docs: JSON list of documents to be updated remotely. Each document must have `docname` property""" docs = json.loads(docs) failed_docs = [] for doc in docs: @@ -290,41 +325,40 @@ def bulk_update(docs): existing_doc.update(doc) existing_doc.save() except Exception: - failed_docs.append({ - 'doc': doc, - 'exc': frappe.utils.get_traceback() - }) + failed_docs.append({"doc": doc, "exc": frappe.utils.get_traceback()}) + + return {"failed_docs": failed_docs} - return {'failed_docs': failed_docs} @frappe.whitelist() def has_permission(doctype, docname, perm_type="read"): - '''Returns a JSON with data whether the document has the requested permission + """Returns a JSON with data whether the document has the requested permission :param doctype: DocType of the document to be checked :param docname: `name` of the document to be checked - :param perm_type: one of `read`, `write`, `create`, `submit`, `cancel`, `report`. Default is `read`''' + :param perm_type: one of `read`, `write`, `create`, `submit`, `cancel`, `report`. Default is `read`""" # perm_type can be one of read, write, create, submit, cancel, report return {"has_permission": frappe.has_permission(doctype, perm_type.lower(), docname)} + @frappe.whitelist() def get_password(doctype, name, fieldname): - '''Return a password type property. Only applicable for System Managers + """Return a password type property. Only applicable for System Managers :param doctype: DocType of the document that holds the password :param name: `name` of the document that holds the password :param fieldname: `fieldname` of the password property - ''' + """ frappe.only_for("System Manager") return frappe.get_doc(doctype, name).get_password(fieldname) @frappe.whitelist() def get_js(items): - '''Load JS code files. Will also append translations + """Load JS code files. Will also append translations and extend `frappe._messages` - :param items: JSON list of paths of the js files to be loaded.''' + :param items: JSON list of paths of the js files to be loaded.""" items = json.loads(items) out = [] for src in items: @@ -346,14 +380,25 @@ def get_js(items): return out + @frappe.whitelist(allow_guest=True) def get_time_zone(): - '''Returns default time zone''' + """Returns default time zone""" return {"time_zone": frappe.defaults.get_defaults().get("time_zone")} -@frappe.whitelist(methods=['POST', 'PUT']) -def attach_file(filename=None, filedata=None, doctype=None, docname=None, folder=None, decode_base64=False, is_private=None, docfield=None): - '''Attach a file to Document (POST) + +@frappe.whitelist(methods=["POST", "PUT"]) +def attach_file( + filename=None, + filedata=None, + doctype=None, + docname=None, + folder=None, + decode_base64=False, + is_private=None, + docfield=None, +): + """Attach a file to Document (POST) :param filename: filename e.g. test-file.txt :param filedata: base64 encode filedata which must be urlencoded @@ -362,7 +407,7 @@ def attach_file(filename=None, filedata=None, doctype=None, docname=None, folder :param folder: Folder to add File into :param decode_base64: decode filedata from base64 encode, default is False :param is_private: Attach file as private file (1 or 0) - :param docfield: file to attach to (optional)''' + :param docfield: file to attach to (optional)""" request_method = frappe.local.request.environ.get("REQUEST_METHOD") @@ -374,16 +419,19 @@ def attach_file(filename=None, filedata=None, doctype=None, docname=None, folder if not doc.has_permission(): frappe.throw(_("Not permitted"), frappe.PermissionError) - _file = frappe.get_doc({ - "doctype": "File", - "file_name": filename, - "attached_to_doctype": doctype, - "attached_to_name": docname, - "attached_to_field": docfield, - "folder": folder, - "is_private": is_private, - "content": filedata, - "decode": decode_base64}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": filename, + "attached_to_doctype": doctype, + "attached_to_name": docname, + "attached_to_field": docfield, + "folder": folder, + "is_private": is_private, + "content": filedata, + "decode": decode_base64, + } + ) _file.save() if docfield and doctype: @@ -392,22 +440,23 @@ def attach_file(filename=None, filedata=None, doctype=None, docname=None, folder return _file.as_dict() + @frappe.whitelist() def get_hooks(hook, app_name=None): return frappe.get_hooks(hook, app_name) + @frappe.whitelist() def is_document_amended(doctype, docname): if frappe.permissions.has_permission(doctype): try: - return frappe.db.exists(doctype, { - 'amended_from': docname - }) + return frappe.db.exists(doctype, {"amended_from": docname}) except frappe.db.InternalError: pass return False + @frappe.whitelist() def validate_link(doctype: str, docname: str, fields=None): if not isinstance(doctype, str): @@ -417,13 +466,11 @@ def validate_link(doctype: str, docname: str, fields=None): frappe.throw(_("Document Name must be a string")) if doctype != "DocType" and not ( - frappe.has_permission(doctype, "select") - or frappe.has_permission(doctype, "read") + frappe.has_permission(doctype, "select") or frappe.has_permission(doctype, "read") ): frappe.throw( - _("You do not have Read or Select Permissions for {}") - .format(frappe.bold(doctype)), - frappe.PermissionError + _("You do not have Read or Select Permissions for {}").format(frappe.bold(doctype)), + frappe.PermissionError, ) values = frappe._dict() @@ -438,14 +485,11 @@ def validate_link(doctype: str, docname: str, fields=None): except frappe.PermissionError: frappe.clear_last_message() frappe.msgprint( - _("You need {0} permission to fetch values from {1} {2}") - .format( - frappe.bold(_("Read")), - frappe.bold(doctype), - frappe.bold(docname) + _("You need {0} permission to fetch values from {1} {2}").format( + frappe.bold(_("Read")), frappe.bold(doctype), frappe.bold(docname) ), title=_("Cannot Fetch Values"), - indicator="orange" + indicator="orange", ) return values diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index 82a71ce7b4..f61b3b9d34 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -1,23 +1,26 @@ # Copyright (c) 2015, Web Notes Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import sys -import click import cProfile import pstats -import frappe -import frappe.utils -import subprocess # nosec +import subprocess # nosec +import sys from functools import wraps from io import StringIO from os import environ +import click + +import frappe +import frappe.utils + click.disable_unicode_literals_warning = True + def pass_context(f): @wraps(f) def _func(ctx, *args, **kwargs): - profile = ctx.obj['profile'] + profile = ctx.obj["profile"] if profile: pr = cProfile.Profile() pr.enable() @@ -25,18 +28,17 @@ def pass_context(f): try: ret = f(frappe._dict(ctx.obj), *args, **kwargs) except frappe.exceptions.SiteNotSpecifiedError as e: - click.secho(str(e), fg='yellow') + click.secho(str(e), fg="yellow") sys.exit(1) except frappe.exceptions.IncorrectSitePath: site = ctx.obj.get("sites", "")[0] - click.secho(f'Site {site} does not exist!', fg='yellow') + click.secho(f"Site {site} does not exist!", fg="yellow") sys.exit(1) if profile: pr.disable() s = StringIO() - ps = pstats.Stats(pr, stream=s)\ - .sort_stats('cumtime', 'tottime', 'ncalls') + ps = pstats.Stats(pr, stream=s).sort_stats("cumtime", "tottime", "ncalls") ps.print_stats() # print the top-100 @@ -47,6 +49,7 @@ def pass_context(f): return click.pass_context(_func) + def get_site(context, raise_err=True): try: site = context.sites[0] @@ -56,17 +59,19 @@ def get_site(context, raise_err=True): raise frappe.SiteNotSpecifiedError return None + def popen(command, *args, **kwargs): - output = kwargs.get('output', True) - cwd = kwargs.get('cwd') - shell = kwargs.get('shell', True) - raise_err = kwargs.get('raise_err') - env = kwargs.get('env') + output = kwargs.get("output", True) + cwd = kwargs.get("cwd") + shell = kwargs.get("shell", True) + raise_err = kwargs.get("raise_err") + env = kwargs.get("env") if env: env = dict(environ, **env) def set_low_prio(): import psutil + if psutil.LINUX: psutil.Process().nice(19) psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE) @@ -77,13 +82,14 @@ def popen(command, *args, **kwargs): psutil.Process().nice(19) # ionice not supported - proc = subprocess.Popen(command, + proc = subprocess.Popen( + command, stdout=None if output else subprocess.PIPE, stderr=None if output else subprocess.PIPE, shell=shell, cwd=cwd, preexec_fn=set_low_prio, - env=env + env=env, ) return_ = proc.wait() @@ -93,26 +99,22 @@ def popen(command, *args, **kwargs): return return_ + def call_command(cmd, context): return click.Context(cmd, obj=context).forward(cmd) + def get_commands(): # prevent circular imports + from .redis_utils import commands as redis_commands from .scheduler import commands as scheduler_commands from .site import commands as site_commands from .translate import commands as translate_commands from .utils import commands as utils_commands - from .redis_utils import commands as redis_commands - clickable_link = ( - "\x1b]8;;https://frappeframework.com/docs\afrappeframework.com\x1b]8;;\a" - ) + clickable_link = "\x1b]8;;https://frappeframework.com/docs\afrappeframework.com\x1b]8;;\a" all_commands = ( - scheduler_commands - + site_commands - + translate_commands - + utils_commands - + redis_commands + scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands ) for command in all_commands: diff --git a/frappe/commands/redis_utils.py b/frappe/commands/redis_utils.py index 3556050782..1c3292b7fa 100644 --- a/frappe/commands/redis_utils.py +++ b/frappe/commands/redis_utils.py @@ -3,51 +3,71 @@ import os import click import frappe -from frappe.utils.redis_queue import RedisQueue from frappe.installer import update_site_config +from frappe.utils.redis_queue import RedisQueue -@click.command('create-rq-users') -@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password') -@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites') + +@click.command("create-rq-users") +@click.option( + "--set-admin-password", + is_flag=True, + default=False, + help="Set new Redis admin(default user) password", +) +@click.option( + "--use-rq-auth", is_flag=True, default=False, help="Enable Redis authentication for sites" +) def create_rq_users(set_admin_password=False, use_rq_auth=False): """Create Redis Queue users and add to acl and app configs. acl config file will be used by redis server while starting the server and app config is used by app while connecting to redis server. """ - acl_file_path = os.path.abspath('../config/redis_queue.acl') + acl_file_path = os.path.abspath("../config/redis_queue.acl") with frappe.init_site(): - acl_list, user_credentials = RedisQueue.gen_acl_list( - set_admin_password=set_admin_password) + acl_list, user_credentials = RedisQueue.gen_acl_list(set_admin_password=set_admin_password) - with open(acl_file_path, 'w') as f: - f.writelines([acl+'\n' for acl in acl_list]) + with open(acl_file_path, "w") as f: + f.writelines([acl + "\n" for acl in acl_list]) sites_path = os.getcwd() - common_site_config_path = os.path.join(sites_path, 'common_site_config.json') - update_site_config("rq_username", user_credentials['bench'][0], validate=False, - site_config_path=common_site_config_path) - update_site_config("rq_password", user_credentials['bench'][1], validate=False, - site_config_path=common_site_config_path) - update_site_config("use_rq_auth", use_rq_auth, validate=False, - site_config_path=common_site_config_path) - - click.secho('* ACL and site configs are updated with new user credentials. ' - 'Please restart Redis Queue server to enable namespaces.', - fg='green') + common_site_config_path = os.path.join(sites_path, "common_site_config.json") + update_site_config( + "rq_username", + user_credentials["bench"][0], + validate=False, + site_config_path=common_site_config_path, + ) + update_site_config( + "rq_password", + user_credentials["bench"][1], + validate=False, + site_config_path=common_site_config_path, + ) + update_site_config( + "use_rq_auth", use_rq_auth, validate=False, site_config_path=common_site_config_path + ) + + click.secho( + "* ACL and site configs are updated with new user credentials. " + "Please restart Redis Queue server to enable namespaces.", + fg="green", + ) if set_admin_password: - env_key = 'RQ_ADMIN_PASWORD' - click.secho('* Redis admin password is successfully set up. ' - 'Include below line in .bashrc file for system to use', - fg='green') + env_key = "RQ_ADMIN_PASWORD" + click.secho( + "* Redis admin password is successfully set up. " + "Include below line in .bashrc file for system to use", + fg="green", + ) click.secho(f"`export {env_key}={user_credentials['default'][1]}`") - click.secho('NOTE: Please save the admin password as you ' - 'can not access redis server without the password', - fg='yellow') + click.secho( + "NOTE: Please save the admin password as you " + "can not access redis server without the password", + fg="yellow", + ) -commands = [ - create_rq_users -] +commands = [create_rq_users] diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index f82473fd55..ed6a0dea57 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -1,15 +1,20 @@ -import click import sys + +import click + import frappe -from frappe.utils import cint -from frappe.commands import pass_context, get_site +from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError +from frappe.utils import cint + def _is_scheduler_enabled(): enable_scheduler = False try: frappe.connect() - enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False + enable_scheduler = ( + cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False + ) except: pass finally: @@ -44,11 +49,12 @@ def trigger_scheduler_event(context, event): sys.exit(exit_code) -@click.command('enable-scheduler') +@click.command("enable-scheduler") @pass_context def enable_scheduler(context): "Enable scheduler" import frappe.utils.scheduler + for site in context.sites: try: frappe.init(site=site) @@ -61,11 +67,13 @@ def enable_scheduler(context): if not context.sites: raise SiteNotSpecifiedError -@click.command('disable-scheduler') + +@click.command("disable-scheduler") @pass_context def disable_scheduler(context): "Disable scheduler" import frappe.utils.scheduler + for site in context.sites: try: frappe.init(site=site) @@ -79,13 +87,13 @@ def disable_scheduler(context): raise SiteNotSpecifiedError -@click.command('scheduler') -@click.option('--site', help='site name') -@click.argument('state', type=click.Choice(['pause', 'resume', 'disable', 'enable'])) +@click.command("scheduler") +@click.option("--site", help="site name") +@click.argument("state", type=click.Choice(["pause", "resume", "disable", "enable"])) @pass_context def scheduler(context, state, site=None): - from frappe.installer import update_site_config import frappe.utils.scheduler + from frappe.installer import update_site_config if not site: site = get_site(context) @@ -93,58 +101,64 @@ def scheduler(context, state, site=None): try: frappe.init(site=site) - if state == 'pause': - update_site_config('pause_scheduler', 1) - elif state == 'resume': - update_site_config('pause_scheduler', 0) - elif state == 'disable': + if state == "pause": + update_site_config("pause_scheduler", 1) + elif state == "resume": + update_site_config("pause_scheduler", 0) + elif state == "disable": frappe.connect() frappe.utils.scheduler.disable_scheduler() frappe.db.commit() - elif state == 'enable': + elif state == "enable": frappe.connect() frappe.utils.scheduler.enable_scheduler() frappe.db.commit() - print('Scheduler {0}d for site {1}'.format(state, site)) + print("Scheduler {0}d for site {1}".format(state, site)) finally: frappe.destroy() -@click.command('set-maintenance-mode') -@click.option('--site', help='site name') -@click.argument('state', type=click.Choice(['on', 'off'])) +@click.command("set-maintenance-mode") +@click.option("--site", help="site name") +@click.argument("state", type=click.Choice(["on", "off"])) @pass_context def set_maintenance_mode(context, state, site=None): from frappe.installer import update_site_config + if not site: site = get_site(context) try: frappe.init(site=site) - update_site_config('maintenance_mode', 1 if (state == 'on') else 0) + update_site_config("maintenance_mode", 1 if (state == "on") else 0) finally: frappe.destroy() -@click.command('doctor') #Passing context always gets a site and if there is no use site it breaks -@click.option('--site', help='site name') +@click.command( + "doctor" +) # Passing context always gets a site and if there is no use site it breaks +@click.option("--site", help="site name") @pass_context def doctor(context, site=None): "Get diagnostic info about background workers" from frappe.utils.doctor import doctor as _doctor + if not site: site = get_site(context, raise_err=False) return _doctor(site=site) -@click.command('show-pending-jobs') -@click.option('--site', help='site name') + +@click.command("show-pending-jobs") +@click.option("--site", help="site name") @pass_context def show_pending_jobs(context, site=None): "Get diagnostic info about background jobs" from frappe.utils.doctor import pending_jobs as _pending_jobs + if not site: site = get_site(context) @@ -153,35 +167,45 @@ def show_pending_jobs(context, site=None): return pending_jobs -@click.command('purge-jobs') -@click.option('--site', help='site name') -@click.option('--queue', default=None, help='one of "low", "default", "high') -@click.option('--event', default=None, help='one of "all", "weekly", "monthly", "hourly", "daily", "weekly_long", "daily_long"') + +@click.command("purge-jobs") +@click.option("--site", help="site name") +@click.option("--queue", default=None, help='one of "low", "default", "high') +@click.option( + "--event", + default=None, + help='one of "all", "weekly", "monthly", "hourly", "daily", "weekly_long", "daily_long"', +) def purge_jobs(site=None, queue=None, event=None): "Purge any pending periodic tasks, if event option is not given, it will purge everything for the site" from frappe.utils.doctor import purge_pending_jobs - frappe.init(site or '') + + frappe.init(site or "") count = purge_pending_jobs(event=event, site=site, queue=queue) print("Purged {} jobs".format(count)) -@click.command('schedule') + +@click.command("schedule") def start_scheduler(): from frappe.utils.scheduler import start_scheduler + start_scheduler() -@click.command('worker') -@click.option('--queue', type=str) -@click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs') -@click.option('-u', '--rq-username', default=None, help='Redis ACL user') -@click.option('-p', '--rq-password', default=None, help='Redis ACL user password') -def start_worker(queue, quiet = False, rq_username=None, rq_password=None): - """Site is used to find redis credentals. - """ + +@click.command("worker") +@click.option("--queue", type=str) +@click.option("--quiet", is_flag=True, default=False, help="Hide Log Outputs") +@click.option("-u", "--rq-username", default=None, help="Redis ACL user") +@click.option("-p", "--rq-password", default=None, help="Redis ACL user password") +def start_worker(queue, quiet=False, rq_username=None, rq_password=None): + """Site is used to find redis credentals.""" from frappe.utils.background_jobs import start_worker - start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password) -@click.command('ready-for-migration') -@click.option('--site', help='site name') + start_worker(queue, quiet=quiet, rq_username=rq_username, rq_password=rq_password) + + +@click.command("ready-for-migration") +@click.option("--site", help="site name") @pass_context def ready_for_migration(context, site=None): from frappe.utils.doctor import get_pending_jobs @@ -194,16 +218,17 @@ def ready_for_migration(context, site=None): pending_jobs = get_pending_jobs(site=site) if pending_jobs: - print('NOT READY for migration: site {0} has pending background jobs'.format(site)) + print("NOT READY for migration: site {0} has pending background jobs".format(site)) sys.exit(1) else: - print('READY for migration: site {0} does not have any background jobs'.format(site)) + print("READY for migration: site {0} does not have any background jobs".format(site)) return 0 finally: frappe.destroy() + commands = [ disable_scheduler, doctor, diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 6bee442054..fa9ab4be59 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -12,56 +12,128 @@ from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -@click.command('new-site') -@click.argument('site') -@click.option('--db-name', help='Database name') -@click.option('--db-password', help='Database password') -@click.option('--db-type', default='mariadb', type=click.Choice(['mariadb', 'postgres']), help='Optional "postgres" or "mariadb". Default is "mariadb"') -@click.option('--db-host', help='Database Host') -@click.option('--db-port', type=int, help='Database Port') -@click.option('--db-root-username', '--mariadb-root-username', help='Root username for MariaDB or PostgreSQL, Default is "root"') -@click.option('--db-root-password', '--mariadb-root-password', help='Root password for MariaDB or PostgreSQL') -@click.option('--no-mariadb-socket', is_flag=True, default=False, help='Set MariaDB host to % and use TCP/IP Socket instead of using the UNIX Socket') -@click.option('--admin-password', help='Administrator password for new site', default=None) -@click.option('--verbose', is_flag=True, default=False, help='Verbose') -@click.option('--force', help='Force restore if site/database already exists', is_flag=True, default=False) -@click.option('--source_sql', help='Initiate database with a SQL file') -@click.option('--install-app', multiple=True, help='Install app after installation') -@click.option('--set-default', is_flag=True, default=False, help='Set the new site as default site') -def new_site(site, db_root_username=None, db_root_password=None, admin_password=None, - verbose=False, install_apps=None, source_sql=None, force=None, no_mariadb_socket=False, - install_app=None, db_name=None, db_password=None, db_type=None, db_host=None, db_port=None, - set_default=False): +@click.command("new-site") +@click.argument("site") +@click.option("--db-name", help="Database name") +@click.option("--db-password", help="Database password") +@click.option( + "--db-type", + default="mariadb", + type=click.Choice(["mariadb", "postgres"]), + help='Optional "postgres" or "mariadb". Default is "mariadb"', +) +@click.option("--db-host", help="Database Host") +@click.option("--db-port", type=int, help="Database Port") +@click.option( + "--db-root-username", + "--mariadb-root-username", + help='Root username for MariaDB or PostgreSQL, Default is "root"', +) +@click.option( + "--db-root-password", "--mariadb-root-password", help="Root password for MariaDB or PostgreSQL" +) +@click.option( + "--no-mariadb-socket", + is_flag=True, + default=False, + help="Set MariaDB host to % and use TCP/IP Socket instead of using the UNIX Socket", +) +@click.option("--admin-password", help="Administrator password for new site", default=None) +@click.option("--verbose", is_flag=True, default=False, help="Verbose") +@click.option( + "--force", help="Force restore if site/database already exists", is_flag=True, default=False +) +@click.option("--source_sql", help="Initiate database with a SQL file") +@click.option("--install-app", multiple=True, help="Install app after installation") +@click.option( + "--set-default", is_flag=True, default=False, help="Set the new site as default site" +) +def new_site( + site, + db_root_username=None, + db_root_password=None, + admin_password=None, + verbose=False, + install_apps=None, + source_sql=None, + force=None, + no_mariadb_socket=False, + install_app=None, + db_name=None, + db_password=None, + db_type=None, + db_host=None, + db_port=None, + set_default=False, +): "Create a new site" from frappe.installer import _new_site frappe.init(site=site, new_site=True) - _new_site(db_name, site, db_root_username=db_root_username, - db_root_password=db_root_password, admin_password=admin_password, - verbose=verbose, install_apps=install_app, source_sql=source_sql, force=force, - no_mariadb_socket=no_mariadb_socket, db_password=db_password, db_type=db_type, db_host=db_host, - db_port=db_port, new_site=True) + _new_site( + db_name, + site, + db_root_username=db_root_username, + db_root_password=db_root_password, + admin_password=admin_password, + verbose=verbose, + install_apps=install_app, + source_sql=source_sql, + force=force, + no_mariadb_socket=no_mariadb_socket, + db_password=db_password, + db_type=db_type, + db_host=db_host, + db_port=db_port, + new_site=True, + ) if set_default: use(site) -@click.command('restore') -@click.argument('sql-file-path') -@click.option('--db-root-username', '--mariadb-root-username', help='Root username for MariaDB or PostgreSQL, Default is "root"') -@click.option('--db-root-password', '--mariadb-root-password', help='Root password for MariaDB or PostgreSQL') -@click.option('--db-name', help='Database name for site in case it is a new one') -@click.option('--admin-password', help='Administrator password for new site') -@click.option('--install-app', multiple=True, help='Install app after installation') -@click.option('--with-public-files', help='Restores the public files of the site, given path to its tar file') -@click.option('--with-private-files', help='Restores the private files of the site, given path to its tar file') -@click.option('--force', is_flag=True, default=False, help='Ignore the validations and downgrade warnings. This action is not recommended') -@click.option('--encryption-key', help='Backup encryption key') +@click.command("restore") +@click.argument("sql-file-path") +@click.option( + "--db-root-username", + "--mariadb-root-username", + help='Root username for MariaDB or PostgreSQL, Default is "root"', +) +@click.option( + "--db-root-password", "--mariadb-root-password", help="Root password for MariaDB or PostgreSQL" +) +@click.option("--db-name", help="Database name for site in case it is a new one") +@click.option("--admin-password", help="Administrator password for new site") +@click.option("--install-app", multiple=True, help="Install app after installation") +@click.option( + "--with-public-files", help="Restores the public files of the site, given path to its tar file" +) +@click.option( + "--with-private-files", help="Restores the private files of the site, given path to its tar file" +) +@click.option( + "--force", + is_flag=True, + default=False, + help="Ignore the validations and downgrade warnings. This action is not recommended", +) +@click.option("--encryption-key", help="Backup encryption key") @pass_context -def restore(context, sql_file_path, encryption_key=None, db_root_username=None, db_root_password=None, - db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, - with_private_files=None): +def restore( + context, + sql_file_path, + encryption_key=None, + db_root_username=None, + db_root_password=None, + db_name=None, + verbose=None, + install_app=None, + admin_password=None, + force=None, + with_public_files=None, + with_private_files=None, +): "Restore site database from an sql file" from frappe.installer import ( _new_site, @@ -72,6 +144,7 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, validate_database_sql, ) from frappe.utils.backups import Backup + if not os.path.exists(sql_file_path): print("Invalid path", sql_file_path) sys.exit(1) @@ -87,11 +160,10 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, if is_partial(decompressed_file_name): click.secho( "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", - fg="red" + fg="red", ) click.secho( - "Use `bench partial-restore` to restore a partial backup to an existing site.", - fg="yellow" + "Use `bench partial-restore` to restore a partial backup to an existing site.", fg="yellow" ) _backup.decryption_rollback() sys.exit(1) @@ -99,26 +171,17 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, except UnicodeDecodeError: _backup.decryption_rollback() if encryption_key: - click.secho( - "Encrypted backup file detected. Decrypting using provided key.", - fg="yellow" - ) + click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") _backup.backup_decryption(encryption_key) else: - click.secho( - "Encrypted backup file detected. Decrypting using site config.", - fg="yellow" - ) + click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") encryption_key = frappe.get_site_config().encryption_key _backup.backup_decryption(encryption_key) # Rollback on unsuccessful decryrption if not os.path.exists(sql_file_path): - click.secho( - "Decryption failed. Please provide a valid key and try again.", - fg="red" - ) + click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") _backup.decryption_rollback() sys.exit(1) @@ -128,17 +191,14 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, if is_partial(decompressed_file_name): click.secho( "Partial Backup file detected. You cannot use a partial file to restore a Frappe site.", - fg="red" + fg="red", ) click.secho( - "Use `bench partial-restore` to restore a partial backup to an existing site.", - fg="yellow" + "Use `bench partial-restore` to restore a partial backup to an existing site.", fg="yellow" ) _backup.decryption_rollback() sys.exit(1) - - validate_database_sql(decompressed_file_name, _raise=not force) # dont allow downgrading to older versions of frappe without force @@ -149,13 +209,19 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, ) click.confirm(warn_message, abort=True) - - try: - _new_site(frappe.conf.db_name, site, db_root_username=db_root_username, - db_root_password=db_root_password, admin_password=admin_password, - verbose=context.verbose, install_apps=install_app, source_sql=decompressed_file_name, - force=True, db_type=frappe.conf.db_type) + _new_site( + frappe.conf.db_name, + site, + db_root_username=db_root_username, + db_root_password=db_root_password, + admin_password=admin_password, + verbose=context.verbose, + install_apps=install_app, + source_sql=decompressed_file_name, + force=True, + db_type=frappe.conf.db_type, + ) except Exception as err: print(err.args[1]) @@ -181,7 +247,6 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, os.remove(public) _backup.decryption_rollback() - if with_private_files: # Decrypt data if there is a Key if encryption_key: @@ -196,17 +261,17 @@ def restore(context, sql_file_path, encryption_key=None, db_root_username=None, _backup.decryption_rollback() success_message = "Site {0} has been restored{1}".format( - site, - " with files" if (with_public_files or with_private_files) else "" + site, " with files" if (with_public_files or with_private_files) else "" ) click.secho(success_message, fg="green") -@click.command('partial-restore') -@click.argument('sql-file-path') + +@click.command("partial-restore") +@click.argument("sql-file-path") @click.option("--verbose", "-v", is_flag=True) -@click.option('--encryption-key', help='Backup encryption key') +@click.option("--encryption-key", help="Backup encryption key") @pass_context -def partial_restore(context, sql_file_path, verbose, encryption_key=None): +def partial_restore(context, sql_file_path, verbose, encryption_key=None): from frappe.installer import extract_sql_from_archive, partial_restore from frappe.utils.backups import Backup @@ -228,40 +293,29 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): with open(decompressed_file_name) as f: header = " ".join(f.readline() for _ in range(5)) - #Check for full backup file + # Check for full backup file if "Partial Backup" not in header: click.secho( - "Full backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red" + "Full backup file detected.Use `bench restore` to restore a Frappe Site.", fg="red" ) _backup.decryption_rollback() sys.exit(1) - except UnicodeDecodeError: _backup.decryption_rollback() if encryption_key: - click.secho( - "Encrypted backup file detected. Decrypting using provided key.", - fg="yellow" - ) + click.secho("Encrypted backup file detected. Decrypting using provided key.", fg="yellow") key = encryption_key else: - click.secho( - "Encrypted backup file detected. Decrypting using site config.", - fg="yellow" - ) + click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") key = frappe.get_site_config().encryption_key _backup.backup_decryption(key) # Rollback on unsuccessful decryrption if not os.path.exists(sql_file_path): - click.secho( - "Decryption failed. Please provide a valid key and try again.", - fg="red" - ) + click.secho("Decryption failed. Please provide a valid key and try again.", fg="red") _backup.decryption_rollback() sys.exit(1) @@ -270,16 +324,14 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): with open(decompressed_file_name) as f: header = " ".join(f.readline() for _ in range(5)) - #Check for Full backup file. + # Check for Full backup file. if "Partial Backup" not in header: click.secho( - "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", - fg="red" + "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", fg="red" ) _backup.decryption_rollback() sys.exit(1) - partial_restore(sql_file_path, verbose) # Removing temporarily created file @@ -290,22 +342,33 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): frappe.destroy() -@click.command('reinstall') -@click.option('--admin-password', help='Administrator Password for reinstalled site') -@click.option('--db-root-username', '--mariadb-root-username', help='Root username for MariaDB or PostgreSQL, Default is "root"') -@click.option('--db-root-password', '--mariadb-root-password', help='Root password for MariaDB or PostgreSQL') -@click.option('--yes', is_flag=True, default=False, help='Pass --yes to skip confirmation') +@click.command("reinstall") +@click.option("--admin-password", help="Administrator Password for reinstalled site") +@click.option( + "--db-root-username", + "--mariadb-root-username", + help='Root username for MariaDB or PostgreSQL, Default is "root"', +) +@click.option( + "--db-root-password", "--mariadb-root-password", help="Root password for MariaDB or PostgreSQL" +) +@click.option("--yes", is_flag=True, default=False, help="Pass --yes to skip confirmation") @pass_context -def reinstall(context, admin_password=None, db_root_username=None, db_root_password=None, yes=False): +def reinstall( + context, admin_password=None, db_root_username=None, db_root_password=None, yes=False +): "Reinstall site ie. wipe all data and start over" site = get_site(context) _reinstall(site, admin_password, db_root_username, db_root_password, yes, verbose=context.verbose) -def _reinstall(site, admin_password=None, db_root_username=None, db_root_password=None, yes=False, verbose=False): + +def _reinstall( + site, admin_password=None, db_root_username=None, db_root_password=None, yes=False, verbose=False +): from frappe.installer import _new_site if not yes: - click.confirm('This will wipe your database. Are you sure you want to reinstall?', abort=True) + click.confirm("This will wipe your database. Are you sure you want to reinstall?", abort=True) try: frappe.init(site=site) frappe.connect() @@ -320,16 +383,26 @@ def _reinstall(site, admin_password=None, db_root_username=None, db_root_passwor frappe.destroy() frappe.init(site=site) - _new_site(frappe.conf.db_name, site, verbose=verbose, force=True, reinstall=True, install_apps=installed, - db_root_username=db_root_username, db_root_password=db_root_password, - admin_password=admin_password) + _new_site( + frappe.conf.db_name, + site, + verbose=verbose, + force=True, + reinstall=True, + install_apps=installed, + db_root_username=db_root_username, + db_root_password=db_root_password, + admin_password=admin_password, + ) -@click.command('install-app') -@click.argument('apps', nargs=-1) + +@click.command("install-app") +@click.argument("apps", nargs=-1) @pass_context def install_app(context, apps): "Install a new app to site, supports multiple apps" from frappe.installer import install_app as _install_app + exit_code = 0 if not context.sites: @@ -374,21 +447,15 @@ def list_apps(context, format): for site in context.sites: frappe.init(site=site) frappe.connect() - site_title = ( - click.style(f"{site}", fg="green") if len(context.sites) > 1 else "" - ) + site_title = click.style(f"{site}", fg="green") if len(context.sites) > 1 else "" apps = frappe.get_single("Installed Applications").installed_applications if apps: - name_len, ver_len = [ - max([len(x.get(y)) for x in apps]) - for y in ["app_name", "app_version"] - ] + name_len, ver_len = [max([len(x.get(y)) for x in apps]) for y in ["app_name", "app_version"]] template = "{{0:{0}}} {{1:{1}}} {{2}}".format(name_len, ver_len) installed_applications = [ - template.format(app.app_name, app.app_version, app.git_branch) - for app in apps + template.format(app.app_name, app.app_version, app.git_branch) for app in apps ] applications_summary = "\n".join(installed_applications) summary = f"{site_title}\n{applications_summary}\n" @@ -410,29 +477,31 @@ def list_apps(context, format): if format == "json": click.echo(frappe.as_json(summary_dict)) -@click.command('add-system-manager') -@click.argument('email') -@click.option('--first-name') -@click.option('--last-name') -@click.option('--password') -@click.option('--send-welcome-email', default=False, is_flag=True) + +@click.command("add-system-manager") +@click.argument("email") +@click.option("--first-name") +@click.option("--last-name") +@click.option("--password") +@click.option("--send-welcome-email", default=False, is_flag=True) @pass_context def add_system_manager(context, email, first_name, last_name, send_welcome_email, password): "Add a new system manager to a site" import frappe.utils.user + for site in context.sites: frappe.connect(site=site) try: - frappe.utils.user.add_system_manager(email, first_name, last_name, - send_welcome_email, password) + frappe.utils.user.add_system_manager(email, first_name, last_name, send_welcome_email, password) frappe.db.commit() finally: frappe.destroy() if not context.sites: raise SiteNotSpecifiedError -@click.command('disable-user') -@click.argument('email') + +@click.command("disable-user") +@click.argument("email") @pass_context def disable_user(context, email): site = get_site(context) @@ -443,9 +512,10 @@ def disable_user(context, email): user.save(ignore_permissions=True) frappe.db.commit() -@click.command('migrate') -@click.option('--skip-failing', is_flag=True, help="Skip patches that fail to run") -@click.option('--skip-search-index', is_flag=True, help="Skip search indexing for web documents") + +@click.command("migrate") +@click.option("--skip-failing", is_flag=True, help="Skip patches that fail to run") +@click.option("--skip-search-index", is_flag=True, help="Skip search indexing for web documents") @pass_context def migrate(context, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" @@ -463,12 +533,14 @@ def migrate(context, skip_failing=False, skip_search_index=False): if not context.sites: raise SiteNotSpecifiedError -@click.command('migrate-to') -@click.argument('frappe_provider') + +@click.command("migrate-to") +@click.argument("frappe_provider") @pass_context def migrate_to(context, frappe_provider): "Migrates site to the specified provider" from frappe.integrations.frappe_providers import migrate_to + for site in context.sites: frappe.init(site=site) frappe.connect() @@ -477,13 +549,15 @@ def migrate_to(context, frappe_provider): if not context.sites: raise SiteNotSpecifiedError -@click.command('run-patch') -@click.argument('module') -@click.option('--force', is_flag=True) + +@click.command("run-patch") +@click.argument("module") +@click.option("--force", is_flag=True) @pass_context def run_patch(context, module, force): "Run a particular patch" import frappe.modules.patch_handler + for site in context.sites: frappe.init(site=site) try: @@ -494,10 +568,11 @@ def run_patch(context, module, force): if not context.sites: raise SiteNotSpecifiedError -@click.command('reload-doc') -@click.argument('module') -@click.argument('doctype') -@click.argument('docname') + +@click.command("reload-doc") +@click.argument("module") +@click.argument("doctype") +@click.argument("docname") @pass_context def reload_doc(context, module, doctype, docname): "Reload schema for a DocType" @@ -512,8 +587,9 @@ def reload_doc(context, module, doctype, docname): if not context.sites: raise SiteNotSpecifiedError -@click.command('reload-doctype') -@click.argument('doctype') + +@click.command("reload-doctype") +@click.argument("doctype") @pass_context def reload_doctype(context, doctype): "Reload schema for a DocType" @@ -528,22 +604,25 @@ def reload_doctype(context, doctype): if not context.sites: raise SiteNotSpecifiedError -@click.command('add-to-hosts') + +@click.command("add-to-hosts") @pass_context def add_to_hosts(context): "Add site to hosts" for site in context.sites: - frappe.commands.popen('echo 127.0.0.1\t{0} | sudo tee -a /etc/hosts'.format(site)) + frappe.commands.popen("echo 127.0.0.1\t{0} | sudo tee -a /etc/hosts".format(site)) if not context.sites: raise SiteNotSpecifiedError -@click.command('use') -@click.argument('site') -def _use(site, sites_path='.'): + +@click.command("use") +@click.argument("site") +def _use(site, sites_path="."): "Set a default site" use(site, sites_path=sites_path) -def use(site, sites_path='.'): + +def use(site, sites_path="."): if os.path.exists(os.path.join(sites_path, site)): with open(os.path.join(sites_path, "currentsite.txt"), "w") as sitefile: sitefile.write(site) @@ -551,25 +630,55 @@ def use(site, sites_path='.'): else: print("Site {} does not exist".format(site)) -@click.command('backup') -@click.option('--with-files', default=False, is_flag=True, help="Take backup with files") -@click.option('--include', '--only', '-i', default="", type=str, help="Specify the DocTypes to backup seperated by commas") -@click.option('--exclude', '-e', default="", type=str, help="Specify the DocTypes to not backup seperated by commas") -@click.option('--backup-path', default=None, help="Set path for saving all the files in this operation") -@click.option('--backup-path-db', default=None, help="Set path for saving database file") -@click.option('--backup-path-files', default=None, help="Set path for saving public file") -@click.option('--backup-path-private-files', default=None, help="Set path for saving private file") -@click.option('--backup-path-conf', default=None, help="Set path for saving config file") -@click.option('--ignore-backup-conf', default=False, is_flag=True, help="Ignore excludes/includes set in config") -@click.option('--verbose', default=False, is_flag=True, help="Add verbosity") -@click.option('--compress', default=False, is_flag=True, help="Compress private and public files") + +@click.command("backup") +@click.option("--with-files", default=False, is_flag=True, help="Take backup with files") +@click.option( + "--include", + "--only", + "-i", + default="", + type=str, + help="Specify the DocTypes to backup seperated by commas", +) +@click.option( + "--exclude", + "-e", + default="", + type=str, + help="Specify the DocTypes to not backup seperated by commas", +) +@click.option( + "--backup-path", default=None, help="Set path for saving all the files in this operation" +) +@click.option("--backup-path-db", default=None, help="Set path for saving database file") +@click.option("--backup-path-files", default=None, help="Set path for saving public file") +@click.option("--backup-path-private-files", default=None, help="Set path for saving private file") +@click.option("--backup-path-conf", default=None, help="Set path for saving config file") +@click.option( + "--ignore-backup-conf", default=False, is_flag=True, help="Ignore excludes/includes set in config" +) +@click.option("--verbose", default=False, is_flag=True, help="Add verbosity") +@click.option("--compress", default=False, is_flag=True, help="Compress private and public files") @pass_context -def backup(context, with_files=False, backup_path=None, backup_path_db=None, backup_path_files=None, - backup_path_private_files=None, backup_path_conf=None, ignore_backup_conf=False, verbose=False, - compress=False, include="", exclude=""): +def backup( + context, + with_files=False, + backup_path=None, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + backup_path_conf=None, + ignore_backup_conf=False, + verbose=False, + compress=False, + include="", + exclude="", +): "Backup" from frappe.utils.backups import scheduled_backup + verbose = verbose or context.verbose exit_code = 0 @@ -589,12 +698,12 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac exclude_doctypes=exclude, compress=compress, verbose=verbose, - force=True + force=True, ) except Exception: click.secho( "Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), - fg="red" + fg="red", ) if verbose: print(frappe.get_traceback()) @@ -602,14 +711,15 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac continue if frappe.get_system_settings("encrypt_backup") and frappe.get_site_config().encryption_key: click.secho( - "Backup encryption is turned on. Please note the backup encryption key.", - fg="yellow" + "Backup encryption is turned on. Please note the backup encryption key.", fg="yellow" ) odb.print_summary() click.secho( - "Backup for Site {0} has been successfully completed{1}".format(site, " with files" if with_files else ""), - fg="green" + "Backup for Site {0} has been successfully completed{1}".format( + site, " with files" if with_files else "" + ), + fg="green", ) frappe.destroy() @@ -619,12 +729,13 @@ def backup(context, with_files=False, backup_path=None, backup_path_db=None, bac sys.exit(exit_code) -@click.command('remove-from-installed-apps') -@click.argument('app') +@click.command("remove-from-installed-apps") +@click.argument("app") @pass_context def remove_from_installed_apps(context, app): "Remove app from site's installed-apps list" from frappe.installer import remove_from_installed_apps + for site in context.sites: try: frappe.init(site=site) @@ -635,16 +746,26 @@ def remove_from_installed_apps(context, app): if not context.sites: raise SiteNotSpecifiedError -@click.command('uninstall-app') -@click.argument('app') -@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False) -@click.option('--dry-run', help='List all doctypes that will be deleted', is_flag=True, default=False) -@click.option('--no-backup', help='Do not backup the site', is_flag=True, default=False) -@click.option('--force', help='Force remove app from site', is_flag=True, default=False) + +@click.command("uninstall-app") +@click.argument("app") +@click.option( + "--yes", + "-y", + help="To bypass confirmation prompt for uninstalling the app", + is_flag=True, + default=False, +) +@click.option( + "--dry-run", help="List all doctypes that will be deleted", is_flag=True, default=False +) +@click.option("--no-backup", help="Do not backup the site", is_flag=True, default=False) +@click.option("--force", help="Force remove app from site", is_flag=True, default=False) @pass_context def uninstall(context, app, dry_run, yes, no_backup, force): "Remove app and linked modules from site" from frappe.installer import remove_app + for site in context.sites: try: frappe.init(site=site) @@ -656,18 +777,44 @@ def uninstall(context, app, dry_run, yes, no_backup, force): raise SiteNotSpecifiedError -@click.command('drop-site') -@click.argument('site') -@click.option('--db-root-username', '--mariadb-root-username', '--root-login', help='Root username for MariaDB or PostgreSQL, Default is "root"') -@click.option('--db-root-password', '--mariadb-root-password', '--root-password', help='Root password for MariaDB or PostgreSQL') -@click.option('--archived-sites-path') -@click.option('--no-backup', is_flag=True, default=False) -@click.option('--force', help='Force drop-site even if an error is encountered', is_flag=True, default=False) -def drop_site(site, db_root_username='root', db_root_password=None, archived_sites_path=None, force=False, no_backup=False): +@click.command("drop-site") +@click.argument("site") +@click.option( + "--db-root-username", + "--mariadb-root-username", + "--root-login", + help='Root username for MariaDB or PostgreSQL, Default is "root"', +) +@click.option( + "--db-root-password", + "--mariadb-root-password", + "--root-password", + help="Root password for MariaDB or PostgreSQL", +) +@click.option("--archived-sites-path") +@click.option("--no-backup", is_flag=True, default=False) +@click.option( + "--force", help="Force drop-site even if an error is encountered", is_flag=True, default=False +) +def drop_site( + site, + db_root_username="root", + db_root_password=None, + archived_sites_path=None, + force=False, + no_backup=False, +): _drop_site(site, db_root_username, db_root_password, archived_sites_path, force, no_backup) -def _drop_site(site, db_root_username=None, db_root_password=None, archived_sites_path=None, force=False, no_backup=False): +def _drop_site( + site, + db_root_username=None, + db_root_password=None, + archived_sites_path=None, + force=False, + no_backup=False, +): "Remove site from database and filesystem" from frappe.database import drop_user_and_database from frappe.utils.backups import scheduled_backup @@ -689,7 +836,7 @@ def _drop_site(site, db_root_username=None, db_root_password=None, archived_site "Error: The operation has stopped because backup of {0}'s database failed.".format(site), "Reason: {0}\n".format(str(err)), "Fix the issue and try again.", - "Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site) + "Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site), ] click.echo("\n".join(messages)) sys.exit(1) @@ -697,7 +844,9 @@ def _drop_site(site, db_root_username=None, db_root_password=None, archived_site click.secho("Dropping site database and user", fg="green") drop_user_and_database(frappe.conf.db_name, db_root_username, db_root_password) - archived_sites_path = archived_sites_path or os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived', 'sites') + archived_sites_path = archived_sites_path or os.path.join( + frappe.get_app_path("frappe"), "..", "..", "..", "archived", "sites" + ) os.makedirs(archived_sites_path, exist_ok=True) @@ -725,10 +874,12 @@ def move(dest_dir, site): return final_new_path -@click.command('set-password') -@click.argument('user') -@click.argument('password', required=False) -@click.option('--logout-all-sessions', help='Log out from all sessions', is_flag=True, default=False) +@click.command("set-password") +@click.argument("user") +@click.argument("password", required=False) +@click.option( + "--logout-all-sessions", help="Log out from all sessions", is_flag=True, default=False +) @pass_context def set_password(context, user, password=None, logout_all_sessions=False): "Set password for a user on a site" @@ -739,9 +890,11 @@ def set_password(context, user, password=None, logout_all_sessions=False): set_user_password(site, user, password, logout_all_sessions) -@click.command('set-admin-password') -@click.argument('admin-password', required=False) -@click.option('--logout-all-sessions', help='Log out from all sessions', is_flag=True, default=False) +@click.command("set-admin-password") +@click.argument("admin-password", required=False) +@click.option( + "--logout-all-sessions", help="Log out from all sessions", is_flag=True, default=False +) @pass_context def set_admin_password(context, admin_password=None, logout_all_sessions=False): "Set Administrator password for a site" @@ -775,8 +928,8 @@ def set_user_password(site, user, password, logout_all_sessions=False): frappe.destroy() -@click.command('set-last-active-for-user') -@click.option('--user', help="Setup last active date for user") +@click.command("set-last-active-for-user") +@click.option("--user", help="Setup last active date for user") @pass_context def set_last_active_for_user(context, user=None): "Set users last active date to current datetime" @@ -798,36 +951,45 @@ def set_last_active_for_user(context, user=None): frappe.db.commit() -@click.command('publish-realtime') -@click.argument('event') -@click.option('--message') -@click.option('--room') -@click.option('--user') -@click.option('--doctype') -@click.option('--docname') -@click.option('--after-commit') +@click.command("publish-realtime") +@click.argument("event") +@click.option("--message") +@click.option("--room") +@click.option("--user") +@click.option("--doctype") +@click.option("--docname") +@click.option("--after-commit") @pass_context def publish_realtime(context, event, message, room, user, doctype, docname, after_commit): "Publish realtime event from bench" from frappe import publish_realtime + for site in context.sites: try: frappe.init(site=site) frappe.connect() - publish_realtime(event, message=message, room=room, user=user, doctype=doctype, docname=docname, - after_commit=after_commit) + publish_realtime( + event, + message=message, + room=room, + user=user, + doctype=doctype, + docname=docname, + after_commit=after_commit, + ) frappe.db.commit() finally: frappe.destroy() if not context.sites: raise SiteNotSpecifiedError -@click.command('browse') -@click.argument('site', required=False) -@click.option('--user', required=False, help='Login as user') + +@click.command("browse") +@click.argument("site", required=False) +@click.option("--user", required=False, help="Login as user") @pass_context def browse(context, site, user=None): - '''Opens the site on web browser''' + """Opens the site on web browser""" from frappe.auth import CookieManager, LoginManager site = get_site(context, raise_err=False) or site @@ -842,29 +1004,30 @@ def browse(context, site, user=None): frappe.init(site=site) frappe.connect() - sid = '' + sid = "" if user: if frappe.conf.developer_mode or user == "Administrator": frappe.utils.set_request(path="/") frappe.local.cookie_manager = CookieManager() frappe.local.login_manager = LoginManager() frappe.local.login_manager.login_as(user) - sid = f'/app?sid={frappe.session.sid}' + sid = f"/app?sid={frappe.session.sid}" else: click.echo("Please enable developer mode to login as a user") - url = f'{frappe.utils.get_site_url(site)}{sid}' + url = f"{frappe.utils.get_site_url(site)}{sid}" if user == "Administrator": - click.echo(f'Login URL: {url}') + click.echo(f"Login URL: {url}") click.launch(url) -@click.command('start-recording') +@click.command("start-recording") @pass_context def start_recording(context): import frappe.recorder + for site in context.sites: frappe.init(site=site) frappe.set_user("Administrator") @@ -873,10 +1036,11 @@ def start_recording(context): raise SiteNotSpecifiedError -@click.command('stop-recording') +@click.command("stop-recording") @pass_context def stop_recording(context): import frappe.recorder + for site in context.sites: frappe.init(site=site) frappe.set_user("Administrator") @@ -884,8 +1048,11 @@ def stop_recording(context): if not context.sites: raise SiteNotSpecifiedError -@click.command('ngrok') -@click.option('--bind-tls', is_flag=True, default=False, help='Returns a reference to the https tunnel.') + +@click.command("ngrok") +@click.option( + "--bind-tls", is_flag=True, default=False, help="Returns a reference to the https tunnel." +) @pass_context def start_ngrok(context, bind_tls): from pyngrok import ngrok @@ -895,8 +1062,8 @@ def start_ngrok(context, bind_tls): port = frappe.conf.http_port or frappe.conf.webserver_port tunnel = ngrok.connect(addr=str(port), host_header=site, bind_tls=bind_tls) - print(f'Public URL: {tunnel.public_url}') - print('Inspect logs at http://localhost:4040') + print(f"Public URL: {tunnel.public_url}") + print("Inspect logs at http://localhost:4040") ngrok_process = ngrok.get_ngrok_process() try: @@ -907,15 +1074,17 @@ def start_ngrok(context, bind_tls): frappe.destroy() ngrok.kill() -@click.command('build-search-index') + +@click.command("build-search-index") @pass_context def build_search_index(context): from frappe.search.website_search import build_index_for_all_routes + site = get_site(context) if not site: raise SiteNotSpecifiedError - print('Building search index for {}'.format(site)) + print("Building search index for {}".format(site)) frappe.init(site=site) frappe.connect() try: @@ -923,10 +1092,13 @@ def build_search_index(context): finally: frappe.destroy() -@click.command('trim-database') -@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') -@click.option('--format', '-f', default='text', type=click.Choice(['json', 'text']), help='Output format') -@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site') + +@click.command("trim-database") +@click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted") +@click.option( + "--format", "-f", default="text", type=click.Choice(["json", "text"]), help="Output format" +) +@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the site") @pass_context def trim_database(context, dry_run, format, no_backup): if not context.sites: @@ -945,11 +1117,12 @@ def trim_database(context, dry_run, format, no_backup): information_schema = frappe.qb.Schema("information_schema") table_name = frappe.qb.Field("table_name").as_("name") - queried_result = frappe.qb.from_( - information_schema.tables - ).select(table_name).where( - information_schema.tables.table_schema == frappe.conf.db_name - ).run() + queried_result = ( + frappe.qb.from_(information_schema.tables) + .select(table_name) + .where(information_schema.tables.table_schema == frappe.conf.db_name) + .run() + ) database_tables = [x[0] for x in queried_result] doctype_tables = frappe.get_all("DocType", pluck="name") @@ -988,6 +1161,7 @@ def trim_database(context, dry_run, format, no_backup): if format == "json": import json + print(json.dumps(ALL_DATA, indent=1)) @@ -996,7 +1170,13 @@ def get_standard_tables(): tables = [] sql_file = os.path.join( - "..", "apps", "frappe", "frappe", "database", frappe.conf.db_type, f'framework_{frappe.conf.db_type}.sql' + "..", + "apps", + "frappe", + "frappe", + "database", + frappe.conf.db_type, + f"framework_{frappe.conf.db_type}.sql", ) content = open(sql_file).read().splitlines() @@ -1007,10 +1187,13 @@ def get_standard_tables(): return tables -@click.command('trim-tables') -@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted') -@click.option('--format', '-f', default='table', type=click.Choice(['json', 'table']), help='Output format') -@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site') + +@click.command("trim-tables") +@click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted") +@click.option( + "--format", "-f", default="table", type=click.Choice(["json", "table"]), help="Output format" +) +@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the site") @pass_context def trim_tables(context, dry_run, format, no_backup): if not context.sites: @@ -1029,21 +1212,24 @@ def trim_tables(context, dry_run, format, no_backup): odb.print_summary() try: - trimmed_data = trim_tables(dry_run=dry_run, quiet=format == 'json') + trimmed_data = trim_tables(dry_run=dry_run, quiet=format == "json") - if format == 'table' and not dry_run: - click.secho(f"The following data have been removed from {frappe.local.site}", fg='green') + if format == "table" and not dry_run: + click.secho(f"The following data have been removed from {frappe.local.site}", fg="green") handle_data(trimmed_data, format=format) finally: frappe.destroy() -def handle_data(data: dict, format='json'): - if format == 'json': + +def handle_data(data: dict, format="json"): + if format == "json": import json + print(json.dumps({frappe.local.site: data}, indent=1, sort_keys=True)) else: from frappe.utils.commands import render_table + data = [["DocType", "Fields"]] + [[table, ", ".join(columns)] for table, columns in data.items()] render_table(data) diff --git a/frappe/commands/translate.py b/frappe/commands/translate.py index 68d210eaaa..0b14e03002 100644 --- a/frappe/commands/translate.py +++ b/frappe/commands/translate.py @@ -1,13 +1,16 @@ import click -from frappe.commands import pass_context, get_site + +from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError + # translation -@click.command('build-message-files') +@click.command("build-message-files") @pass_context def build_message_files(context): "Build message files for translation" import frappe.translate + for site in context.sites: try: frappe.init(site=site) @@ -18,32 +21,41 @@ def build_message_files(context): if not context.sites: raise SiteNotSpecifiedError -@click.command('new-language') #, help="Create lang-code.csv for given app") + +@click.command("new-language") # , help="Create lang-code.csv for given app") @pass_context -@click.argument('lang_code') #, help="Language code eg. en") -@click.argument('app') #, help="App name eg. frappe") +@click.argument("lang_code") # , help="Language code eg. en") +@click.argument("app") # , help="App name eg. frappe") def new_language(context, lang_code, app): """Create lang-code.csv for given app""" import frappe.translate - if not context['sites']: - raise Exception('--site is required') + if not context["sites"]: + raise Exception("--site is required") # init site - frappe.connect(site=context['sites'][0]) + frappe.connect(site=context["sites"][0]) frappe.translate.write_translations_file(app, lang_code) - print("File created at ./apps/{app}/{app}/translations/{lang_code}.csv".format(app=app, lang_code=lang_code)) - print("You will need to add the language in frappe/geo/languages.json, if you haven't done it already.") + print( + "File created at ./apps/{app}/{app}/translations/{lang_code}.csv".format( + app=app, lang_code=lang_code + ) + ) + print( + "You will need to add the language in frappe/geo/languages.json, if you haven't done it already." + ) + -@click.command('get-untranslated') -@click.argument('lang') -@click.argument('untranslated_file') -@click.option('--all', default=False, is_flag=True, help='Get all message strings') +@click.command("get-untranslated") +@click.argument("lang") +@click.argument("untranslated_file") +@click.option("--all", default=False, is_flag=True, help="Get all message strings") @pass_context def get_untranslated(context, lang, untranslated_file, all=None): "Get untranslated strings for language" import frappe.translate + site = get_site(context) try: frappe.init(site=site) @@ -52,14 +64,16 @@ def get_untranslated(context, lang, untranslated_file, all=None): finally: frappe.destroy() -@click.command('update-translations') -@click.argument('lang') -@click.argument('untranslated_file') -@click.argument('translated-file') + +@click.command("update-translations") +@click.argument("lang") +@click.argument("untranslated_file") +@click.argument("translated-file") @pass_context def update_translations(context, lang, untranslated_file, translated_file): "Update translated strings" import frappe.translate + site = get_site(context) try: frappe.init(site=site) @@ -68,13 +82,15 @@ def update_translations(context, lang, untranslated_file, translated_file): finally: frappe.destroy() -@click.command('import-translations') -@click.argument('lang') -@click.argument('path') + +@click.command("import-translations") +@click.argument("lang") +@click.argument("path") @pass_context def import_translations(context, lang, path): "Update translated strings" import frappe.translate + site = get_site(context) try: frappe.init(site=site) @@ -83,6 +99,7 @@ def import_translations(context, lang, path): finally: frappe.destroy() + commands = [ build_message_files, get_untranslated, diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index c0bb44efab..2e27b8d6fe 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -8,9 +8,9 @@ import click import frappe from frappe.commands import get_site, pass_context -from frappe.exceptions import SiteNotSpecifiedError -from frappe.utils import update_progress_bar, cint from frappe.coverage import CodeCoverage +from frappe.exceptions import SiteNotSpecifiedError +from frappe.utils import cint, update_progress_bar DATA_IMPORT_DEPRECATION = ( "[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" @@ -18,25 +18,49 @@ DATA_IMPORT_DEPRECATION = ( ) -@click.command('build') -@click.option('--app', help='Build assets for app') -@click.option('--apps', help='Build assets for specific apps') -@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking') -@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking') -@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force') -@click.option('--production', is_flag=True, default=False, help='Build assets in production mode') -@click.option('--verbose', is_flag=True, default=False, help='Verbose') -@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') -def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False): +@click.command("build") +@click.option("--app", help="Build assets for app") +@click.option("--apps", help="Build assets for specific apps") +@click.option( + "--hard-link", is_flag=True, default=False, help="Copy the files instead of symlinking" +) +@click.option( + "--make-copy", + is_flag=True, + default=False, + help="[DEPRECATED] Copy the files instead of symlinking", +) +@click.option( + "--restore", + is_flag=True, + default=False, + help="[DEPRECATED] Copy the files instead of symlinking with force", +) +@click.option("--production", is_flag=True, default=False, help="Build assets in production mode") +@click.option("--verbose", is_flag=True, default=False, help="Verbose") +@click.option( + "--force", is_flag=True, default=False, help="Force build assets instead of downloading available" +) +def build( + app=None, + apps=None, + hard_link=False, + make_copy=False, + restore=False, + production=False, + verbose=False, + force=False, +): "Compile JS and CSS source files" from frappe.build import bundle, download_frappe_assets - frappe.init('') + + frappe.init("") if not apps and app: apps = app # dont try downloading assets if force used, app specified or running via CI - if not (force or apps or os.environ.get('CI')): + if not (force or apps or os.environ.get("CI")): # skip building frappe if assets exist remotely skip_frappe = download_frappe_assets(verbose=verbose) else: @@ -58,23 +82,24 @@ def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) - -@click.command('watch') -@click.option('--apps', help='Watch assets for specific apps') +@click.command("watch") +@click.option("--apps", help="Watch assets for specific apps") def watch(apps=None): "Watch and compile JS and CSS files as and when they change" from frappe.build import watch - frappe.init('') + + frappe.init("") watch(apps) -@click.command('clear-cache') +@click.command("clear-cache") @pass_context def clear_cache(context): "Clear cache, doctype cache and defaults" import frappe.sessions - from frappe.website.utils import clear_website_cache from frappe.desk.notifications import clear_notifications + from frappe.website.utils import clear_website_cache + for site in context.sites: try: frappe.connect(site) @@ -86,11 +111,13 @@ def clear_cache(context): if not context.sites: raise SiteNotSpecifiedError -@click.command('clear-website-cache') + +@click.command("clear-website-cache") @pass_context def clear_website_cache(context): "Clear website cache" from frappe.website.utils import clear_website_cache + for site in context.sites: try: frappe.init(site=site) @@ -101,12 +128,14 @@ def clear_website_cache(context): if not context.sites: raise SiteNotSpecifiedError -@click.command('destroy-all-sessions') -@click.option('--reason') + +@click.command("destroy-all-sessions") +@click.option("--reason") @pass_context def destroy_all_sessions(context, reason=None): "Clear sessions of all users (logs them out)" import frappe.sessions + for site in context.sites: try: frappe.init(site=site) @@ -118,7 +147,8 @@ def destroy_all_sessions(context, reason=None): if not context.sites: raise SiteNotSpecifiedError -@click.command('show-config') + +@click.command("show-config") @click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text") @pass_context def show_config(context, format): @@ -157,7 +187,7 @@ def show_config(context, format): if format == "text": data = transform_config(configuration) - data.insert(0, ['Config','Value']) + data.insert(0, ["Config", "Value"]) render_table(data) if format == "json": @@ -169,29 +199,33 @@ def show_config(context, format): click.echo(frappe.as_json(sites_config)) -@click.command('reset-perms') +@click.command("reset-perms") @pass_context def reset_perms(context): "Reset permissions for all doctypes" from frappe.permissions import reset_perms + for site in context.sites: try: frappe.init(site=site) frappe.connect() - for d in frappe.db.sql_list("""select name from `tabDocType` - where istable=0 and custom=0"""): - frappe.clear_cache(doctype=d) - reset_perms(d) + for d in frappe.db.sql_list( + """select name from `tabDocType` + where istable=0 and custom=0""" + ): + frappe.clear_cache(doctype=d) + reset_perms(d) finally: frappe.destroy() if not context.sites: raise SiteNotSpecifiedError -@click.command('execute') -@click.argument('method') -@click.option('--args') -@click.option('--kwargs') -@click.option('--profile', is_flag=True, default=False) + +@click.command("execute") +@click.argument("method") +@click.option("--args") +@click.option("--kwargs") +@click.option("--profile", is_flag=True, default=False) @pass_context def execute(context, method, args=None, kwargs=None, profile=False): "Execute a function" @@ -216,13 +250,16 @@ def execute(context, method, args=None, kwargs=None, profile=False): if profile: import cProfile + pr = cProfile.Profile() pr.enable() try: ret = frappe.get_attr(method)(*args, **kwargs) except Exception: - ret = frappe.safe_eval(method + "(*args, **kwargs)", eval_globals=globals(), eval_locals=locals()) + ret = frappe.safe_eval( + method + "(*args, **kwargs)", eval_globals=globals(), eval_locals=locals() + ) if profile: import pstats @@ -230,7 +267,7 @@ def execute(context, method, args=None, kwargs=None, profile=False): pr.disable() s = StringIO() - pstats.Stats(pr, stream=s).sort_stats('cumulative').print_stats(.5) + pstats.Stats(pr, stream=s).sort_stats("cumulative").print_stats(0.5) print(s.getvalue()) if frappe.db: @@ -239,14 +276,15 @@ def execute(context, method, args=None, kwargs=None, profile=False): frappe.destroy() if ret: from frappe.utils.response import json_handler + print(json.dumps(ret, default=json_handler)) if not context.sites: raise SiteNotSpecifiedError -@click.command('add-to-email-queue') -@click.argument('email-path') +@click.command("add-to-email-queue") +@click.argument("email-path") @pass_context def add_to_email_queue(context, email_path): "Add an email to the Email Queue" @@ -258,18 +296,19 @@ def add_to_email_queue(context, email_path): for email in os.listdir(email_path): with open(os.path.join(email_path, email)) as email_data: kwargs = json.load(email_data) - kwargs['delayed'] = True + kwargs["delayed"] = True frappe.sendmail(**kwargs) frappe.db.commit() -@click.command('export-doc') -@click.argument('doctype') -@click.argument('docname') +@click.command("export-doc") +@click.argument("doctype") +@click.argument("docname") @pass_context def export_doc(context, doctype, docname): "Export a single document to csv" import frappe.modules + for site in context.sites: try: frappe.init(site=site) @@ -280,14 +319,16 @@ def export_doc(context, doctype, docname): if not context.sites: raise SiteNotSpecifiedError -@click.command('export-json') -@click.argument('doctype') -@click.argument('path') -@click.option('--name', help='Export only one document') + +@click.command("export-json") +@click.argument("doctype") +@click.argument("path") +@click.option("--name", help="Export only one document") @pass_context def export_json(context, doctype, path, name=None): "Export doclist as json to the given path, use '-' as name for Singles." from frappe.core.doctype.data_import.data_import import export_json + for site in context.sites: try: frappe.init(site=site) @@ -298,13 +339,15 @@ def export_json(context, doctype, path, name=None): if not context.sites: raise SiteNotSpecifiedError -@click.command('export-csv') -@click.argument('doctype') -@click.argument('path') + +@click.command("export-csv") +@click.argument("doctype") +@click.argument("path") @pass_context def export_csv(context, doctype, path): "Export data import template with data for DocType" from frappe.core.doctype.data_import.data_import import export_csv + for site in context.sites: try: frappe.init(site=site) @@ -315,12 +358,14 @@ def export_csv(context, doctype, path): if not context.sites: raise SiteNotSpecifiedError -@click.command('export-fixtures') -@click.option('--app', default=None, help='Export fixtures of a specific app') + +@click.command("export-fixtures") +@click.option("--app", default=None, help="Export fixtures of a specific app") @pass_context def export_fixtures(context, app=None): "Export fixtures" from frappe.utils.fixtures import export_fixtures + for site in context.sites: try: frappe.init(site=site) @@ -331,17 +376,18 @@ def export_fixtures(context, app=None): if not context.sites: raise SiteNotSpecifiedError -@click.command('import-doc') -@click.argument('path') + +@click.command("import-doc") +@click.argument("path") @pass_context def import_doc(context, path, force=False): "Import (insert/update) doclist. If the argument is a directory, all files ending with .json are imported" from frappe.core.doctype.data_import.data_import import import_doc if not os.path.exists(path): - path = os.path.join('..', path) + path = os.path.join("..", path) if not os.path.exists(path): - print('Invalid path {0}'.format(path)) + print("Invalid path {0}".format(path)) sys.exit(1) for site in context.sites: @@ -355,28 +401,57 @@ def import_doc(context, path, force=False): raise SiteNotSpecifiedError -@click.command('import-csv', help=DATA_IMPORT_DEPRECATION) -@click.argument('path') -@click.option('--only-insert', default=False, is_flag=True, help='Do not overwrite existing records') -@click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it') -@click.option('--ignore-encoding-errors', default=False, is_flag=True, help='Ignore encoding errors while coverting to unicode') -@click.option('--no-email', default=True, is_flag=True, help='Send email if applicable') +@click.command("import-csv", help=DATA_IMPORT_DEPRECATION) +@click.argument("path") +@click.option( + "--only-insert", default=False, is_flag=True, help="Do not overwrite existing records" +) +@click.option( + "--submit-after-import", default=False, is_flag=True, help="Submit document after importing it" +) +@click.option( + "--ignore-encoding-errors", + default=False, + is_flag=True, + help="Ignore encoding errors while coverting to unicode", +) +@click.option("--no-email", default=True, is_flag=True, help="Send email if applicable") @pass_context -def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): +def import_csv( + context, + path, + only_insert=False, + submit_after_import=False, + ignore_encoding_errors=False, + no_email=True, +): click.secho(DATA_IMPORT_DEPRECATION, fg="yellow") sys.exit(1) -@click.command('data-import') -@click.option('--file', 'file_path', type=click.Path(), required=True, help="Path to import file (.csv, .xlsx)") -@click.option('--doctype', type=str, required=True) -@click.option('--type', 'import_type', type=click.Choice(['Insert', 'Update'], case_sensitive=False), default='Insert', help="Insert New Records or Update Existing Records") -@click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it') -@click.option('--mute-emails', default=True, is_flag=True, help='Mute emails during import') +@click.command("data-import") +@click.option( + "--file", "file_path", type=click.Path(), required=True, help="Path to import file (.csv, .xlsx)" +) +@click.option("--doctype", type=str, required=True) +@click.option( + "--type", + "import_type", + type=click.Choice(["Insert", "Update"], case_sensitive=False), + default="Insert", + help="Insert New Records or Update Existing Records", +) +@click.option( + "--submit-after-import", default=False, is_flag=True, help="Submit document after importing it" +) +@click.option("--mute-emails", default=True, is_flag=True, help="Mute emails during import") @pass_context -def data_import(context, file_path, doctype, import_type=None, submit_after_import=False, mute_emails=True): +def data_import( + context, file_path, doctype, import_type=None, submit_after_import=False, mute_emails=True +): "Import documents in bulk from CSV or XLSX using data import" from frappe.core.doctype.data_import.data_import import import_file + site = get_site(context) frappe.init(site=site) @@ -385,9 +460,9 @@ def data_import(context, file_path, doctype, import_type=None, submit_after_impo frappe.destroy() -@click.command('bulk-rename') -@click.argument('doctype') -@click.argument('path') +@click.command("bulk-rename") +@click.argument("doctype") +@click.argument("path") @pass_context def bulk_rename(context, doctype, path): "Rename multiple records via CSV file" @@ -396,22 +471,22 @@ def bulk_rename(context, doctype, path): site = get_site(context) - with open(path, 'r') as csvfile: + with open(path, "r") as csvfile: rows = read_csv_content(csvfile.read()) frappe.init(site=site) frappe.connect() - bulk_rename(doctype, rows, via_console = True) + bulk_rename(doctype, rows, via_console=True) frappe.destroy() -@click.command('db-console') +@click.command("db-console") @pass_context def database(context): """ - Enter into the Database console for given site. + Enter into the Database console for given site. """ site = get_site(context) if not site: @@ -423,69 +498,79 @@ def database(context): _psql() -@click.command('mariadb') +@click.command("mariadb") @pass_context def mariadb(context): """ - Enter into mariadb console for a given site. + Enter into mariadb console for a given site. """ - site = get_site(context) + site = get_site(context) if not site: raise SiteNotSpecifiedError frappe.init(site=site) _mariadb() -@click.command('postgres') +@click.command("postgres") @pass_context def postgres(context): """ - Enter into postgres console for a given site. + Enter into postgres console for a given site. """ - site = get_site(context) + site = get_site(context) frappe.init(site=site) _psql() def _mariadb(): - mysql = find_executable('mysql') - os.execv(mysql, [ + mysql = find_executable("mysql") + os.execv( mysql, - '-u', frappe.conf.db_name, - '-p'+frappe.conf.db_password, - frappe.conf.db_name, - '-h', frappe.conf.db_host or "localhost", - '--pager=less -SFX', - '--safe-updates', - "-A"]) + [ + mysql, + "-u", + frappe.conf.db_name, + "-p" + frappe.conf.db_password, + frappe.conf.db_name, + "-h", + frappe.conf.db_host or "localhost", + "--pager=less -SFX", + "--safe-updates", + "-A", + ], + ) def _psql(): - psql = find_executable('psql') - subprocess.run([ psql, '-d', frappe.conf.db_name]) + psql = find_executable("psql") + subprocess.run([psql, "-d", frappe.conf.db_name]) -@click.command('jupyter') +@click.command("jupyter") @pass_context def jupyter(context): - installed_packages = (r.split('==')[0] for r in subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'], encoding='utf8')) + installed_packages = ( + r.split("==")[0] + for r in subprocess.check_output([sys.executable, "-m", "pip", "freeze"], encoding="utf8") + ) - if 'jupyter' not in installed_packages: - subprocess.check_output([sys.executable, '-m', 'pip', 'install', 'jupyter']) + if "jupyter" not in installed_packages: + subprocess.check_output([sys.executable, "-m", "pip", "install", "jupyter"]) site = get_site(context) frappe.init(site=site) - jupyter_notebooks_path = os.path.abspath(frappe.get_site_path('jupyter_notebooks')) - sites_path = os.path.abspath(frappe.get_site_path('..')) + jupyter_notebooks_path = os.path.abspath(frappe.get_site_path("jupyter_notebooks")) + sites_path = os.path.abspath(frappe.get_site_path("..")) try: os.stat(jupyter_notebooks_path) except OSError: - print('Creating folder to keep jupyter notebooks at {}'.format(jupyter_notebooks_path)) + print("Creating folder to keep jupyter notebooks at {}".format(jupyter_notebooks_path)) os.mkdir(jupyter_notebooks_path) - bin_path = os.path.abspath('../env/bin') - print(''' + bin_path = os.path.abspath("../env/bin") + print( + """ Starting Jupyter notebook Run the following in your first cell to connect notebook to frappe ``` @@ -495,12 +580,18 @@ frappe.connect() frappe.local.lang = frappe.db.get_default('lang') frappe.db.connect() ``` - '''.format(site=site, sites_path=sites_path)) - os.execv('{0}/jupyter'.format(bin_path), [ - '{0}/jupyter'.format(bin_path), - 'notebook', - jupyter_notebooks_path, - ]) + """.format( + site=site, sites_path=sites_path + ) + ) + os.execv( + "{0}/jupyter".format(bin_path), + [ + "{0}/jupyter".format(bin_path), + "notebook", + jupyter_notebooks_path, + ], + ) def _console_cleanup(): @@ -509,12 +600,8 @@ def _console_cleanup(): frappe.destroy() -@click.command('console') -@click.option( - '--autoreload', - is_flag=True, - help="Reload changes to code automatically" -) +@click.command("console") +@click.option("--autoreload", is_flag=True, help="Reload changes to code automatically") @pass_context def console(context, autoreload=False): "Start ipython console for a site" @@ -523,9 +610,10 @@ def console(context, autoreload=False): frappe.connect() frappe.local.lang = frappe.db.get_default("lang") - from IPython.terminal.embed import InteractiveShellEmbed from atexit import register + from IPython.terminal.embed import InteractiveShellEmbed + register(_console_cleanup) terminal = InteractiveShellEmbed() @@ -552,11 +640,27 @@ def console(context, autoreload=False): terminal() -@click.command('transform-database', help="Change tables' internal settings changing engine and row formats") -@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'") -@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)") -@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)") -@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred") +@click.command( + "transform-database", help="Change tables' internal settings changing engine and row formats" +) +@click.option( + "--table", + required=True, + help="Comma separated name of tables to convert. To convert all tables, pass 'all'", +) +@click.option( + "--engine", + default=None, + type=click.Choice(["InnoDB", "MyISAM"]), + help="Choice of storage engine for said table(s)", +) +@click.option( + "--row_format", + default=None, + type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), + help="Set ROW_FORMAT parameter for said table(s)", +) +@click.option("--failfast", is_flag=True, default=False, help="Exit on first failure occurred") @pass_context def transform_database(context, table, engine, row_format, failfast): "Transform site database through given parameters" @@ -578,12 +682,15 @@ def transform_database(context, table, engine, row_format, failfast): if table == "all": information_schema = frappe.qb.Schema("information_schema") - queried_tables = frappe.qb.from_( - information_schema.tables - ).select("table_name").where( - (information_schema.tables.row_format != row_format) - & (information_schema.tables.table_schema == frappe.conf.db_name) - ).run() + queried_tables = ( + frappe.qb.from_(information_schema.tables) + .select("table_name") + .where( + (information_schema.tables.row_format != row_format) + & (information_schema.tables.table_schema == frappe.conf.db_name) + ) + .run() + ) tables = [x[0] for x in queried_tables] else: tables = [x.strip() for x in table.split(",")] @@ -620,37 +727,58 @@ def transform_database(context, table, engine, row_format, failfast): frappe.destroy() -@click.command('run-tests') -@click.option('--app', help="For App") -@click.option('--doctype', help="For DocType") -@click.option('--case', help="Select particular TestCase") -@click.option('--doctype-list-path', help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt") -@click.option('--test', multiple=True, help="Specific test") -@click.option('--ui-tests', is_flag=True, default=False, help="Run UI Tests") -@click.option('--module', help="Run tests in a module") -@click.option('--profile', is_flag=True, default=False) -@click.option('--coverage', is_flag=True, default=False) -@click.option('--skip-test-records', is_flag=True, default=False, help="Don't create test records") -@click.option('--skip-before-tests', is_flag=True, default=False, help="Don't run before tests hook") -@click.option('--junit-xml-output', help="Destination file path for junit xml report") -@click.option('--failfast', is_flag=True, default=False, help="Stop the test run on the first error or failure") +@click.command("run-tests") +@click.option("--app", help="For App") +@click.option("--doctype", help="For DocType") +@click.option("--case", help="Select particular TestCase") +@click.option( + "--doctype-list-path", + help="Path to .txt file for list of doctypes. Example erpnext/tests/server/agriculture.txt", +) +@click.option("--test", multiple=True, help="Specific test") +@click.option("--ui-tests", is_flag=True, default=False, help="Run UI Tests") +@click.option("--module", help="Run tests in a module") +@click.option("--profile", is_flag=True, default=False) +@click.option("--coverage", is_flag=True, default=False) +@click.option("--skip-test-records", is_flag=True, default=False, help="Don't create test records") +@click.option( + "--skip-before-tests", is_flag=True, default=False, help="Don't run before tests hook" +) +@click.option("--junit-xml-output", help="Destination file path for junit xml report") +@click.option( + "--failfast", is_flag=True, default=False, help="Stop the test run on the first error or failure" +) @pass_context -def run_tests(context, app=None, module=None, doctype=None, test=(), profile=False, - coverage=False, junit_xml_output=False, ui_tests = False, doctype_list_path=None, - skip_test_records=False, skip_before_tests=False, failfast=False, case=None): +def run_tests( + context, + app=None, + module=None, + doctype=None, + test=(), + profile=False, + coverage=False, + junit_xml_output=False, + ui_tests=False, + doctype_list_path=None, + skip_test_records=False, + skip_before_tests=False, + failfast=False, + case=None, +): with CodeCoverage(coverage, app): import frappe import frappe.test_runner + tests = test site = get_site(context) allow_tests = frappe.get_conf(site).allow_tests - if not (allow_tests or os.environ.get('CI')): - click.secho('Testing is disabled for the site!', bold=True) - click.secho('You can enable tests by entering following command:') - click.secho('bench --site {0} set-config allow_tests true'.format(site), fg='green') + if not (allow_tests or os.environ.get("CI")): + click.secho("Testing is disabled for the site!", bold=True) + click.secho("You can enable tests by entering following command:") + click.secho("bench --site {0} set-config allow_tests true".format(site), fg="green") return frappe.init(site=site) @@ -658,51 +786,70 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal frappe.flags.skip_before_tests = skip_before_tests frappe.flags.skip_test_records = skip_test_records - ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, - force=context.force, profile=profile, junit_xml_output=junit_xml_output, - ui_tests=ui_tests, doctype_list_path=doctype_list_path, failfast=failfast, case=case) + ret = frappe.test_runner.main( + app, + module, + doctype, + context.verbose, + tests=tests, + force=context.force, + profile=profile, + junit_xml_output=junit_xml_output, + ui_tests=ui_tests, + doctype_list_path=doctype_list_path, + failfast=failfast, + case=case, + ) if len(ret.failures) == 0 and len(ret.errors) == 0: ret = 0 - if os.environ.get('CI'): + if os.environ.get("CI"): sys.exit(ret) -@click.command('run-parallel-tests') -@click.option('--app', help="For App", default='frappe') -@click.option('--build-number', help="Build number", default=1) -@click.option('--total-builds', help="Total number of builds", default=1) -@click.option('--with-coverage', is_flag=True, help="Build coverage file") -@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests") + +@click.command("run-parallel-tests") +@click.option("--app", help="For App", default="frappe") +@click.option("--build-number", help="Build number", default=1) +@click.option("--total-builds", help="Total number of builds", default=1) +@click.option("--with-coverage", is_flag=True, help="Build coverage file") +@click.option("--use-orchestrator", is_flag=True, help="Use orchestrator to run parallel tests") @pass_context -def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False): +def run_parallel_tests( + context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False +): with CodeCoverage(with_coverage, app): site = get_site(context) if use_orchestrator: from frappe.parallel_test_runner import ParallelTestWithOrchestrator + ParallelTestWithOrchestrator(app, site=site) else: from frappe.parallel_test_runner import ParallelTestRunner + ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds) -@click.command('run-ui-tests') -@click.argument('app') -@click.option('--headless', is_flag=True, help="Run UI Test in headless mode") -@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode") -@click.option('--with-coverage', is_flag=True, help="Generate coverage report") -@click.option('--ci-build-id') + +@click.command("run-ui-tests") +@click.argument("app") +@click.option("--headless", is_flag=True, help="Run UI Test in headless mode") +@click.option("--parallel", is_flag=True, help="Run UI Test in parallel mode") +@click.option("--with-coverage", is_flag=True, help="Generate coverage report") +@click.option("--ci-build-id") @pass_context -def run_ui_tests(context, app, headless=False, parallel=True, with_coverage=False, ci_build_id=None): +def run_ui_tests( + context, app, headless=False, parallel=True, with_coverage=False, ci_build_id=None +): "Run UI tests" site = get_site(context) - app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..')) + app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), "..")) site_url = frappe.utils.get_site_url(site) admin_password = frappe.get_conf(site).admin_password # override baseUrl using env variable - site_env = f'CYPRESS_baseUrl={site_url}' - password_env = f'CYPRESS_adminPassword={admin_password}' if admin_password else '' - coverage_env = f'CYPRESS_coverage={str(with_coverage).lower()}' + site_env = f"CYPRESS_baseUrl={site_url}" + password_env = f"CYPRESS_adminPassword={admin_password}" if admin_password else "" + coverage_env = f"CYPRESS_coverage={str(with_coverage).lower()}" os.chdir(app_base_path) @@ -722,30 +869,41 @@ def run_ui_tests(context, app, headless=False, parallel=True, with_coverage=Fals ): # install cypress click.secho("Installing Cypress...", fg="yellow") - frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile") + frappe.commands.popen( + "yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile" + ) # run for headless mode - run_or_open = 'run --browser chrome --record' if headless else 'open' - formatted_command = f'{site_env} {password_env} {coverage_env} {cypress_path} {run_or_open}' + run_or_open = "run --browser chrome --record" if headless else "open" + formatted_command = f"{site_env} {password_env} {coverage_env} {cypress_path} {run_or_open}" if parallel: - formatted_command += ' --parallel' + formatted_command += " --parallel" if ci_build_id: - formatted_command += f' --ci-build-id {ci_build_id}' + formatted_command += f" --ci-build-id {ci_build_id}" click.secho("Running Cypress...", fg="yellow") frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True) -@click.command('serve') -@click.option('--port', default=8000) -@click.option('--profile', is_flag=True, default=False) -@click.option('--noreload', "no_reload", is_flag=True, default=False) -@click.option('--nothreading', "no_threading", is_flag=True, default=False) -@click.option('--with-coverage', is_flag=True, default=False) +@click.command("serve") +@click.option("--port", default=8000) +@click.option("--profile", is_flag=True, default=False) +@click.option("--noreload", "no_reload", is_flag=True, default=False) +@click.option("--nothreading", "no_threading", is_flag=True, default=False) +@click.option("--with-coverage", is_flag=True, default=False) @pass_context -def serve(context, port=None, profile=False, no_reload=False, no_threading=False, sites_path='.', site=None, with_coverage=False): +def serve( + context, + port=None, + profile=False, + no_reload=False, + no_threading=False, + sites_path=".", + site=None, + with_coverage=False, +): "Start development web server" import frappe.app @@ -753,22 +911,30 @@ def serve(context, port=None, profile=False, no_reload=False, no_threading=False site = None else: site = context.sites[0] - with CodeCoverage(with_coverage, 'frappe'): + with CodeCoverage(with_coverage, "frappe"): if with_coverage: # unable to track coverage with threading enabled no_threading = True no_reload = True - frappe.app.serve(port=port, profile=profile, no_reload=no_reload, no_threading=no_threading, site=site, sites_path='.') + frappe.app.serve( + port=port, + profile=profile, + no_reload=no_reload, + no_threading=no_threading, + site=site, + sites_path=".", + ) -@click.command('request') -@click.option('--args', help='arguments like `?cmd=test&key=value` or `/api/request/method?..`') -@click.option('--path', help='path to request JSON') +@click.command("request") +@click.option("--args", help="arguments like `?cmd=test&key=value` or `/api/request/method?..`") +@click.option("--path", help="path to request JSON") @pass_context def request(context, args=None, path=None): "Run a request as an admin" - import frappe.handler import frappe.api + import frappe.handler + for site in context.sites: try: frappe.init(site=site) @@ -782,7 +948,7 @@ def request(context, args=None, path=None): if args.startswith("/api/method"): frappe.local.form_dict.cmd = args.split("?")[0].split("/")[-1] elif path: - with open(os.path.join('..', path), 'r') as f: + with open(os.path.join("..", path), "r") as f: args = json.loads(f.read()) frappe.local.form_dict = frappe._dict(args) @@ -795,22 +961,28 @@ def request(context, args=None, path=None): if not context.sites: raise SiteNotSpecifiedError -@click.command('make-app') -@click.argument('destination') -@click.argument('app_name') -@click.option('--no-git', is_flag=True, default=False, help='Do not initialize git repository for the app') + +@click.command("make-app") +@click.argument("destination") +@click.argument("app_name") +@click.option( + "--no-git", is_flag=True, default=False, help="Do not initialize git repository for the app" +) def make_app(destination, app_name, no_git=False): "Creates a boilerplate app" from frappe.utils.boilerplate import make_boilerplate + make_boilerplate(destination, app_name, no_git=no_git) -@click.command('set-config') -@click.argument('key') -@click.argument('value') -@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config') -@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object') -@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object') +@click.command("set-config") +@click.argument("key") +@click.argument("value") +@click.option( + "-g", "--global", "global_", is_flag=True, default=False, help="Set value in bench config" +) +@click.option("-p", "--parse", is_flag=True, default=False, help="Evaluate as Python Object") +@click.option("--as-dict", is_flag=True, default=False, help="Legacy: Evaluate as Python Object") @pass_context def set_config(context, key, value, global_=False, parse=False, as_dict=False): "Insert/Update a value in site_config.json" @@ -818,16 +990,20 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): if as_dict: from frappe.utils.commands import warn - warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning) + + warn( + "--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning + ) parse = as_dict if parse: import ast + value = ast.literal_eval(value) if global_: sites_path = os.getcwd() - common_site_config_path = os.path.join(sites_path, 'common_site_config.json') + common_site_config_path = os.path.join(sites_path, "common_site_config.json") update_site_config(key, value, validate=False, site_config_path=common_site_config_path) else: for site in context.sites: @@ -837,13 +1013,20 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): @click.command("version") -@click.option("-f", "--format", "output", - type=click.Choice(["plain", "table", "json", "legacy"]), help="Output format", default="legacy") +@click.option( + "-f", + "--format", + "output", + type=click.Choice(["plain", "table", "json", "legacy"]), + help="Output format", + default="legacy", +) def get_version(output): """Show the versions of all the installed apps.""" from git import Repo - from frappe.utils.commands import render_table + from frappe.utils.change_log import get_app_branch + from frappe.utils.commands import render_table frappe.init("") data = [] @@ -862,32 +1045,33 @@ def get_version(output): data.append(app_info) { - "legacy": lambda: [ - click.echo(f"{app_info.app} {app_info.version}") - for app_info in data - ], + "legacy": lambda: [click.echo(f"{app_info.app} {app_info.version}") for app_info in data], "plain": lambda: [ click.echo(f"{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})") for app_info in data ], "table": lambda: render_table( - [["App", "Version", "Branch", "Commit"]] + - [ - [app_info.app, app_info.version, app_info.branch, app_info.commit] - for app_info in data - ] + [["App", "Version", "Branch", "Commit"]] + + [[app_info.app, app_info.version, app_info.branch, app_info.commit] for app_info in data] ), "json": lambda: click.echo(json.dumps(data, indent=4)), }[output]() -@click.command('rebuild-global-search') -@click.option('--static-pages', is_flag=True, default=False, help='Rebuild global search for static pages') +@click.command("rebuild-global-search") +@click.option( + "--static-pages", is_flag=True, default=False, help="Rebuild global search for static pages" +) @pass_context def rebuild_global_search(context, static_pages=False): - '''Setup help table in the current site (called after migrate)''' - from frappe.utils.global_search import (get_doctypes_with_global_search, rebuild_for_doctype, - get_routes_to_index, add_route_to_global_search, sync_global_search) + """Setup help table in the current site (called after migrate)""" + from frappe.utils.global_search import ( + add_route_to_global_search, + get_doctypes_with_global_search, + get_routes_to_index, + rebuild_for_doctype, + sync_global_search, + ) for site in context.sites: try: @@ -899,13 +1083,13 @@ def rebuild_global_search(context, static_pages=False): for i, route in enumerate(routes): add_route_to_global_search(route) frappe.local.request = None - update_progress_bar('Rebuilding Global Search', i, len(routes)) + update_progress_bar("Rebuilding Global Search", i, len(routes)) sync_global_search() else: doctypes = get_doctypes_with_global_search() for i, doctype in enumerate(doctypes): rebuild_for_doctype(doctype) - update_progress_bar('Rebuilding Global Search', i, len(doctypes)) + update_progress_bar("Rebuilding Global Search", i, len(doctypes)) finally: frappe.destroy() @@ -945,5 +1129,5 @@ commands = [ bulk_rename, add_to_email_queue, rebuild_global_search, - run_parallel_tests + run_parallel_tests, ] diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index aa441b7d71..ebd75cd70a 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -1,14 +1,20 @@ import frappe from frappe import _ -from frappe.desk.moduleview import (get_data, get_onboard_items, config_exists, get_module_link_items_from_list) +from frappe.desk.moduleview import ( + config_exists, + get_data, + get_module_link_items_from_list, + get_onboard_items, +) + def get_modules_from_all_apps_for_user(user=None): if not user: user = frappe.session.user all_modules = get_modules_from_all_apps() - global_blocked_modules = frappe.get_doc('User', 'Administrator').get_blocked_modules() - user_blocked_modules = frappe.get_doc('User', user).get_blocked_modules() + global_blocked_modules = frappe.get_doc("User", "Administrator").get_blocked_modules() + user_blocked_modules = frappe.get_doc("User", user).get_blocked_modules() blocked_modules = global_blocked_modules + user_blocked_modules allowed_modules_list = [m for m in all_modules if m.get("module_name") not in blocked_modules] @@ -22,31 +28,31 @@ def get_modules_from_all_apps_for_user(user=None): module["onboard_present"] = 1 # Set defaults links - module["links"] = get_onboard_items(module["app"], frappe.scrub(module_name))[:5] + module["links"] = get_onboard_items(module["app"], frappe.scrub(module_name))[:5] return allowed_modules_list + def get_modules_from_all_apps(): modules_list = [] for app in frappe.get_installed_apps(): modules_list += get_modules_from_app(app) return modules_list + def get_modules_from_app(app): - return frappe.get_all('Module Def', - filters={'app_name': app}, - fields=['module_name', 'app_name as app'] + return frappe.get_all( + "Module Def", filters={"app_name": app}, fields=["module_name", "app_name as app"] ) + def get_all_empty_tables_by_module(): table_rows = frappe.qb.Field("table_rows") table_name = frappe.qb.Field("table_name") information_schema = frappe.qb.Schema("information_schema") empty_tables = ( - frappe.qb.from_(information_schema.tables) - .select(table_name) - .where(table_rows == 0) + frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0) ).run() empty_tables = {r[0] for r in empty_tables} @@ -62,8 +68,10 @@ def get_all_empty_tables_by_module(): empty_tables_by_module[module] = [doctype] return empty_tables_by_module + def is_domain(module): return module.get("category") == "Domains" + def is_module(module): - return module.get("type") == "module" \ No newline at end of file + return module.get("type") == "module" diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 7824568a43..1eb9f1cf33 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -1,12 +1,13 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe - -from frappe import _ import functools import re +import frappe +from frappe import _ + + def load_address_and_contact(doc, key=None): """Loads address list and contact list in `__onload`""" from frappe.contacts.doctype.address.address import get_address_display, get_condensed_address @@ -18,15 +19,18 @@ def load_address_and_contact(doc, key=None): ] address_list = frappe.get_list("Address", filters=filters, fields=["*"]) - address_list = [a.update({"display": get_address_display(a)}) - for a in address_list] + address_list = [a.update({"display": get_address_display(a)}) for a in address_list] - address_list = sorted(address_list, - key = functools.cmp_to_key(lambda a, b: - (int(a.is_primary_address - b.is_primary_address)) or - (1 if a.modified - b.modified else 0)), reverse=True) + address_list = sorted( + address_list, + key=functools.cmp_to_key( + lambda a, b: (int(a.is_primary_address - b.is_primary_address)) + or (1 if a.modified - b.modified else 0) + ), + reverse=True, + ) - doc.set_onload('addr_list', address_list) + doc.set_onload("addr_list", address_list) contact_list = [] filters = [ @@ -37,29 +41,38 @@ def load_address_and_contact(doc, key=None): contact_list = frappe.get_list("Contact", filters=filters, fields=["*"]) for contact in contact_list: - contact["email_ids"] = frappe.get_all("Contact Email", filters={ - "parenttype": "Contact", - "parent": contact.name, - "is_primary": 0 - }, fields=["email_id"]) - - contact["phone_nos"] = frappe.get_all("Contact Phone", filters={ + contact["email_ids"] = frappe.get_all( + "Contact Email", + filters={"parenttype": "Contact", "parent": contact.name, "is_primary": 0}, + fields=["email_id"], + ) + + contact["phone_nos"] = frappe.get_all( + "Contact Phone", + filters={ "parenttype": "Contact", "parent": contact.name, "is_primary_phone": 0, - "is_primary_mobile_no": 0 - }, fields=["phone"]) + "is_primary_mobile_no": 0, + }, + fields=["phone"], + ) if contact.address: address = frappe.get_doc("Address", contact.address) contact["address"] = get_condensed_address(address) - contact_list = sorted(contact_list, - key = functools.cmp_to_key(lambda a, b: - (int(a.is_primary_contact - b.is_primary_contact)) or - (1 if a.modified - b.modified else 0)), reverse=True) + contact_list = sorted( + contact_list, + key=functools.cmp_to_key( + lambda a, b: (int(a.is_primary_contact - b.is_primary_contact)) + or (1 if a.modified - b.modified else 0) + ), + reverse=True, + ) + + doc.set_onload("contact_list", contact_list) - doc.set_onload('contact_list', contact_list) def has_permission(doc, ptype, user): links = get_permitted_and_not_permitted_links(doc.doctype) @@ -69,7 +82,7 @@ def has_permission(doc, ptype, user): # True if any one is True or all are empty names = [] - for df in (links.get("permitted_links") + links.get("not_permitted_links")): + for df in links.get("permitted_links") + links.get("not_permitted_links"): doctype = df.options name = doc.get(df.fieldname) names.append(name) @@ -81,12 +94,15 @@ def has_permission(doc, ptype, user): return True return False + def get_permission_query_conditions_for_contact(user): return get_permission_query_conditions("Contact") + def get_permission_query_conditions_for_address(user): return get_permission_query_conditions("Address") + def get_permission_query_conditions(doctype): links = get_permitted_and_not_permitted_links(doctype) @@ -100,7 +116,9 @@ def get_permission_query_conditions(doctype): # when everything is not permitted for df in links.get("not_permitted_links"): # like ifnull(customer, '')='' and ifnull(supplier, '')='' - conditions.append("ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname)) + conditions.append( + "ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname) + ) return "( " + " and ".join(conditions) + " )" @@ -109,10 +127,13 @@ def get_permission_query_conditions(doctype): for df in links.get("permitted_links"): # like ifnull(customer, '')!='' or ifnull(supplier, '')!='' - conditions.append("ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname)) + conditions.append( + "ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname) + ) return "( " + " or ".join(conditions) + " )" + def get_permitted_and_not_permitted_links(doctype): permitted_links = [] not_permitted_links = [] @@ -129,40 +150,40 @@ def get_permitted_and_not_permitted_links(doctype): else: not_permitted_links.append(df) - return { - "permitted_links": permitted_links, - "not_permitted_links": not_permitted_links - } + return {"permitted_links": permitted_links, "not_permitted_links": not_permitted_links} + def delete_contact_and_address(doctype, docname): - for parenttype in ('Contact', 'Address'): - items = frappe.db.sql_list("""select parent from `tabDynamic Link` + for parenttype in ("Contact", "Address"): + items = frappe.db.sql_list( + """select parent from `tabDynamic Link` where parenttype=%s and link_doctype=%s and link_name=%s""", - (parenttype, doctype, docname)) + (parenttype, doctype, docname), + ) for name in items: doc = frappe.get_doc(parenttype, name) - if len(doc.links)==1: + if len(doc.links) == 1: doc.delete() + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): - if not txt: txt = "" + if not txt: + txt = "" - doctypes = frappe.db.get_all("DocField", filters=filters, fields=["parent"], - distinct=True, as_list=True) + doctypes = frappe.db.get_all( + "DocField", filters=filters, fields=["parent"], distinct=True, as_list=True + ) - doctypes = tuple(d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE)) + doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)) - filters.update({ - "dt": ("not in", [d[0] for d in doctypes]) - }) + filters.update({"dt": ("not in", [d[0] for d in doctypes])}) - _doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], - as_list=True) + _doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True) - _doctypes = tuple([d for d in _doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE)]) + _doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)]) all_doctypes = [d[0] for d in doctypes + _doctypes] allowed_doctypes = frappe.permissions.get_doctypes_with_read() @@ -172,6 +193,7 @@ def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, fil return valid_doctypes + def set_link_title(doc): if not doc.links: return diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 5d0ed18d5f..d9ba31d474 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -2,16 +2,15 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe - -from frappe import throw, _ -from frappe.utils import cstr +from jinja2 import TemplateSyntaxError +import frappe +from frappe import _, throw +from frappe.contacts.address_and_contact import set_link_title +from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links from frappe.model.document import Document -from jinja2 import TemplateSyntaxError from frappe.model.naming import make_autoname -from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links -from frappe.contacts.address_and_contact import set_link_title +from frappe.utils import cstr class Address(Document): @@ -24,10 +23,11 @@ class Address(Document): self.address_title = self.links[0].link_name if self.address_title: - self.name = (cstr(self.address_title).strip() + "-" + cstr(_(self.address_type)).strip()) + self.name = cstr(self.address_title).strip() + "-" + cstr(_(self.address_type)).strip() if frappe.db.exists("Address", self.name): - self.name = make_autoname(cstr(self.address_title).strip() + "-" + - cstr(self.address_type).strip() + "-.#") + self.name = make_autoname( + cstr(self.address_title).strip() + "-" + cstr(self.address_type).strip() + "-.#" + ) else: throw(_("Address Title is mandatory.")) @@ -42,15 +42,15 @@ class Address(Document): if not self.links: contact_name = frappe.db.get_value("Contact", {"email_id": self.owner}) if contact_name: - contact = frappe.get_cached_doc('Contact', contact_name) + contact = frappe.get_cached_doc("Contact", contact_name) for link in contact.links: - self.append('links', dict(link_doctype=link.link_doctype, link_name=link.link_name)) + self.append("links", dict(link_doctype=link.link_doctype, link_name=link.link_name)) return True return False def validate_preferred_address(self): - preferred_fields = ['is_primary_address', 'is_shipping_address'] + preferred_fields = ["is_primary_address", "is_shipping_address"] for field in preferred_fields: if self.get(field): @@ -76,9 +76,11 @@ class Address(Document): return False -def get_preferred_address(doctype, name, preferred_key='is_primary_address'): - if preferred_key in ['is_shipping_address', 'is_primary_address']: - address = frappe.db.sql(""" SELECT + +def get_preferred_address(doctype, name, preferred_key="is_primary_address"): + if preferred_key in ["is_shipping_address", "is_primary_address"]: + address = frappe.db.sql( + """ SELECT addr.name FROM `tabAddress` addr, `tabDynamic Link` dl @@ -86,27 +88,37 @@ def get_preferred_address(doctype, name, preferred_key='is_primary_address'): dl.parent = addr.name and dl.link_doctype = %s and dl.link_name = %s and ifnull(addr.disabled, 0) = 0 and %s = %s - """ % ('%s', '%s', preferred_key, '%s'), (doctype, name, 1), as_dict=1) + """ + % ("%s", "%s", preferred_key, "%s"), + (doctype, name, 1), + as_dict=1, + ) if address: return address[0].name return + @frappe.whitelist() -def get_default_address(doctype, name, sort_key='is_primary_address'): - '''Returns default Address name for the given doctype, name''' - if sort_key not in ['is_shipping_address', 'is_primary_address']: +def get_default_address(doctype, name, sort_key="is_primary_address"): + """Returns default Address name for the given doctype, name""" + if sort_key not in ["is_shipping_address", "is_primary_address"]: return None - out = frappe.db.sql(""" SELECT + out = frappe.db.sql( + """ SELECT addr.name, addr.%s FROM `tabAddress` addr, `tabDynamic Link` dl WHERE dl.parent = addr.name and dl.link_doctype = %s and dl.link_name = %s and ifnull(addr.disabled, 0) = 0 - """ %(sort_key, '%s', '%s'), (doctype, name), as_dict=True) + """ + % (sort_key, "%s", "%s"), + (doctype, name), + as_dict=True, + ) if out: for contact in out: @@ -150,84 +162,96 @@ def get_territory_from_address(address): return territory + def get_list_context(context=None): return { "title": _("Addresses"), "get_list": get_address_list, "row_template": "templates/includes/address_row.html", - 'no_breadcrumbs': True, + "no_breadcrumbs": True, } -def get_address_list(doctype, txt, filters, limit_start, limit_page_length = 20, order_by = None): + +def get_address_list(doctype, txt, filters, limit_start, limit_page_length=20, order_by=None): from frappe.www.list import get_list + user = frappe.session.user ignore_permissions = True - if not filters: filters = [] + if not filters: + filters = [] filters.append(("Address", "owner", "=", user)) - return get_list(doctype, txt, filters, limit_start, limit_page_length, ignore_permissions=ignore_permissions) + return get_list( + doctype, txt, filters, limit_start, limit_page_length, ignore_permissions=ignore_permissions + ) + def has_website_permission(doc, ptype, user, verbose=False): """Returns true if there is a related lead or contact related to this document""" contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user}) if contact_name: - contact = frappe.get_doc('Contact', contact_name) + contact = frappe.get_doc("Contact", contact_name) return contact.has_common_link(doc) return False + def get_address_templates(address): - result = frappe.db.get_value("Address Template", \ - {"country": address.get("country")}, ["name", "template"]) + result = frappe.db.get_value( + "Address Template", {"country": address.get("country")}, ["name", "template"] + ) if not result: - result = frappe.db.get_value("Address Template", \ - {"is_default": 1}, ["name", "template"]) + result = frappe.db.get_value("Address Template", {"is_default": 1}, ["name", "template"]) if not result: - frappe.throw(_("No default Address Template found. Please create a new one from Setup > Printing and Branding > Address Template.")) + frappe.throw( + _( + "No default Address Template found. Please create a new one from Setup > Printing and Branding > Address Template." + ) + ) else: return result + def get_company_address(company): ret = frappe._dict() - ret.company_address = get_default_address('Company', company) + ret.company_address = get_default_address("Company", company) ret.company_address_display = get_address_display(ret.company_address) return ret + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def address_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond - link_doctype = filters.pop('link_doctype') - link_name = filters.pop('link_name') + link_doctype = filters.pop("link_doctype") + link_name = filters.pop("link_name") condition = "" meta = frappe.get_meta("Address") for fieldname, value in filters.items(): if meta.get_field(fieldname) or fieldname in frappe.db.DEFAULT_COLUMNS: - condition += " and {field}={value}".format( - field=fieldname, - value=frappe.db.escape(value)) + condition += " and {field}={value}".format(field=fieldname, value=frappe.db.escape(value)) searchfields = meta.get_search_fields() - if searchfield and (meta.get_field(searchfield)\ - or searchfield in frappe.db.DEFAULT_COLUMNS): + if searchfield and (meta.get_field(searchfield) or searchfield in frappe.db.DEFAULT_COLUMNS): searchfields.append(searchfield) - search_condition = '' + search_condition = "" for field in searchfields: - if search_condition == '': - search_condition += '`tabAddress`.`{field}` like %(txt)s'.format(field=field) + if search_condition == "": + search_condition += "`tabAddress`.`{field}` like %(txt)s".format(field=field) else: - search_condition += ' or `tabAddress`.`{field}` like %(txt)s'.format(field=field) + search_condition += " or `tabAddress`.`{field}` like %(txt)s".format(field=field) - return frappe.db.sql("""select + return frappe.db.sql( + """select `tabAddress`.name, `tabAddress`.city, `tabAddress`.country from `tabAddress`, `tabDynamic Link` @@ -245,19 +269,24 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), key=searchfield, - search_condition = search_condition, - condition=condition or ""), { - 'txt': '%' + txt + '%', - '_txt': txt.replace("%", ""), - 'start': start, - 'page_len': page_len, - 'link_name': link_name, - 'link_doctype': link_doctype - }) + search_condition=search_condition, + condition=condition or "", + ), + { + "txt": "%" + txt + "%", + "_txt": txt.replace("%", ""), + "start": start, + "page_len": page_len, + "link_name": link_name, + "link_doctype": link_doctype, + }, + ) + def get_condensed_address(doc): fields = ["address_title", "address_line1", "address_line2", "city", "county", "state", "country"] return ", ".join(doc.get(d) for d in fields if doc.get(d)) + def update_preferred_address(address, field): - frappe.db.set_value('Address', address, field, 0) + frappe.db.set_value("Address", address, field, 0) diff --git a/frappe/contacts/doctype/address/test_address.py b/frappe/contacts/doctype/address/test_address.py index dd6cd1ca83..4a6e6e53f7 100644 --- a/frappe/contacts/doctype/address/test_address.py +++ b/frappe/contacts/doctype/address/test_address.py @@ -1,31 +1,32 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + +import frappe from frappe.contacts.doctype.address.address import get_address_display + class TestAddress(unittest.TestCase): def test_template_works(self): - if not frappe.db.exists('Address Template', 'India'): - frappe.get_doc({ - "doctype": "Address Template", - "country": 'India', - "is_default": 1 - }).insert() + if not frappe.db.exists("Address Template", "India"): + frappe.get_doc({"doctype": "Address Template", "country": "India", "is_default": 1}).insert() - if not frappe.db.exists('Address', '_Test Address-Office'): - frappe.get_doc({ - "address_line1": "_Test Address Line 1", - "address_title": "_Test Address", - "address_type": "Office", - "city": "_Test City", - "state": "Test State", - "country": "India", - "doctype": "Address", - "is_primary_address": 1, - "phone": "+91 0000000000" - }).insert() + if not frappe.db.exists("Address", "_Test Address-Office"): + frappe.get_doc( + { + "address_line1": "_Test Address Line 1", + "address_title": "_Test Address", + "address_type": "Office", + "city": "_Test City", + "state": "Test State", + "country": "India", + "doctype": "Address", + "is_primary_address": 1, + "phone": "+91 0000000000", + } + ).insert() address = frappe.get_list("Address")[0].name display = get_address_display(frappe.get_doc("Address", address).as_dict()) - self.assertTrue(display) \ No newline at end of file + self.assertTrue(display) diff --git a/frappe/contacts/doctype/address_template/address_template.py b/frappe/contacts/doctype/address_template/address_template.py index 005f414303..85e9a986ef 100644 --- a/frappe/contacts/doctype/address_template/address_template.py +++ b/frappe/contacts/doctype/address_template/address_template.py @@ -3,21 +3,24 @@ # License: MIT. See LICENSE import frappe +from frappe import _ from frappe.model.document import Document from frappe.utils import cint from frappe.utils.jinja import validate_template -from frappe import _ + class AddressTemplate(Document): def validate(self): if not self.template: self.template = get_default_address_template() - self.defaults = frappe.db.get_values("Address Template", {"is_default":1, "name":("!=", self.name)}) + self.defaults = frappe.db.get_values( + "Address Template", {"is_default": 1, "name": ("!=", self.name)} + ) if not self.is_default: if not self.defaults: self.is_default = 1 - if cint(frappe.db.get_single_value('System Settings', 'setup_complete')): + if cint(frappe.db.get_single_value("System Settings", "setup_complete")): frappe.msgprint(_("Setting this Address Template as default as there is no other default")) validate_template(self.template) @@ -31,14 +34,23 @@ class AddressTemplate(Document): if self.is_default: frappe.throw(_("Default Address Template cannot be deleted")) + @frappe.whitelist() def get_default_address_template(): - '''Get default address template (translated)''' - return '''{{ address_line1 }}
{% if address_line2 %}{{ address_line2 }}
{% endif -%}\ + """Get default address template (translated)""" + return ( + """{{ address_line1 }}
{% if address_line2 %}{{ address_line2 }}
{% endif -%}\ {{ city }}
{% if state %}{{ state }}
{% endif -%} {% if pincode %}{{ pincode }}
{% endif -%} {{ country }}
-{% if phone %}'''+_('Phone')+''': {{ phone }}
{% endif -%} -{% if fax %}'''+_('Fax')+''': {{ fax }}
{% endif -%} -{% if email_id %}'''+_('Email')+''': {{ email_id }}
{% endif -%}''' +{% if phone %}""" + + _("Phone") + + """: {{ phone }}
{% endif -%} +{% if fax %}""" + + _("Fax") + + """: {{ fax }}
{% endif -%} +{% if email_id %}""" + + _("Email") + + """: {{ email_id }}
{% endif -%}""" + ) diff --git a/frappe/contacts/doctype/address_template/test_address_template.py b/frappe/contacts/doctype/address_template/test_address_template.py index b86623b548..699de5ada0 100644 --- a/frappe/contacts/doctype/address_template/test_address_template.py +++ b/frappe/contacts/doctype/address_template/test_address_template.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + +import frappe + class TestAddressTemplate(unittest.TestCase): def setUp(self): @@ -27,17 +30,12 @@ class TestAddressTemplate(unittest.TestCase): def make_default_address_template(self): template = """{{ address_line1 }}
{% if address_line2 %}{{ address_line2 }}
{% endif -%}{{ city }}
{% if state %}{{ state }}
{% endif -%}{% if pincode %}{{ pincode }}
{% endif -%}{{ country }}
{% if phone %}Phone: {{ phone }}
{% endif -%}{% if fax %}Fax: {{ fax }}
{% endif -%}{% if email_id %}Email: {{ email_id }}
{% endif -%}""" - if not frappe.db.exists('Address Template', 'India'): - frappe.get_doc({ - "doctype": "Address Template", - "country": 'India', - "is_default": 1, - "template": template - }).insert() - - if not frappe.db.exists('Address Template', 'Brazil'): - frappe.get_doc({ - "doctype": "Address Template", - "country": 'Brazil', - "template": template - }).insert() \ No newline at end of file + if not frappe.db.exists("Address Template", "India"): + frappe.get_doc( + {"doctype": "Address Template", "country": "India", "is_default": 1, "template": template} + ).insert() + + if not frappe.db.exists("Address Template", "Brazil"): + frappe.get_doc( + {"doctype": "Address Template", "country": "Brazil", "template": template} + ).insert() diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 9152655b85..4036cda853 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -1,26 +1,27 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import frappe -from frappe.utils import cstr, has_gravatar from frappe import _ -from frappe.model.document import Document +from frappe.contacts.address_and_contact import set_link_title from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links +from frappe.model.document import Document from frappe.model.naming import append_number_if_name_exists -from frappe.contacts.address_and_contact import set_link_title +from frappe.utils import cstr, has_gravatar class Contact(Document): def autoname(self): # concat first and last name - self.name = " ".join(filter(None, - [cstr(self.get(f)).strip() for f in ["first_name", "last_name"]])) + self.name = " ".join( + filter(None, [cstr(self.get(f)).strip() for f in ["first_name", "last_name"]]) + ) if frappe.db.exists("Contact", self.name): - self.name = append_number_if_name_exists('Contact', self.name) + self.name = append_number_if_name_exists("Contact", self.name) # concat party name if reqd for link in self.links: - self.name = self.name + '-' + link.link_name.strip() + self.name = self.name + "-" + link.link_name.strip() break def validate(self): @@ -45,7 +46,7 @@ class Contact(Document): self.user = frappe.db.get_value("User", {"email": self.email_id}) def get_link_for(self, link_doctype): - '''Return the link name, if exists for the given link DocType''' + """Return the link name, if exists for the given link DocType""" for link in self.links: if link.link_doctype == link_doctype: return link.link_name @@ -65,21 +66,21 @@ class Contact(Document): def add_email(self, email_id, is_primary=0, autosave=False): if not frappe.db.exists("Contact Email", {"email_id": email_id, "parent": self.name}): - self.append("email_ids", { - "email_id": email_id, - "is_primary": is_primary - }) + self.append("email_ids", {"email_id": email_id, "is_primary": is_primary}) if autosave: self.save(ignore_permissions=True) def add_phone(self, phone, is_primary_phone=0, is_primary_mobile_no=0, autosave=False): if not frappe.db.exists("Contact Phone", {"phone": phone, "parent": self.name}): - self.append("phone_nos", { - "phone": phone, - "is_primary_phone": is_primary_phone, - "is_primary_mobile_no": is_primary_mobile_no - }) + self.append( + "phone_nos", + { + "phone": phone, + "is_primary_phone": is_primary_phone, + "is_primary_mobile_no": is_primary_mobile_no, + }, + ) if autosave: self.save(ignore_permissions=True) @@ -113,7 +114,9 @@ class Contact(Document): is_primary = [phone.phone for phone in self.phone_nos if phone.get(field_name)] if len(is_primary) > 1: - frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname)))) + frappe.throw( + _("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname))) + ) primary_number_exists = False for d in self.phone_nos: @@ -125,9 +128,11 @@ class Contact(Document): if not primary_number_exists: setattr(self, fieldname, "") + def get_default_contact(doctype, name): - '''Returns default contact for the given doctype, name''' - out = frappe.db.sql('''select parent, + """Returns default contact for the given doctype, name""" + out = frappe.db.sql( + '''select parent, IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0) as is_primary_contact from @@ -135,7 +140,10 @@ def get_default_contact(doctype, name): where dl.link_doctype=%s and dl.link_name=%s and - dl.parenttype = "Contact"''', (doctype, name), as_dict=True) + dl.parenttype = "Contact"''', + (doctype, name), + as_dict=True, + ) if out: for contact in out: @@ -145,6 +153,7 @@ def get_default_contact(doctype, name): else: return None + @frappe.whitelist() def invite_user(contact): contact = frappe.get_doc("Contact", contact) @@ -153,34 +162,39 @@ def invite_user(contact): frappe.throw(_("Please set Email Address")) if contact.has_permission("write"): - user = frappe.get_doc({ - "doctype": "User", - "first_name": contact.first_name, - "last_name": contact.last_name, - "email": contact.email_id, - "user_type": "Website User", - "send_welcome_email": 1 - }).insert(ignore_permissions = True) + user = frappe.get_doc( + { + "doctype": "User", + "first_name": contact.first_name, + "last_name": contact.last_name, + "email": contact.email_id, + "user_type": "Website User", + "send_welcome_email": 1, + } + ).insert(ignore_permissions=True) return user.name + @frappe.whitelist() def get_contact_details(contact): contact = frappe.get_doc("Contact", contact) out = { "contact_person": contact.get("name"), - "contact_display": " ".join(filter(None, - [contact.get("salutation"), contact.get("first_name"), contact.get("last_name")])), + "contact_display": " ".join( + filter(None, [contact.get("salutation"), contact.get("first_name"), contact.get("last_name")]) + ), "contact_email": contact.get("email_id"), "contact_mobile": contact.get("mobile_no"), "contact_phone": contact.get("phone"), "contact_designation": contact.get("designation"), - "contact_department": contact.get("department") + "contact_department": contact.get("department"), } return out + def update_contact(doc, method): - '''Update contact when user is updated, if contact is found. Called via hooks''' + """Update contact when user is updated, if contact is found. Called via hooks""" contact_name = frappe.db.get_value("Contact", {"email_id": doc.name}) if contact_name: contact = frappe.get_doc("Contact", contact_name) @@ -190,19 +204,23 @@ def update_contact(doc, method): contact.flags.ignore_mandatory = True contact.save(ignore_permissions=True) + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def contact_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond - if not frappe.get_meta("Contact").get_field(searchfield)\ - and searchfield not in frappe.db.DEFAULT_COLUMNS: + if ( + not frappe.get_meta("Contact").get_field(searchfield) + and searchfield not in frappe.db.DEFAULT_COLUMNS + ): return [] - link_doctype = filters.pop('link_doctype') - link_name = filters.pop('link_name') + link_doctype = filters.pop("link_doctype") + link_name = filters.pop("link_name") - return frappe.db.sql("""select + return frappe.db.sql( + """select `tabContact`.name, `tabContact`.first_name, `tabContact`.last_name from `tabContact`, `tabDynamic Link` @@ -216,68 +234,90 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters): order by if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999), `tabContact`.idx desc, `tabContact`.name - limit %(start)s, %(page_len)s """.format(mcond=get_match_cond(doctype), key=searchfield), { - 'txt': '%' + txt + '%', - '_txt': txt.replace("%", ""), - 'start': start, - 'page_len': page_len, - 'link_name': link_name, - 'link_doctype': link_doctype - }) + limit %(start)s, %(page_len)s """.format( + mcond=get_match_cond(doctype), key=searchfield + ), + { + "txt": "%" + txt + "%", + "_txt": txt.replace("%", ""), + "start": start, + "page_len": page_len, + "link_name": link_name, + "link_doctype": link_doctype, + }, + ) + @frappe.whitelist() def address_query(links): import json - links = [{"link_doctype": d.get("link_doctype"), "link_name": d.get("link_name")} for d in json.loads(links)] + links = [ + {"link_doctype": d.get("link_doctype"), "link_name": d.get("link_name")} + for d in json.loads(links) + ] result = [] for link in links: - if not frappe.has_permission(doctype=link.get("link_doctype"), ptype="read", doc=link.get("link_name")): + if not frappe.has_permission( + doctype=link.get("link_doctype"), ptype="read", doc=link.get("link_name") + ): continue - res = frappe.db.sql(""" + res = frappe.db.sql( + """ SELECT `tabAddress`.name FROM `tabAddress`, `tabDynamic Link` WHERE `tabDynamic Link`.parenttype='Address' AND `tabDynamic Link`.parent=`tabAddress`.name AND `tabDynamic Link`.link_doctype = %(link_doctype)s AND `tabDynamic Link`.link_name = %(link_name)s - """, { - "link_doctype": link.get("link_doctype"), - "link_name": link.get("link_name"), - }, as_dict=True) + """, + { + "link_doctype": link.get("link_doctype"), + "link_name": link.get("link_name"), + }, + as_dict=True, + ) result.extend([l.name for l in res]) return result + def get_contact_with_phone_number(number): - if not number: return + if not number: + return - contacts = frappe.get_all('Contact Phone', filters=[ - ['phone', 'like', '%{0}'.format(number)] - ], fields=["parent"], limit=1) + contacts = frappe.get_all( + "Contact Phone", filters=[["phone", "like", "%{0}".format(number)]], fields=["parent"], limit=1 + ) return contacts[0].parent if contacts else None + def get_contact_name(email_id): - contact = frappe.get_all("Contact Email", filters={"email_id": email_id}, fields=["parent"], limit=1) + contact = frappe.get_all( + "Contact Email", filters={"email_id": email_id}, fields=["parent"], limit=1 + ) return contact[0].parent if contact else None + def get_contacts_linking_to(doctype, docname, fields=None): """Return a list of contacts containing a link to the given document.""" - return frappe.get_list('Contact', fields=fields, filters=[ - ['Dynamic Link', 'link_doctype', '=', doctype], - ['Dynamic Link', 'link_name', '=', docname] - ]) + return frappe.get_list( + "Contact", + fields=fields, + filters=[ + ["Dynamic Link", "link_doctype", "=", doctype], + ["Dynamic Link", "link_name", "=", docname], + ], + ) + def get_contacts_linked_from(doctype, docname, fields=None): """Return a list of contacts that are contained in (linked from) the given document.""" - link_fields = frappe.get_meta(doctype).get('fields', { - 'fieldtype': 'Link', - 'options': 'Contact' - }) + link_fields = frappe.get_meta(doctype).get("fields", {"fieldtype": "Link", "options": "Contact"}) if not link_fields: return [] @@ -285,6 +325,4 @@ def get_contacts_linked_from(doctype, docname, fields=None): if not contact_names: return [] - return frappe.get_list('Contact', fields=fields, filters={ - 'name': ('in', contact_names) - }) + return frappe.get_list("Contact", fields=fields, filters={"name": ("in", contact_names)}) diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py index 1170ba843a..7ca47476e8 100644 --- a/frappe/contacts/doctype/contact/test_contact.py +++ b/frappe/contacts/doctype/contact/test_contact.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest -test_dependencies = ['Contact', 'Salutation'] +import frappe -class TestContact(unittest.TestCase): +test_dependencies = ["Contact", "Salutation"] + +class TestContact(unittest.TestCase): def test_check_default_email(self): emails = [ {"email": "test1@example.com", "is_primary": 0}, @@ -32,13 +33,11 @@ class TestContact(unittest.TestCase): self.assertEqual(contact.phone, "+91 0000000002") self.assertEqual(contact.mobile_no, "+91 0000000003") + def create_contact(name, salutation, emails=None, phones=None, save=True): - doc = frappe.get_doc({ - "doctype": "Contact", - "first_name": name, - "status": "Open", - "salutation": salutation - }) + doc = frappe.get_doc( + {"doctype": "Contact", "first_name": name, "status": "Open", "salutation": salutation} + ) if emails: for d in emails: diff --git a/frappe/contacts/doctype/contact_email/contact_email.py b/frappe/contacts/doctype/contact_email/contact_email.py index 58d37376b8..ed794ac06c 100644 --- a/frappe/contacts/doctype/contact_email/contact_email.py +++ b/frappe/contacts/doctype/contact_email/contact_email.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ContactEmail(Document): pass diff --git a/frappe/contacts/doctype/contact_phone/contact_phone.py b/frappe/contacts/doctype/contact_phone/contact_phone.py index ed7d3b9911..2a842c9c9e 100644 --- a/frappe/contacts/doctype/contact_phone/contact_phone.py +++ b/frappe/contacts/doctype/contact_phone/contact_phone.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ContactPhone(Document): pass diff --git a/frappe/contacts/doctype/gender/gender.py b/frappe/contacts/doctype/gender/gender.py index b4efcb64b9..8e9951eaf9 100644 --- a/frappe/contacts/doctype/gender/gender.py +++ b/frappe/contacts/doctype/gender/gender.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class Gender(Document): pass diff --git a/frappe/contacts/doctype/gender/test_gender.py b/frappe/contacts/doctype/gender/test_gender.py index 8549cc2130..6b795749ee 100644 --- a/frappe/contacts/doctype/gender/test_gender.py +++ b/frappe/contacts/doctype/gender/test_gender.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestGender(unittest.TestCase): pass diff --git a/frappe/contacts/doctype/salutation/salutation.py b/frappe/contacts/doctype/salutation/salutation.py index 380af6de28..57fb2d6abc 100644 --- a/frappe/contacts/doctype/salutation/salutation.py +++ b/frappe/contacts/doctype/salutation/salutation.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class Salutation(Document): pass diff --git a/frappe/contacts/doctype/salutation/test_salutation.py b/frappe/contacts/doctype/salutation/test_salutation.py index 59333fb61e..5149ced3dd 100644 --- a/frappe/contacts/doctype/salutation/test_salutation.py +++ b/frappe/contacts/doctype/salutation/test_salutation.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestSalutation(unittest.TestCase): pass diff --git a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py index 671e1c6bc8..9848e81b63 100644 --- a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py +++ b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py @@ -4,17 +4,37 @@ import frappe from frappe import _ field_map = { - "Contact": ["first_name", "last_name", "address", "phone", "mobile_no", "email_id", "is_primary_contact"], - "Address": ["address_line1", "address_line2", "city", "state", "pincode", "country", "is_primary_address"] + "Contact": [ + "first_name", + "last_name", + "address", + "phone", + "mobile_no", + "email_id", + "is_primary_contact", + ], + "Address": [ + "address_line1", + "address_line2", + "city", + "state", + "pincode", + "country", + "is_primary_address", + ], } + def execute(filters=None): columns, data = get_columns(filters), get_data(filters) return columns, data + def get_columns(filters): return [ - "{reference_doctype}:Link/{reference_doctype}".format(reference_doctype=filters.get("reference_doctype")), + "{reference_doctype}:Link/{reference_doctype}".format( + reference_doctype=filters.get("reference_doctype") + ), "Address Line 1", "Address Line 2", "City", @@ -27,9 +47,10 @@ def get_columns(filters): "Address", "Phone", "Email Id", - "Is Primary Contact:Check" + "Is Primary Contact:Check", ] + def get_data(filters): data = [] reference_doctype = filters.get("reference_doctype") @@ -37,6 +58,7 @@ def get_data(filters): return get_reference_addresses_and_contact(reference_doctype, reference_name) + def get_reference_addresses_and_contact(reference_doctype, reference_name): data = [] filters = None @@ -48,16 +70,22 @@ def get_reference_addresses_and_contact(reference_doctype, reference_name): if reference_name: filters = {"name": reference_name} - reference_list = [d[0] for d in frappe.get_list(reference_doctype, filters=filters, fields=["name"], as_list=True)] + reference_list = [ + d[0] for d in frappe.get_list(reference_doctype, filters=filters, fields=["name"], as_list=True) + ] for d in reference_list: reference_details.setdefault(d, frappe._dict()) - reference_details = get_reference_details(reference_doctype, "Address", reference_list, reference_details) - reference_details = get_reference_details(reference_doctype, "Contact", reference_list, reference_details) + reference_details = get_reference_details( + reference_doctype, "Address", reference_list, reference_details + ) + reference_details = get_reference_details( + reference_doctype, "Contact", reference_list, reference_details + ) for reference_name, details in reference_details.items(): addresses = details.get("address", []) - contacts = details.get("contact", []) + contacts = details.get("contact", []) if not any([addresses, contacts]): result = [reference_name] result.extend(add_blank_columns_for("Address")) @@ -78,10 +106,11 @@ def get_reference_addresses_and_contact(reference_doctype, reference_name): return data + def get_reference_details(reference_doctype, doctype, reference_list, reference_details): - filters = [ + filters = [ ["Dynamic Link", "link_doctype", "=", reference_doctype], - ["Dynamic Link", "link_name", "in", reference_list] + ["Dynamic Link", "link_name", "in", reference_list], ] fields = ["`tabDynamic Link`.link_name"] + field_map.get(doctype, []) @@ -97,5 +126,6 @@ def get_reference_details(reference_doctype, doctype, reference_list, reference_ reference_details[reference_list[0]][frappe.scrub(doctype)] = temp_records return reference_details + def add_blank_columns_for(doctype): return ["" for field in field_map.get(doctype, [])] diff --git a/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py b/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py index f539722175..2ad8bfaba3 100644 --- a/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py +++ b/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py @@ -1,95 +1,87 @@ +import unittest import frappe import frappe.defaults -import unittest - from frappe.contacts.report.addresses_and_contacts.addresses_and_contacts import get_data + def get_custom_linked_doctype(): - if bool(frappe.get_all("DocType", filters={'name':'Test Custom Doctype'})): + if bool(frappe.get_all("DocType", filters={"name": "Test Custom Doctype"})): return - doc = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "fields": [{ - "label": "Test Field", - "fieldname": "test_field", - "fieldtype": "Data" - }, - { - "label": "Contact HTML", - "fieldname": "contact_html", - "fieldtype": "HTML" - }, + doc = frappe.get_doc( { - "label": "Address HTML", - "fieldname": "address_html", - "fieldtype": "HTML" - }], - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": "Test Custom Doctype", - }) + "doctype": "DocType", + "module": "Core", + "custom": 1, + "fields": [ + {"label": "Test Field", "fieldname": "test_field", "fieldtype": "Data"}, + {"label": "Contact HTML", "fieldname": "contact_html", "fieldtype": "HTML"}, + {"label": "Address HTML", "fieldname": "address_html", "fieldtype": "HTML"}, + ], + "permissions": [{"role": "System Manager", "read": 1}], + "name": "Test Custom Doctype", + } + ) doc.insert() + def get_custom_doc_for_address_and_contacts(): get_custom_linked_doctype() - linked_doc = frappe.get_doc({ - "doctype": "Test Custom Doctype", - "test_field": "Hello", - }).insert() + linked_doc = frappe.get_doc( + { + "doctype": "Test Custom Doctype", + "test_field": "Hello", + } + ).insert() return linked_doc + def create_linked_address(link_list): if frappe.flags.test_address_created: return - address = frappe.get_doc({ - "doctype": "Address", - "address_title": "_Test Address", - "address_type": "Billing", - "address_line1": "test address line 1", - "address_line2": "test address line 2", - "city": "Milan", - "country": "Italy" - }) + address = frappe.get_doc( + { + "doctype": "Address", + "address_title": "_Test Address", + "address_type": "Billing", + "address_line1": "test address line 1", + "address_line2": "test address line 2", + "city": "Milan", + "country": "Italy", + } + ) for name in link_list: - address.append("links",{ - 'link_doctype': 'Test Custom Doctype', - 'link_name': name - }) + address.append("links", {"link_doctype": "Test Custom Doctype", "link_name": name}) address.insert() frappe.flags.test_address_created = True return address.name + def create_linked_contact(link_list, address): if frappe.flags.test_contact_created: return - contact = frappe.get_doc({ - "doctype": "Contact", - "salutation": "Mr", - "first_name": "_Test First Name", - "last_name": "_Test Last Name", - "is_primary_contact": 1, - "address": address, - "status": "Open" - }) + contact = frappe.get_doc( + { + "doctype": "Contact", + "salutation": "Mr", + "first_name": "_Test First Name", + "last_name": "_Test Last Name", + "is_primary_contact": 1, + "address": address, + "status": "Open", + } + ) contact.add_email("test_contact@example.com", is_primary=True) contact.add_phone("+91 0000000000", is_primary_phone=True) for name in link_list: - contact.append("links",{ - 'link_doctype': 'Test Custom Doctype', - 'link_name': name - }) + contact.append("links", {"link_doctype": "Test Custom Doctype", "link_name": name}) contact.insert(ignore_permissions=True) frappe.flags.test_contact_created = True @@ -103,7 +95,23 @@ class TestAddressesAndContacts(unittest.TestCase): create_linked_contact(links_list, d) report_data = get_data({"reference_doctype": "Test Custom Doctype"}) for idx, link in enumerate(links_list): - test_item = [link, 'test address line 1', 'test address line 2', 'Milan', None, None, 'Italy', 0, '_Test First Name', '_Test Last Name', '_Test Address-Billing', '+91 0000000000', '', 'test_contact@example.com', 1] + test_item = [ + link, + "test address line 1", + "test address line 2", + "Milan", + None, + None, + "Italy", + 0, + "_Test First Name", + "_Test Last Name", + "_Test Address-Billing", + "+91 0000000000", + "", + "test_contact@example.com", + 1, + ] self.assertListEqual(test_item, report_data[idx]) def tearDown(self): diff --git a/frappe/core/doctype/__init__.py b/frappe/core/doctype/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/__init__.py +++ b/frappe/core/doctype/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index db2e64e868..b7a6d77206 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -1,9 +1,10 @@ # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe -from frappe.utils import cstr from tenacity import retry, retry_if_exception_type, stop_after_attempt + +import frappe from frappe.model.document import Document +from frappe.utils import cstr class AccessLog(Document): @@ -22,14 +23,19 @@ def make_access_log( columns=None, ): _make_access_log( - doctype, document, method, file_type, report_name, filters, page, columns, + doctype, + document, + method, + file_type, + report_name, + filters, + page, + columns, ) @frappe.write_only() -@retry( - stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError) -) +@retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError)) def _make_access_log( doctype=None, document=None, @@ -43,18 +49,20 @@ def _make_access_log( user = frappe.session.user in_request = frappe.request and frappe.request.method == "GET" - frappe.get_doc({ - "doctype": "Access Log", - "user": user, - "export_from": doctype, - "reference_document": document, - "file_type": file_type, - "report_name": report_name, - "page": page, - "method": method, - "filters": cstr(filters) or None, - "columns": columns, - }).db_insert() + frappe.get_doc( + { + "doctype": "Access Log", + "user": user, + "export_from": doctype, + "reference_document": document, + "file_type": file_type, + "report_name": report_name, + "page": page, + "method": method, + "filters": cstr(filters) or None, + "columns": columns, + } + ).db_insert() # `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview` # dont commit in test mode. It must be tempting to put this block along with the in_request in the diff --git a/frappe/core/doctype/access_log/test_access_log.py b/frappe/core/doctype/access_log/test_access_log.py index 42878d0eb4..983e3cb5e4 100644 --- a/frappe/core/doctype/access_log/test_access_log.py +++ b/frappe/core/doctype/access_log/test_access_log.py @@ -2,20 +2,21 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -# imports - standard imports -import unittest import base64 import os +# imports - standard imports +import unittest + +# imports - third party imports +import requests + # imports - module imports import frappe from frappe.core.doctype.access_log.access_log import make_access_log -from frappe.utils import cstr, get_site_url from frappe.core.doctype.data_import.data_import import export_csv from frappe.core.doctype.user.user import generate_keys - -# imports - third party imports -import requests +from frappe.utils import cstr, get_site_url class TestAccessLog(unittest.TestCase): @@ -23,8 +24,9 @@ class TestAccessLog(unittest.TestCase): # generate keys for current user to send requests for the following tests generate_keys(frappe.session.user) frappe.db.commit() - generated_secret = frappe.utils.password.get_decrypted_password("User", - frappe.session.user, fieldname='api_secret') + generated_secret = frappe.utils.password.get_decrypted_password( + "User", frappe.session.user, fieldname="api_secret" + ) api_key = frappe.db.get_value("User", "Administrator", "api_key") self.header = {"Authorization": "token {}:{}".format(api_key, generated_secret)} @@ -101,54 +103,55 @@ class TestAccessLog(unittest.TestCase): "party": [], "group_by": "Group by Voucher (Consolidated)", "cost_center": [], - "project": [] + "project": [], } - self.test_doctype = 'File' - self.test_document = 'Test Document' - self.test_report_name = 'General Ledger' - self.test_file_type = 'CSV' - self.test_method = 'Test Method' - self.file_name = frappe.utils.random_string(10) + '.txt' + self.test_doctype = "File" + self.test_document = "Test Document" + self.test_report_name = "General Ledger" + self.test_file_type = "CSV" + self.test_method = "Test Method" + self.file_name = frappe.utils.random_string(10) + ".txt" self.test_content = frappe.utils.random_string(1024) - def test_make_full_access_log(self): self.maxDiff = None # test if all fields maintain data: html page and filters are converted? - make_access_log(doctype=self.test_doctype, + make_access_log( + doctype=self.test_doctype, document=self.test_document, report_name=self.test_report_name, page=self.test_html_template, file_type=self.test_file_type, method=self.test_method, - filters=self.test_filters) + filters=self.test_filters, + ) - last_doc = frappe.get_last_doc('Access Log') + last_doc = frappe.get_last_doc("Access Log") self.assertEqual(last_doc.filters, cstr(self.test_filters)) self.assertEqual(self.test_doctype, last_doc.export_from) self.assertEqual(self.test_document, last_doc.reference_document) - def test_make_export_log(self): # export data and delete temp file generated on disk export_csv(self.test_doctype, self.file_name) os.remove(self.file_name) # test if the exported data is logged - last_doc = frappe.get_last_doc('Access Log') + last_doc = frappe.get_last_doc("Access Log") self.assertEqual(self.test_doctype, last_doc.export_from) - def test_private_file_download(self): # create new private file - new_private_file = frappe.get_doc({ - 'doctype': self.test_doctype, - 'file_name': self.file_name, - 'content': base64.b64encode(self.test_content.encode('utf-8')), - 'is_private': 1, - }) + new_private_file = frappe.get_doc( + { + "doctype": self.test_doctype, + "file_name": self.file_name, + "content": base64.b64encode(self.test_content.encode("utf-8")), + "is_private": 1, + } + ) new_private_file.insert() # access the created file @@ -156,7 +159,7 @@ class TestAccessLog(unittest.TestCase): try: request = requests.post(private_file_link, headers=self.header) - last_doc = frappe.get_last_doc('Access Log') + last_doc = frappe.get_last_doc("Access Log") if request.ok: # check for the access log of downloaded file @@ -169,6 +172,5 @@ class TestAccessLog(unittest.TestCase): # cleanup new_private_file.delete() - def tearDown(self): pass diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 70d4ca3ffe..84c908d0ae 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -26,20 +26,25 @@ class ActivityLog(Document): if self.reference_doctype and self.reference_name: self.status = "Linked" + def on_doctype_update(): """Add indexes in `tabActivity Log`""" frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"]) frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"]) frappe.db.add_index("Activity Log", ["link_doctype", "link_name"]) + def add_authentication_log(subject, user, operation="Login", status="Success"): - frappe.get_doc({ - "doctype": "Activity Log", - "user": user, - "status": status, - "subject": subject, - "operation": operation, - }).insert(ignore_permissions=True, ignore_links=True) + frappe.get_doc( + { + "doctype": "Activity Log", + "user": user, + "status": status, + "subject": subject, + "operation": operation, + } + ).insert(ignore_permissions=True, ignore_links=True) + def clear_activity_logs(days=None): """clear 90 day old authentication logs or configured in log settings""" @@ -47,6 +52,4 @@ def clear_activity_logs(days=None): if not days: days = 90 doctype = DocType("Activity Log") - frappe.db.delete(doctype, filters=( - doctype.creation < (Now() - Interval(days=days)) - )) + frappe.db.delete(doctype, filters=(doctype.creation < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index 358272ac63..3e29154242 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -3,15 +3,16 @@ import frappe import frappe.permissions -from frappe.utils import get_fullname from frappe import _ from frappe.core.doctype.activity_log.activity_log import add_authentication_log +from frappe.utils import get_fullname + def update_feed(doc, method=None): if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import: return - if doc._action!="save" or doc.flags.ignore_feed: + if doc._action != "save" or doc.flags.ignore_feed: return if doc.doctype == "Activity Log" or doc.meta.issingle: @@ -29,65 +30,75 @@ def update_feed(doc, method=None): name = feed.name or doc.name # delete earlier feed - frappe.db.delete("Activity Log", { - "reference_doctype": doctype, - "reference_name": name, - "link_doctype": feed.link_doctype - }) - - frappe.get_doc({ - "doctype": "Activity Log", - "reference_doctype": doctype, - "reference_name": name, - "subject": feed.subject, - "full_name": get_fullname(doc.owner), - "reference_owner": frappe.db.get_value(doctype, name, "owner"), - "link_doctype": feed.link_doctype, - "link_name": feed.link_name - }).insert(ignore_permissions=True) + frappe.db.delete( + "Activity Log", + {"reference_doctype": doctype, "reference_name": name, "link_doctype": feed.link_doctype}, + ) + + frappe.get_doc( + { + "doctype": "Activity Log", + "reference_doctype": doctype, + "reference_name": name, + "subject": feed.subject, + "full_name": get_fullname(doc.owner), + "reference_owner": frappe.db.get_value(doctype, name, "owner"), + "link_doctype": feed.link_doctype, + "link_name": feed.link_name, + } + ).insert(ignore_permissions=True) + def login_feed(login_manager): if login_manager.user != "Guest": subject = _("{0} logged in").format(get_fullname(login_manager.user)) add_authentication_log(subject, login_manager.user) + def logout_feed(user, reason): if user and user != "Guest": subject = _("{0} logged out: {1}").format(get_fullname(user), frappe.bold(reason)) add_authentication_log(subject, user, operation="Logout") -def get_feed_match_conditions(user=None, doctype='Comment'): - if not user: user = frappe.session.user - conditions = ['`tab{doctype}`.owner={user} or `tab{doctype}`.reference_owner={user}'.format( - user = frappe.db.escape(user), - doctype = doctype - )] +def get_feed_match_conditions(user=None, doctype="Comment"): + if not user: + user = frappe.session.user + + conditions = [ + "`tab{doctype}`.owner={user} or `tab{doctype}`.reference_owner={user}".format( + user=frappe.db.escape(user), doctype=doctype + ) + ] user_permissions = frappe.permissions.get_user_permissions(user) can_read = frappe.get_user().get_can_read() - can_read_doctypes = ["'{}'".format(dt) for dt in - list(set(can_read) - set(list(user_permissions)))] + can_read_doctypes = [ + "'{}'".format(dt) for dt in list(set(can_read) - set(list(user_permissions))) + ] if can_read_doctypes: - conditions += ["""(`tab{doctype}`.reference_doctype is null + conditions += [ + """(`tab{doctype}`.reference_doctype is null or `tab{doctype}`.reference_doctype = '' or `tab{doctype}`.reference_doctype in ({values}))""".format( - doctype = doctype, - values =", ".join(can_read_doctypes) - )] + doctype=doctype, values=", ".join(can_read_doctypes) + ) + ] if user_permissions: can_read_docs = [] for dt, obj in user_permissions.items(): for n in obj: - can_read_docs.append('{}|{}'.format(frappe.db.escape(dt), frappe.db.escape(n.get('doc', '')))) + can_read_docs.append("{}|{}".format(frappe.db.escape(dt), frappe.db.escape(n.get("doc", "")))) if can_read_docs: - conditions.append("concat_ws('|', `tab{doctype}`.reference_doctype, `tab{doctype}`.reference_name) in ({values})".format( - doctype = doctype, - values = ", ".join(can_read_docs))) + conditions.append( + "concat_ws('|', `tab{doctype}`.reference_doctype, `tab{doctype}`.reference_name) in ({values})".format( + doctype=doctype, values=", ".join(can_read_docs) + ) + ) - return "(" + " or ".join(conditions) + ")" + return "(" + " or ".join(conditions) + ")" diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index 87d3538cc7..f5f94dc44b 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -1,77 +1,74 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -import unittest import time -from frappe.auth import LoginManager, CookieManager +import unittest + +import frappe +from frappe.auth import CookieManager, LoginManager + class TestActivityLog(unittest.TestCase): def test_activity_log(self): # test user login log - frappe.local.form_dict = frappe._dict({ - 'cmd': 'login', - 'sid': 'Guest', - 'pwd': 'admin', - 'usr': 'Administrator' - }) + frappe.local.form_dict = frappe._dict( + {"cmd": "login", "sid": "Guest", "pwd": "admin", "usr": "Administrator"} + ) frappe.local.cookie_manager = CookieManager() frappe.local.login_manager = LoginManager() auth_log = self.get_auth_log() - self.assertEqual(auth_log.status, 'Success') + self.assertEqual(auth_log.status, "Success") # test user logout log frappe.local.login_manager.logout() - auth_log = self.get_auth_log(operation='Logout') - self.assertEqual(auth_log.status, 'Success') + auth_log = self.get_auth_log(operation="Logout") + self.assertEqual(auth_log.status, "Success") # test invalid login - frappe.form_dict.update({ 'pwd': 'password' }) + frappe.form_dict.update({"pwd": "password"}) self.assertRaises(frappe.AuthenticationError, LoginManager) auth_log = self.get_auth_log() - self.assertEqual(auth_log.status, 'Failed') + self.assertEqual(auth_log.status, "Failed") frappe.local.form_dict = frappe._dict() - def get_auth_log(self, operation='Login'): - names = frappe.db.get_all('Activity Log', filters={ - 'user': 'Administrator', - 'operation': operation, - }, order_by='`creation` DESC') + def get_auth_log(self, operation="Login"): + names = frappe.db.get_all( + "Activity Log", + filters={ + "user": "Administrator", + "operation": operation, + }, + order_by="`creation` DESC", + ) name = names[0] - auth_log = frappe.get_doc('Activity Log', name) + auth_log = frappe.get_doc("Activity Log", name) return auth_log def test_brute_security(self): - update_system_settings({ - 'allow_consecutive_login_attempts': 3, - 'allow_login_after_fail': 5 - }) - - frappe.local.form_dict = frappe._dict({ - 'cmd': 'login', - 'sid': 'Guest', - 'pwd': 'admin', - 'usr': 'Administrator' - }) + update_system_settings({"allow_consecutive_login_attempts": 3, "allow_login_after_fail": 5}) + + frappe.local.form_dict = frappe._dict( + {"cmd": "login", "sid": "Guest", "pwd": "admin", "usr": "Administrator"} + ) frappe.local.cookie_manager = CookieManager() frappe.local.login_manager = LoginManager() auth_log = self.get_auth_log() - self.assertEqual(auth_log.status, 'Success') + self.assertEqual(auth_log.status, "Success") # test user logout log frappe.local.login_manager.logout() - auth_log = self.get_auth_log(operation='Logout') - self.assertEqual(auth_log.status, 'Success') + auth_log = self.get_auth_log(operation="Logout") + self.assertEqual(auth_log.status, "Success") # test invalid login - frappe.form_dict.update({ 'pwd': 'password' }) + frappe.form_dict.update({"pwd": "password"}) self.assertRaises(frappe.AuthenticationError, LoginManager) self.assertRaises(frappe.AuthenticationError, LoginManager) self.assertRaises(frappe.AuthenticationError, LoginManager) @@ -85,8 +82,9 @@ class TestActivityLog(unittest.TestCase): frappe.local.form_dict = frappe._dict() + def update_system_settings(args): - doc = frappe.get_doc('System Settings') + doc = frappe.get_doc("System Settings") doc.update(args) doc.flags.ignore_mandatory = 1 doc.save() diff --git a/frappe/core/doctype/block_module/block_module.py b/frappe/core/doctype/block_module/block_module.py index cc6c222a04..3158e3e6a5 100644 --- a/frappe/core/doctype/block_module/block_module.py +++ b/frappe/core/doctype/block_module/block_module.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class BlockModule(Document): pass diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index e28d350d04..1456f1ec93 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -1,22 +1,27 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -import json -from frappe.model.document import Document from frappe.core.doctype.user.user import extract_mentions -from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\ - get_title, get_title_html -from frappe.utils import get_fullname -from frappe.website.utils import clear_cache from frappe.database.schema import add_column +from frappe.desk.doctype.notification_log.notification_log import ( + enqueue_create_notification, + get_title, + get_title_html, +) from frappe.exceptions import ImplicitCommitError +from frappe.model.document import Document +from frappe.utils import get_fullname +from frappe.website.utils import clear_cache + class Comment(Document): def after_insert(self): self.notify_mentions() - self.notify_change('add') + self.notify_change("add") def validate(self): if not self.comment_email: @@ -26,34 +31,35 @@ class Comment(Document): def on_update(self): update_comment_in_doc(self) if self.is_new(): - self.notify_change('update') + self.notify_change("update") def on_trash(self): self.remove_comment_from_cache() - self.notify_change('delete') + self.notify_change("delete") def notify_change(self, action): key_map = { - 'Like': 'like_logs', - 'Assigned': 'assignment_logs', - 'Assignment Completed': 'assignment_logs', - 'Comment': 'comments', - 'Attachment': 'attachment_logs', - 'Attachment Removed': 'attachment_logs', + "Like": "like_logs", + "Assigned": "assignment_logs", + "Assignment Completed": "assignment_logs", + "Comment": "comments", + "Attachment": "attachment_logs", + "Attachment Removed": "attachment_logs", } key = key_map.get(self.comment_type) - if not key: return + if not key: + return - frappe.publish_realtime('update_docinfo_for_{}_{}'.format(self.reference_doctype, self.reference_name), { - 'doc': self.as_dict(), - 'key': key, - 'action': action - }, after_commit=True) + frappe.publish_realtime( + "update_docinfo_for_{}_{}".format(self.reference_doctype, self.reference_name), + {"doc": self.as_dict(), "key": key, "action": action}, + after_commit=True, + ) def remove_comment_from_cache(self): _comments = get_comments_from_parent(self) for c in _comments: - if c.get("name")==self.name: + if c.get("name") == self.name: _comments.remove(c) update_comments_in_parent(self.reference_doctype, self.reference_name, _comments) @@ -68,19 +74,26 @@ class Comment(Document): sender_fullname = get_fullname(frappe.session.user) title = get_title(self.reference_doctype, self.reference_name) - recipients = [frappe.db.get_value("User", {"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1}, "email") - for name in mentions] + recipients = [ + frappe.db.get_value( + "User", + {"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1}, + "email", + ) + for name in mentions + ] - notification_message = _('''{0} mentioned you in a comment in {1} {2}''')\ - .format(frappe.bold(sender_fullname), frappe.bold(self.reference_doctype), get_title_html(title)) + notification_message = _("""{0} mentioned you in a comment in {1} {2}""").format( + frappe.bold(sender_fullname), frappe.bold(self.reference_doctype), get_title_html(title) + ) notification_doc = { - 'type': 'Mention', - 'document_type': self.reference_doctype, - 'document_name': self.reference_name, - 'subject': notification_message, - 'from_user': frappe.session.user, - 'email_content': self.content + "type": "Mention", + "document_type": self.reference_doctype, + "document_name": self.reference_name, + "subject": notification_message, + "from_user": frappe.session.user, + "email_content": self.content, } enqueue_create_notification(recipients, notification_doc) @@ -99,45 +112,46 @@ def update_comment_in_doc(doc): `_comments` format - { - "comment": [String], - "by": [user], - "name": [Comment Document name] - }""" + { + "comment": [String], + "by": [user], + "name": [Comment Document name] + }""" # only comments get updates, not likes, assignments etc. - if doc.doctype == 'Comment' and doc.comment_type != 'Comment': + if doc.doctype == "Comment" and doc.comment_type != "Comment": return def get_truncated(content): - return (content[:97] + '...') if len(content) > 100 else content + return (content[:97] + "...") if len(content) > 100 else content if doc.reference_doctype and doc.reference_name and doc.content: _comments = get_comments_from_parent(doc) updated = False for c in _comments: - if c.get("name")==doc.name: + if c.get("name") == doc.name: c["comment"] = get_truncated(doc.content) updated = True if not updated: - _comments.append({ - "comment": get_truncated(doc.content), - - # "comment_email" for Comment and "sender" for Communication - "by": getattr(doc, 'comment_email', None) or getattr(doc, 'sender', None) or doc.owner, - "name": doc.name - }) + _comments.append( + { + "comment": get_truncated(doc.content), + # "comment_email" for Comment and "sender" for Communication + "by": getattr(doc, "comment_email", None) or getattr(doc, "sender", None) or doc.owner, + "name": doc.name, + } + ) update_comments_in_parent(doc.reference_doctype, doc.reference_name, _comments) def get_comments_from_parent(doc): - ''' + """ get the list of comments cached in the document record in the column `_comments` - ''' + """ try: _comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]" @@ -153,23 +167,32 @@ def get_comments_from_parent(doc): except ValueError: return [] + def update_comments_in_parent(reference_doctype, reference_name, _comments): """Updates `_comments` property in parent Document with given dict. :param _comments: Dict of comments.""" - if not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle") or frappe.db.get_value("DocType", reference_doctype, "is_virtual"): + if ( + not reference_doctype + or not reference_name + or frappe.db.get_value("DocType", reference_doctype, "issingle") + or frappe.db.get_value("DocType", reference_doctype, "is_virtual") + ): return try: # use sql, so that we do not mess with the timestamp - frappe.db.sql("""update `tab{0}` set `_comments`=%s where name=%s""".format(reference_doctype), # nosec - (json.dumps(_comments[-100:]), reference_name)) + frappe.db.sql( + """update `tab{0}` set `_comments`=%s where name=%s""".format(reference_doctype), # nosec + (json.dumps(_comments[-100:]), reference_name), + ) except Exception as e: - if frappe.db.is_column_missing(e) and getattr(frappe.local, 'request', None): + if frappe.db.is_column_missing(e) and getattr(frappe.local, "request", None): # missing column and in request, add column and update after commit - frappe.local._comments = (getattr(frappe.local, "_comments", []) - + [(reference_doctype, reference_name, _comments)]) + frappe.local._comments = getattr(frappe.local, "_comments", []) + [ + (reference_doctype, reference_name, _comments) + ] elif frappe.db.is_data_too_long(e): raise frappe.DataTooLongException @@ -183,6 +206,7 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): if getattr(reference_doc, "route", None): clear_cache(reference_doc.route) + def update_comments_in_parent_after_request(): """update _comments in parent if _comments column is missing""" if hasattr(frappe.local, "_comments"): diff --git a/frappe/core/doctype/comment/test_comment.py b/frappe/core/doctype/comment/test_comment.py index 33672a7dea..3072f8b5b9 100644 --- a/frappe/core/doctype/comment/test_comment.py +++ b/frappe/core/doctype/comment/test_comment.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe, json +import json import unittest +import frappe + + class TestComment(unittest.TestCase): def tearDown(self): frappe.form_dict.comment = None @@ -15,75 +18,88 @@ class TestComment(unittest.TestCase): frappe.local.request_ip = None def test_comment_creation(self): - test_doc = frappe.get_doc(dict(doctype = 'ToDo', description = 'test')) + test_doc = frappe.get_doc(dict(doctype="ToDo", description="test")) test_doc.insert() - comment = test_doc.add_comment('Comment', 'test comment') + comment = test_doc.add_comment("Comment", "test comment") test_doc.reload() # check if updated in _comments cache - comments = json.loads(test_doc.get('_comments')) - self.assertEqual(comments[0].get('name'), comment.name) - self.assertEqual(comments[0].get('comment'), comment.content) + comments = json.loads(test_doc.get("_comments")) + self.assertEqual(comments[0].get("name"), comment.name) + self.assertEqual(comments[0].get("comment"), comment.content) # check document creation - comment_1 = frappe.get_all('Comment', fields = ['*'], filters = dict( - reference_doctype = test_doc.doctype, - reference_name = test_doc.name - ))[0] + comment_1 = frappe.get_all( + "Comment", + fields=["*"], + filters=dict(reference_doctype=test_doc.doctype, reference_name=test_doc.name), + )[0] - self.assertEqual(comment_1.content, 'test comment') + self.assertEqual(comment_1.content, "test comment") # test via blog def test_public_comment(self): from frappe.website.doctype.blog_post.test_blog_post import make_test_blog + test_blog = make_test_blog() frappe.db.delete("Comment", {"reference_doctype": "Blog Post"}) from frappe.templates.includes.comments.comments import add_comment - frappe.form_dict.comment = 'Good comment with 10 chars' - frappe.form_dict.comment_email = 'test@test.com' - frappe.form_dict.comment_by = 'Good Tester' - frappe.form_dict.reference_doctype = 'Blog Post' + frappe.form_dict.comment = "Good comment with 10 chars" + frappe.form_dict.comment_email = "test@test.com" + frappe.form_dict.comment_by = "Good Tester" + frappe.form_dict.reference_doctype = "Blog Post" frappe.form_dict.reference_name = test_blog.name frappe.form_dict.route = test_blog.route - frappe.local.request_ip = '127.0.0.1' + frappe.local.request_ip = "127.0.0.1" add_comment() - self.assertEqual(frappe.get_all('Comment', fields = ['*'], filters = dict( - reference_doctype = test_blog.doctype, - reference_name = test_blog.name - ))[0].published, 1) + self.assertEqual( + frappe.get_all( + "Comment", + fields=["*"], + filters=dict(reference_doctype=test_blog.doctype, reference_name=test_blog.name), + )[0].published, + 1, + ) frappe.db.delete("Comment", {"reference_doctype": "Blog Post"}) - frappe.form_dict.comment = 'pleez vizits my site http://mysite.com' - frappe.form_dict.comment_by = 'bad commentor' + frappe.form_dict.comment = "pleez vizits my site http://mysite.com" + frappe.form_dict.comment_by = "bad commentor" add_comment() - self.assertEqual(len(frappe.get_all('Comment', fields = ['*'], filters = dict( - reference_doctype = test_blog.doctype, - reference_name = test_blog.name - ))), 0) + self.assertEqual( + len( + frappe.get_all( + "Comment", + fields=["*"], + filters=dict(reference_doctype=test_blog.doctype, reference_name=test_blog.name), + ) + ), + 0, + ) # test for filtering html and css injection elements frappe.db.delete("Comment", {"reference_doctype": "Blog Post"}) - frappe.form_dict.comment = 'Comment' - frappe.form_dict.comment_by = 'hacker' + frappe.form_dict.comment = "Comment" + frappe.form_dict.comment_by = "hacker" add_comment() - self.assertEqual(frappe.get_all('Comment', fields = ['content'], filters = dict( - reference_doctype = test_blog.doctype, - reference_name = test_blog.name - ))[0]['content'], 'Comment') + self.assertEqual( + frappe.get_all( + "Comment", + fields=["content"], + filters=dict(reference_doctype=test_blog.doctype, reference_name=test_blog.name), + )[0]["content"], + "Comment", + ) test_blog.delete() - - - diff --git a/frappe/core/doctype/communication/__init__.py b/frappe/core/doctype/communication/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/communication/__init__.py +++ b/frappe/core/doctype/communication/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 38fb0fd757..409c4c0956 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -2,49 +2,67 @@ # License: MIT. See LICENSE from collections import Counter +from email.utils import getaddresses from typing import List +from urllib.parse import unquote + +from parse import compile + import frappe from frappe import _ -from frappe.model.document import Document -from frappe.utils import validate_email_address, strip_html, cstr, time_diff_in_seconds +from frappe.automation.doctype.assignment_rule.assignment_rule import ( + apply as apply_assignment_rule, +) +from frappe.contacts.doctype.contact.contact import get_contact_name +from frappe.core.doctype.comment.comment import update_comment_in_doc from frappe.core.doctype.communication.email import validate_email from frappe.core.doctype.communication.mixins import CommunicationEmailMixin from frappe.core.utils import get_parent_doc -from frappe.utils import parse_addr, split_emails -from frappe.core.doctype.comment.comment import update_comment_in_doc -from email.utils import getaddresses -from urllib.parse import unquote +from frappe.model.document import Document +from frappe.utils import ( + cstr, + parse_addr, + split_emails, + strip_html, + time_diff_in_seconds, + validate_email_address, +) from frappe.utils.user import is_system_user -from frappe.contacts.doctype.contact.contact import get_contact_name -from frappe.automation.doctype.assignment_rule.assignment_rule import apply as apply_assignment_rule -from parse import compile exclude_from_linked_with = True + class Communication(Document, CommunicationEmailMixin): - """Communication represents an external communication like Email. - """ + """Communication represents an external communication like Email.""" + no_feed_on_delete = True - DOCTYPE = 'Communication' + DOCTYPE = "Communication" def onload(self): """create email flag queue""" - if self.communication_type == "Communication" and self.communication_medium == "Email" \ - and self.sent_or_received == "Received" and self.uid and self.uid != -1: - - email_flag_queue = frappe.db.get_value("Email Flag Queue", { - "communication": self.name, - "is_completed": 0}) + if ( + self.communication_type == "Communication" + and self.communication_medium == "Email" + and self.sent_or_received == "Received" + and self.uid + and self.uid != -1 + ): + + email_flag_queue = frappe.db.get_value( + "Email Flag Queue", {"communication": self.name, "is_completed": 0} + ) if email_flag_queue: return - frappe.get_doc({ - "doctype": "Email Flag Queue", - "action": "Read", - "communication": self.name, - "uid": self.uid, - "email_account": self.email_account - }).insert(ignore_permissions=True) + frappe.get_doc( + { + "doctype": "Email Flag Queue", + "action": "Read", + "communication": self.name, + "uid": self.uid, + "email_account": self.email_account, + } + ).insert(ignore_permissions=True) frappe.db.commit() def validate(self): @@ -74,25 +92,33 @@ class Communication(Document, CommunicationEmailMixin): def validate_reference(self): if self.reference_doctype and self.reference_name: if not self.reference_owner: - self.reference_owner = frappe.db.get_value(self.reference_doctype, self.reference_name, "owner") + self.reference_owner = frappe.db.get_value( + self.reference_doctype, self.reference_name, "owner" + ) # prevent communication against a child table if frappe.get_meta(self.reference_doctype).istable: - frappe.throw(_("Cannot create a {0} against a child document: {1}") - .format(_(self.communication_type), _(self.reference_doctype))) + frappe.throw( + _("Cannot create a {0} against a child document: {1}").format( + _(self.communication_type), _(self.reference_doctype) + ) + ) # Prevent circular linking of Communication DocTypes if self.reference_doctype == "Communication": circular_linking = False doc = get_parent_doc(self) while doc.reference_doctype == "Communication": - if get_parent_doc(doc).name==self.name: + if get_parent_doc(doc).name == self.name: circular_linking = True break doc = get_parent_doc(doc) if circular_linking: - frappe.throw(_("Please make sure the Reference Communication Docs are not circularly linked."), frappe.CircularLinkingError) + frappe.throw( + _("Please make sure the Reference Communication Docs are not circularly linked."), + frappe.CircularLinkingError, + ) def after_insert(self): if not (self.reference_doctype and self.reference_name): @@ -102,21 +128,21 @@ class Communication(Document, CommunicationEmailMixin): frappe.db.set_value("Communication", self.reference_name, "status", "Replied") if self.communication_type == "Communication": - self.notify_change('add') + self.notify_change("add") elif self.communication_type in ("Chat", "Notification"): if self.reference_name == frappe.session.user: message = self.as_dict() - message['broadcast'] = True - frappe.publish_realtime('new_message', message, after_commit=True) + message["broadcast"] = True + frappe.publish_realtime("new_message", message, after_commit=True) else: # reference_name contains the user who is addressed in the messages' page comment - frappe.publish_realtime('new_message', self.as_dict(), - user=self.reference_name, after_commit=True) + frappe.publish_realtime( + "new_message", self.as_dict(), user=self.reference_name, after_commit=True + ) def set_signature_in_email_content(self): - """Set sender's User.email_signature or default outgoing's EmailAccount.signature to the email - """ + """Set sender's User.email_signature or default outgoing's EmailAccount.signature to the email""" if not self.content: return @@ -128,11 +154,15 @@ class Communication(Document, CommunicationEmailMixin): email_body = email_body[0] - user_email_signature = frappe.db.get_value( - "User", - self.sender, - "email_signature", - ) if self.sender else None + user_email_signature = ( + frappe.db.get_value( + "User", + self.sender, + "email_signature", + ) + if self.sender + else None + ) signature = user_email_signature or frappe.db.get_value( "Email Account", @@ -157,19 +187,19 @@ class Communication(Document, CommunicationEmailMixin): # comments count for the list view update_comment_in_doc(self) - if self.comment_type != 'Updated': + if self.comment_type != "Updated": update_parent_document_on_communication(self) def on_trash(self): if self.communication_type == "Communication": - self.notify_change('delete') + self.notify_change("delete") @property def sender_mailid(self): return parse_addr(self.sender)[1] if self.sender else "" @staticmethod - def _get_emails_list(emails=None, exclude_displayname = False): + def _get_emails_list(emails=None, exclude_displayname=False): """Returns list of emails from given email string. * Removes duplicate mailids @@ -180,35 +210,32 @@ class Communication(Document, CommunicationEmailMixin): return [email.lower() for email in set([parse_addr(email)[1] for email in emails]) if email] return [email.lower() for email in set(emails) if email] - def to_list(self, exclude_displayname = True): - """Returns to list. - """ + def to_list(self, exclude_displayname=True): + """Returns to list.""" return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname) - def cc_list(self, exclude_displayname = True): - """Returns cc list. - """ + def cc_list(self, exclude_displayname=True): + """Returns cc list.""" return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname) - def bcc_list(self, exclude_displayname = True): - """Returns bcc list. - """ + def bcc_list(self, exclude_displayname=True): + """Returns bcc list.""" return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname) def get_attachments(self): attachments = frappe.get_all( "File", fields=["name", "file_name", "file_url", "is_private"], - filters = {"attached_to_name": self.name, "attached_to_doctype": self.DOCTYPE} + filters={"attached_to_name": self.name, "attached_to_doctype": self.DOCTYPE}, ) return attachments def notify_change(self, action): - frappe.publish_realtime('update_docinfo_for_{}_{}'.format(self.reference_doctype, self.reference_name), { - 'doc': self.as_dict(), - 'key': 'communications', - 'action': action - }, after_commit=True) + frappe.publish_realtime( + "update_docinfo_for_{}_{}".format(self.reference_doctype, self.reference_name), + {"doc": self.as_dict(), "key": "communications", "action": action}, + after_commit=True, + ) def set_status(self): if not self.is_new(): @@ -216,15 +243,19 @@ class Communication(Document, CommunicationEmailMixin): if self.reference_doctype and self.reference_name: self.status = "Linked" - elif self.communication_type=="Communication": + elif self.communication_type == "Communication": self.status = "Open" else: self.status = "Closed" # set email status to spam - email_rule = frappe.db.get_value("Email Rule", { "email_id": self.sender, "is_spam":1 }) - if self.communication_type == "Communication" and self.communication_medium == "Email" \ - and self.sent_or_received == "Sent" and email_rule: + email_rule = frappe.db.get_value("Email Rule", {"email_id": self.sender, "is_spam": 1}) + if ( + self.communication_type == "Communication" + and self.communication_medium == "Email" + and self.sent_or_received == "Sent" + and email_rule + ): self.email_status = "Spam" @@ -254,7 +285,7 @@ class Communication(Document, CommunicationEmailMixin): self.sender_full_name = self.sender self.sender = None else: - if self.sent_or_received=='Sent': + if self.sent_or_received == "Sent": validate_email_address(self.sender, throw=True) sender_name, sender_email = parse_addr(self.sender) if sender_name == sender_email: @@ -264,40 +295,41 @@ class Communication(Document, CommunicationEmailMixin): self.sender_full_name = sender_name if not self.sender_full_name: - self.sender_full_name = frappe.db.get_value('User', self.sender, 'full_name') + self.sender_full_name = frappe.db.get_value("User", self.sender, "full_name") if not self.sender_full_name: - first_name, last_name = frappe.db.get_value('Contact', - filters={'email_id': sender_email}, - fieldname=['first_name', 'last_name'] + first_name, last_name = frappe.db.get_value( + "Contact", filters={"email_id": sender_email}, fieldname=["first_name", "last_name"] ) or [None, None] - self.sender_full_name = (first_name or '') + (last_name or '') + self.sender_full_name = (first_name or "") + (last_name or "") if not self.sender_full_name: self.sender_full_name = sender_email def set_delivery_status(self, commit=False): - '''Look into the status of Email Queue linked to this Communication and set the Delivery Status of this Communication''' + """Look into the status of Email Queue linked to this Communication and set the Delivery Status of this Communication""" delivery_status = None - status_counts = Counter(frappe.get_all("Email Queue", pluck="status", filters={"communication": self.name})) + status_counts = Counter( + frappe.get_all("Email Queue", pluck="status", filters={"communication": self.name}) + ) if self.sent_or_received == "Received": return - if status_counts.get('Not Sent') or status_counts.get('Sending'): - delivery_status = 'Sending' + if status_counts.get("Not Sent") or status_counts.get("Sending"): + delivery_status = "Sending" - elif status_counts.get('Error'): - delivery_status = 'Error' + elif status_counts.get("Error"): + delivery_status = "Error" - elif status_counts.get('Expired'): - delivery_status = 'Expired' + elif status_counts.get("Expired"): + delivery_status = "Expired" - elif status_counts.get('Sent'): - delivery_status = 'Sent' + elif status_counts.get("Sent"): + delivery_status = "Sent" if delivery_status: - self.db_set('delivery_status', delivery_status) - self.notify_change('update') + self.db_set("delivery_status", delivery_status) + self.notify_change("update") # for list views and forms self.notify_update() @@ -311,13 +343,17 @@ class Communication(Document, CommunicationEmailMixin): # Timeline Links def set_timeline_links(self): contacts = [] - create_contact_enabled = self.email_account and frappe.db.get_value("Email Account", self.email_account, "create_contact") - contacts = get_contacts([self.sender, self.recipients, self.cc, self.bcc], auto_create_contact=create_contact_enabled) + create_contact_enabled = self.email_account and frappe.db.get_value( + "Email Account", self.email_account, "create_contact" + ) + contacts = get_contacts( + [self.sender, self.recipients, self.cc, self.bcc], auto_create_contact=create_contact_enabled + ) for contact_name in contacts: - self.add_link('Contact', contact_name) + self.add_link("Contact", contact_name) - #link contact's dynamic links to communication + # link contact's dynamic links to communication add_contact_links_to_communication(self, contact_name) def deduplicate_timeline_links(self): @@ -332,17 +368,12 @@ class Communication(Document, CommunicationEmailMixin): duplicate = True if duplicate: - del self.timeline_links[:] # make it python 2 compatible as list.clear() is python 3 only + del self.timeline_links[:] # make it python 2 compatible as list.clear() is python 3 only for l in links: self.add_link(link_doctype=l[0], link_name=l[1]) def add_link(self, link_doctype, link_name, autosave=False): - self.append("timeline_links", - { - "link_doctype": link_doctype, - "link_name": link_name - } - ) + self.append("timeline_links", {"link_doctype": link_doctype, "link_name": link_name}) if autosave: self.save(ignore_permissions=True) @@ -358,13 +389,15 @@ class Communication(Document, CommunicationEmailMixin): if autosave: self.save(ignore_permissions=ignore_permissions) + def on_doctype_update(): """Add indexes in `tabCommunication`""" frappe.db.add_index("Communication", ["reference_doctype", "reference_name"]) frappe.db.add_index("Communication", ["status", "communication_type"]) + def has_permission(doc, ptype, user): - if ptype=="read": + if ptype == "read": if doc.reference_doctype == "Communication" and doc.reference_name == doc.name: return @@ -372,24 +405,28 @@ def has_permission(doc, ptype, user): if frappe.has_permission(doc.reference_doctype, ptype="read", doc=doc.reference_name): return True + def get_permission_query_conditions_for_communication(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user roles = frappe.get_roles(user) if "Super Email User" in roles or "System Manager" in roles: return None else: - accounts = frappe.get_all("User Email", filters={ "parent": user }, - fields=["email_account"], - distinct=True, order_by="idx") + accounts = frappe.get_all( + "User Email", filters={"parent": user}, fields=["email_account"], distinct=True, order_by="idx" + ) if not accounts: return """`tabCommunication`.communication_medium!='Email'""" - email_accounts = [ '"%s"'%account.get("email_account") for account in accounts ] - return """`tabCommunication`.email_account in ({email_accounts})"""\ - .format(email_accounts=','.join(email_accounts)) + email_accounts = ['"%s"' % account.get("email_account") for account in accounts] + return """`tabCommunication`.email_account in ({email_accounts})""".format( + email_accounts=",".join(email_accounts) + ) + def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[str]: email_addrs = get_emails(email_strings) @@ -403,12 +440,12 @@ def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[st first_name = frappe.unscrub(email_parts[0]) try: - contact_name = '{0}-{1}'.format(first_name, email_parts[1]) if first_name == 'Contact' else first_name - contact = frappe.get_doc({ - "doctype": "Contact", - "first_name": contact_name, - "name": contact_name - }) + contact_name = ( + "{0}-{1}".format(first_name, email_parts[1]) if first_name == "Contact" else first_name + ) + contact = frappe.get_doc( + {"doctype": "Contact", "first_name": contact_name, "name": contact_name} + ) contact.add_email(email_id=email, is_primary=True) contact.insert(ignore_permissions=True) contact_name = contact.name @@ -421,6 +458,7 @@ def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[st return contacts + def get_emails(email_strings: List[str]) -> List[str]: email_addrs = [] @@ -432,22 +470,25 @@ def get_emails(email_strings: List[str]) -> List[str]: return email_addrs + def add_contact_links_to_communication(communication, contact_name): - contact_links = frappe.get_all("Dynamic Link", filters={ - "parenttype": "Contact", - "parent": contact_name - }, fields=["link_doctype", "link_name"]) + contact_links = frappe.get_all( + "Dynamic Link", + filters={"parenttype": "Contact", "parent": contact_name}, + fields=["link_doctype", "link_name"], + ) if contact_links: for contact_link in contact_links: communication.add_link(contact_link.link_doctype, contact_link.link_name) + def parse_email(communication, email_strings): """ - Parse email to add timeline links. - When automatic email linking is enabled, an email from email_strings can contain - a doctype and docname ie in the format `admin+doctype+docname@example.com`, - the email is parsed and doctype and docname is extracted and timeline link is added. + Parse email to add timeline links. + When automatic email linking is enabled, an email from email_strings can contain + a doctype and docname ie in the format `admin+doctype+docname@example.com`, + the email is parsed and doctype and docname is extracted and timeline link is added. """ if not frappe.get_all("Email Account", filters={"enable_automatic_linking": 1}): return @@ -469,10 +510,11 @@ def parse_email(communication, email_strings): if doctype and docname and frappe.db.exists(doctype, docname): communication.add_link(doctype, docname) + def get_email_without_link(email): """ - returns email address without doctype links - returns admin@example.com for email admin+doctype+docname@example.com + returns email address without doctype links + returns admin@example.com for email admin+doctype+docname@example.com """ if not frappe.get_all("Email Account", filters={"enable_automatic_linking": 1}): return email @@ -486,6 +528,7 @@ def get_email_without_link(email): return "{0}@{1}".format(email_id, email_host) + def update_parent_document_on_communication(doc): """Update mins_to_first_communication of parent document based on who is replying.""" @@ -516,6 +559,7 @@ def update_parent_document_on_communication(doc): parent.run_method("notify_communication", doc) parent.notify_update() + def update_first_response_time(parent, communication): if parent.meta.has_field("first_response_time") and not parent.get("first_response_time"): if is_system_user(communication.sender): @@ -526,25 +570,29 @@ def update_first_response_time(parent, communication): first_response_time = round(time_diff_in_seconds(first_responded_on, parent.creation), 2) parent.db_set("first_response_time", first_response_time) + def set_avg_response_time(parent, communication): if parent.meta.has_field("avg_response_time") and communication.sent_or_received == "Sent": # avg response time for all the responses - communications = frappe.get_list("Communication", filters={ - "reference_doctype": parent.doctype, - "reference_name": parent.name - }, + communications = frappe.get_list( + "Communication", + filters={"reference_doctype": parent.doctype, "reference_name": parent.name}, fields=["sent_or_received", "name", "creation"], - order_by="creation" + order_by="creation", ) if len(communications): response_times = [] for i in range(len(communications)): - if communications[i].sent_or_received == "Sent" and communications[i-1].sent_or_received == "Received": - response_time = round(time_diff_in_seconds(communications[i].creation, communications[i-1].creation), 2) + if ( + communications[i].sent_or_received == "Sent" + and communications[i - 1].sent_or_received == "Received" + ): + response_time = round( + time_diff_in_seconds(communications[i].creation, communications[i - 1].creation), 2 + ) if response_time > 0: response_times.append(response_time) if response_times: avg_response_time = sum(response_times) / len(response_times) parent.db_set("avg_response_time", avg_response_time) - diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index b51749ccb7..5737572194 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -8,17 +8,25 @@ import frappe import frappe.email.smtp from frappe import _ from frappe.email.email_body import get_message_id -from frappe.utils import (cint, get_datetime, get_formatted_email, - list_to_str, split_emails, validate_email_address) +from frappe.utils import ( + cint, + get_datetime, + get_formatted_email, + list_to_str, + split_emails, + validate_email_address, +) if TYPE_CHECKING: from frappe.core.doctype.communication.communication import Communication -OUTGOING_EMAIL_ACCOUNT_MISSING = _(""" +OUTGOING_EMAIL_ACCOUNT_MISSING = _( + """ Unable to send mail because of a missing email account. Please setup default Email Account from Setup > Email > Email Account -""") +""" +) @frappe.whitelist() @@ -64,16 +72,15 @@ def make( """ if kwargs: from frappe.utils.commands import warn + warn( f"Options {kwargs} used in frappe.core.doctype.communication.email.make " "are deprecated or unsupported", - category=DeprecationWarning + category=DeprecationWarning, ) if doctype and name and not frappe.has_permission(doctype=doctype, ptype="email", doc=name): - raise frappe.PermissionError( - f"You are not allowed to send emails related to: {doctype} {name}" - ) + raise frappe.PermissionError(f"You are not allowed to send emails related to: {doctype} {name}") return _make( doctype=doctype, @@ -123,33 +130,34 @@ def _make( communication_type=None, add_signature=True, ) -> Dict[str, str]: - """Internal method to make a new communication that ignores Permission checks. - """ + """Internal method to make a new communication that ignores Permission checks.""" sender = sender or get_formatted_email(frappe.session.user) recipients = list_to_str(recipients) if isinstance(recipients, list) else recipients cc = list_to_str(cc) if isinstance(cc, list) else cc bcc = list_to_str(bcc) if isinstance(bcc, list) else bcc - comm: "Communication" = frappe.get_doc({ - "doctype":"Communication", - "subject": subject, - "content": content, - "sender": sender, - "sender_full_name":sender_full_name, - "recipients": recipients, - "cc": cc or None, - "bcc": bcc or None, - "communication_medium": communication_medium, - "sent_or_received": sent_or_received, - "reference_doctype": doctype, - "reference_name": name, - "email_template": email_template, - "message_id":get_message_id().strip(" <>"), - "read_receipt":read_receipt, - "has_attachment": 1 if attachments else 0, - "communication_type": communication_type, - }) + comm: "Communication" = frappe.get_doc( + { + "doctype": "Communication", + "subject": subject, + "content": content, + "sender": sender, + "sender_full_name": sender_full_name, + "recipients": recipients, + "cc": cc or None, + "bcc": bcc or None, + "communication_medium": communication_medium, + "sent_or_received": sent_or_received, + "reference_doctype": doctype, + "reference_name": name, + "email_template": email_template, + "message_id": get_message_id().strip(" <>"), + "read_receipt": read_receipt, + "has_attachment": 1 if attachments else 0, + "communication_type": communication_type, + } + ) comm.flags.skip_add_signature = not add_signature comm.insert(ignore_permissions=True) @@ -161,9 +169,7 @@ def _make( if cint(send_email): if not comm.get_outgoing_email_account(): - frappe.throw( - msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError - ) + frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError) comm.send_email( print_html=print_html, @@ -179,7 +185,10 @@ def _make( def validate_email(doc: "Communication") -> None: """Validate Email Addresses of Recipients and CC""" - if not (doc.communication_type=="Communication" and doc.communication_medium == "Email") or doc.flags.in_receive: + if ( + not (doc.communication_type == "Communication" and doc.communication_medium == "Email") + or doc.flags.in_receive + ): return # validate recipients @@ -193,36 +202,45 @@ def validate_email(doc: "Communication") -> None: for email in split_emails(doc.bcc): validate_email_address(email, throw=True) + def set_incoming_outgoing_accounts(doc): from frappe.email.doctype.email_account.email_account import EmailAccount + incoming_email_account = EmailAccount.find_incoming( - match_by_email=doc.sender, match_by_doctype=doc.reference_doctype) + match_by_email=doc.sender, match_by_doctype=doc.reference_doctype + ) doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None doc.outgoing_email_account = EmailAccount.find_outgoing( - match_by_email=doc.sender, match_by_doctype=doc.reference_doctype) + match_by_email=doc.sender, match_by_doctype=doc.reference_doctype + ) if doc.sent_or_received == "Sent": doc.db_set("email_account", doc.outgoing_email_account.name) + def add_attachments(name, attachments): - '''Add attachments to the given Communication''' + """Add attachments to the given Communication""" # loop through attachments for a in attachments: if isinstance(a, str): - attach = frappe.db.get_value("File", {"name":a}, - ["file_name", "file_url", "is_private"], as_dict=1) + attach = frappe.db.get_value( + "File", {"name": a}, ["file_name", "file_url", "is_private"], as_dict=1 + ) # save attachments to new doc - _file = frappe.get_doc({ - "doctype": "File", - "file_url": attach.file_url, - "attached_to_doctype": "Communication", - "attached_to_name": name, - "folder": "Home/Attachments", - "is_private": attach.is_private - }) + _file = frappe.get_doc( + { + "doctype": "File", + "file_url": attach.file_url, + "attached_to_doctype": "Communication", + "attached_to_name": name, + "folder": "Home/Attachments", + "is_private": attach.is_private, + } + ) _file.save(ignore_permissions=True) + @frappe.whitelist(allow_guest=True, methods=("GET",)) def mark_email_as_seen(name: str = None): try: @@ -233,33 +251,31 @@ def mark_email_as_seen(name: str = None): frappe.log_error(frappe.get_traceback()) finally: - frappe.response.update({ - "type": "binary", - "filename": "imaginary_pixel.png", - "filecontent": ( - b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" - b"\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\r" - b"IDATx\x9cc\xf8\xff\xff?\x03\x00\x08\xfc\x02\xfe\xa7\x9a\xa0" - b"\xa0\x00\x00\x00\x00IEND\xaeB`\x82" - ) - }) + frappe.response.update( + { + "type": "binary", + "filename": "imaginary_pixel.png", + "filecontent": ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" + b"\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\r" + b"IDATx\x9cc\xf8\xff\xff?\x03\x00\x08\xfc\x02\xfe\xa7\x9a\xa0" + b"\xa0\x00\x00\x00\x00IEND\xaeB`\x82" + ), + } + ) + def update_communication_as_read(name): if not name or not isinstance(name, str): return - communication = frappe.db.get_value( - "Communication", - name, - "read_by_recipient", - as_dict=True - ) + communication = frappe.db.get_value("Communication", name, "read_by_recipient", as_dict=True) if not communication or communication.read_by_recipient: return - frappe.db.set_value("Communication", name, { - "read_by_recipient": 1, - "delivery_status": "Read", - "read_by_recipient_on": get_datetime() - }) + frappe.db.set_value( + "Communication", + name, + {"read_by_recipient": 1, "delivery_status": "Read", "read_by_recipient_on": get_datetime()}, + ) diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index dd9f58342e..68abba3c13 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -1,33 +1,34 @@ from typing import List + import frappe from frappe import _ from frappe.core.utils import get_parent_doc -from frappe.utils import parse_addr, get_formatted_email, get_url -from frappe.email.doctype.email_account.email_account import EmailAccount from frappe.desk.doctype.todo.todo import ToDo +from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.utils import get_formatted_email, get_url, parse_addr + class CommunicationEmailMixin: - """Mixin class to handle communication mails. - """ + """Mixin class to handle communication mails.""" + def is_email_communication(self): - return self.communication_type=="Communication" and self.communication_medium == "Email" + return self.communication_type == "Communication" and self.communication_medium == "Email" def get_owner(self): - """Get owner of the communication docs parent. - """ + """Get owner of the communication docs parent.""" parent_doc = get_parent_doc(self) return parent_doc.owner if parent_doc else None def get_all_email_addresses(self, exclude_displayname=False): - """Get all Email addresses mentioned in the doc along with display name. - """ - return self.to_list(exclude_displayname=exclude_displayname) + \ - self.cc_list(exclude_displayname=exclude_displayname) + \ - self.bcc_list(exclude_displayname=exclude_displayname) + """Get all Email addresses mentioned in the doc along with display name.""" + return ( + self.to_list(exclude_displayname=exclude_displayname) + + self.cc_list(exclude_displayname=exclude_displayname) + + self.bcc_list(exclude_displayname=exclude_displayname) + ) def get_email_with_displayname(self, email_address): - """Returns email address after adding displayname. - """ + """Returns email address after adding displayname.""" display_name, email = parse_addr(email_address) if display_name and display_name != email: return email_address @@ -37,26 +38,24 @@ class CommunicationEmailMixin: return email_map.get(email, email) def mail_recipients(self, is_inbound_mail_communcation=False): - """Build to(recipient) list to send an email. - """ + """Build to(recipient) list to send an email.""" # Incase of inbound mail, recipients already received the mail, no need to send again. if is_inbound_mail_communcation: return [] - if hasattr(self, '_final_recipients'): + if hasattr(self, "_final_recipients"): return self._final_recipients to = self.to_list() - self._final_recipients = list(filter(lambda id: id != 'Administrator', to)) + self._final_recipients = list(filter(lambda id: id != "Administrator", to)) return self._final_recipients def get_mail_recipients_with_displayname(self, is_inbound_mail_communcation=False): - """Build to(recipient) list to send an email including displayname in email. - """ + """Build to(recipient) list to send an email including displayname in email.""" to_list = self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation) return [self.get_email_with_displayname(email) for email in to_list] - def mail_cc(self, is_inbound_mail_communcation=False, include_sender = False): + def mail_cc(self, is_inbound_mail_communcation=False, include_sender=False): """Build cc list to send an email. * if email copy is requested by sender, then add sender to CC. @@ -67,7 +66,7 @@ class CommunicationEmailMixin: * FixMe: Removed adding TODO owners to cc list. Check if that is needed. """ - if hasattr(self, '_final_cc'): + if hasattr(self, "_final_cc"): return self._final_cc cc = self.cc_list() @@ -88,11 +87,13 @@ class CommunicationEmailMixin: if is_inbound_mail_communcation: cc = cc - set(self.cc_list() + self.to_list()) - self._final_cc = list(filter(lambda id: id != 'Administrator', cc)) + self._final_cc = list(filter(lambda id: id != "Administrator", cc)) return self._final_cc - def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender = False): - cc_list = self.mail_cc(is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender = include_sender) + def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender=False): + cc_list = self.mail_cc( + is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender + ) return [self.get_email_with_displayname(email) for email in cc_list] def mail_bcc(self, is_inbound_mail_communcation=False): @@ -102,7 +103,7 @@ class CommunicationEmailMixin: * User must be enabled in the system * remove_administrator_from_email_list """ - if hasattr(self, '_final_bcc'): + if hasattr(self, "_final_bcc"): return self._final_bcc bcc = set(self.bcc_list()) @@ -116,7 +117,7 @@ class CommunicationEmailMixin: if is_inbound_mail_communcation: bcc = bcc - set(self.bcc_list() + self.to_list()) - self._final_bcc = list(filter(lambda id: id != 'Administrator', bcc)) + self._final_bcc = list(filter(lambda id: id != "Administrator", bcc)) return self._final_bcc def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False): @@ -145,22 +146,23 @@ class CommunicationEmailMixin: def get_attach_link(self, print_format): """Returns public link for the attachment via `templates/emails/print_link.html`.""" - return frappe.get_template("templates/emails/print_link.html").render({ - "url": get_url(), - "doctype": self.reference_doctype, - "name": self.reference_name, - "print_format": print_format, - "key": get_parent_doc(self).get_signature() - }) + return frappe.get_template("templates/emails/print_link.html").render( + { + "url": get_url(), + "doctype": self.reference_doctype, + "name": self.reference_name, + "print_format": print_format, + "key": get_parent_doc(self).get_signature(), + } + ) def get_outgoing_email_account(self): - if not hasattr(self, '_outgoing_email_account'): + if not hasattr(self, "_outgoing_email_account"): if self.email_account: self._outgoing_email_account = EmailAccount.find(self.email_account) else: self._outgoing_email_account = EmailAccount.find_outgoing( - match_by_email=self.sender_mailid, - match_by_doctype=self.reference_doctype + match_by_email=self.sender_mailid, match_by_doctype=self.reference_doctype ) if self.sent_or_received == "Sent" and self._outgoing_email_account: @@ -169,10 +171,9 @@ class CommunicationEmailMixin: return self._outgoing_email_account def get_incoming_email_account(self): - if not hasattr(self, '_incoming_email_account'): + if not hasattr(self, "_incoming_email_account"): self._incoming_email_account = EmailAccount.find_incoming( - match_by_email=self.sender_mailid, - match_by_doctype=self.reference_doctype + match_by_email=self.sender_mailid, match_by_doctype=self.reference_doctype ) return self._incoming_email_account @@ -180,12 +181,17 @@ class CommunicationEmailMixin: final_attachments = [] if print_format or print_html: - d = {'print_format': print_format, 'html': print_html, 'print_format_attachment': 1, - 'doctype': self.reference_doctype, 'name': self.reference_name} + d = { + "print_format": print_format, + "html": print_html, + "print_format_attachment": 1, + "doctype": self.reference_doctype, + "name": self.reference_name, + } final_attachments.append(d) for a in self.get_attachments() or []: - final_attachments.append({"fid": a['name']}) + final_attachments.append({"fid": a["name"]}) return final_attachments @@ -193,48 +199,57 @@ class CommunicationEmailMixin: email_account = self.get_outgoing_email_account() if email_account and email_account.send_unsubscribe_message: return _("Leave this conversation") - return '' + return "" def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False) -> List: - """List of mail id's excluded while sending mail. - """ + """List of mail id's excluded while sending mail.""" all_ids = self.get_all_email_addresses(exclude_displayname=True) final_ids = ( self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation) + self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation) - + self.mail_cc(is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender) + + self.mail_cc( + is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender + ) ) return list(set(all_ids) - set(final_ids)) def get_assignees(self): - """Get owners of the reference document. - """ - filters = {'status': 'Open', 'reference_name': self.reference_name, - 'reference_type': self.reference_doctype} + """Get owners of the reference document.""" + filters = { + "status": "Open", + "reference_name": self.reference_name, + "reference_type": self.reference_doctype, + } return ToDo.get_owners(filters) @staticmethod def filter_thread_notification_disbled_users(emails): - """Filter users based on notifications for email threads setting is disabled. - """ + """Filter users based on notifications for email threads setting is disabled.""" if not emails: return [] - return frappe.get_all("User", pluck="email", filters={"email": ["in", emails], "thread_notify": 0}) + return frappe.get_all( + "User", pluck="email", filters={"email": ["in", emails], "thread_notify": 0} + ) @staticmethod def filter_disabled_users(emails): - """ - """ + """ """ if not emails: return [] return frappe.get_all("User", pluck="email", filters={"email": ["in", emails], "enabled": 0}) - def sendmail_input_dict(self, print_html=None, print_format=None, - send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None): + def sendmail_input_dict( + self, + print_html=None, + print_format=None, + send_me_a_copy=None, + print_letterhead=None, + is_inbound_mail_communcation=None, + ): outgoing_email_account = self.get_outgoing_email_account() if not outgoing_email_account: @@ -244,8 +259,7 @@ class CommunicationEmailMixin: is_inbound_mail_communcation=is_inbound_mail_communcation ) cc = self.get_mail_cc_with_displayname( - is_inbound_mail_communcation=is_inbound_mail_communcation, - include_sender = send_me_a_copy + is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=send_me_a_copy ) bcc = self.get_mail_bcc_with_displayname( is_inbound_mail_communcation=is_inbound_mail_communcation @@ -273,18 +287,24 @@ class CommunicationEmailMixin: "delayed": True, "communication": self.name, "read_receipt": self.read_receipt, - "is_notification": (self.sent_or_received =="Received" and True) or False, - "print_letterhead": print_letterhead + "is_notification": (self.sent_or_received == "Received" and True) or False, + "print_letterhead": print_letterhead, } - def send_email(self, print_html=None, print_format=None, - send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None): + def send_email( + self, + print_html=None, + print_format=None, + send_me_a_copy=None, + print_letterhead=None, + is_inbound_mail_communcation=None, + ): input_dict = self.sendmail_input_dict( print_html=print_html, print_format=print_format, send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead, - is_inbound_mail_communcation=is_inbound_mail_communcation + is_inbound_mail_communcation=is_inbound_mail_communcation, ) if input_dict: diff --git a/frappe/core/doctype/communication/test_communication.py b/frappe/core/doctype/communication/test_communication.py index 8012d8facf..a338295374 100644 --- a/frappe/core/doctype/communication/test_communication.py +++ b/frappe/core/doctype/communication/test_communication.py @@ -7,20 +7,30 @@ import frappe from frappe.core.doctype.communication.communication import get_emails from frappe.email.doctype.email_queue.email_queue import EmailQueue -test_records = frappe.get_test_records('Communication') +test_records = frappe.get_test_records("Communication") -class TestCommunication(unittest.TestCase): +class TestCommunication(unittest.TestCase): def test_email(self): - valid_email_list = ["Full Name ", - '"Full Name with quotes and " ', - "Surname, Name ", - "Purchase@ABC ", "xyz@abc2.com ", - "Name [something else] "] - - invalid_email_list = ["[invalid!email]", "invalid-email", - "tes2", "e", "rrrrrrrr", "manas","[[[sample]]]", - "[invalid!email].com"] + valid_email_list = [ + "Full Name ", + '"Full Name with quotes and " ', + "Surname, Name ", + "Purchase@ABC ", + "xyz@abc2.com ", + "Name [something else] ", + ] + + invalid_email_list = [ + "[invalid!email]", + "invalid-email", + "tes2", + "e", + "rrrrrrrr", + "manas", + "[[[sample]]]", + "[invalid!email].com", + ] for x in valid_email_list: self.assertTrue(frappe.utils.parse_addr(x)[1]) @@ -29,15 +39,25 @@ class TestCommunication(unittest.TestCase): self.assertFalse(frappe.utils.parse_addr(x)[0]) def test_name(self): - valid_email_list = ["Full Name ", - '"Full Name with quotes and " ', - "Surname, Name ", - "Purchase@ABC ", "xyz@abc2.com ", - "Name [something else] "] - - invalid_email_list = ["[invalid!email]", "invalid-email", - "tes2", "e", "rrrrrrrr", "manas","[[[sample]]]", - "[invalid!email].com"] + valid_email_list = [ + "Full Name ", + '"Full Name with quotes and " ', + "Surname, Name ", + "Purchase@ABC ", + "xyz@abc2.com ", + "Name [something else] ", + ] + + invalid_email_list = [ + "[invalid!email]", + "invalid-email", + "tes2", + "e", + "rrrrrrrr", + "manas", + "[[[sample]]]", + "[invalid!email].com", + ] for x in valid_email_list: self.assertTrue(frappe.utils.parse_addr(x)[0]) @@ -46,27 +66,33 @@ class TestCommunication(unittest.TestCase): self.assertFalse(frappe.utils.parse_addr(x)[0]) def test_circular_linking(self): - a = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "This was created to test circular linking: Communication A", - }).insert(ignore_permissions=True) - - b = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "This was created to test circular linking: Communication B", - "reference_doctype": "Communication", - "reference_name": a.name - }).insert(ignore_permissions=True) - - c = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "This was created to test circular linking: Communication C", - "reference_doctype": "Communication", - "reference_name": b.name - }).insert(ignore_permissions=True) + a = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "This was created to test circular linking: Communication A", + } + ).insert(ignore_permissions=True) + + b = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "This was created to test circular linking: Communication B", + "reference_doctype": "Communication", + "reference_name": a.name, + } + ).insert(ignore_permissions=True) + + c = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "This was created to test circular linking: Communication C", + "reference_doctype": "Communication", + "reference_name": b.name, + } + ).insert(ignore_permissions=True) a = frappe.get_doc("Communication", a.name) a.reference_doctype = "Communication" @@ -77,20 +103,24 @@ class TestCommunication(unittest.TestCase): def test_deduplication_timeline_links(self): frappe.delete_doc_if_exists("Note", "deduplication timeline links") - note = frappe.get_doc({ - "doctype": "Note", - "title": "deduplication timeline links", - "content": "deduplication timeline links" - }).insert(ignore_permissions=True) - - comm = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "Deduplication of Links", - "communication_medium": "Email" - }).insert(ignore_permissions=True) - - #adding same link twice + note = frappe.get_doc( + { + "doctype": "Note", + "title": "deduplication timeline links", + "content": "deduplication timeline links", + } + ).insert(ignore_permissions=True) + + comm = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "Deduplication of Links", + "communication_medium": "Email", + } + ).insert(ignore_permissions=True) + + # adding same link twice comm.add_link(link_doctype="Note", link_name=note.name, autosave=True) comm.add_link(link_doctype="Note", link_name=note.name, autosave=True) @@ -99,35 +129,43 @@ class TestCommunication(unittest.TestCase): self.assertNotEqual(2, len(comm.timeline_links)) def test_contacts_attached(self): - contact_sender = frappe.get_doc({ - "doctype": "Contact", - "first_name": "contact_sender", - }) + contact_sender = frappe.get_doc( + { + "doctype": "Contact", + "first_name": "contact_sender", + } + ) contact_sender.add_email("comm_sender@example.com") contact_sender.insert(ignore_permissions=True) - contact_recipient = frappe.get_doc({ - "doctype": "Contact", - "first_name": "contact_recipient", - }) + contact_recipient = frappe.get_doc( + { + "doctype": "Contact", + "first_name": "contact_recipient", + } + ) contact_recipient.add_email("comm_recipient@example.com") contact_recipient.insert(ignore_permissions=True) - contact_cc = frappe.get_doc({ - "doctype": "Contact", - "first_name": "contact_cc", - }) + contact_cc = frappe.get_doc( + { + "doctype": "Contact", + "first_name": "contact_cc", + } + ) contact_cc.add_email("comm_cc@example.com") contact_cc.insert(ignore_permissions=True) - comm = frappe.get_doc({ - "doctype": "Communication", - "communication_medium": "Email", - "subject": "Contacts Attached Test", - "sender": "comm_sender@example.com", - "recipients": "comm_recipient@example.com", - "cc": "comm_cc@example.com" - }).insert(ignore_permissions=True) + comm = frappe.get_doc( + { + "doctype": "Communication", + "communication_medium": "Email", + "subject": "Contacts Attached Test", + "sender": "comm_sender@example.com", + "recipients": "comm_recipient@example.com", + "cc": "comm_cc@example.com", + } + ).insert(ignore_permissions=True) comm = frappe.get_doc("Communication", comm.name) @@ -144,27 +182,29 @@ class TestCommunication(unittest.TestCase): frappe.delete_doc_if_exists("Note", "get communication data") - note = frappe.get_doc({ - "doctype": "Note", - "title": "get communication data", - "content": "get communication data" - }).insert(ignore_permissions=True) + note = frappe.get_doc( + {"doctype": "Note", "title": "get communication data", "content": "get communication data"} + ).insert(ignore_permissions=True) - comm_note_1 = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "Test Get Communication Data 1", - "communication_medium": "Email" - }).insert(ignore_permissions=True) + comm_note_1 = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "Test Get Communication Data 1", + "communication_medium": "Email", + } + ).insert(ignore_permissions=True) comm_note_1.add_link(link_doctype="Note", link_name=note.name, autosave=True) - comm_note_2 = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "Test Get Communication Data 2", - "communication_medium": "Email" - }).insert(ignore_permissions=True) + comm_note_2 = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "Test Get Communication Data 2", + "communication_medium": "Email", + } + ).insert(ignore_permissions=True) comm_note_2.add_link(link_doctype="Note", link_name=note.name, autosave=True) @@ -182,19 +222,23 @@ class TestCommunication(unittest.TestCase): create_email_account() - note = frappe.get_doc({ - "doctype": "Note", - "title": "test document link in email", - "content": "test document link in email" - }).insert(ignore_permissions=True) - - comm = frappe.get_doc({ - "doctype": "Communication", - "communication_medium": "Email", - "subject": "Document Link in Email", - "sender": "comm_sender@example.com", - "recipients": "comm_recipient+{0}+{1}@example.com".format(quote("Note"), quote(note.name)), - }).insert(ignore_permissions=True) + note = frappe.get_doc( + { + "doctype": "Note", + "title": "test document link in email", + "content": "test document link in email", + } + ).insert(ignore_permissions=True) + + comm = frappe.get_doc( + { + "doctype": "Communication", + "communication_medium": "Email", + "subject": "Document Link in Email", + "sender": "comm_sender@example.com", + "recipients": "comm_recipient+{0}+{1}@example.com".format(quote("Note"), quote(note.name)), + } + ).insert(ignore_permissions=True) doc_links = [] for timeline_link in comm.timeline_links: @@ -205,9 +249,9 @@ class TestCommunication(unittest.TestCase): def test_parse_emails(self): emails = get_emails( [ - 'comm_recipient+DocType+DocName@example.com', + "comm_recipient+DocType+DocName@example.com", '"First, LastName" ', - 'test@user.com' + "test@user.com", ] ) @@ -215,99 +259,108 @@ class TestCommunication(unittest.TestCase): self.assertEqual(emails[1], "first.lastname@email.com") self.assertEqual(emails[2], "test@user.com") + class TestCommunicationEmailMixin(unittest.TestCase): def new_communication(self, recipients=None, cc=None, bcc=None): - recipients = ', '.join(recipients or []) - cc = ', '.join(cc or []) - bcc = ', '.join(bcc or []) - - comm = frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "communication_medium": "Email", - "content": "Test content", - "recipients": recipients, - "cc": cc, - "bcc": bcc - }).insert(ignore_permissions=True) + recipients = ", ".join(recipients or []) + cc = ", ".join(cc or []) + bcc = ", ".join(bcc or []) + + comm = frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "communication_medium": "Email", + "content": "Test content", + "recipients": recipients, + "cc": cc, + "bcc": bcc, + } + ).insert(ignore_permissions=True) return comm def new_user(self, email, **user_data): - user_data.setdefault('first_name', 'first_name') - user = frappe.new_doc('User') + user_data.setdefault("first_name", "first_name") + user = frappe.new_doc("User") user.email = email user.update(user_data) user.insert(ignore_permissions=True, ignore_if_duplicate=True) return user def test_recipients(self): - to_list = ['to@test.com', 'receiver ', 'to@test.com'] - comm = self.new_communication(recipients = to_list) + to_list = ["to@test.com", "receiver ", "to@test.com"] + comm = self.new_communication(recipients=to_list) res = comm.get_mail_recipients_with_displayname() - self.assertCountEqual(res, ['to@test.com', 'receiver ']) + self.assertCountEqual(res, ["to@test.com", "receiver "]) comm.delete() def test_cc(self): - to_list = ['to@test.com'] - cc_list = ['cc+1@test.com', 'cc ', 'to@test.com'] - user = self.new_user(email='cc+1@test.com', thread_notify=0) + to_list = ["to@test.com"] + cc_list = ["cc+1@test.com", "cc ", "to@test.com"] + user = self.new_user(email="cc+1@test.com", thread_notify=0) comm = self.new_communication(recipients=to_list, cc=cc_list) res = comm.get_mail_cc_with_displayname() - self.assertCountEqual(res, ['cc ']) + self.assertCountEqual(res, ["cc "]) user.delete() comm.delete() def test_bcc(self): - bcc_list = ['bcc+1@test.com', 'cc ', ] - user = self.new_user(email='bcc+2@test.com', enabled=0) + bcc_list = [ + "bcc+1@test.com", + "cc ", + ] + user = self.new_user(email="bcc+2@test.com", enabled=0) comm = self.new_communication(bcc=bcc_list) res = comm.get_mail_bcc_with_displayname() - self.assertCountEqual(res, ['bcc+1@test.com']) + self.assertCountEqual(res, ["bcc+1@test.com"]) user.delete() comm.delete() def test_sendmail(self): - to_list = ['to '] - cc_list = ['cc ', 'cc '] + to_list = ["to "] + cc_list = ["cc ", "cc "] comm = self.new_communication(recipients=to_list, cc=cc_list) comm.send_email() doc = EmailQueue.find_one_by_filters(communication=comm.name) mail_receivers = [each.recipient for each in doc.recipients] self.assertIsNotNone(doc) - self.assertCountEqual(to_list+cc_list, mail_receivers) + self.assertCountEqual(to_list + cc_list, mail_receivers) doc.delete() comm.delete() + def create_email_account(): frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1") frappe.flags.mute_emails = False frappe.flags.sent_mail = None - email_account = frappe.get_doc({ - "is_default": 1, - "is_global": 1, - "doctype": "Email Account", - "domain":"example.com", - "append_to": "ToDo", - "email_account_name": "_Test Comm Account 1", - "enable_outgoing": 1, - "smtp_server": "test.example.com", - "email_id": "test_comm@example.com", - "password": "password", - "add_signature": 1, - "signature": "\nBest Wishes\nTest Signature", - "enable_auto_reply": 1, - "auto_reply_message": "", - "enable_incoming": 1, - "notify_if_unreplied": 1, - "unreplied_for_mins": 20, - "send_notification_to": "test_comm@example.com", - "pop3_server": "pop.test.example.com", - "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], - "no_remaining":"0", - "enable_automatic_linking": 1 - }).insert(ignore_permissions=True) + email_account = frappe.get_doc( + { + "is_default": 1, + "is_global": 1, + "doctype": "Email Account", + "domain": "example.com", + "append_to": "ToDo", + "email_account_name": "_Test Comm Account 1", + "enable_outgoing": 1, + "smtp_server": "test.example.com", + "email_id": "test_comm@example.com", + "password": "password", + "add_signature": 1, + "signature": "\nBest Wishes\nTest Signature", + "enable_auto_reply": 1, + "auto_reply_message": "", + "enable_incoming": 1, + "notify_if_unreplied": 1, + "unreplied_for_mins": 20, + "send_notification_to": "test_comm@example.com", + "pop3_server": "pop.test.example.com", + "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], + "no_remaining": "0", + "enable_automatic_linking": 1, + } + ).insert(ignore_permissions=True) return email_account diff --git a/frappe/core/doctype/communication_link/communication_link.py b/frappe/core/doctype/communication_link/communication_link.py index a895ad3df5..21b6a7828a 100644 --- a/frappe/core/doctype/communication_link/communication_link.py +++ b/frappe/core/doctype/communication_link/communication_link.py @@ -5,8 +5,10 @@ import frappe from frappe.model.document import Document + class CommunicationLink(Document): pass + def on_doctype_update(): - frappe.db.add_index("Communication Link", ["link_doctype", "link_name"]) \ No newline at end of file + frappe.db.add_index("Communication Link", ["link_doctype", "link_name"]) diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.py b/frappe/core/doctype/custom_docperm/custom_docperm.py index 1790344776..23815e9bf6 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/custom_docperm.py @@ -5,6 +5,7 @@ import frappe from frappe.model.document import Document + class CustomDocPerm(Document): def on_update(self): - frappe.clear_cache(doctype = self.parent) + frappe.clear_cache(doctype=self.parent) diff --git a/frappe/core/doctype/custom_docperm/test_custom_docperm.py b/frappe/core/doctype/custom_docperm/test_custom_docperm.py index 422b711e5b..e3a831bb8d 100644 --- a/frappe/core/doctype/custom_docperm/test_custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/test_custom_docperm.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Custom DocPerm') + class TestCustomDocPerm(unittest.TestCase): pass diff --git a/frappe/core/doctype/custom_role/custom_role.py b/frappe/core/doctype/custom_role/custom_role.py index c6630baf6d..dd215dea17 100644 --- a/frappe/core/doctype/custom_role/custom_role.py +++ b/frappe/core/doctype/custom_role/custom_role.py @@ -5,16 +5,18 @@ import frappe from frappe.model.document import Document + class CustomRole(Document): def validate(self): if self.report and not self.ref_doctype: - self.ref_doctype = frappe.db.get_value('Report', self.report, 'ref_doctype') + self.ref_doctype = frappe.db.get_value("Report", self.report, "ref_doctype") + def get_custom_allowed_roles(field, name): allowed_roles = [] - custom_role = frappe.db.get_value('Custom Role', {field: name}, 'name') + custom_role = frappe.db.get_value("Custom Role", {field: name}, "name") if custom_role: - custom_role_doc = frappe.get_doc('Custom Role', custom_role) + custom_role_doc = frappe.get_doc("Custom Role", custom_role) allowed_roles = [d.role for d in custom_role_doc.roles] - return allowed_roles \ No newline at end of file + return allowed_roles diff --git a/frappe/core/doctype/custom_role/test_custom_role.py b/frappe/core/doctype/custom_role/test_custom_role.py index 21511a7408..01956ceda3 100644 --- a/frappe/core/doctype/custom_role/test_custom_role.py +++ b/frappe/core/doctype/custom_role/test_custom_role.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Custom Role') + class TestCustomRole(unittest.TestCase): pass diff --git a/frappe/core/doctype/data_export/data_export.py b/frappe/core/doctype/data_export/data_export.py index 46fe3570a1..268182fbd4 100644 --- a/frappe/core/doctype/data_export/data_export.py +++ b/frappe/core/doctype/data_export/data_export.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class DataExport(Document): pass diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index 9f1492af19..514be16694 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -1,47 +1,78 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import csv +import os +import re + import frappe -from frappe import _ import frappe.permissions -import re, csv, os -from frappe.utils.csvutils import UnicodeWriter -from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint, format_duration +from frappe import _ from frappe.core.doctype.access_log.access_log import make_access_log +from frappe.utils import cint, cstr, format_datetime, format_duration, formatdate, parse_json +from frappe.utils.csvutils import UnicodeWriter + +reflags = {"I": re.I, "L": re.L, "M": re.M, "U": re.U, "S": re.S, "X": re.X, "D": re.DEBUG} -reflags = { - "I":re.I, - "L":re.L, - "M":re.M, - "U":re.U, - "S":re.S, - "X":re.X, - "D": re.DEBUG -} def get_data_keys(): - return frappe._dict({ - "data_separator": _('Start entering data below this line'), - "main_table": _("Table") + ":", - "parent_table": _("Parent Table") + ":", - "columns": _("Column Name") + ":", - "doctype": _("DocType") + ":" - }) + return frappe._dict( + { + "data_separator": _("Start entering data below this line"), + "main_table": _("Table") + ":", + "parent_table": _("Parent Table") + ":", + "columns": _("Column Name") + ":", + "doctype": _("DocType") + ":", + } + ) + @frappe.whitelist() -def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False, - select_columns=None, file_type='CSV', template=False, filters=None): +def export_data( + doctype=None, + parent_doctype=None, + all_doctypes=True, + with_data=False, + select_columns=None, + file_type="CSV", + template=False, + filters=None, +): _doctype = doctype if isinstance(_doctype, list): _doctype = _doctype[0] - make_access_log(doctype=_doctype, file_type=file_type, columns=select_columns, filters=filters, method=parent_doctype) - exporter = DataExporter(doctype=doctype, parent_doctype=parent_doctype, all_doctypes=all_doctypes, with_data=with_data, - select_columns=select_columns, file_type=file_type, template=template, filters=filters) + make_access_log( + doctype=_doctype, + file_type=file_type, + columns=select_columns, + filters=filters, + method=parent_doctype, + ) + exporter = DataExporter( + doctype=doctype, + parent_doctype=parent_doctype, + all_doctypes=all_doctypes, + with_data=with_data, + select_columns=select_columns, + file_type=file_type, + template=template, + filters=filters, + ) exporter.build_response() + class DataExporter: - def __init__(self, doctype=None, parent_doctype=None, all_doctypes=True, with_data=False, - select_columns=None, file_type='CSV', template=False, filters=None): + def __init__( + self, + doctype=None, + parent_doctype=None, + all_doctypes=True, + with_data=False, + select_columns=None, + file_type="CSV", + template=False, + filters=None, + ): self.doctype = doctype self.parent_doctype = parent_doctype self.all_doctypes = all_doctypes @@ -81,18 +112,18 @@ class DataExporter: def build_response(self): self.writer = UnicodeWriter() - self.name_field = 'parent' if self.parent_doctype != self.doctype else 'name' + self.name_field = "parent" if self.parent_doctype != self.doctype else "name" if self.template: self.add_main_header() - self.writer.writerow(['']) + self.writer.writerow([""]) self.tablerow = [self.data_keys.doctype] self.labelrow = [_("Column Labels:")] self.fieldrow = [self.data_keys.columns] self.mandatoryrow = [_("Mandatory:")] - self.typerow = [_('Type:')] - self.inforow = [_('Info:')] + self.typerow = [_("Type:")] + self.inforow = [_("Info:")] self.columns = [] self.build_field_columns(self.doctype) @@ -100,74 +131,99 @@ class DataExporter: if self.all_doctypes: for d in self.child_doctypes: self.append_empty_field_column() - if (self.select_columns and self.select_columns.get(d['doctype'], None)) or not self.select_columns: + if ( + self.select_columns and self.select_columns.get(d["doctype"], None) + ) or not self.select_columns: # if atleast one column is selected for this doctype - self.build_field_columns(d['doctype'], d['parentfield']) + self.build_field_columns(d["doctype"], d["parentfield"]) self.add_field_headings() self.add_data() if self.with_data and not self.data: - frappe.respond_as_web_page(_('No Data'), _('There is no data to be exported'), indicator_color='orange') + frappe.respond_as_web_page( + _("No Data"), _("There is no data to be exported"), indicator_color="orange" + ) - if self.file_type == 'Excel': + if self.file_type == "Excel": self.build_response_as_excel() else: # write out response as a type csv - frappe.response['result'] = cstr(self.writer.getvalue()) - frappe.response['type'] = 'csv' - frappe.response['doctype'] = self.doctype + frappe.response["result"] = cstr(self.writer.getvalue()) + frappe.response["type"] = "csv" + frappe.response["doctype"] = self.doctype def add_main_header(self): - self.writer.writerow([_('Data Import Template')]) + self.writer.writerow([_("Data Import Template")]) self.writer.writerow([self.data_keys.main_table, self.doctype]) if self.parent_doctype != self.doctype: self.writer.writerow([self.data_keys.parent_table, self.parent_doctype]) else: - self.writer.writerow(['']) - - self.writer.writerow(['']) - self.writer.writerow([_('Notes:')]) - self.writer.writerow([_('Please do not change the template headings.')]) - self.writer.writerow([_('First data column must be blank.')]) - self.writer.writerow([_('If you are uploading new records, leave the "name" (ID) column blank.')]) - self.writer.writerow([_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')]) - self.writer.writerow([_('Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.')]) - self.writer.writerow([_('For updating, you can update only selective columns.')]) - self.writer.writerow([_('You can only upload upto 5000 records in one go. (may be less in some cases)')]) + self.writer.writerow([""]) + + self.writer.writerow([""]) + self.writer.writerow([_("Notes:")]) + self.writer.writerow([_("Please do not change the template headings.")]) + self.writer.writerow([_("First data column must be blank.")]) + self.writer.writerow( + [_('If you are uploading new records, leave the "name" (ID) column blank.')] + ) + self.writer.writerow( + [_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')] + ) + self.writer.writerow( + [ + _( + "Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish." + ) + ] + ) + self.writer.writerow([_("For updating, you can update only selective columns.")]) + self.writer.writerow( + [_("You can only upload upto 5000 records in one go. (may be less in some cases)")] + ) if self.name_field == "parent": self.writer.writerow([_('"Parent" signifies the parent table in which this row must be added')]) - self.writer.writerow([_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')]) + self.writer.writerow( + [_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')] + ) def build_field_columns(self, dt, parentfield=None): meta = frappe.get_meta(dt) # build list of valid docfields tablecolumns = [] - table_name = 'tab' + dt + table_name = "tab" + dt for f in frappe.db.get_table_columns_description(table_name): field = meta.get_field(f.name) - if field and ((self.select_columns and f.name in self.select_columns[dt]) or not self.select_columns): + if field and ( + (self.select_columns and f.name in self.select_columns[dt]) or not self.select_columns + ): tablecolumns.append(field) - tablecolumns.sort(key = lambda a: int(a.idx)) + tablecolumns.sort(key=lambda a: int(a.idx)) _column_start_end = frappe._dict(start=0) - if dt==self.doctype: - if (meta.get('autoname') and meta.get('autoname').lower()=='prompt') or (self.with_data): + if dt == self.doctype: + if (meta.get("autoname") and meta.get("autoname").lower() == "prompt") or (self.with_data): self._append_name_column() # if importing only child table for new record, add parent field - if meta.get('istable') and not self.with_data: - self.append_field_column(frappe._dict({ - "fieldname": "parent", - "parent": "", - "label": "Parent", - "fieldtype": "Data", - "reqd": 1, - "info": _("Parent is the name of the document to which the data will get added to.") - }), True) + if meta.get("istable") and not self.with_data: + self.append_field_column( + frappe._dict( + { + "fieldname": "parent", + "parent": "", + "label": "Parent", + "fieldtype": "Data", + "reqd": 1, + "info": _("Parent is the name of the document to which the data will get added to."), + } + ), + True, + ) _column_start_end = frappe._dict(start=0) else: @@ -184,7 +240,7 @@ class DataExporter: self.append_field_column(docfield, False) # if there is one column, add a blank column (?) - if len(self.columns)-_column_start_end.start == 1: + if len(self.columns) - _column_start_end.start == 1: self.append_empty_field_column() # append DocType name @@ -204,18 +260,21 @@ class DataExporter: return if not for_mandatory and docfield.reqd: return - if docfield.fieldname in ('parenttype', 'trash_reason'): + if docfield.fieldname in ("parenttype", "trash_reason"): return if docfield.hidden: return - if self.select_columns and docfield.fieldname not in self.select_columns.get(docfield.parent, []) \ - and docfield.fieldname!="name": + if ( + self.select_columns + and docfield.fieldname not in self.select_columns.get(docfield.parent, []) + and docfield.fieldname != "name" + ): return self.tablerow.append("") self.fieldrow.append(docfield.fieldname) self.labelrow.append(_(docfield.label)) - self.mandatoryrow.append(docfield.reqd and 'Yes' or 'No') + self.mandatoryrow.append(docfield.reqd and "Yes" or "No") self.typerow.append(docfield.fieldtype) self.inforow.append(self.getinforow(docfield)) self.columns.append(docfield.fieldname) @@ -232,15 +291,15 @@ class DataExporter: @staticmethod def getinforow(docfield): """make info comment for options, links etc.""" - if docfield.fieldtype == 'Select': + if docfield.fieldtype == "Select": if not docfield.options: - return '' + return "" else: - return _("One of") + ': %s' % ', '.join(filter(None, docfield.options.split('\n'))) - elif docfield.fieldtype == 'Link': - return 'Valid %s' % docfield.options - elif docfield.fieldtype == 'Int': - return 'Integer' + return _("One of") + ": %s" % ", ".join(filter(None, docfield.options.split("\n"))) + elif docfield.fieldtype == "Link": + return "Valid %s" % docfield.options + elif docfield.fieldtype == "Int": + return "Integer" elif docfield.fieldtype == "Check": return "0 or 1" elif docfield.fieldtype in ["Date", "Datetime"]: @@ -248,7 +307,7 @@ class DataExporter: elif hasattr(docfield, "info"): return docfield.info else: - return '' + return "" def add_field_headings(self): self.writer.writerow(self.tablerow) @@ -262,6 +321,7 @@ class DataExporter: def add_data(self): from frappe.query_builder import DocType + if self.template and not self.with_data: return @@ -270,26 +330,28 @@ class DataExporter: # sort nested set doctypes by `lft asc` order_by = None table_columns = frappe.db.get_table_columns(self.parent_doctype) - if 'lft' in table_columns and 'rgt' in table_columns: - order_by = '`tab{doctype}`.`lft` asc'.format(doctype=self.parent_doctype) + if "lft" in table_columns and "rgt" in table_columns: + order_by = "`tab{doctype}`.`lft` asc".format(doctype=self.parent_doctype) # get permitted data only - self.data = frappe.get_list(self.doctype, fields=["*"], filters=self.filters, limit_page_length=None, order_by=order_by) + self.data = frappe.get_list( + self.doctype, fields=["*"], filters=self.filters, limit_page_length=None, order_by=order_by + ) for doc in self.data: op = self.docs_to_export.get("op") names = self.docs_to_export.get("name") if names and op: - if op == '=' and doc.name not in names: + if op == "=" and doc.name not in names: continue - elif op == '!=' and doc.name in names: + elif op == "!=" and doc.name in names: continue elif names: try: sflags = self.docs_to_export.get("flags", "I,U").upper() flags = 0 - for a in re.split(r'\W+', sflags): - flags = flags | reflags.get(a,0) + for a in re.split(r"\W+", sflags): + flags = flags | reflags.get(a, 0) c = re.compile(names, flags) m = c.match(doc.name) @@ -315,7 +377,7 @@ class DataExporter: .orderby(child_doctype_table.idx) ) for ci, child in enumerate(data_row.run(as_dict=True)): - self.add_data_row(rows, c['doctype'], c['parentfield'], child, ci) + self.add_data_row(rows, c["doctype"], c["parentfield"], child, ci) for row in rows: self.writer.writerow(row) @@ -333,7 +395,7 @@ class DataExporter: _column_start_end = self.column_start_end.get((dt, parentfield)) if _column_start_end: - for i, c in enumerate(self.columns[_column_start_end.start:_column_start_end.end]): + for i, c in enumerate(self.columns[_column_start_end.start : _column_start_end.end]): df = meta.get_field(c) fieldtype = df.fieldtype if df else "Data" value = d.get(c, "") @@ -349,27 +411,33 @@ class DataExporter: def build_response_as_excel(self): filename = frappe.generate_hash("", 10) - with open(filename, 'wb') as f: - f.write(cstr(self.writer.getvalue()).encode('utf-8')) + with open(filename, "wb") as f: + f.write(cstr(self.writer.getvalue()).encode("utf-8")) f = open(filename) reader = csv.reader(f) from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else 'Data Export') + + xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else "Data Export") f.close() os.remove(filename) # write out response as a xlsx type - frappe.response['filename'] = self.doctype + '.xlsx' - frappe.response['filecontent'] = xlsx_file.getvalue() - frappe.response['type'] = 'binary' + frappe.response["filename"] = self.doctype + ".xlsx" + frappe.response["filecontent"] = xlsx_file.getvalue() + frappe.response["type"] = "binary" def _append_name_column(self, dt=None): - self.append_field_column(frappe._dict({ - "fieldname": "name" if dt else self.name_field, - "parent": dt or "", - "label": "ID", - "fieldtype": "Data", - "reqd": 1, - }), True) + self.append_field_column( + frappe._dict( + { + "fieldname": "name" if dt else self.name_field, + "parent": dt or "", + "label": "ID", + "fieldtype": "Data", + "reqd": 1, + } + ), + True, + ) diff --git a/frappe/core/doctype/data_export/test_data_exporter.py b/frappe/core/doctype/data_export/test_data_exporter.py index 8d05707cf1..7b7dfc0069 100644 --- a/frappe/core/doctype/data_export/test_data_exporter.py +++ b/frappe/core/doctype/data_export/test_data_exporter.py @@ -2,13 +2,15 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest + import frappe from frappe.core.doctype.data_export.exporter import DataExporter + class TestDataExporter(unittest.TestCase): def setUp(self): - self.doctype_name = 'Test DocType for Export Tool' - self.doc_name = 'Test Data for Export Tool' + self.doctype_name = "Test DocType for Export Tool" + self.doc_name = "Test Data for Export Tool" self.create_doctype_if_not_exists(doctype_name=self.doctype_name) self.create_test_data() @@ -17,42 +19,49 @@ class TestDataExporter(unittest.TestCase): Helper Function for setting up doctypes """ if force: - frappe.delete_doc_if_exists('DocType', doctype_name) - frappe.delete_doc_if_exists('DocType', 'Child 1 of ' + doctype_name) + frappe.delete_doc_if_exists("DocType", doctype_name) + frappe.delete_doc_if_exists("DocType", "Child 1 of " + doctype_name) - if frappe.db.exists('DocType', doctype_name): + if frappe.db.exists("DocType", doctype_name): return # Child Table 1 - table_1_name = 'Child 1 of ' + doctype_name - frappe.get_doc({ - 'doctype': 'DocType', - 'name': table_1_name, - 'module': 'Custom', - 'custom': 1, - 'istable': 1, - 'fields': [ - {'label': 'Child Title', 'fieldname': 'child_title', 'reqd': 1, 'fieldtype': 'Data'}, - {'label': 'Child Number', 'fieldname': 'child_number', 'fieldtype': 'Int'}, - ] - }).insert() + table_1_name = "Child 1 of " + doctype_name + frappe.get_doc( + { + "doctype": "DocType", + "name": table_1_name, + "module": "Custom", + "custom": 1, + "istable": 1, + "fields": [ + {"label": "Child Title", "fieldname": "child_title", "reqd": 1, "fieldtype": "Data"}, + {"label": "Child Number", "fieldname": "child_number", "fieldtype": "Int"}, + ], + } + ).insert() # Main Table - frappe.get_doc({ - 'doctype': 'DocType', - 'name': doctype_name, - 'module': 'Custom', - 'custom': 1, - 'autoname': 'field:title', - 'fields': [ - {'label': 'Title', 'fieldname': 'title', 'reqd': 1, 'fieldtype': 'Data'}, - {'label': 'Number', 'fieldname': 'number', 'fieldtype': 'Int'}, - {'label': 'Table Field 1', 'fieldname': 'table_field_1', 'fieldtype': 'Table', 'options': table_1_name}, - ], - 'permissions': [ - {'role': 'System Manager'} - ] - }).insert() + frappe.get_doc( + { + "doctype": "DocType", + "name": doctype_name, + "module": "Custom", + "custom": 1, + "autoname": "field:title", + "fields": [ + {"label": "Title", "fieldname": "title", "reqd": 1, "fieldtype": "Data"}, + {"label": "Number", "fieldname": "number", "fieldtype": "Int"}, + { + "label": "Table Field 1", + "fieldname": "table_field_1", + "fieldtype": "Table", + "options": table_1_name, + }, + ], + "permissions": [{"role": "System Manager"}], + } + ).insert() def create_test_data(self, force=False): """ @@ -69,37 +78,38 @@ class TestDataExporter(unittest.TestCase): table_field_1=[ {"child_title": "Child Title 1", "child_number": "50"}, {"child_title": "Child Title 2", "child_number": "51"}, - ] + ], ).insert() else: self.doc = frappe.get_doc(self.doctype_name, self.doc_name) def test_export_content(self): - exp = DataExporter(doctype=self.doctype_name, file_type='CSV') + exp = DataExporter(doctype=self.doctype_name, file_type="CSV") exp.build_response() - self.assertEqual(frappe.response['type'],'csv') - self.assertEqual(frappe.response['doctype'], self.doctype_name) - self.assertTrue(frappe.response['result']) - self.assertIn('Child Title 1\",50',frappe.response['result']) - self.assertIn('Child Title 2\",51',frappe.response['result']) + self.assertEqual(frappe.response["type"], "csv") + self.assertEqual(frappe.response["doctype"], self.doctype_name) + self.assertTrue(frappe.response["result"]) + self.assertIn('Child Title 1",50', frappe.response["result"]) + self.assertIn('Child Title 2",51', frappe.response["result"]) def test_export_type(self): - for type in ['csv', 'Excel']: + for type in ["csv", "Excel"]: with self.subTest(type=type): exp = DataExporter(doctype=self.doctype_name, file_type=type) exp.build_response() - self.assertEqual(frappe.response['doctype'], self.doctype_name) - self.assertTrue(frappe.response['result']) + self.assertEqual(frappe.response["doctype"], self.doctype_name) + self.assertTrue(frappe.response["result"]) - if type == 'csv': - self.assertEqual(frappe.response['type'],'csv') - elif type == 'Excel': - self.assertEqual(frappe.response['type'],'binary') - self.assertEqual(frappe.response['filename'], self.doctype_name+'.xlsx') # 'Test DocType for Export Tool.xlsx') - self.assertTrue(frappe.response['filecontent']) + if type == "csv": + self.assertEqual(frappe.response["type"], "csv") + elif type == "Excel": + self.assertEqual(frappe.response["type"], "binary") + self.assertEqual( + frappe.response["filename"], self.doctype_name + ".xlsx" + ) # 'Test DocType for Export Tool.xlsx') + self.assertTrue(frappe.response["filecontent"]) def tearDown(self): pass - diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5972e79b4d..295f7e79ba 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -64,9 +64,7 @@ class DataImport(Document): from frappe.utils.scheduler import is_scheduler_inactive if is_scheduler_inactive() and not frappe.flags.in_test: - frappe.throw( - _("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive") - ) + frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive")) enqueued_jobs = [d.get("job_name") for d in get_info()] @@ -100,6 +98,7 @@ def get_preview_from_template(data_import, import_file=None, google_sheets_url=N import_file, google_sheets_url ) + @frappe.whitelist() def form_start_import(data_import): return frappe.get_doc("Data Import", data_import).start_import() @@ -127,11 +126,11 @@ def download_template( ): """ Download template from Exporter - :param doctype: Document Type - :param export_fields=None: Fields to export as dict {'Sales Invoice': ['name', 'customer'], 'Sales Invoice Item': ['item_code']} - :param export_records=None: One of 'all', 'by_filter', 'blank_template' - :param export_filters: Filter dict - :param file_type: File type to export into + :param doctype: Document Type + :param export_fields=None: Fields to export as dict {'Sales Invoice': ['name', 'customer'], 'Sales Invoice Item': ['item_code']} + :param export_records=None: One of 'all', 'by_filter', 'blank_template' + :param export_filters: Filter dict + :param file_type: File type to export into """ export_fields = frappe.parse_json(export_fields) @@ -154,34 +153,38 @@ def download_errored_template(data_import_name): data_import = frappe.get_doc("Data Import", data_import_name) data_import.export_errored_rows() + @frappe.whitelist() def download_import_log(data_import_name): data_import = frappe.get_doc("Data Import", data_import_name) data_import.download_import_log() + @frappe.whitelist() def get_import_status(data_import_name): import_status = {} - logs = frappe.get_all('Data Import Log', fields=['count(*) as count', 'success'], - filters={'data_import': data_import_name}, - group_by='success') + logs = frappe.get_all( + "Data Import Log", + fields=["count(*) as count", "success"], + filters={"data_import": data_import_name}, + group_by="success", + ) - total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count') + total_payload_count = frappe.db.get_value("Data Import", data_import_name, "payload_count") for log in logs: - if log.get('success'): - import_status['success'] = log.get('count') + if log.get("success"): + import_status["success"] = log.get("count") else: - import_status['failed'] = log.get('count') + import_status["failed"] = log.get("count") - import_status['total_records'] = total_payload_count + import_status["total_records"] = total_payload_count return import_status -def import_file( - doctype, file_path, import_type, submit_after_import=False, console=False -): + +def import_file(doctype, file_path, import_type, submit_after_import=False, console=False): """ Import documents in from CSV or XLSX using data import. @@ -198,9 +201,7 @@ def import_file( "Insert New Records" if import_type.lower() == "insert" else "Update Existing Records" ) - i = Importer( - doctype=doctype, file_path=file_path, data_import=data_import, console=console - ) + i = Importer(doctype=doctype, file_path=file_path, data_import=data_import, console=console) i.import_data() @@ -214,11 +215,7 @@ def import_doc(path, pre_process=None): if f.endswith(".json"): frappe.flags.mute_emails = True import_file_by_path( - f, - data_import=True, - force=True, - pre_process=pre_process, - reset_permissions=True + f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True ) frappe.flags.mute_emails = False frappe.db.commit() @@ -226,9 +223,7 @@ def import_doc(path, pre_process=None): raise NotImplementedError("Only .json files can be imported") -def export_json( - doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc" -): +def export_json(doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"): def post_process(out): # Note on Tree DocTypes: # The tree structure is maintained in the database via the fields "lft" diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py index c09bd58c25..b647bdcb62 100644 --- a/frappe/core/doctype/data_import/exporter.py +++ b/frappe/core/doctype/data_import/exporter.py @@ -6,11 +6,8 @@ import typing import frappe from frappe import _ -from frappe.model import ( - display_fieldtypes, - no_value_fields, - table_fields as table_fieldtypes, -) +from frappe.model import display_fieldtypes, no_value_fields +from frappe.model import table_fields as table_fieldtypes from frappe.utils import flt, format_duration, groupby_metric from frappe.utils.csvutils import build_csv_response from frappe.utils.xlsxutils import build_xlsx_response @@ -28,11 +25,11 @@ class Exporter: ): """ Exports records of a DocType for use with Importer - :param doctype: Document Type to export - :param export_fields=None: One of 'All', 'Mandatory' or {'DocType': ['field1', 'field2'], 'Child DocType': ['childfield1']} - :param export_data=False: Whether to export data as well - :param export_filters=None: The filters (dict or list) which is used to query the records - :param file_type: One of 'Excel' or 'CSV' + :param doctype: Document Type to export + :param export_fields=None: One of 'All', 'Mandatory' or {'DocType': ['field1', 'field2'], 'Child DocType': ['childfield1']} + :param export_data=False: Whether to export data as well + :param export_filters=None: The filters (dict or list) which is used to query the records + :param file_type: One of 'Excel' or 'CSV' """ self.doctype = doctype self.meta = frappe.get_meta(doctype) @@ -168,9 +165,7 @@ class Exporter: else: order_by = "`tab{0}`.`creation` DESC".format(self.doctype) - parent_fields = [ - format_column_name(df) for df in self.fields if df.parent == self.doctype - ] + parent_fields = [format_column_name(df) for df in self.fields if df.parent == self.doctype] parent_data = frappe.db.get_list( self.doctype, filters=filters, @@ -188,9 +183,7 @@ class Exporter: child_table_df = self.meta.get_field(key) child_table_doctype = child_table_df.options child_fields = ["name", "idx", "parent", "parentfield"] + list( - set( - [format_column_name(df) for df in self.fields if df.parent == child_table_doctype] - ) + set([format_column_name(df) for df in self.fields if df.parent == child_table_doctype]) ) data = frappe.db.get_all( child_table_doctype, @@ -261,4 +254,4 @@ class Exporter: build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype)) def group_children_data_by_parent(self, children_data: typing.Dict[str, list]): - return groupby_metric(children_data, key='parent') + return groupby_metric(children_data, key="parent") diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index b10b84f048..9a801cfc19 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -1,21 +1,23 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import os import io -import frappe -import timeit import json -from datetime import datetime, date +import os +import timeit +from datetime import date, datetime + +import frappe from frappe import _ -from frappe.utils import cint, flt, update_progress_bar, cstr, duration_to_seconds -from frappe.utils.csvutils import read_csv_content, get_csv_content_from_google_sheets +from frappe.core.doctype.version.version import get_diff +from frappe.model import no_value_fields +from frappe.model import table_fields as table_fieldtypes +from frappe.utils import cint, cstr, duration_to_seconds, flt, update_progress_bar +from frappe.utils.csvutils import get_csv_content_from_google_sheets, read_csv_content from frappe.utils.xlsxutils import ( - read_xlsx_file_from_attached_file, read_xls_file_from_attached_file, + read_xlsx_file_from_attached_file, ) -from frappe.model import no_value_fields, table_fields as table_fieldtypes -from frappe.core.doctype.version.version import get_diff INVALID_VALUES = ("", None) MAX_ROWS_IN_PREVIEW = 10 @@ -24,9 +26,7 @@ UPDATE = "Update Existing Records" class Importer: - def __init__( - self, doctype, data_import=None, file_path=None, import_type=None, console=False - ): + def __init__(self, doctype, data_import=None, file_path=None, import_type=None, console=False): self.doctype = doctype self.console = console @@ -49,9 +49,13 @@ class Importer: def get_data_for_import_preview(self): out = self.import_file.get_data_for_import_preview() - out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], + out.import_log = frappe.db.get_all( + "Data Import Log", + fields=["row_indexes", "success"], filters={"data_import": self.data_import.name}, - order_by="log_index", limit=10) + order_by="log_index", + limit=10, + ) return out @@ -84,14 +88,23 @@ class Importer: return # setup import log - import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], - filters={"data_import": self.data_import.name}, - order_by="log_index") or [] + import_log = ( + frappe.db.get_all( + "Data Import Log", + fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index", + ) + or [] + ) log_index = 0 # Do not remove rows in case of retry after an error or pending data import - if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count: + if ( + self.data_import.status == "Partial Success" + and len(import_log) >= self.data_import.payload_count + ): # remove previous failures from import log only in case of retry after partial success import_log = [log for log in import_log if log.get("success")] @@ -108,9 +121,7 @@ class Importer: total_payload_count = len(payloads) batch_size = frappe.conf.data_import_batch_size or 1000 - for batch_index, batched_payloads in enumerate( - frappe.utils.create_batch(payloads, batch_size) - ): + for batch_index, batched_payloads in enumerate(frappe.utils.create_batch(payloads, batch_size)): for i, payload in enumerate(batched_payloads): doc = payload.doc row_indexes = [row.row_number for row in payload.rows] @@ -156,11 +167,11 @@ class Importer: }, ) - create_import_log(self.data_import.name, log_index, { - 'success': True, - 'docname': doc.name, - 'row_indexes': row_indexes - }) + create_import_log( + self.data_import.name, + log_index, + {"success": True, "docname": doc.name, "row_indexes": row_indexes}, + ) log_index += 1 @@ -177,19 +188,29 @@ class Importer: # rollback if exception frappe.db.rollback() - create_import_log(self.data_import.name, log_index, { - 'success': False, - 'exception': frappe.get_traceback(), - 'messages': messages, - 'row_indexes': row_indexes - }) + create_import_log( + self.data_import.name, + log_index, + { + "success": False, + "exception": frappe.get_traceback(), + "messages": messages, + "row_indexes": row_indexes, + }, + ) log_index += 1 # Logs are db inserted directly so will have to be fetched again - import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"], - filters={"data_import": self.data_import.name}, - order_by="log_index") or [] + import_log = ( + frappe.db.get_all( + "Data Import Log", + fields=["row_indexes", "success", "log_index"], + filters={"data_import": self.data_import.name}, + order_by="log_index", + ) + or [] + ) # set status failures = [log for log in import_log if not log.get("success")] @@ -274,9 +295,15 @@ class Importer: if not self.data_import: return - import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"], - filters={"data_import": self.data_import.name}, - order_by="log_index") or [] + import_log = ( + frappe.db.get_all( + "Data Import Log", + fields=["row_indexes", "success"], + filters={"data_import": self.data_import.name}, + order_by="log_index", + ) + or [] + ) failures = [log for log in import_log if not log.get("success")] row_indexes = [] @@ -299,9 +326,12 @@ class Importer: if not self.data_import: return - import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + import_log = frappe.db.get_all( + "Data Import Log", + fields=["row_indexes", "success", "messages", "exception", "docname"], filters={"data_import": self.data_import.name}, - order_by="log_index") + order_by="log_index", + ) header_row = ["Row Numbers", "Status", "Message", "Exception"] @@ -309,10 +339,13 @@ class Importer: for log in import_log: row_number = json.loads(log.get("row_indexes"))[0] - status = "Success" if log.get('success') else "Failure" - message = "Successfully Imported {0}".format(log.get('docname')) if log.get('success') else \ - log.get("messages") - exception = frappe.utils.cstr(log.get("exception", '')) + status = "Success" if log.get("success") else "Failure" + message = ( + "Successfully Imported {0}".format(log.get("docname")) + if log.get("success") + else log.get("messages") + ) + exception = frappe.utils.cstr(log.get("exception", "")) rows += [[row_number, status, message, exception]] build_csv_response(rows, self.doctype) @@ -324,9 +357,7 @@ class Importer: if successful_records: print() print( - "Successfully imported {0} records out of {1}".format( - len(successful_records), len(import_log) - ) + "Successfully imported {0} records out of {1}".format(len(successful_records), len(import_log)) ) if failed_records: @@ -363,9 +394,7 @@ class Importer: class ImportFile: def __init__(self, doctype, file, template_options=None, import_type=None): self.doctype = doctype - self.template_options = template_options or frappe._dict( - column_to_field_map=frappe._dict() - ) + self.template_options = template_options or frappe._dict(column_to_field_map=frappe._dict()) self.column_to_field_map = self.template_options.column_to_field_map self.import_type = import_type self.warnings = [] @@ -556,9 +585,7 @@ class ImportFile: def read_content(self, content, extension): error_title = _("Template Error") if extension not in ("csv", "xlsx", "xls"): - frappe.throw( - _("Import template should be of type .csv, .xlsx or .xls"), title=error_title - ) + frappe.throw(_("Import template should be of type .csv, .xlsx or .xls"), title=error_title) if extension == "csv": data = read_csv_content(content) @@ -587,12 +614,13 @@ class Row: if len_row != len_columns: less_than_columns = len_row < len_columns message = ( - "Row has less values than columns" - if less_than_columns - else "Row has more values than columns" + "Row has less values than columns" if less_than_columns else "Row has more values than columns" ) self.warnings.append( - {"row": self.row_number, "message": message,} + { + "row": self.row_number, + "message": message, + } ) def parse_doc(self, doctype, parent_doc=None, table_df=None): @@ -662,18 +690,24 @@ class Row: options_string = ", ".join(frappe.bold(d) for d in select_options) msg = _("Value must be one of {0}").format(options_string) self.warnings.append( - {"row": self.row_number, "field": df_as_json(df), "message": msg,} + { + "row": self.row_number, + "field": df_as_json(df), + "message": msg, + } ) return elif df.fieldtype == "Link": exists = self.link_exists(value, df) if not exists: - msg = _("Value {0} missing for {1}").format( - frappe.bold(value), frappe.bold(df.options) - ) + msg = _("Value {0} missing for {1}").format(frappe.bold(value), frappe.bold(df.options)) self.warnings.append( - {"row": self.row_number, "field": df_as_json(df), "message": msg,} + { + "row": self.row_number, + "field": df_as_json(df), + "message": msg, + } ) return elif df.fieldtype in ["Date", "Datetime"]: @@ -693,6 +727,7 @@ class Row: return elif df.fieldtype == "Duration": import re + is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) if not is_valid_duration: self.warnings.append( @@ -702,7 +737,7 @@ class Row: "field": df_as_json(df), "message": _("Value {0} must be in the valid duration format: d h m s").format( frappe.bold(value) - ) + ), } ) @@ -789,9 +824,7 @@ class Header(Row): else: doctypes.append((col.df.parent, col.df.child_table_df)) - self.doctypes = sorted( - list(set(doctypes)), key=lambda x: -1 if x[0] == self.doctype else 1 - ) + self.doctypes = sorted(list(set(doctypes)), key=lambda x: -1 if x[0] == self.doctype else 1) def get_column_indexes(self, doctype, tablefield=None): def is_table_field(df): @@ -802,10 +835,7 @@ class Header(Row): return [ col.index for col in self.columns - if not col.skip_import - and col.df - and col.df.parent == doctype - and is_table_field(col.df) + if not col.skip_import and col.df and col.df.parent == doctype and is_table_field(col.df) ] def get_columns(self, indexes): @@ -893,9 +923,7 @@ class Column: self.warnings.append( { "col": column_number, - "message": _("Cannot match column {0} with any field").format( - frappe.bold(header_title) - ), + "message": _("Cannot match column {0} with any field").format(frappe.bold(header_title)), "type": "info", } ) @@ -958,9 +986,7 @@ class Column: if self.df.fieldtype == "Link": # find all values that dont exist values = list({cstr(v) for v in self.column_values[1:] if v}) - exists = [ - d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)}) - ] + exists = [d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)})] not_exists = list(set(values) - set(exists)) if not_exists: missing_values = ", ".join(not_exists) @@ -968,9 +994,7 @@ class Column: { "col": self.column_number, "message": ( - "The following values do not exist for {}: {}".format( - self.df.options, missing_values - ) + "The following values do not exist for {}: {}".format(self.df.options, missing_values) ), "type": "warning", } @@ -983,7 +1007,9 @@ class Column: self.warnings.append( { "col": self.column_number, - "message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."), + "message": _( + "Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd." + ), "type": "info", } ) @@ -1027,12 +1053,12 @@ def build_fields_dict_for_column_matching(parent_doctype): Build a dict with various keys to match with column headers and value as docfield The keys can be label or fieldname { - 'Customer': df1, - 'customer': df1, - 'Due Date': df2, - 'due_date': df2, - 'Item Code (Sales Invoice Item)': df3, - 'Sales Invoice Item:item_code': df3, + 'Customer': df1, + 'customer': df1, + 'Due Date': df2, + 'due_date': df2, + 'Item Code (Sales Invoice Item)': df3, + 'Sales Invoice Item:item_code': df3, } """ @@ -1062,9 +1088,7 @@ def build_fields_dict_for_column_matching(parent_doctype): out = {} # doctypes and fieldname if it is a child doctype - doctypes = [(parent_doctype, None)] + [ - (df.options, df) for df in parent_meta.get_table_fields() - ] + doctypes = [(parent_doctype, None)] + [(df.options, df) for df in parent_meta.get_table_fields()] for doctype, table_df in doctypes: translated_table_label = _(table_df.label) if table_df else None @@ -1082,15 +1106,15 @@ def build_fields_dict_for_column_matching(parent_doctype): if doctype == parent_doctype: name_headers = ( - "name", # fieldname - "ID", # label - _("ID"), # translated label + "name", # fieldname + "ID", # label + _("ID"), # translated label ) else: name_headers = ( - "{0}.name".format(table_df.fieldname), # fieldname - "ID ({0})".format(table_df.label), # label - "{0} ({1})".format(_("ID"), translated_table_label), # translated label + "{0}.name".format(table_df.fieldname), # fieldname + "ID ({0})".format(table_df.label), # label + "{0} ({1})".format(_("ID"), translated_table_label), # translated label ) name_df.is_child_table_field = True @@ -1122,7 +1146,7 @@ def build_fields_dict_for_column_matching(parent_doctype): for header in ( df.fieldname, f"{label} ({df.fieldname})", - f"{translated_label} ({df.fieldname})" + f"{translated_label} ({df.fieldname})", ): out[header] = df @@ -1155,9 +1179,8 @@ def build_fields_dict_for_column_matching(parent_doctype): autoname_field = get_autoname_field(parent_doctype) if autoname_field: for header in ( - "ID ({})".format(autoname_field.label), # label - "{0} ({1})".format(_("ID"), _(autoname_field.label)), # translated label - + "ID ({})".format(autoname_field.label), # label + "{0} ({1})".format(_("ID"), _(autoname_field.label)), # translated label # ID field should also map to the autoname field "ID", _("ID"), @@ -1205,10 +1228,7 @@ def get_item_at_index(_list, i, default=None): def get_user_format(date_format): return ( - date_format.replace("%Y", "yyyy") - .replace("%y", "yy") - .replace("%m", "mm") - .replace("%d", "dd") + date_format.replace("%Y", "yyyy").replace("%y", "yy").replace("%m", "mm").replace("%d", "dd") ) @@ -1226,16 +1246,17 @@ def df_as_json(df): def get_select_options(df): return [d for d in (df.options or "").split("\n") if d] -def create_import_log(data_import, log_index, log_details): - frappe.get_doc({ - 'doctype': 'Data Import Log', - 'log_index': log_index, - 'success': log_details.get('success'), - 'data_import': data_import, - 'row_indexes': json.dumps(log_details.get('row_indexes')), - 'docname': log_details.get('docname'), - 'messages': json.dumps(log_details.get('messages', '[]')), - 'exception': log_details.get('exception') - }).db_insert() - +def create_import_log(data_import, log_index, log_details): + frappe.get_doc( + { + "doctype": "Data Import Log", + "log_index": log_index, + "success": log_details.get("success"), + "data_import": data_import, + "row_indexes": json.dumps(log_details.get("row_indexes")), + "docname": log_details.get("docname"), + "messages": json.dumps(log_details.get("messages", "[]")), + "exception": log_details.get("exception"), + } + ).db_insert() diff --git a/frappe/core/doctype/data_import/test_data_import.py b/frappe/core/doctype/data_import/test_data_import.py index c0e4f50d6d..fb15b3ad52 100644 --- a/frappe/core/doctype/data_import/test_data_import.py +++ b/frappe/core/doctype/data_import/test_data_import.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDataImport(unittest.TestCase): pass diff --git a/frappe/core/doctype/data_import/test_exporter.py b/frappe/core/doctype/data_import/test_exporter.py index cb9461451f..ed01a2648c 100644 --- a/frappe/core/doctype/data_import/test_exporter.py +++ b/frappe/core/doctype/data_import/test_exporter.py @@ -2,13 +2,13 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest + import frappe from frappe.core.doctype.data_import.exporter import Exporter -from frappe.core.doctype.data_import.test_importer import ( - create_doctype_if_not_exists, -) +from frappe.core.doctype.data_import.test_importer import create_doctype_if_not_exists + +doctype_name = "DocType for Export" -doctype_name = 'DocType for Export' class TestExporter(unittest.TestCase): def setUp(self): @@ -93,10 +93,10 @@ class TestExporter(unittest.TestCase): doctype_name, export_fields={doctype_name: ["title", "description"]}, export_data=True, - file_type="CSV" + file_type="CSV", ) e.build_response() - self.assertTrue(frappe.response['result']) - self.assertEqual(frappe.response['doctype'], doctype_name) - self.assertEqual(frappe.response['type'], "csv") + self.assertTrue(frappe.response["result"]) + self.assertEqual(frappe.response["doctype"], doctype_name) + self.assertEqual(frappe.response["type"], "csv") diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index 489b8caa58..46b3c352ca 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -2,53 +2,57 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest + import frappe from frappe.core.doctype.data_import.importer import Importer from frappe.tests.test_query_builder import db_type_is, run_only_if -from frappe.utils import getdate, format_duration +from frappe.utils import format_duration, getdate + +doctype_name = "DocType for Import" -doctype_name = 'DocType for Import' class TestImporter(unittest.TestCase): @classmethod def setUpClass(cls): - create_doctype_if_not_exists(doctype_name,) + create_doctype_if_not_exists( + doctype_name, + ) def test_data_import_from_file(self): - import_file = get_import_file('sample_import_file') + import_file = get_import_file("sample_import_file") data_import = self.get_importer(doctype_name, import_file) data_import.start_import() - doc1 = frappe.get_doc(doctype_name, 'Test') - doc2 = frappe.get_doc(doctype_name, 'Test 2') - doc3 = frappe.get_doc(doctype_name, 'Test 3') + doc1 = frappe.get_doc(doctype_name, "Test") + doc2 = frappe.get_doc(doctype_name, "Test 2") + doc3 = frappe.get_doc(doctype_name, "Test 3") - self.assertEqual(doc1.description, 'test description') + self.assertEqual(doc1.description, "test description") self.assertEqual(doc1.number, 1) - self.assertEqual(format_duration(doc1.duration), '3h') + self.assertEqual(format_duration(doc1.duration), "3h") - self.assertEqual(doc1.table_field_1[0].child_title, 'child title') - self.assertEqual(doc1.table_field_1[0].child_description, 'child description') + self.assertEqual(doc1.table_field_1[0].child_title, "child title") + self.assertEqual(doc1.table_field_1[0].child_description, "child description") - self.assertEqual(doc1.table_field_1[1].child_title, 'child title 2') - self.assertEqual(doc1.table_field_1[1].child_description, 'child description 2') + self.assertEqual(doc1.table_field_1[1].child_title, "child title 2") + self.assertEqual(doc1.table_field_1[1].child_description, "child description 2") - self.assertEqual(doc1.table_field_2[1].child_2_title, 'title child') - self.assertEqual(doc1.table_field_2[1].child_2_date, getdate('2019-10-30')) + self.assertEqual(doc1.table_field_2[1].child_2_title, "title child") + self.assertEqual(doc1.table_field_2[1].child_2_date, getdate("2019-10-30")) self.assertEqual(doc1.table_field_2[1].child_2_another_number, 5) - self.assertEqual(doc1.table_field_1_again[0].child_title, 'child title again') - self.assertEqual(doc1.table_field_1_again[1].child_title, 'child title again 2') - self.assertEqual(doc1.table_field_1_again[1].child_date, getdate('2021-09-22')) + self.assertEqual(doc1.table_field_1_again[0].child_title, "child title again") + self.assertEqual(doc1.table_field_1_again[1].child_title, "child title again 2") + self.assertEqual(doc1.table_field_1_again[1].child_date, getdate("2021-09-22")) - self.assertEqual(doc2.description, 'test description 2') - self.assertEqual(format_duration(doc2.duration), '4d 3h') + self.assertEqual(doc2.description, "test description 2") + self.assertEqual(format_duration(doc2.duration), "4d 3h") self.assertEqual(doc3.another_number, 5) - self.assertEqual(format_duration(doc3.duration), '5d 5h 45m') + self.assertEqual(format_duration(doc3.duration), "5d 5h 45m") def test_data_import_preview(self): - import_file = get_import_file('sample_import_file') + import_file = get_import_file("sample_import_file") data_import = self.get_importer(doctype_name, import_file) preview = data_import.get_preview_from_template() @@ -58,35 +62,49 @@ class TestImporter(unittest.TestCase): # ignored on postgres because myisam doesn't exist on pg @run_only_if(db_type_is.MARIADB) def test_data_import_without_mandatory_values(self): - import_file = get_import_file('sample_import_file_without_mandatory') + import_file = get_import_file("sample_import_file_without_mandatory") data_import = self.get_importer(doctype_name, import_file) frappe.local.message_log = [] data_import.start_import() data_import.reload() - import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], + import_log = frappe.db.get_all( + "Data Import Log", + fields=["row_indexes", "success", "messages", "exception", "docname"], filters={"data_import": data_import.name}, - order_by="log_index") + order_by="log_index", + ) - self.assertEqual(frappe.parse_json(import_log[0]['row_indexes']), [2,3]) - expected_error = "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[0])['message'], expected_error) - expected_error = "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" - self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[0]['messages'])[1])['message'], expected_error) + self.assertEqual(frappe.parse_json(import_log[0]["row_indexes"]), [2, 3]) + expected_error = ( + "Error: Child 1 of DocType for Import Row #1: Value missing for: Child Title" + ) + self.assertEqual( + frappe.parse_json(frappe.parse_json(import_log[0]["messages"])[0])["message"], expected_error + ) + expected_error = ( + "Error: Child 1 of DocType for Import Row #2: Value missing for: Child Title" + ) + self.assertEqual( + frappe.parse_json(frappe.parse_json(import_log[0]["messages"])[1])["message"], expected_error + ) - self.assertEqual(frappe.parse_json(import_log[1]['row_indexes']), [4]) - self.assertEqual(frappe.parse_json(frappe.parse_json(import_log[1]['messages'])[0])['message'], "Title is required") + self.assertEqual(frappe.parse_json(import_log[1]["row_indexes"]), [4]) + self.assertEqual( + frappe.parse_json(frappe.parse_json(import_log[1]["messages"])[0])["message"], + "Title is required", + ) def test_data_import_update(self): existing_doc = frappe.get_doc( doctype=doctype_name, title=frappe.generate_hash(doctype_name, 8), - table_field_1=[{'child_title': 'child title to update'}] + table_field_1=[{"child_title": "child title to update"}], ) existing_doc.save() frappe.db.commit() - import_file = get_import_file('sample_import_file_for_update') + import_file = get_import_file("sample_import_file_for_update") data_import = self.get_importer(doctype_name, import_file, update=True) i = Importer(data_import.reference_doctype, data_import=data_import) @@ -104,15 +122,15 @@ class TestImporter(unittest.TestCase): updated_doc = frappe.get_doc(doctype_name, existing_doc.name) self.assertEqual(existing_doc.title, updated_doc.title) - self.assertEqual(updated_doc.description, 'test description') - self.assertEqual(updated_doc.table_field_1[0].child_title, 'child title') + self.assertEqual(updated_doc.description, "test description") + self.assertEqual(updated_doc.table_field_1[0].child_title, "child title") self.assertEqual(updated_doc.table_field_1[0].name, existing_doc.table_field_1[0].name) - self.assertEqual(updated_doc.table_field_1[0].child_description, 'child description') - self.assertEqual(updated_doc.table_field_1_again[0].child_title, 'child title again') + self.assertEqual(updated_doc.table_field_1[0].child_description, "child description") + self.assertEqual(updated_doc.table_field_1_again[0].child_title, "child title again") def get_importer(self, doctype, import_file, update=False): - data_import = frappe.new_doc('Data Import') - data_import.import_type = 'Insert New Records' if not update else 'Update Existing Records' + data_import = frappe.new_doc("Data Import") + data_import.import_type = "Insert New Records" if not update else "Update Existing Records" data_import.reference_doctype = doctype data_import.import_file = import_file.file_url data_import.insert() @@ -121,88 +139,109 @@ class TestImporter(unittest.TestCase): return data_import + def create_doctype_if_not_exists(doctype_name, force=False): if force: - frappe.delete_doc_if_exists('DocType', doctype_name) - frappe.delete_doc_if_exists('DocType', 'Child 1 of ' + doctype_name) - frappe.delete_doc_if_exists('DocType', 'Child 2 of ' + doctype_name) + frappe.delete_doc_if_exists("DocType", doctype_name) + frappe.delete_doc_if_exists("DocType", "Child 1 of " + doctype_name) + frappe.delete_doc_if_exists("DocType", "Child 2 of " + doctype_name) - if frappe.db.exists('DocType', doctype_name): + if frappe.db.exists("DocType", doctype_name): return # Child Table 1 - table_1_name = 'Child 1 of ' + doctype_name - frappe.get_doc({ - 'doctype': 'DocType', - 'name': table_1_name, - 'module': 'Custom', - 'custom': 1, - 'istable': 1, - 'fields': [ - {'label': 'Child Title', 'fieldname': 'child_title', 'reqd': 1, 'fieldtype': 'Data'}, - {'label': 'Child Description', 'fieldname': 'child_description', 'fieldtype': 'Small Text'}, - {'label': 'Child Date', 'fieldname': 'child_date', 'fieldtype': 'Date'}, - {'label': 'Child Number', 'fieldname': 'child_number', 'fieldtype': 'Int'}, - {'label': 'Child Number', 'fieldname': 'child_another_number', 'fieldtype': 'Int'}, - ] - }).insert() + table_1_name = "Child 1 of " + doctype_name + frappe.get_doc( + { + "doctype": "DocType", + "name": table_1_name, + "module": "Custom", + "custom": 1, + "istable": 1, + "fields": [ + {"label": "Child Title", "fieldname": "child_title", "reqd": 1, "fieldtype": "Data"}, + {"label": "Child Description", "fieldname": "child_description", "fieldtype": "Small Text"}, + {"label": "Child Date", "fieldname": "child_date", "fieldtype": "Date"}, + {"label": "Child Number", "fieldname": "child_number", "fieldtype": "Int"}, + {"label": "Child Number", "fieldname": "child_another_number", "fieldtype": "Int"}, + ], + } + ).insert() # Child Table 2 - table_2_name = 'Child 2 of ' + doctype_name - frappe.get_doc({ - 'doctype': 'DocType', - 'name': table_2_name, - 'module': 'Custom', - 'custom': 1, - 'istable': 1, - 'fields': [ - {'label': 'Child 2 Title', 'fieldname': 'child_2_title', 'reqd': 1, 'fieldtype': 'Data'}, - {'label': 'Child 2 Description', 'fieldname': 'child_2_description', 'fieldtype': 'Small Text'}, - {'label': 'Child 2 Date', 'fieldname': 'child_2_date', 'fieldtype': 'Date'}, - {'label': 'Child 2 Number', 'fieldname': 'child_2_number', 'fieldtype': 'Int'}, - {'label': 'Child 2 Number', 'fieldname': 'child_2_another_number', 'fieldtype': 'Int'}, - ] - }).insert() + table_2_name = "Child 2 of " + doctype_name + frappe.get_doc( + { + "doctype": "DocType", + "name": table_2_name, + "module": "Custom", + "custom": 1, + "istable": 1, + "fields": [ + {"label": "Child 2 Title", "fieldname": "child_2_title", "reqd": 1, "fieldtype": "Data"}, + { + "label": "Child 2 Description", + "fieldname": "child_2_description", + "fieldtype": "Small Text", + }, + {"label": "Child 2 Date", "fieldname": "child_2_date", "fieldtype": "Date"}, + {"label": "Child 2 Number", "fieldname": "child_2_number", "fieldtype": "Int"}, + {"label": "Child 2 Number", "fieldname": "child_2_another_number", "fieldtype": "Int"}, + ], + } + ).insert() # Main Table - frappe.get_doc({ - 'doctype': 'DocType', - 'name': doctype_name, - 'module': 'Custom', - 'custom': 1, - 'autoname': 'field:title', - 'fields': [ - {'label': 'Title', 'fieldname': 'title', 'reqd': 1, 'fieldtype': 'Data'}, - {'label': 'Description', 'fieldname': 'description', 'fieldtype': 'Small Text'}, - {'label': 'Date', 'fieldname': 'date', 'fieldtype': 'Date'}, - {'label': 'Duration', 'fieldname': 'duration', 'fieldtype': 'Duration'}, - {'label': 'Number', 'fieldname': 'number', 'fieldtype': 'Int'}, - {'label': 'Number', 'fieldname': 'another_number', 'fieldtype': 'Int'}, - {'label': 'Table Field 1', 'fieldname': 'table_field_1', 'fieldtype': 'Table', 'options': table_1_name}, - {'label': 'Table Field 2', 'fieldname': 'table_field_2', 'fieldtype': 'Table', 'options': table_2_name}, - {'label': 'Table Field 1 Again', 'fieldname': 'table_field_1_again', 'fieldtype': 'Table', 'options': table_1_name}, - ], - 'permissions': [ - {'role': 'System Manager'} - ] - }).insert() + frappe.get_doc( + { + "doctype": "DocType", + "name": doctype_name, + "module": "Custom", + "custom": 1, + "autoname": "field:title", + "fields": [ + {"label": "Title", "fieldname": "title", "reqd": 1, "fieldtype": "Data"}, + {"label": "Description", "fieldname": "description", "fieldtype": "Small Text"}, + {"label": "Date", "fieldname": "date", "fieldtype": "Date"}, + {"label": "Duration", "fieldname": "duration", "fieldtype": "Duration"}, + {"label": "Number", "fieldname": "number", "fieldtype": "Int"}, + {"label": "Number", "fieldname": "another_number", "fieldtype": "Int"}, + { + "label": "Table Field 1", + "fieldname": "table_field_1", + "fieldtype": "Table", + "options": table_1_name, + }, + { + "label": "Table Field 2", + "fieldname": "table_field_2", + "fieldtype": "Table", + "options": table_2_name, + }, + { + "label": "Table Field 1 Again", + "fieldname": "table_field_1_again", + "fieldtype": "Table", + "options": table_1_name, + }, + ], + "permissions": [{"role": "System Manager"}], + } + ).insert() def get_import_file(csv_file_name, force=False): - file_name = csv_file_name + '.csv' - _file = frappe.db.exists('File', {'file_name': file_name}) + file_name = csv_file_name + ".csv" + _file = frappe.db.exists("File", {"file_name": file_name}) if force and _file: - frappe.delete_doc_if_exists('File', _file) + frappe.delete_doc_if_exists("File", _file) - if frappe.db.exists('File', {'file_name': file_name}): - f = frappe.get_doc('File', {'file_name': file_name}) + if frappe.db.exists("File", {"file_name": file_name}): + f = frappe.get_doc("File", {"file_name": file_name}) else: full_path = get_csv_file_path(file_name) f = frappe.get_doc( - doctype='File', - content=frappe.read_file(full_path), - file_name=file_name, - is_private=1 + doctype="File", content=frappe.read_file(full_path), file_name=file_name, is_private=1 ) f.save(ignore_permissions=True) @@ -210,4 +249,4 @@ def get_import_file(csv_file_name, force=False): def get_csv_file_path(file_name): - return frappe.get_app_path('frappe', 'core', 'doctype', 'data_import', 'fixtures', file_name) + return frappe.get_app_path("frappe", "core", "doctype", "data_import", "fixtures", file_name) diff --git a/frappe/core/doctype/data_import_log/data_import_log.py b/frappe/core/doctype/data_import_log/data_import_log.py index a71aefa8bc..8c778babde 100644 --- a/frappe/core/doctype/data_import_log/data_import_log.py +++ b/frappe/core/doctype/data_import_log/data_import_log.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class DataImportLog(Document): pass diff --git a/frappe/core/doctype/data_import_log/test_data_import_log.py b/frappe/core/doctype/data_import_log/test_data_import_log.py index 244404936e..6ae7c532bf 100644 --- a/frappe/core/doctype/data_import_log/test_data_import_log.py +++ b/frappe/core/doctype/data_import_log/test_data_import_log.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDataImportLog(unittest.TestCase): pass diff --git a/frappe/core/doctype/defaultvalue/__init__.py b/frappe/core/doctype/defaultvalue/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/defaultvalue/__init__.py +++ b/frappe/core/doctype/defaultvalue/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/defaultvalue/defaultvalue.py b/frappe/core/doctype/defaultvalue/defaultvalue.py index 1d597c7fc4..da15787769 100644 --- a/frappe/core/doctype/defaultvalue/defaultvalue.py +++ b/frappe/core/doctype/defaultvalue/defaultvalue.py @@ -2,19 +2,24 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class DefaultValue(Document): pass + def on_doctype_update(): """Create indexes for `tabDefaultValue` on `(parent, defkey)`""" frappe.db.commit() - frappe.db.add_index(doctype='DefaultValue', - fields=['parent', 'defkey'], - index_name='defaultvalue_parent_defkey_index') + frappe.db.add_index( + doctype="DefaultValue", + fields=["parent", "defkey"], + index_name="defaultvalue_parent_defkey_index", + ) - frappe.db.add_index(doctype='DefaultValue', - fields=['parent', 'parenttype'], - index_name='defaultvalue_parent_parenttype_index') + frappe.db.add_index( + doctype="DefaultValue", + fields=["parent", "parenttype"], + index_name="defaultvalue_parent_parenttype_index", + ) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index b398ec5410..f2c98a41b8 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -2,11 +2,12 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import json + +import frappe +from frappe import _ from frappe.desk.doctype.bulk_update.bulk_update import show_progress from frappe.model.document import Document -from frappe import _ class DeletedDocument(Document): @@ -15,7 +16,7 @@ class DeletedDocument(Document): @frappe.whitelist() def restore(name, alert=True): - deleted = frappe.get_doc('Deleted Document', name) + deleted = frappe.get_doc("Deleted Document", name) if deleted.restored: frappe.throw(_("Document {0} Already Restored").format(name), exc=frappe.DocumentAlreadyRestored) @@ -29,20 +30,20 @@ def restore(name, alert=True): doc.docstatus = 0 doc.insert() - doc.add_comment('Edit', _('restored {0} as {1}').format(deleted.deleted_name, doc.name)) + doc.add_comment("Edit", _("restored {0} as {1}").format(deleted.deleted_name, doc.name)) deleted.new_name = doc.name deleted.restored = 1 deleted.db_update() if alert: - frappe.msgprint(_('Document Restored')) + frappe.msgprint(_("Document Restored")) @frappe.whitelist() def bulk_restore(docnames): docnames = frappe.parse_json(docnames) - message = _('Restoring Deleted Document') + message = _("Restoring Deleted Document") restored, invalid, failed = [], [], [] for i, d in enumerate(docnames): @@ -61,8 +62,4 @@ def bulk_restore(docnames): failed.append(d) frappe.db.rollback() - return { - "restored": restored, - "invalid": invalid, - "failed": failed - } + return {"restored": restored, "invalid": invalid, "failed": failed} diff --git a/frappe/core/doctype/deleted_document/test_deleted_document.py b/frappe/core/doctype/deleted_document/test_deleted_document.py index fb2376de90..0c8e88a32f 100644 --- a/frappe/core/doctype/deleted_document/test_deleted_document.py +++ b/frappe/core/doctype/deleted_document/test_deleted_document.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Deleted Document') + class TestDeletedDocument(unittest.TestCase): pass diff --git a/frappe/core/doctype/docfield/__init__.py b/frappe/core/doctype/docfield/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/docfield/__init__.py +++ b/frappe/core/doctype/docfield/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py index 4dd49631ae..9fe55df5fe 100644 --- a/frappe/core/doctype/docfield/docfield.py +++ b/frappe/core/doctype/docfield/docfield.py @@ -4,28 +4,28 @@ import frappe from frappe.model.document import Document + class DocField(Document): def get_link_doctype(self): - '''Returns the Link doctype for the docfield (if applicable) + """Returns the Link doctype for the docfield (if applicable) if fieldtype is Link: Returns "options" if fieldtype is Table MultiSelect: Returns "options" of the Link field in the Child Table - ''' - if self.fieldtype == 'Link': + """ + if self.fieldtype == "Link": return self.options - if self.fieldtype == 'Table MultiSelect': + if self.fieldtype == "Table MultiSelect": table_doctype = self.options - link_doctype = frappe.db.get_value('DocField', { - 'fieldtype': 'Link', - 'parenttype': 'DocType', - 'parent': table_doctype, - 'in_list_view': 1 - }, 'options') + link_doctype = frappe.db.get_value( + "DocField", + {"fieldtype": "Link", "parenttype": "DocType", "parent": table_doctype, "in_list_view": 1}, + "options", + ) return link_doctype def get_select_options(self): - if self.fieldtype == 'Select': - options = self.options or '' - return [d for d in options.split('\n') if d] + if self.fieldtype == "Select": + options = self.options or "" + return [d for d in options.split("\n") if d] diff --git a/frappe/core/doctype/docperm/__init__.py b/frappe/core/doctype/docperm/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/docperm/__init__.py +++ b/frappe/core/doctype/docperm/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/docperm/docperm.py b/frappe/core/doctype/docperm/docperm.py index 4751816dc5..68ef21e770 100644 --- a/frappe/core/doctype/docperm/docperm.py +++ b/frappe/core/doctype/docperm/docperm.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class DocPerm(Document): pass diff --git a/frappe/core/doctype/docshare/docshare.py b/frappe/core/doctype/docshare/docshare.py index 6320fba60b..641b118f28 100644 --- a/frappe/core/doctype/docshare/docshare.py +++ b/frappe/core/doctype/docshare/docshare.py @@ -2,12 +2,13 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document from frappe import _ -from frappe.utils import get_fullname, cint +from frappe.model.document import Document +from frappe.utils import cint, get_fullname exclude_from_linked_with = True + class DocShare(Document): no_feed_on_delete = True @@ -36,15 +37,21 @@ class DocShare(Document): frappe.throw(_("User is mandatory for Share"), frappe.MandatoryError) def check_share_permission(self): - if (not self.flags.ignore_share_permission and - not frappe.has_permission(self.share_doctype, "share", self.get_doc())): + if not self.flags.ignore_share_permission and not frappe.has_permission( + self.share_doctype, "share", self.get_doc() + ): frappe.throw(_('You need to have "Share" permission'), frappe.PermissionError) def check_is_submittable(self): - if self.submit and not cint(frappe.db.get_value("DocType", self.share_doctype, "is_submittable")): - frappe.throw(_("Cannot share {0} with submit permission as the doctype {1} is not submittable").format( - frappe.bold(self.share_name), frappe.bold(self.share_doctype))) + if self.submit and not cint( + frappe.db.get_value("DocType", self.share_doctype, "is_submittable") + ): + frappe.throw( + _("Cannot share {0} with submit permission as the doctype {1} is not submittable").format( + frappe.bold(self.share_name), frappe.bold(self.share_doctype) + ) + ) def after_insert(self): doc = self.get_doc() @@ -53,14 +60,21 @@ class DocShare(Document): if self.everyone: doc.add_comment("Shared", _("{0} shared this document with everyone").format(owner)) else: - doc.add_comment("Shared", _("{0} shared this document with {1}").format(owner, get_fullname(self.user))) + doc.add_comment( + "Shared", _("{0} shared this document with {1}").format(owner, get_fullname(self.user)) + ) def on_trash(self): if not self.flags.ignore_share_permission: self.check_share_permission() - self.get_doc().add_comment("Unshared", - _("{0} un-shared this document with {1}").format(get_fullname(self.owner), get_fullname(self.user))) + self.get_doc().add_comment( + "Unshared", + _("{0} un-shared this document with {1}").format( + get_fullname(self.owner), get_fullname(self.user) + ), + ) + def on_doctype_update(): """Add index in `tabDocShare` for `(user, share_doctype)`""" diff --git a/frappe/core/doctype/docshare/test_docshare.py b/frappe/core/doctype/docshare/test_docshare.py index cbdaa8ebaf..e374e4069d 100644 --- a/frappe/core/doctype/docshare/test_docshare.py +++ b/frappe/core/doctype/docshare/test_docshare.py @@ -1,20 +1,26 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import unittest + import frappe import frappe.share -import unittest from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype -test_dependencies = ['User'] +test_dependencies = ["User"] + class TestDocShare(unittest.TestCase): def setUp(self): self.user = "test@example.com" - self.event = frappe.get_doc({"doctype": "Event", - "subject": "test share event", - "starts_on": "2015-01-01 10:00:00", - "event_type": "Private"}).insert() + self.event = frappe.get_doc( + { + "doctype": "Event", + "subject": "test share event", + "starts_on": "2015-01-01 10:00:00", + "event_type": "Private", + } + ).insert() def tearDown(self): frappe.set_user("Administrator") @@ -98,7 +104,9 @@ class TestDocShare(unittest.TestCase): doctype = "Test DocShare with Submit" create_submittable_doctype(doctype, submit_perms=0) - submittable_doc = frappe.get_doc(dict(doctype=doctype, test="test docshare with submit")).insert() + submittable_doc = frappe.get_doc( + dict(doctype=doctype, test="test docshare with submit") + ).insert() frappe.set_user(self.user) self.assertFalse(frappe.has_permission(doctype, "submit", user=self.user)) @@ -107,10 +115,14 @@ class TestDocShare(unittest.TestCase): frappe.share.add(doctype, submittable_doc.name, self.user, submit=1) frappe.set_user(self.user) - self.assertTrue(frappe.has_permission(doctype, "submit", doc=submittable_doc.name, user=self.user)) + self.assertTrue( + frappe.has_permission(doctype, "submit", doc=submittable_doc.name, user=self.user) + ) # test cascade self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user)) - self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user)) + self.assertTrue( + frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user) + ) frappe.share.remove(doctype, submittable_doc.name, self.user) diff --git a/frappe/core/doctype/doctype/__init__.py b/frappe/core/doctype/doctype/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/doctype/__init__.py +++ b/frappe/core/doctype/doctype/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 29b56fbff6..1d02f09820 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -1,46 +1,78 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -# imports - standard imports -import re, copy, os, shutil +import copy import json -from frappe.cache_manager import clear_user_cache, clear_controller_cache +import os + +# imports - standard imports +import re +import shutil # imports - module imports import frappe from frappe import _ -from frappe.utils import now, cint +from frappe.cache_manager import clear_controller_cache, clear_user_cache +from frappe.custom.doctype.custom_field.custom_field import create_custom_field +from frappe.custom.doctype.property_setter.property_setter import make_property_setter +from frappe.database.schema import validate_column_length, validate_column_name +from frappe.desk.notifications import delete_notification_count_for +from frappe.desk.utils import validate_route_conflict from frappe.model import ( - no_value_fields, default_fields, table_fields, data_field_options, child_table_fields + child_table_fields, + data_field_options, + default_fields, + no_value_fields, + table_fields, ) -from frappe.model.document import Document from frappe.model.base_document import get_controller -from frappe.custom.doctype.property_setter.property_setter import make_property_setter -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.desk.notifications import delete_notification_count_for -from frappe.modules import make_boilerplate, get_doc_path -from frappe.database.schema import validate_column_name, validate_column_length from frappe.model.docfield import supports_translation -from frappe.modules.import_file import get_file_path +from frappe.model.document import Document from frappe.model.meta import Meta -from frappe.desk.utils import validate_route_conflict -from frappe.website.utils import clear_cache +from frappe.modules import get_doc_path, make_boilerplate +from frappe.modules.import_file import get_file_path from frappe.query_builder.functions import Concat +from frappe.utils import cint, now +from frappe.website.utils import clear_cache + + +class InvalidFieldNameError(frappe.ValidationError): + pass + + +class UniqueFieldnameError(frappe.ValidationError): + pass + + +class IllegalMandatoryError(frappe.ValidationError): + pass + + +class DoctypeLinkError(frappe.ValidationError): + pass + + +class WrongOptionsDoctypeLinkError(frappe.ValidationError): + pass + + +class HiddenAndMandatoryWithoutDefaultError(frappe.ValidationError): + pass + + +class NonUniqueError(frappe.ValidationError): + pass + -class InvalidFieldNameError(frappe.ValidationError): pass -class UniqueFieldnameError(frappe.ValidationError): pass -class IllegalMandatoryError(frappe.ValidationError): pass -class DoctypeLinkError(frappe.ValidationError): pass -class WrongOptionsDoctypeLinkError(frappe.ValidationError): pass -class HiddenAndMandatoryWithoutDefaultError(frappe.ValidationError): pass -class NonUniqueError(frappe.ValidationError): pass -class CannotIndexedError(frappe.ValidationError): pass -class CannotCreateStandardDoctypeError(frappe.ValidationError): pass +class CannotIndexedError(frappe.ValidationError): + pass -form_grid_templates = { - "fields": "templates/form_grid/fields.html" -} +class CannotCreateStandardDoctypeError(frappe.ValidationError): + pass + + +form_grid_templates = {"fields": "templates/form_grid/fields.html"} class DocType(Document): @@ -83,14 +115,14 @@ class DocType(Document): validate_links_table_fieldnames(self) if not self.is_new(): - self.before_update = frappe.get_doc('DocType', self.name) + self.before_update = frappe.get_doc("DocType", self.name) self.setup_fields_to_fetch() self.validate_field_name_conflicts() check_email_append_to(self) if self.default_print_format and not self.custom: - frappe.throw(_('Standard DocType cannot have default print format, use Customize Form')) + frappe.throw(_("Standard DocType cannot have default print format, use Customize Form")) def validate_field_name_conflicts(self): """Check if field names dont conflict with controller properties and methods""" @@ -133,8 +165,9 @@ class DocType(Document): if conflict_type: frappe.throw( - _("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}") - .format(field_label, conflict_type, field, self.name) + _("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}").format( + field_label, conflict_type, field, self.name + ) ) def after_insert(self): @@ -152,17 +185,18 @@ class DocType(Document): self.permissions = [] def set_default_in_list_view(self): - '''Set default in-list-view for first 4 mandatory fields''' + """Set default in-list-view for first 4 mandatory fields""" if not [d.fieldname for d in self.fields if d.in_list_view]: cnt = 0 for d in self.fields: if d.reqd and not d.hidden and not d.fieldtype in table_fields: d.in_list_view = 1 cnt += 1 - if cnt == 4: break + if cnt == 4: + break def set_default_translatable(self): - '''Ensure that non-translatable never will be translatable''' + """Ensure that non-translatable never will be translatable""" for d in self.fields: if d.translatable and not supports_translation(d.fieldtype): d.translatable = 0 @@ -173,19 +207,24 @@ class DocType(Document): return if not frappe.conf.get("developer_mode") and not self.custom: - frappe.throw(_("Not in Developer Mode! Set in site_config.json or make 'Custom' DocType."), CannotCreateStandardDoctypeError) + frappe.throw( + _("Not in Developer Mode! Set in site_config.json or make 'Custom' DocType."), + CannotCreateStandardDoctypeError, + ) if self.is_virtual and self.custom: - frappe.throw(_("Not allowed to create custom Virtual DocType."), CannotCreateStandardDoctypeError) + frappe.throw( + _("Not allowed to create custom Virtual DocType."), CannotCreateStandardDoctypeError + ) - if frappe.conf.get('developer_mode'): - self.owner = 'Administrator' - self.modified_by = 'Administrator' + if frappe.conf.get("developer_mode"): + self.owner = "Administrator" + self.modified_by = "Administrator" def setup_fields_to_fetch(self): - '''Setup query to update values for newly set fetch values''' + """Setup query to update values for newly set fetch values""" try: - old_meta = frappe.get_meta(frappe.get_doc('DocType', self.name), cached=False) + old_meta = frappe.get_meta(frappe.get_doc("DocType", self.name), cached=False) old_fields_to_fetch = [df.fieldname for df in old_meta.get_fields_to_fetch()] except frappe.DoesNotExistError: old_fields_to_fetch = [] @@ -197,56 +236,57 @@ class DocType(Document): if set(old_fields_to_fetch) != set(df.fieldname for df in new_meta.get_fields_to_fetch()): for df in new_meta.get_fields_to_fetch(): if df.fieldname not in old_fields_to_fetch: - link_fieldname, source_fieldname = df.fetch_from.split('.', 1) + link_fieldname, source_fieldname = df.fetch_from.split(".", 1) link_df = new_meta.get_field(link_fieldname) - if frappe.db.db_type == 'postgres': - update_query = ''' + if frappe.db.db_type == "postgres": + update_query = """ UPDATE `tab{doctype}` SET `{fieldname}` = source.`{source_fieldname}` FROM `tab{link_doctype}` as source WHERE `{link_fieldname}` = source.name AND ifnull(`{fieldname}`, '')='' - ''' + """ else: - update_query = ''' + update_query = """ UPDATE `tab{doctype}` as target INNER JOIN `tab{link_doctype}` as source ON `target`.`{link_fieldname}` = `source`.`name` SET `target`.`{fieldname}` = `source`.`{source_fieldname}` WHERE ifnull(`target`.`{fieldname}`, '')="" - ''' - - self.flags.update_fields_to_fetch_queries.append(update_query.format( - link_doctype = link_df.options, - source_fieldname = source_fieldname, - doctype = self.name, - fieldname = df.fieldname, - link_fieldname = link_fieldname + """ + + self.flags.update_fields_to_fetch_queries.append( + update_query.format( + link_doctype=link_df.options, + source_fieldname=source_fieldname, + doctype=self.name, + fieldname=df.fieldname, + link_fieldname=link_fieldname, ) ) def update_fields_to_fetch(self): - '''Update fetch values based on queries setup''' + """Update fetch values based on queries setup""" if self.flags.update_fields_to_fetch_queries and not self.issingle: for query in self.flags.update_fields_to_fetch_queries: frappe.db.sql(query) def validate_document_type(self): - if self.document_type=="Transaction": + if self.document_type == "Transaction": self.document_type = "Document" - if self.document_type=="Master": + if self.document_type == "Master": self.document_type = "Setup" def validate_website(self): """Ensure that website generator has field 'route'""" if self.route: - self.route = self.route.strip('/') + self.route = self.route.strip("/") if self.has_web_view: # route field must be present - if not 'route' in [d.fieldname for d in self.fields]: - frappe.throw(_('Field "route" is mandatory for Web Views'), title='Missing Field') + if not "route" in [d.fieldname for d in self.fields]: + frappe.throw(_('Field "route" is mandatory for Web Views'), title="Missing Field") # clear website cache clear_cache() @@ -255,7 +295,6 @@ class DocType(Document): """Ensure that max_attachments is *at least* bigger than number of attach fields.""" from frappe.model import attachment_fieldtypes - if not self.max_attachments: return @@ -263,49 +302,65 @@ class DocType(Document): if total_attach_fields > self.max_attachments: self.max_attachments = total_attach_fields field_label = frappe.bold(self.meta.get_field("max_attachments").label) - frappe.msgprint(_("Number of attachment fields are more than {}, limit updated to {}.") - .format(field_label, total_attach_fields), - title=_("Insufficient attachment limit"), alert=True) + frappe.msgprint( + _("Number of attachment fields are more than {}, limit updated to {}.").format( + field_label, total_attach_fields + ), + title=_("Insufficient attachment limit"), + alert=True, + ) def change_modified_of_parent(self): """Change the timestamp of parent DocType if the current one is a child to clear caches.""" if frappe.flags.in_import: return - parent_list = frappe.db.get_all('DocField', 'parent', - dict(fieldtype=['in', frappe.model.table_fields], options=self.name)) + parent_list = frappe.db.get_all( + "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name) + ) for p in parent_list: frappe.db.update("DocType", p.parent, {}, for_update=False) def scrub_field_names(self): """Sluggify fieldnames if not set from Label.""" - restricted = ('name','parent','creation','modified','modified_by', - 'parentfield','parenttype','file_list', 'flags', 'docstatus') + restricted = ( + "name", + "parent", + "creation", + "modified", + "modified_by", + "parentfield", + "parenttype", + "file_list", + "flags", + "docstatus", + ) for d in self.get("fields"): if d.fieldtype: - if (not getattr(d, "fieldname", None)): + if not getattr(d, "fieldname", None): if d.label: - d.fieldname = d.label.strip().lower().replace(' ','_').strip('?') + d.fieldname = d.label.strip().lower().replace(" ", "_").strip("?") if d.fieldname in restricted: - d.fieldname = d.fieldname + '1' - if d.fieldtype=='Section Break': - d.fieldname = d.fieldname + '_section' - elif d.fieldtype=='Column Break': - d.fieldname = d.fieldname + '_column' - elif d.fieldtype=='Tab Break': - d.fieldname = d.fieldname + '_tab' + d.fieldname = d.fieldname + "1" + if d.fieldtype == "Section Break": + d.fieldname = d.fieldname + "_section" + elif d.fieldtype == "Column Break": + d.fieldname = d.fieldname + "_column" + elif d.fieldtype == "Tab Break": + d.fieldname = d.fieldname + "_tab" else: - d.fieldname = d.fieldtype.lower().replace(" ","_") + "_" + str(d.idx) + d.fieldname = d.fieldtype.lower().replace(" ", "_") + "_" + str(d.idx) else: if d.fieldname in restricted: frappe.throw(_("Fieldname {0} is restricted").format(d.fieldname), InvalidFieldNameError) - d.fieldname = re.sub('''['",./%@()<>{}]''', '', d.fieldname) + d.fieldname = re.sub("""['",./%@()<>{}]""", "", d.fieldname) # fieldnames should be lowercase d.fieldname = d.fieldname.lower() # unique is automatically an index - if d.unique: d.search_index = 0 + if d.unique: + d.search_index = 0 def on_update(self): """Update database schema, make controller templates if `custom` is not set and clear cache.""" @@ -323,10 +378,7 @@ class DocType(Document): allow_doctype_export = ( not self.custom and not frappe.flags.in_import - and ( - frappe.conf.developer_mode - or frappe.flags.allow_doctype_export - ) + and (frappe.conf.developer_mode or frappe.flags.allow_doctype_export) ) if allow_doctype_export: self.export_doc() @@ -342,7 +394,7 @@ class DocType(Document): delete_notification_count_for(doctype=self.name) frappe.clear_cache(doctype=self.name) - if not frappe.flags.in_install and hasattr(self, 'before_update'): + if not frappe.flags.in_install and hasattr(self, "before_update"): self.sync_global_search() # clear from local cache @@ -352,21 +404,20 @@ class DocType(Document): clear_linked_doctype_cache() def sync_global_search(self): - '''If global search settings are changed, rebuild search properties for this table''' - global_search_fields_before_update = [d.fieldname for d in - self.before_update.fields if d.in_global_search] + """If global search settings are changed, rebuild search properties for this table""" + global_search_fields_before_update = [ + d.fieldname for d in self.before_update.fields if d.in_global_search + ] if self.before_update.show_name_in_global_search: - global_search_fields_before_update.append('name') + global_search_fields_before_update.append("name") - global_search_fields_after_update = [d.fieldname for d in - self.fields if d.in_global_search] + global_search_fields_after_update = [d.fieldname for d in self.fields if d.in_global_search] if self.show_name_in_global_search: - global_search_fields_after_update.append('name') + global_search_fields_after_update.append("name") if set(global_search_fields_before_update) != set(global_search_fields_after_update): now = (not frappe.request) or frappe.flags.in_test or frappe.flags.in_install - frappe.enqueue('frappe.utils.global_search.rebuild_for_doctype', - now=now, doctype=self.name) + frappe.enqueue("frappe.utils.global_search.rebuild_for_doctype", now=now, doctype=self.name) def set_base_class_for_controller(self): """If DocType.has_web_view has been changed, updates the controller class and import @@ -391,21 +442,16 @@ class DocType(Document): code = f.read() updated_code = code - is_website_generator_class = all([ - website_generator_cls_tag in code, - website_generator_import_tag in code - ]) + is_website_generator_class = all( + [website_generator_cls_tag in code, website_generator_import_tag in code] + ) if self.has_web_view and not is_website_generator_class: - updated_code = updated_code.replace( - document_import_tag, website_generator_import_tag - ).replace( + updated_code = updated_code.replace(document_import_tag, website_generator_import_tag).replace( document_cls_tag, website_generator_cls_tag ) elif not self.has_web_view and is_website_generator_class: - updated_code = updated_code.replace( - website_generator_import_tag, document_import_tag - ).replace( + updated_code = updated_code.replace(website_generator_import_tag, document_import_tag).replace( website_generator_cls_tag, document_cls_tag ) @@ -415,6 +461,7 @@ class DocType(Document): def run_module_method(self, method): from frappe.modules import load_doctype_module + module = load_doctype_module(self.name, self.module) if hasattr(module, method): getattr(module, method)() @@ -436,8 +483,11 @@ class DocType(Document): if self.issingle: frappe.db.sql("""update tabSingles set doctype=%s where doctype=%s""", (new, old)) - frappe.db.sql("""update tabSingles set value=%s - where doctype=%s and field='name' and value = %s""", (new, new, old)) + frappe.db.sql( + """update tabSingles set value=%s + where doctype=%s and field='name' and value = %s""", + (new, new, old), + ) else: frappe.db.rename_table(old, new) frappe.db.commit() @@ -455,8 +505,8 @@ class DocType(Document): def rename_files_and_folders(self, old, new): # move files - new_path = get_doc_path(self.module, 'doctype', new) - old_path = get_doc_path(self.module, 'doctype', old) + new_path = get_doc_path(self.module, "doctype", new) + old_path = get_doc_path(self.module, "doctype", old) shutil.move(old_path, new_path) # rename files @@ -467,30 +517,34 @@ class DocType(Document): shutil.move(old_file_name, new_file_name) self.rename_inside_controller(new, old, new_path) - frappe.msgprint(_('Renamed files and replaced code in controllers, please check!')) + frappe.msgprint(_("Renamed files and replaced code in controllers, please check!")) def rename_inside_controller(self, new, old, new_path): - for fname in ('{}.js', '{}.py', '{}_list.js', '{}_calendar.js', 'test_{}.py', 'test_{}.js'): + for fname in ("{}.js", "{}.py", "{}_list.js", "{}_calendar.js", "test_{}.py", "test_{}.js"): fname = os.path.join(new_path, fname.format(frappe.scrub(new))) if os.path.exists(fname): - with open(fname, 'r') as f: + with open(fname, "r") as f: code = f.read() - with open(fname, 'w') as f: - if fname.endswith('.js'): - file_content = code.replace(old, new) # replace str with full str (js controllers) - - elif fname.endswith('.py'): - file_content = code.replace(frappe.scrub(old), frappe.scrub(new)) # replace str with _ (py imports) - file_content = file_content.replace(old.replace(' ', ''), new.replace(' ', '')) # replace str (py controllers) + with open(fname, "w") as f: + if fname.endswith(".js"): + file_content = code.replace(old, new) # replace str with full str (js controllers) + + elif fname.endswith(".py"): + file_content = code.replace( + frappe.scrub(old), frappe.scrub(new) + ) # replace str with _ (py imports) + file_content = file_content.replace( + old.replace(" ", ""), new.replace(" ", "") + ) # replace str (py controllers) f.write(file_content) # updating json file with new name - doctype_json_path = os.path.join(new_path, '{}.json'.format(frappe.scrub(new))) + doctype_json_path = os.path.join(new_path, "{}.json".format(frappe.scrub(new))) current_data = frappe.get_file_json(doctype_json_path) - current_data['name'] = new + current_data["name"] = new - with open(doctype_json_path, 'w') as f: + with open(doctype_json_path, "w") as f: json.dump(current_data, f, indent=1) def before_reload(self): @@ -506,16 +560,34 @@ class DocType(Document): return # check if atleast 1 record exists - if not (frappe.db.table_exists(self.name) and frappe.get_all(self.name, fields=["name"], limit=1, as_list=True)): + if not ( + frappe.db.table_exists(self.name) + and frappe.get_all(self.name, fields=["name"], limit=1, as_list=True) + ): return - existing_property_setter = frappe.db.get_value("Property Setter", {"doc_type": self.name, - "property": "options", "field_name": "naming_series"}) + existing_property_setter = frappe.db.get_value( + "Property Setter", {"doc_type": self.name, "property": "options", "field_name": "naming_series"} + ) if not existing_property_setter: - make_property_setter(self.name, "naming_series", "options", naming_series[0].options, "Text", validate_fields_for_doctype=False) + make_property_setter( + self.name, + "naming_series", + "options", + naming_series[0].options, + "Text", + validate_fields_for_doctype=False, + ) if naming_series[0].default: - make_property_setter(self.name, "naming_series", "default", naming_series[0].default, "Text", validate_fields_for_doctype=False) + make_property_setter( + self.name, + "naming_series", + "default", + naming_series[0].default, + "Text", + validate_fields_for_doctype=False, + ) def before_export(self, docdict): # remove null and empty fields @@ -542,26 +614,26 @@ class DocType(Document): path = get_file_path(self.module, "DocType", self.name) if os.path.exists(path): try: - with open(path, 'r') as txtfile: + with open(path, "r") as txtfile: olddoc = json.loads(txtfile.read()) - old_field_names = [f['fieldname'] for f in olddoc.get("fields", [])] + old_field_names = [f["fieldname"] for f in olddoc.get("fields", [])] if old_field_names: new_field_dicts = [] remaining_field_names = [f.fieldname for f in self.fields] for fieldname in old_field_names: - field_dict = list(filter(lambda d: d['fieldname'] == fieldname, docdict['fields'])) + field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict["fields"])) if field_dict: new_field_dicts.append(field_dict[0]) if fieldname in remaining_field_names: remaining_field_names.remove(fieldname) for fieldname in remaining_field_names: - field_dict = list(filter(lambda d: d['fieldname'] == fieldname, docdict['fields'])) + field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict["fields"])) new_field_dicts.append(field_dict[0]) - docdict['fields'] = new_field_dicts + docdict["fields"] = new_field_dicts except ValueError: pass @@ -570,20 +642,20 @@ class DocType(Document): # set order of fields from field_order if docdict.get("field_order"): new_field_dicts = [] - remaining_field_names = [f['fieldname'] for f in docdict.get('fields', [])] + remaining_field_names = [f["fieldname"] for f in docdict.get("fields", [])] - for fieldname in docdict.get('field_order'): - field_dict = list(filter(lambda d: d['fieldname'] == fieldname, docdict.get('fields', []))) + for fieldname in docdict.get("field_order"): + field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict.get("fields", []))) if field_dict: new_field_dicts.append(field_dict[0]) if fieldname in remaining_field_names: remaining_field_names.remove(fieldname) for fieldname in remaining_field_names: - field_dict = list(filter(lambda d: d['fieldname'] == fieldname, docdict.get('fields', []))) + field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict.get("fields", []))) new_field_dicts.append(field_dict[0]) - docdict['fields'] = new_field_dicts + docdict["fields"] = new_field_dicts if "field_order" in docdict: del docdict["field_order"] @@ -591,7 +663,8 @@ class DocType(Document): def export_doc(self): """Export to standard folder `[module]/doctype/[name]/[name].json`.""" from frappe.modules.export_file import export_to_files - export_to_files(record_list=[['DocType', self.name]], create_init=True) + + export_to_files(record_list=[["DocType", self.name]], create_init=True) def make_controller_template(self): """Make boilerplate controller template.""" @@ -600,41 +673,60 @@ class DocType(Document): if not self.istable: make_boilerplate("test_controller._py", self.as_dict()) make_boilerplate("controller.js", self.as_dict()) - #make_boilerplate("controller_list.js", self.as_dict()) + # make_boilerplate("controller_list.js", self.as_dict()) if self.has_web_view: - templates_path = frappe.get_module_path(frappe.scrub(self.module), 'doctype', frappe.scrub(self.name), 'templates') + templates_path = frappe.get_module_path( + frappe.scrub(self.module), "doctype", frappe.scrub(self.name), "templates" + ) if not os.path.exists(templates_path): os.makedirs(templates_path) - make_boilerplate('templates/controller.html', self.as_dict()) - make_boilerplate('templates/controller_row.html', self.as_dict()) + make_boilerplate("templates/controller.html", self.as_dict()) + make_boilerplate("templates/controller_row.html", self.as_dict()) def make_amendable(self): """If is_submittable is set, add amended_from docfields.""" if self.is_submittable: - docfield_exists = frappe.get_all("DocField", filters={"fieldname": "amended_from", "parent": self.name}, pluck="name", limit=1) + docfield_exists = frappe.get_all( + "DocField", filters={"fieldname": "amended_from", "parent": self.name}, pluck="name", limit=1 + ) if not docfield_exists: - self.append("fields", { - "label": "Amended From", - "fieldtype": "Link", - "fieldname": "amended_from", - "options": self.name, - "read_only": 1, - "print_hide": 1, - "no_copy": 1 - }) + self.append( + "fields", + { + "label": "Amended From", + "fieldtype": "Link", + "fieldname": "amended_from", + "options": self.name, + "read_only": 1, + "print_hide": 1, + "no_copy": 1, + }, + ) def make_repeatable(self): """If allow_auto_repeat is set, add auto_repeat custom field.""" if self.allow_auto_repeat: - if not frappe.db.exists('Custom Field', {'fieldname': 'auto_repeat', 'dt': self.name}) and \ - not frappe.db.exists('DocField', {'fieldname': 'auto_repeat', 'parent': self.name}): + if not frappe.db.exists( + "Custom Field", {"fieldname": "auto_repeat", "dt": self.name} + ) and not frappe.db.exists( + "DocField", {"fieldname": "auto_repeat", "parent": self.name} + ): insert_after = self.fields[len(self.fields) - 1].fieldname - df = dict(fieldname='auto_repeat', label='Auto Repeat', fieldtype='Link', options='Auto Repeat', insert_after=insert_after, read_only=1, no_copy=1, print_hide=1) + df = dict( + fieldname="auto_repeat", + label="Auto Repeat", + fieldtype="Link", + options="Auto Repeat", + insert_after=insert_after, + read_only=1, + no_copy=1, + print_hide=1, + ) create_custom_field(self.name, df) def validate_nestedset(self): - if not self.get('is_tree'): + if not self.get("is_tree"): return self.add_nestedset_fields() @@ -650,47 +742,50 @@ class DocType(Document): def add_nestedset_fields(self): """If is_tree is set, add parent_field, lft, rgt, is_group fields.""" fieldnames = [df.fieldname for df in self.fields] - if 'lft' in fieldnames: + if "lft" in fieldnames: return - self.append("fields", { - "label": "Left", - "fieldtype": "Int", - "fieldname": "lft", - "read_only": 1, - "hidden": 1, - "no_copy": 1 - }) - - self.append("fields", { - "label": "Right", - "fieldtype": "Int", - "fieldname": "rgt", - "read_only": 1, - "hidden": 1, - "no_copy": 1 - }) - - self.append("fields", { - "label": "Is Group", - "fieldtype": "Check", - "fieldname": "is_group" - }) - self.append("fields", { - "label": "Old Parent", - "fieldtype": "Link", - "options": self.name, - "fieldname": "old_parent" - }) + self.append( + "fields", + { + "label": "Left", + "fieldtype": "Int", + "fieldname": "lft", + "read_only": 1, + "hidden": 1, + "no_copy": 1, + }, + ) + + self.append( + "fields", + { + "label": "Right", + "fieldtype": "Int", + "fieldname": "rgt", + "read_only": 1, + "hidden": 1, + "no_copy": 1, + }, + ) + + self.append("fields", {"label": "Is Group", "fieldtype": "Check", "fieldname": "is_group"}) + self.append( + "fields", + {"label": "Old Parent", "fieldtype": "Link", "options": self.name, "fieldname": "old_parent"}, + ) parent_field_label = "Parent {}".format(self.name) parent_field_name = frappe.scrub(parent_field_label) - self.append("fields", { - "label": parent_field_label, - "fieldtype": "Link", - "options": self.name, - "fieldname": parent_field_name - }) + self.append( + "fields", + { + "label": parent_field_label, + "fieldtype": "Link", + "options": self.name, + "fieldname": parent_field_name, + }, + ) self.nsm_parent_field = parent_field_name def validate_child_table(self): @@ -711,16 +806,16 @@ class DocType(Document): def get_max_idx(self): """Returns the highest `idx`""" - max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", - self.name) + max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", self.name) return max_idx and max_idx[0][0] or 0 def validate_autoname(self): if not self.is_new(): doc_before_save = self.get_doc_before_save() if doc_before_save: - if (self.autoname == "autoincrement" and doc_before_save.autoname != "autoincrement") \ - or (self.autoname != "autoincrement" and doc_before_save.autoname == "autoincrement"): + if (self.autoname == "autoincrement" and doc_before_save.autoname != "autoincrement") or ( + self.autoname != "autoincrement" and doc_before_save.autoname == "autoincrement" + ): frappe.throw(_("Cannot change to/from Autoincrement naming rule")) else: @@ -736,7 +831,9 @@ class DocType(Document): max_length = frappe.db.MAX_COLUMN_LENGTH - 3 if len(name) > max_length: # length(tab + ) should be equal to 64 characters hence doctype should be 61 characters - frappe.throw(_("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError) + frappe.throw( + _("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError + ) flags = {"flags": re.ASCII} @@ -747,13 +844,18 @@ class DocType(Document): # a DocType's name should not start with a number or underscore # and should only contain letters, numbers, underscore, and hyphen if not re.match(r"^(?![\W])[^\d_\s][\w -]+$", name, **flags): - frappe.throw(_( - "A DocType's name should start with a letter and can only " - "consist of letters, numbers, spaces, underscores and hyphens" - ), frappe.NameError, title="Invalid Name") + frappe.throw( + _( + "A DocType's name should start with a letter and can only " + "consist of letters, numbers, spaces, underscores and hyphens" + ), + frappe.NameError, + title="Invalid Name", + ) validate_route_conflict(self.doctype, self.name) + def validate_series(dt, autoname=None, name=None): """Validate if `autoname` property is correctly set.""" if not autoname: @@ -761,14 +863,14 @@ def validate_series(dt, autoname=None, name=None): if not name: name = dt.name - if not autoname and dt.get("fields", {"fieldname":"naming_series"}): + if not autoname and dt.get("fields", {"fieldname": "naming_series"}): dt.autoname = "naming_series:" - elif dt.autoname == "naming_series:" and not dt.get("fields", {"fieldname":"naming_series"}): + elif dt.autoname == "naming_series:" and not dt.get("fields", {"fieldname": "naming_series"}): frappe.throw(_("Invalid fieldname '{0}' in autoname").format(dt.autoname)) # validate field name if autoname field:fieldname is used # Create unique index on autoname field automatically. - if autoname and autoname.startswith('field:'): + if autoname and autoname.startswith("field:"): field = autoname.split(":")[1] if not field or field not in [df.fieldname for df in dt.fields]: frappe.throw(_("Invalid fieldname '{0}' in autoname").format(field)) @@ -778,23 +880,27 @@ def validate_series(dt, autoname=None, name=None): df.unique = 1 break - if autoname and (not autoname.startswith('field:')) \ - and (not autoname.startswith('eval:')) \ - and (not autoname.lower() in ('prompt', 'hash')) \ - and (not autoname.startswith('naming_series:')) \ - and (not autoname.startswith('format:')): + if ( + autoname + and (not autoname.startswith("field:")) + and (not autoname.startswith("eval:")) + and (not autoname.lower() in ("prompt", "hash")) + and (not autoname.startswith("naming_series:")) + and (not autoname.startswith("format:")) + ): - prefix = autoname.split('.')[0] + prefix = autoname.split(".")[0] doctype = frappe.qb.DocType("DocType") - used_in = (frappe.qb - .from_(doctype) - .select(doctype.name) - .where(doctype.autoname.like(Concat(prefix,".%"))) - .where(doctype.name != name) - ).run() + used_in = ( + frappe.qb.from_(doctype) + .select(doctype.name) + .where(doctype.autoname.like(Concat(prefix, ".%"))) + .where(doctype.name != name) + ).run() if used_in: frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0])) + def validate_links_table_fieldnames(meta): """Validate fieldnames in Links table""" if not meta.links or frappe.flags.in_patch or frappe.flags.in_fixtures: @@ -812,11 +918,15 @@ def validate_links_table_fieldnames(meta): continue if not link.parent_doctype: - message = _("Document Links Row #{0}: Parent DocType is mandatory for internal links").format(index) + message = _("Document Links Row #{0}: Parent DocType is mandatory for internal links").format( + index + ) frappe.throw(message, frappe.ValidationError, _("Parent Missing")) if not link.table_fieldname: - message = _("Document Links Row #{0}: Table Fieldname is mandatory for internal links").format(index) + message = _("Document Links Row #{0}: Table Fieldname is mandatory for internal links").format( + index + ) frappe.throw(message, frappe.ValidationError, _("Table Fieldname Missing")) if meta.name == link.parent_doctype: @@ -830,11 +940,13 @@ def validate_links_table_fieldnames(meta): ) frappe.throw(message, frappe.ValidationError, _("Invalid Table Fieldname")) + def validate_fields_for_doctype(doctype): meta = frappe.get_meta(doctype, cached=False) validate_links_table_fieldnames(meta) validate_fields(meta) + # this is separate because it is also called via custom field def validate_fields(meta): """Validate doctype fields. Checks @@ -859,47 +971,81 @@ def validate_fields(meta): validate_column_name(fieldname) def check_invalid_fieldnames(docname, fieldname): - invalid_fields = ('doctype',) + invalid_fields = ("doctype",) if fieldname in invalid_fields: - frappe.throw(_("{0}: Fieldname cannot be one of {1}") - .format(docname, ", ".join(frappe.bold(d) for d in invalid_fields))) + frappe.throw( + _("{0}: Fieldname cannot be one of {1}").format( + docname, ", ".join(frappe.bold(d) for d in invalid_fields) + ) + ) def check_unique_fieldname(docname, fieldname): - duplicates = list(filter(None, map(lambda df: df.fieldname==fieldname and str(df.idx) or None, fields))) + duplicates = list( + filter(None, map(lambda df: df.fieldname == fieldname and str(df.idx) or None, fields)) + ) if len(duplicates) > 1: - frappe.throw(_("{0}: Fieldname {1} appears multiple times in rows {2}").format(docname, fieldname, ", ".join(duplicates)), UniqueFieldnameError) + frappe.throw( + _("{0}: Fieldname {1} appears multiple times in rows {2}").format( + docname, fieldname, ", ".join(duplicates) + ), + UniqueFieldnameError, + ) def check_fieldname_length(fieldname): validate_column_length(fieldname) def check_illegal_mandatory(docname, d): if (d.fieldtype in no_value_fields) and d.fieldtype not in table_fields and d.reqd: - frappe.throw(_("{0}: Field {1} of type {2} cannot be mandatory").format(docname, d.label, d.fieldtype), IllegalMandatoryError) + frappe.throw( + _("{0}: Field {1} of type {2} cannot be mandatory").format(docname, d.label, d.fieldtype), + IllegalMandatoryError, + ) def check_link_table_options(docname, d): - if frappe.flags.in_patch: return + if frappe.flags.in_patch: + return - if frappe.flags.in_fixtures: return + if frappe.flags.in_fixtures: + return if d.fieldtype in ("Link",) + table_fields: if not d.options: - frappe.throw(_("{0}: Options required for Link or Table type field {1} in row {2}").format(docname, d.label, d.idx), DoctypeLinkError) - if d.options=="[Select]" or d.options==d.parent: + frappe.throw( + _("{0}: Options required for Link or Table type field {1} in row {2}").format( + docname, d.label, d.idx + ), + DoctypeLinkError, + ) + if d.options == "[Select]" or d.options == d.parent: return if d.options != d.parent: options = frappe.db.get_value("DocType", d.options, "name") if not options: - frappe.throw(_("{0}: Options must be a valid DocType for field {1} in row {2}").format(docname, d.label, d.idx), WrongOptionsDoctypeLinkError) + frappe.throw( + _("{0}: Options must be a valid DocType for field {1} in row {2}").format( + docname, d.label, d.idx + ), + WrongOptionsDoctypeLinkError, + ) elif not (options == d.options): - frappe.throw(_("{0}: Options {1} must be the same as doctype name {2} for the field {3}") - .format(docname, d.options, options, d.label), DoctypeLinkError) + frappe.throw( + _("{0}: Options {1} must be the same as doctype name {2} for the field {3}").format( + docname, d.options, options, d.label + ), + DoctypeLinkError, + ) else: # fix case d.options = options def check_hidden_and_mandatory(docname, d): if d.hidden and d.reqd and not d.default: - frappe.throw(_("{0}: Field {1} in row {2} cannot be hidden and mandatory without default").format(docname, d.label, d.idx), HiddenAndMandatoryWithoutDefaultError) + frappe.throw( + _("{0}: Field {1} in row {2} cannot be hidden and mandatory without default").format( + docname, d.label, d.idx + ), + HiddenAndMandatoryWithoutDefaultError, + ) def check_width(d): if d.fieldtype == "Currency" and cint(d.width) < 100: @@ -907,34 +1053,58 @@ def validate_fields(meta): def check_in_list_view(is_table, d): if d.in_list_view and (d.fieldtype in not_allowed_in_list_view): - property_label = 'In Grid View' if is_table else 'In List View' - frappe.throw(_("'{0}' not allowed for type {1} in row {2}").format(property_label, d.fieldtype, d.idx)) + property_label = "In Grid View" if is_table else "In List View" + frappe.throw( + _("'{0}' not allowed for type {1} in row {2}").format(property_label, d.fieldtype, d.idx) + ) def check_in_global_search(d): if d.in_global_search and d.fieldtype in no_value_fields: - frappe.throw(_("'In Global Search' not allowed for type {0} in row {1}") - .format(d.fieldtype, d.idx)) + frappe.throw( + _("'In Global Search' not allowed for type {0} in row {1}").format(d.fieldtype, d.idx) + ) def check_dynamic_link_options(d): - if d.fieldtype=="Dynamic Link": - doctype_pointer = list(filter(lambda df: df.fieldname==d.options, fields)) - if not doctype_pointer or (doctype_pointer[0].fieldtype not in ("Link", "Select")) \ - or (doctype_pointer[0].fieldtype=="Link" and doctype_pointer[0].options!="DocType"): - frappe.throw(_("Options 'Dynamic Link' type of field must point to another Link Field with options as 'DocType'")) + if d.fieldtype == "Dynamic Link": + doctype_pointer = list(filter(lambda df: df.fieldname == d.options, fields)) + if ( + not doctype_pointer + or (doctype_pointer[0].fieldtype not in ("Link", "Select")) + or (doctype_pointer[0].fieldtype == "Link" and doctype_pointer[0].options != "DocType") + ): + frappe.throw( + _( + "Options 'Dynamic Link' type of field must point to another Link Field with options as 'DocType'" + ) + ) def check_illegal_default(d): if d.fieldtype == "Check" and not d.default: - d.default = '0' + d.default = "0" if d.fieldtype == "Check" and cint(d.default) not in (0, 1): - frappe.throw(_("Default for 'Check' type of field {0} must be either '0' or '1'").format(frappe.bold(d.fieldname))) + frappe.throw( + _("Default for 'Check' type of field {0} must be either '0' or '1'").format( + frappe.bold(d.fieldname) + ) + ) if d.fieldtype == "Select" and d.default: if not d.options: - frappe.throw(_("Options for {0} must be set before setting the default value.").format(frappe.bold(d.fieldname))) + frappe.throw( + _("Options for {0} must be set before setting the default value.").format( + frappe.bold(d.fieldname) + ) + ) elif d.default not in d.options.split("\n"): - frappe.throw(_("Default value for {0} must be in the list of options.").format(frappe.bold(d.fieldname))) + frappe.throw( + _("Default value for {0} must be in the list of options.").format(frappe.bold(d.fieldname)) + ) def check_precision(d): - if d.fieldtype in ("Currency", "Float", "Percent") and d.precision is not None and not (1 <= cint(d.precision) <= 6): + if ( + d.fieldtype in ("Currency", "Float", "Percent") + and d.precision is not None + and not (1 <= cint(d.precision) <= 6) + ): frappe.throw(_("Precision should be between 1 and 6")) def check_unique_and_text(docname, d): @@ -944,29 +1114,43 @@ def validate_fields(meta): if getattr(d, "unique", False): if d.fieldtype not in ("Data", "Link", "Read Only"): - frappe.throw(_("{0}: Fieldtype {1} for {2} cannot be unique").format(docname, d.fieldtype, d.label), NonUniqueError) + frappe.throw( + _("{0}: Fieldtype {1} for {2} cannot be unique").format(docname, d.fieldtype, d.label), + NonUniqueError, + ) if not d.get("__islocal") and frappe.db.has_column(d.parent, d.fieldname): - has_non_unique_values = frappe.db.sql("""select `{fieldname}`, count(*) + has_non_unique_values = frappe.db.sql( + """select `{fieldname}`, count(*) from `tab{doctype}` where ifnull(`{fieldname}`, '') != '' group by `{fieldname}` having count(*) > 1 limit 1""".format( - doctype=d.parent, fieldname=d.fieldname)) + doctype=d.parent, fieldname=d.fieldname + ) + ) if has_non_unique_values and has_non_unique_values[0][0]: - frappe.throw(_("{0}: Field '{1}' cannot be set as Unique as it has non-unique values").format(docname, d.label), NonUniqueError) + frappe.throw( + _("{0}: Field '{1}' cannot be set as Unique as it has non-unique values").format( + docname, d.label + ), + NonUniqueError, + ) if d.search_index and d.fieldtype in ("Text", "Long Text", "Small Text", "Code", "Text Editor"): - frappe.throw(_("{0}:Fieldtype {1} for {2} cannot be indexed").format(docname, d.fieldtype, d.label), CannotIndexedError) + frappe.throw( + _("{0}:Fieldtype {1} for {2} cannot be indexed").format(docname, d.fieldtype, d.label), + CannotIndexedError, + ) def check_fold(fields): fold_exists = False for i, f in enumerate(fields): - if f.fieldtype=="Fold": + if f.fieldtype == "Fold": if fold_exists: frappe.throw(_("There can be only one Fold in a form")) fold_exists = True - if i < len(fields)-1: - nxt = fields[i+1] + if i < len(fields) - 1: + nxt = fields[i + 1] if nxt.fieldtype != "Section Break": frappe.throw(_("Fold must come before a Section Break")) else: @@ -979,13 +1163,14 @@ def validate_fields(meta): # No value fields should not be included in search field search_fields = [field.strip() for field in (meta.search_fields or "").split(",")] - fieldtype_mapper = { field.fieldname: field.fieldtype \ - for field in filter(lambda field: field.fieldname in search_fields, fields) } + fieldtype_mapper = { + field.fieldname: field.fieldtype + for field in filter(lambda field: field.fieldname in search_fields, fields) + } for fieldname in search_fields: fieldname = fieldname.strip() - if (fieldtype_mapper.get(fieldname) in no_value_fields) or \ - (fieldname not in fieldname_list): + if (fieldtype_mapper.get(fieldname) in no_value_fields) or (fieldname not in fieldname_list): frappe.throw(_("Search field {0} is not valid").format(fieldname)) def check_title_field(meta): @@ -1006,8 +1191,12 @@ def validate_fields(meta): continue if fieldname not in fieldname_list: - frappe.throw(_("{{{0}}} is not a valid fieldname pattern. It should be {{field_name}}.").format(fieldname), - InvalidFieldNameError) + frappe.throw( + _("{{{0}}} is not a valid fieldname pattern. It should be {{field_name}}.").format( + fieldname + ), + InvalidFieldNameError, + ) df = meta.get("fields", filters={"fieldname": meta.title_field})[0] if df: @@ -1022,7 +1211,7 @@ def validate_fields(meta): df = meta.get("fields", {"fieldname": meta.image_field}) if not df: frappe.throw(_("Image field must be a valid fieldname"), InvalidFieldNameError) - if df[0].fieldtype != 'Attach Image': + if df[0].fieldtype != "Attach Image": frappe.throw(_("Image field must be of type Attach Image"), InvalidFieldNameError) def check_is_published_field(meta): @@ -1040,7 +1229,9 @@ def validate_fields(meta): frappe.throw(_("Website Search Field must be a valid fieldname"), InvalidFieldNameError) if "title" not in fieldname_list: - frappe.throw(_('Field "title" is mandatory if "Website Search Field" is set.'), title=_("Missing Field")) + frappe.throw( + _('Field "title" is mandatory if "Website Search Field" is set.'), title=_("Missing Field") + ) def check_timeline_field(meta): if not meta.timeline_field: @@ -1054,37 +1245,49 @@ def validate_fields(meta): frappe.throw(_("Timeline field must be a Link or Dynamic Link"), InvalidFieldNameError) def check_sort_field(meta): - '''Validate that sort_field(s) is a valid field''' + """Validate that sort_field(s) is a valid field""" if meta.sort_field: sort_fields = [meta.sort_field] - if ',' in meta.sort_field: - sort_fields = [d.split()[0] for d in meta.sort_field.split(',')] + if "," in meta.sort_field: + sort_fields = [d.split()[0] for d in meta.sort_field.split(",")] for fieldname in sort_fields: if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)): - frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname), - InvalidFieldNameError) + frappe.throw( + _("Sort field {0} must be a valid fieldname").format(fieldname), InvalidFieldNameError + ) def check_illegal_depends_on_conditions(docfield): - ''' assignment operation should not be allowed in the depends on condition.''' - depends_on_fields = ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"] + """assignment operation should not be allowed in the depends on condition.""" + depends_on_fields = [ + "depends_on", + "collapsible_depends_on", + "mandatory_depends_on", + "read_only_depends_on", + ] for field in depends_on_fields: depends_on = docfield.get(field, None) - if depends_on and ("=" in depends_on) and \ - re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on): + if ( + depends_on and ("=" in depends_on) and re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on) + ): frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError) def check_table_multiselect_option(docfield): - '''check if the doctype provided in Option has atleast 1 Link field''' - if not docfield.fieldtype == 'Table MultiSelect': return + """check if the doctype provided in Option has atleast 1 Link field""" + if not docfield.fieldtype == "Table MultiSelect": + return doctype = docfield.options meta = frappe.get_meta(doctype) - link_field = [df for df in meta.fields if df.fieldtype == 'Link'] + link_field = [df for df in meta.fields if df.fieldtype == "Link"] if not link_field: - frappe.throw(_('DocType {0} provided for the field {1} must have atleast one Link field') - .format(doctype, docfield.fieldname), frappe.ValidationError) + frappe.throw( + _( + "DocType {0} provided for the field {1} must have atleast one Link field" + ).format(doctype, docfield.fieldname), + frappe.ValidationError, + ) def scrub_options_in_select(field): """Strip options for whitespaces""" @@ -1093,46 +1296,59 @@ def validate_fields(meta): options_list = [] for i, option in enumerate(field.options.split("\n")): _option = option.strip() - if i==0 or _option: + if i == 0 or _option: options_list.append(_option) - field.options = '\n'.join(options_list) + field.options = "\n".join(options_list) def scrub_fetch_from(field): - if hasattr(field, 'fetch_from') and getattr(field, 'fetch_from'): - field.fetch_from = field.fetch_from.strip('\n').strip() + if hasattr(field, "fetch_from") and getattr(field, "fetch_from"): + field.fetch_from = field.fetch_from.strip("\n").strip() def validate_data_field_type(docfield): if docfield.get("is_virtual"): return - if docfield.fieldtype == "Data" and not (docfield.oldfieldtype and docfield.oldfieldtype != "Data"): + if docfield.fieldtype == "Data" and not ( + docfield.oldfieldtype and docfield.oldfieldtype != "Data" + ): if docfield.options and (docfield.options not in data_field_options): df_str = frappe.bold(_(docfield.label)) - text_str = _("{0} is an invalid Data field.").format(df_str) + "
" * 2 + _("Only Options allowed for Data field are:") + "
" + text_str = ( + _("{0} is an invalid Data field.").format(df_str) + + "
" * 2 + + _("Only Options allowed for Data field are:") + + "
" + ) df_options_str = "
  • " + "
  • ".join(_(x) for x in data_field_options) + "
" frappe.msgprint(text_str + df_options_str, title="Invalid Data Field", raise_exception=True) def check_child_table_option(docfield): - if frappe.flags.in_fixtures: return - if docfield.fieldtype not in ['Table MultiSelect', 'Table']: return + if frappe.flags.in_fixtures: + return + if docfield.fieldtype not in ["Table MultiSelect", "Table"]: + return doctype = docfield.options meta = frappe.get_meta(doctype) if not meta.istable: - frappe.throw(_('Option {0} for field {1} is not a child table') - .format(frappe.bold(doctype), frappe.bold(docfield.fieldname)), title=_("Invalid Option")) + frappe.throw( + _("Option {0} for field {1} is not a child table").format( + frappe.bold(doctype), frappe.bold(docfield.fieldname) + ), + title=_("Invalid Option"), + ) def check_max_height(docfield): - if getattr(docfield, 'max_height', None) and (docfield.max_height[-2:] not in ('px', 'em')): - frappe.throw('Max for {} height must be in px, em, rem'.format(frappe.bold(docfield.fieldname))) + if getattr(docfield, "max_height", None) and (docfield.max_height[-2:] not in ("px", "em")): + frappe.throw("Max for {} height must be in px, em, rem".format(frappe.bold(docfield.fieldname))) def check_no_of_ratings(docfield): if docfield.fieldtype == "Rating": if docfield.options and (int(docfield.options) > 10 or int(docfield.options) < 3): - frappe.throw(_('Options for Rating field can range from 3 to 10')) + frappe.throw(_("Options for Rating field can range from 3 to 10")) fields = meta.get("fields") fieldname_list = [d.fieldname for d in fields] @@ -1140,13 +1356,15 @@ def validate_fields(meta): not_allowed_in_list_view = list(copy.copy(no_value_fields)) not_allowed_in_list_view.append("Attach Image") if meta.istable: - not_allowed_in_list_view.remove('Button') + not_allowed_in_list_view.remove("Button") for d in fields: - if not d.permlevel: d.permlevel = 0 - if d.fieldtype not in table_fields: d.allow_bulk_edit = 0 + if not d.permlevel: + d.permlevel = 0 + if d.fieldtype not in table_fields: + d.allow_bulk_edit = 0 if not d.fieldname: - d.fieldname = d.fieldname.lower().strip('?') + d.fieldname = d.fieldname.lower().strip("?") check_illegal_characters(d.fieldname) check_invalid_fieldnames(meta.get("name"), d.fieldname) @@ -1156,7 +1374,7 @@ def validate_fields(meta): check_link_table_options(meta.get("name"), d) check_dynamic_link_options(d) check_hidden_and_mandatory(meta.get("name"), d) - check_in_list_view(meta.get('istable'), d) + check_in_list_view(meta.get("istable"), d) check_in_global_search(d) check_illegal_default(d) check_unique_and_text(meta.get("name"), d) @@ -1178,6 +1396,7 @@ def validate_fields(meta): check_sort_field(meta) check_image_field(meta) + def validate_permissions_for_doctype(doctype, for_remove=False, alert=False): """Validates if permissions are set correctly.""" doctype = frappe.get_doc("DocType", doctype) @@ -1189,10 +1408,12 @@ def validate_permissions_for_doctype(doctype, for_remove=False, alert=False): clear_permissions_cache(doctype.name) + def clear_permissions_cache(doctype): frappe.clear_cache(doctype=doctype) delete_notification_count_for(doctype) - for user in frappe.db.sql_list(""" + for user in frappe.db.sql_list( + """ SELECT DISTINCT `tabHas Role`.`parent` FROM @@ -1201,14 +1422,17 @@ def clear_permissions_cache(doctype): WHERE `tabDocPerm`.`parent` = %s AND `tabDocPerm`.`role` = `tabHas Role`.`role` AND `tabHas Role`.`parenttype` = 'User' - """, doctype): + """, + doctype, + ): frappe.clear_cache(user=user) + def validate_permissions(doctype, for_remove=False, alert=False): permissions = doctype.get("permissions") # Some DocTypes may not have permissions by default, don't show alert for them if not permissions and alert: - frappe.msgprint(_('No Permissions Specified'), alert=True, indicator='orange') + frappe.msgprint(_("No Permissions Specified"), alert=True, indicator="orange") issingle = issubmittable = isimportable = False if doctype: issingle = cint(doctype.issingle) @@ -1219,36 +1443,44 @@ def validate_permissions(doctype, for_remove=False, alert=False): return _("For {0} at level {1} in {2} in row {3}").format(d.role, d.permlevel, d.parent, d.idx) def check_atleast_one_set(d): - if not d.select and not d.read and not d.write and not d.submit and not d.cancel and not d.create: + if ( + not d.select and not d.read and not d.write and not d.submit and not d.cancel and not d.create + ): frappe.throw(_("{0}: No basic permissions set").format(get_txt(d))) def check_double(d): has_similar = False similar_because_of = "" for p in permissions: - if p.role==d.role and p.permlevel==d.permlevel and p!=d: - if p.if_owner==d.if_owner: + if p.role == d.role and p.permlevel == d.permlevel and p != d: + if p.if_owner == d.if_owner: similar_because_of = _("If Owner") has_similar = True break if has_similar: - frappe.throw(_("{0}: Only one rule allowed with the same Role, Level and {1}")\ - .format(get_txt(d), similar_because_of)) + frappe.throw( + _("{0}: Only one rule allowed with the same Role, Level and {1}").format( + get_txt(d), similar_because_of + ) + ) def check_level_zero_is_set(d): - if cint(d.permlevel) > 0 and d.role != 'All': + if cint(d.permlevel) > 0 and d.role != "All": has_zero_perm = False for p in permissions: - if p.role==d.role and (p.permlevel or 0)==0 and p!=d: + if p.role == d.role and (p.permlevel or 0) == 0 and p != d: has_zero_perm = True break if not has_zero_perm: - frappe.throw(_("{0}: Permission at level 0 must be set before higher levels are set").format(get_txt(d))) + frappe.throw( + _("{0}: Permission at level 0 must be set before higher levels are set").format(get_txt(d)) + ) for invalid in ("create", "submit", "cancel", "amend"): - if d.get(invalid): d.set(invalid, 0) + if d.get(invalid): + d.set(invalid, 0) def check_permission_dependency(d): if d.cancel and not d.submit: @@ -1287,23 +1519,31 @@ def validate_permissions(doctype, for_remove=False, alert=False): frappe.throw(_("{0}: Cannot set import as {1} is not importable").format(get_txt(d), doctype)) def validate_permission_for_all_role(d): - if frappe.session.user == 'Administrator': + if frappe.session.user == "Administrator": return if doctype.custom: - if d.role == 'All': - frappe.throw(_('Row # {0}: Non administrator user can not set the role {1} to the custom doctype') - .format(d.idx, frappe.bold(_('All'))), title=_('Permissions Error')) + if d.role == "All": + frappe.throw( + _("Row # {0}: Non administrator user can not set the role {1} to the custom doctype").format( + d.idx, frappe.bold(_("All")) + ), + title=_("Permissions Error"), + ) - roles = [row.name for row in frappe.get_all('Role', filters={'is_custom': 1})] + roles = [row.name for row in frappe.get_all("Role", filters={"is_custom": 1})] if d.role in roles: - frappe.throw(_('Row # {0}: Non administrator user can not set the role {1} to the custom doctype') - .format(d.idx, frappe.bold(_(d.role))), title=_('Permissions Error')) + frappe.throw( + _("Row # {0}: Non administrator user can not set the role {1} to the custom doctype").format( + d.idx, frappe.bold(_(d.role)) + ), + title=_("Permissions Error"), + ) for d in permissions: if not d.permlevel: - d.permlevel=0 + d.permlevel = 0 check_atleast_one_set(d) if not for_remove: check_double(d) @@ -1314,20 +1554,23 @@ def validate_permissions(doctype, for_remove=False, alert=False): remove_rights_for_single(d) validate_permission_for_all_role(d) + def make_module_and_roles(doc, perm_fieldname="permissions"): """Make `Module Def` and `Role` records if already not made. Called while installing.""" try: - if hasattr(doc,'restrict_to_domain') and doc.restrict_to_domain and \ - not frappe.db.exists('Domain', doc.restrict_to_domain): - frappe.get_doc(dict(doctype='Domain', domain=doc.restrict_to_domain)).insert() - - if ("tabModule Def" in frappe.db.get_tables() - and not frappe.db.exists("Module Def", doc.module)): + if ( + hasattr(doc, "restrict_to_domain") + and doc.restrict_to_domain + and not frappe.db.exists("Domain", doc.restrict_to_domain) + ): + frappe.get_doc(dict(doctype="Domain", domain=doc.restrict_to_domain)).insert() + + if "tabModule Def" in frappe.db.get_tables() and not frappe.db.exists("Module Def", doc.module): m = frappe.get_doc({"doctype": "Module Def", "module_name": doc.module}) if frappe.scrub(doc.module) in frappe.local.module_app: m.app_name = frappe.local.module_app[frappe.scrub(doc.module)] else: - m.app_name = 'frappe' + m.app_name = "frappe" m.flags.ignore_mandatory = m.flags.ignore_permissions = True if frappe.flags.package: m.package = frappe.flags.package.name @@ -1339,7 +1582,7 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): for role in list(set(roles)): if frappe.db.table_exists("Role", cached=False) and not frappe.db.exists("Role", role): - r = frappe.get_doc(dict(doctype= "Role", role_name=role, desk_access=1)) + r = frappe.get_doc(dict(doctype="Role", role_name=role, desk_access=1)) r.flags.ignore_mandatory = r.flags.ignore_permissions = True r.insert() except frappe.DoesNotExistError as e: @@ -1350,6 +1593,7 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): else: raise + def check_fieldname_conflicts(docfield): """Checks if fieldname conflicts with methods or properties""" doc = frappe.get_doc({"doctype": docfield.dt}) @@ -1365,8 +1609,10 @@ def check_fieldname_conflicts(docfield): if docfield.fieldname in method_list + property_list: frappe.msgprint(msg, raise_exception=not docfield.is_virtual) + def clear_linked_doctype_cache(): - frappe.cache().delete_value('linked_doctypes_without_ignore_user_permissions_enabled') + frappe.cache().delete_value("linked_doctypes_without_ignore_user_permissions_enabled") + def check_email_append_to(doc): if not hasattr(doc, "email_append_to") or not doc.email_append_to: @@ -1379,7 +1625,13 @@ def check_email_append_to(doc): if doc.subject_field and not subject_field: frappe.throw(_("Select a valid Subject field for creating documents from Email")) - if subject_field and subject_field.fieldtype not in ["Data", "Text", "Long Text", "Small Text", "Text Editor"]: + if subject_field and subject_field.fieldtype not in [ + "Data", + "Text", + "Long Text", + "Small Text", + "Text Editor", + ]: frappe.throw(_("Subject Field type should be Data, Text, Long Text, Small Text, Text Editor")) # Sender Field is mandatory diff --git a/frappe/core/doctype/doctype/patches/set_route.py b/frappe/core/doctype/doctype/patches/set_route.py index c052a51f38..9012f5ce2c 100644 --- a/frappe/core/doctype/doctype/patches/set_route.py +++ b/frappe/core/doctype/doctype/patches/set_route.py @@ -1,7 +1,8 @@ import frappe from frappe.desk.utils import slug + def execute(): - for doctype in frappe.get_all('DocType', ['name', 'route'], dict(istable=0)): - if not doctype.route: - frappe.db.set_value('DocType', doctype.name, 'route', slug(doctype.name), update_modified = False) \ No newline at end of file + for doctype in frappe.get_all("DocType", ["name", "route"], dict(istable=0)): + if not doctype.route: + frappe.db.set_value("DocType", doctype.name, "route", slug(doctype.name), update_modified=False) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index dc6d14b451..135172e8da 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -1,21 +1,24 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest -from frappe.core.doctype.doctype.doctype import (UniqueFieldnameError, - IllegalMandatoryError, + +import frappe +from frappe.core.doctype.doctype.doctype import ( + CannotIndexedError, DoctypeLinkError, - WrongOptionsDoctypeLinkError, HiddenAndMandatoryWithoutDefaultError, - CannotIndexedError, + IllegalMandatoryError, InvalidFieldNameError, - validate_links_table_fieldnames) + UniqueFieldnameError, + WrongOptionsDoctypeLinkError, + validate_links_table_fieldnames, +) # test_records = frappe.get_test_records('DocType') -class TestDocType(unittest.TestCase): +class TestDocType(unittest.TestCase): def tearDown(self): frappe.db.rollback() @@ -23,7 +26,10 @@ class TestDocType(unittest.TestCase): self.assertRaises(frappe.NameError, new_doctype("_Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("8Some DocType").insert) self.assertRaises(frappe.NameError, new_doctype("Some (DocType)").insert) - self.assertRaises(frappe.NameError, new_doctype("Some Doctype with a name whose length is more than 61 characters").insert) + self.assertRaises( + frappe.NameError, + new_doctype("Some Doctype with a name whose length is more than 61 characters").insert, + ) for name in ("Some DocType", "Some_DocType", "Some-DocType"): if frappe.db.exists("DocType", name): frappe.delete_doc("DocType", name) @@ -86,19 +92,33 @@ class TestDocType(unittest.TestCase): def test_all_depends_on_fields_conditions(self): import re - docfields = frappe.get_all("DocField", - or_filters={ - "ifnull(depends_on, '')": ("!=", ''), - "ifnull(collapsible_depends_on, '')": ("!=", ''), - "ifnull(mandatory_depends_on, '')": ("!=", ''), - "ifnull(read_only_depends_on, '')": ("!=", '') + docfields = frappe.get_all( + "DocField", + or_filters={ + "ifnull(depends_on, '')": ("!=", ""), + "ifnull(collapsible_depends_on, '')": ("!=", ""), + "ifnull(mandatory_depends_on, '')": ("!=", ""), + "ifnull(read_only_depends_on, '')": ("!=", ""), }, - fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\ - "read_only_depends_on", "fieldname", "fieldtype"]) + fields=[ + "parent", + "depends_on", + "collapsible_depends_on", + "mandatory_depends_on", + "read_only_depends_on", + "fieldname", + "fieldtype", + ], + ) pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+' for field in docfields: - for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]: + for depends_on in [ + "depends_on", + "collapsible_depends_on", + "mandatory_depends_on", + "read_only_depends_on", + ]: condition = field.get(depends_on) if condition: self.assertFalse(re.match(pattern, condition)) @@ -108,18 +128,18 @@ class TestDocType(unittest.TestCase): valid_data_field_options = frappe.model.data_field_options + ("",) invalid_data_field_options = ("Invalid Option 1", frappe.utils.random_string(5)) - for field_option in (valid_data_field_options + invalid_data_field_options): - test_doctype = frappe.get_doc({ - "doctype": "DocType", - "name": doctype_name, - "module": "Core", - "custom": 1, - "fields": [{ - "fieldname": "{0}_field".format(field_option), - "fieldtype": "Data", - "options": field_option - }] - }) + for field_option in valid_data_field_options + invalid_data_field_options: + test_doctype = frappe.get_doc( + { + "doctype": "DocType", + "name": doctype_name, + "module": "Core", + "custom": 1, + "fields": [ + {"fieldname": "{0}_field".format(field_option), "fieldtype": "Data", "options": field_option} + ], + } + ) if field_option in invalid_data_field_options: # assert that only data options in frappe.model.data_field_options are valid @@ -130,45 +150,29 @@ class TestDocType(unittest.TestCase): test_doctype.delete() def test_sync_field_order(self): - from frappe.modules.import_file import get_file_path import os + from frappe.modules.import_file import get_file_path + # create test doctype - test_doctype = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "fields": [ - { - "label": "Field 1", - "fieldname": "field_1", - "fieldtype": "Data" - }, - { - "label": "Field 2", - "fieldname": "field_2", - "fieldtype": "Data" - }, - { - "label": "Field 3", - "fieldname": "field_3", - "fieldtype": "Data" - }, - { - "label": "Field 4", - "fieldname": "field_4", - "fieldtype": "Data" - } - ], - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": "Test Field Order DocType", - "__islocal": 1 - }) + test_doctype = frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "fields": [ + {"label": "Field 1", "fieldname": "field_1", "fieldtype": "Data"}, + {"label": "Field 2", "fieldname": "field_2", "fieldtype": "Data"}, + {"label": "Field 3", "fieldname": "field_3", "fieldtype": "Data"}, + {"label": "Field 4", "fieldname": "field_4", "fieldtype": "Data"}, + ], + "permissions": [{"role": "System Manager", "read": 1}], + "name": "Test Field Order DocType", + "__islocal": 1, + } + ) path = get_file_path(test_doctype.module, test_doctype.doctype, test_doctype.name) - initial_fields_order = ['field_1', 'field_2', 'field_3', 'field_4'] + initial_fields_order = ["field_1", "field_2", "field_3", "field_4"] frappe.delete_doc_if_exists("DocType", "Test Field Order DocType") if os.path.isfile(path): @@ -181,14 +185,18 @@ class TestDocType(unittest.TestCase): # assert that field_order list is being created with the default order test_doctype_json = frappe.get_file_json(path) self.assertTrue(test_doctype_json.get("field_order")) - self.assertEqual(len(test_doctype_json['fields']), len(test_doctype_json['field_order'])) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], test_doctype_json['field_order']) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], initial_fields_order) - self.assertListEqual(test_doctype_json['field_order'], initial_fields_order) + self.assertEqual(len(test_doctype_json["fields"]), len(test_doctype_json["field_order"])) + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], test_doctype_json["field_order"] + ) + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], initial_fields_order + ) + self.assertListEqual(test_doctype_json["field_order"], initial_fields_order) # remove field_order to test reload_doc/sync/migrate is backwards compatible without field_order - del test_doctype_json['field_order'] - with open(path, 'w+') as txtfile: + del test_doctype_json["field_order"] + with open(path, "w+") as txtfile: txtfile.write(frappe.as_json(test_doctype_json)) # assert that field_order is actually removed from the json file @@ -203,10 +211,14 @@ class TestDocType(unittest.TestCase): test_doctype.save() test_doctype_json = frappe.get_file_json(path) self.assertTrue(test_doctype_json.get("field_order")) - self.assertEqual(len(test_doctype_json['fields']), len(test_doctype_json['field_order'])) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], test_doctype_json['field_order']) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], initial_fields_order) - self.assertListEqual(test_doctype_json['field_order'], initial_fields_order) + self.assertEqual(len(test_doctype_json["fields"]), len(test_doctype_json["field_order"])) + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], test_doctype_json["field_order"] + ) + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], initial_fields_order + ) + self.assertListEqual(test_doctype_json["field_order"], initial_fields_order) # reorder fields: swap row 1 and 3 test_doctype.fields[0], test_doctype.fields[2] = test_doctype.fields[2], test_doctype.fields[0] @@ -216,25 +228,30 @@ class TestDocType(unittest.TestCase): # assert that reordering fields only affects `field_order` rather than `fields` attr test_doctype.save() test_doctype_json = frappe.get_file_json(path) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], initial_fields_order) - self.assertListEqual(test_doctype_json['field_order'], ['field_3', 'field_2', 'field_1', 'field_4']) + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], initial_fields_order + ) + self.assertListEqual( + test_doctype_json["field_order"], ["field_3", "field_2", "field_1", "field_4"] + ) # reorder `field_order` in the json file: swap row 2 and 4 - test_doctype_json['field_order'][1], test_doctype_json['field_order'][3] = test_doctype_json['field_order'][3], test_doctype_json['field_order'][1] - with open(path, 'w+') as txtfile: + test_doctype_json["field_order"][1], test_doctype_json["field_order"][3] = ( + test_doctype_json["field_order"][3], + test_doctype_json["field_order"][1], + ) + with open(path, "w+") as txtfile: txtfile.write(frappe.as_json(test_doctype_json)) # assert that reordering `field_order` from json file is reflected in DocType upon migrate/sync frappe.reload_doctype(test_doctype.name, force=True) test_doctype.reload() - self.assertListEqual([f.fieldname for f in test_doctype.fields], ['field_3', 'field_4', 'field_1', 'field_2']) + self.assertListEqual( + [f.fieldname for f in test_doctype.fields], ["field_3", "field_4", "field_1", "field_2"] + ) # insert row in the middle and remove first row (field 3) - test_doctype.append("fields", { - "label": "Field 5", - "fieldname": "field_5", - "fieldtype": "Data" - }) + test_doctype.append("fields", {"label": "Field 5", "fieldname": "field_5", "fieldtype": "Data"}) test_doctype.fields[4], test_doctype.fields[3] = test_doctype.fields[3], test_doctype.fields[4] test_doctype.fields[3], test_doctype.fields[2] = test_doctype.fields[2], test_doctype.fields[3] test_doctype.remove(test_doctype.fields[0]) @@ -243,115 +260,121 @@ class TestDocType(unittest.TestCase): test_doctype.save() test_doctype_json = frappe.get_file_json(path) - self.assertListEqual([f['fieldname'] for f in test_doctype_json['fields']], ['field_1', 'field_2', 'field_4', 'field_5']) - self.assertListEqual(test_doctype_json['field_order'], ['field_4', 'field_5', 'field_1', 'field_2']) + self.assertListEqual( + [f["fieldname"] for f in test_doctype_json["fields"]], + ["field_1", "field_2", "field_4", "field_5"], + ) + self.assertListEqual( + test_doctype_json["field_order"], ["field_4", "field_5", "field_1", "field_2"] + ) except: raise finally: frappe.flags.allow_doctype_export = 0 def test_unique_field_name_for_two_fields(self): - doc = new_doctype('Test Unique Field') - field_1 = doc.append('fields', {}) - field_1.fieldname = 'some_fieldname_1' - field_1.fieldtype = 'Data' + doc = new_doctype("Test Unique Field") + field_1 = doc.append("fields", {}) + field_1.fieldname = "some_fieldname_1" + field_1.fieldtype = "Data" - field_2 = doc.append('fields', {}) - field_2.fieldname = 'some_fieldname_1' - field_2.fieldtype = 'Data' + field_2 = doc.append("fields", {}) + field_2.fieldname = "some_fieldname_1" + field_2.fieldtype = "Data" self.assertRaises(UniqueFieldnameError, doc.insert) def test_fieldname_is_not_name(self): - doc = new_doctype('Test Name Field') - field_1 = doc.append('fields', {}) - field_1.label = 'Name' - field_1.fieldtype = 'Data' + doc = new_doctype("Test Name Field") + field_1 = doc.append("fields", {}) + field_1.label = "Name" + field_1.fieldtype = "Data" doc.insert() self.assertEqual(doc.fields[1].fieldname, "name1") - doc.fields[1].fieldname = 'name' + doc.fields[1].fieldname = "name" self.assertRaises(InvalidFieldNameError, doc.save) def test_illegal_mandatory_validation(self): - doc = new_doctype('Test Illegal mandatory') - field_1 = doc.append('fields', {}) - field_1.fieldname = 'some_fieldname_1' - field_1.fieldtype = 'Section Break' + doc = new_doctype("Test Illegal mandatory") + field_1 = doc.append("fields", {}) + field_1.fieldname = "some_fieldname_1" + field_1.fieldtype = "Section Break" field_1.reqd = 1 self.assertRaises(IllegalMandatoryError, doc.insert) def test_link_with_wrong_and_no_options(self): - doc = new_doctype('Test link') - field_1 = doc.append('fields', {}) - field_1.fieldname = 'some_fieldname_1' - field_1.fieldtype = 'Link' + doc = new_doctype("Test link") + field_1 = doc.append("fields", {}) + field_1.fieldname = "some_fieldname_1" + field_1.fieldtype = "Link" self.assertRaises(DoctypeLinkError, doc.insert) - field_1.options = 'wrongdoctype' + field_1.options = "wrongdoctype" self.assertRaises(WrongOptionsDoctypeLinkError, doc.insert) def test_hidden_and_mandatory_without_default(self): - doc = new_doctype('Test hidden and mandatory') - field_1 = doc.append('fields', {}) - field_1.fieldname = 'some_fieldname_1' - field_1.fieldtype = 'Data' + doc = new_doctype("Test hidden and mandatory") + field_1 = doc.append("fields", {}) + field_1.fieldname = "some_fieldname_1" + field_1.fieldtype = "Data" field_1.reqd = 1 field_1.hidden = 1 self.assertRaises(HiddenAndMandatoryWithoutDefaultError, doc.insert) def test_field_can_not_be_indexed_validation(self): - doc = new_doctype('Test index') - field_1 = doc.append('fields', {}) - field_1.fieldname = 'some_fieldname_1' - field_1.fieldtype = 'Long Text' + doc = new_doctype("Test index") + field_1 = doc.append("fields", {}) + field_1.fieldname = "some_fieldname_1" + field_1.fieldtype = "Long Text" field_1.search_index = 1 self.assertRaises(CannotIndexedError, doc.insert) def test_cancel_link_doctype(self): import json - from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs - #create doctype - link_doc = new_doctype('Test Linked Doctype') + from frappe.desk.form.linked_with import cancel_all_linked_docs, get_submitted_linked_docs + + # create doctype + link_doc = new_doctype("Test Linked Doctype") link_doc.is_submittable = 1 - for data in link_doc.get('permissions'): + for data in link_doc.get("permissions"): data.submit = 1 data.cancel = 1 link_doc.insert() - doc = new_doctype('Test Doctype') + doc = new_doctype("Test Doctype") doc.is_submittable = 1 - field_2 = doc.append('fields', {}) - field_2.label = 'Test Linked Doctype' - field_2.fieldname = 'test_linked_doctype' - field_2.fieldtype = 'Link' - field_2.options = 'Test Linked Doctype' - for data in link_doc.get('permissions'): + field_2 = doc.append("fields", {}) + field_2.label = "Test Linked Doctype" + field_2.fieldname = "test_linked_doctype" + field_2.fieldtype = "Link" + field_2.options = "Test Linked Doctype" + for data in link_doc.get("permissions"): data.submit = 1 data.cancel = 1 doc.insert() # create doctype data - data_link_doc = frappe.new_doc('Test Linked Doctype') - data_link_doc.some_fieldname = 'Data1' + data_link_doc = frappe.new_doc("Test Linked Doctype") + data_link_doc.some_fieldname = "Data1" data_link_doc.insert() data_link_doc.save() data_link_doc.submit() - data_doc = frappe.new_doc('Test Doctype') - data_doc.some_fieldname = 'Data1' + data_doc = frappe.new_doc("Test Doctype") + data_doc.some_fieldname = "Data1" data_doc.test_linked_doctype = data_link_doc.name data_doc.insert() data_doc.save() data_doc.submit() docs = get_submitted_linked_docs(link_doc.name, data_link_doc.name) - dump_docs = json.dumps(docs.get('docs')) + dump_docs = json.dumps(docs.get("docs")) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() data_doc.load_from_db() @@ -369,69 +392,70 @@ class TestDocType(unittest.TestCase): def test_ignore_cancelation_of_linked_doctype_during_cancel(self): import json - from frappe.desk.form.linked_with import get_submitted_linked_docs, cancel_all_linked_docs - #create linked doctype - link_doc = new_doctype('Test Linked Doctype 1') + from frappe.desk.form.linked_with import cancel_all_linked_docs, get_submitted_linked_docs + + # create linked doctype + link_doc = new_doctype("Test Linked Doctype 1") link_doc.is_submittable = 1 - for data in link_doc.get('permissions'): + for data in link_doc.get("permissions"): data.submit = 1 data.cancel = 1 link_doc.insert() - #create first parent doctype - test_doc_1 = new_doctype('Test Doctype 1') + # create first parent doctype + test_doc_1 = new_doctype("Test Doctype 1") test_doc_1.is_submittable = 1 - field_2 = test_doc_1.append('fields', {}) - field_2.label = 'Test Linked Doctype 1' - field_2.fieldname = 'test_linked_doctype_a' - field_2.fieldtype = 'Link' - field_2.options = 'Test Linked Doctype 1' + field_2 = test_doc_1.append("fields", {}) + field_2.label = "Test Linked Doctype 1" + field_2.fieldname = "test_linked_doctype_a" + field_2.fieldtype = "Link" + field_2.options = "Test Linked Doctype 1" - for data in test_doc_1.get('permissions'): + for data in test_doc_1.get("permissions"): data.submit = 1 data.cancel = 1 test_doc_1.insert() - #crete second parent doctype - doc = new_doctype('Test Doctype 2') + # crete second parent doctype + doc = new_doctype("Test Doctype 2") doc.is_submittable = 1 - field_2 = doc.append('fields', {}) - field_2.label = 'Test Linked Doctype 1' - field_2.fieldname = 'test_linked_doctype_a' - field_2.fieldtype = 'Link' - field_2.options = 'Test Linked Doctype 1' + field_2 = doc.append("fields", {}) + field_2.label = "Test Linked Doctype 1" + field_2.fieldname = "test_linked_doctype_a" + field_2.fieldtype = "Link" + field_2.options = "Test Linked Doctype 1" - for data in link_doc.get('permissions'): + for data in link_doc.get("permissions"): data.submit = 1 data.cancel = 1 doc.insert() # create doctype data - data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') - data_link_doc_1.some_fieldname = 'Data1' + data_link_doc_1 = frappe.new_doc("Test Linked Doctype 1") + data_link_doc_1.some_fieldname = "Data1" data_link_doc_1.insert() data_link_doc_1.save() data_link_doc_1.submit() - data_doc_2 = frappe.new_doc('Test Doctype 1') - data_doc_2.some_fieldname = 'Data1' + data_doc_2 = frappe.new_doc("Test Doctype 1") + data_doc_2.some_fieldname = "Data1" data_doc_2.test_linked_doctype_a = data_link_doc_1.name data_doc_2.insert() data_doc_2.save() data_doc_2.submit() - data_doc = frappe.new_doc('Test Doctype 2') - data_doc.some_fieldname = 'Data1' + data_doc = frappe.new_doc("Test Doctype 2") + data_doc.some_fieldname = "Data1" data_doc.test_linked_doctype_a = data_link_doc_1.name data_doc.insert() data_doc.save() data_doc.submit() docs = get_submitted_linked_docs(link_doc.name, data_link_doc_1.name) - dump_docs = json.dumps(docs.get('docs')) + dump_docs = json.dumps(docs.get("docs")) cancel_all_linked_docs(dump_docs, ignore_doctypes_on_cancel_all=["Test Doctype 2"]) @@ -442,10 +466,10 @@ class TestDocType(unittest.TestCase): data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) - #linked doc is canceled + # linked doc is canceled self.assertEqual(data_doc_2.docstatus, 2) - #ignored doctype 2 during cancel + # ignored doctype 2 during cancel self.assertEqual(data_doc.docstatus, 1) # delete doctype record @@ -464,42 +488,35 @@ class TestDocType(unittest.TestCase): doc = new_doctype("Test Links Table Validation") # check valid data - doc.append("links", { - 'link_doctype': "User", - 'link_fieldname': "first_name" - }) - validate_links_table_fieldnames(doc) # no error - doc.links = [] # reset links table + doc.append("links", {"link_doctype": "User", "link_fieldname": "first_name"}) + validate_links_table_fieldnames(doc) # no error + doc.links = [] # reset links table # check invalid doctype - doc.append("links", { - 'link_doctype': "User2", - 'link_fieldname': "first_name" - }) + doc.append("links", {"link_doctype": "User2", "link_fieldname": "first_name"}) self.assertRaises(frappe.DoesNotExistError, validate_links_table_fieldnames, doc) - doc.links = [] # reset links table + doc.links = [] # reset links table # check invalid fieldname - doc.append("links", { - 'link_doctype': "User", - 'link_fieldname': "a_field_that_does_not_exists" - }) + doc.append("links", {"link_doctype": "User", "link_fieldname": "a_field_that_does_not_exists"}) self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc) def test_create_virtual_doctype(self): """Test virtual DOcTYpe.""" - virtual_doc = new_doctype('Test Virtual Doctype') + virtual_doc = new_doctype("Test Virtual Doctype") virtual_doc.is_virtual = 1 virtual_doc.insert() virtual_doc.save() doc = frappe.get_doc("DocType", "Test Virtual Doctype") self.assertEqual(doc.is_virtual, 1) - self.assertFalse(frappe.db.table_exists('Test Virtual Doctype')) + self.assertFalse(frappe.db.table_exists("Test Virtual Doctype")) def test_default_fieldname(self): - fields = [{"label": "title", "fieldname": "title", "fieldtype": "Data", "default": "{some_fieldname}"}] + fields = [ + {"label": "title", "fieldname": "title", "fieldtype": "Data", "default": "{some_fieldname}"} + ] dt = new_doctype("DT with default field", fields=fields) dt.insert() @@ -521,28 +538,34 @@ class TestDocType(unittest.TestCase): dt.delete(ignore_permissions=True) -def new_doctype(name, unique=0, depends_on='', fields=None, autoincremented=False): - doc = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "fields": [{ - "label": "Some Field", - "fieldname": "some_fieldname", - "fieldtype": "Data", - "unique": unique, - "depends_on": depends_on, - }], - "permissions": [{ - "role": "System Manager", - "read": 1, - }], - "name": name, - "autoname": "autoincrement" if autoincremented else "" - }) +def new_doctype(name, unique=0, depends_on="", fields=None, autoincremented=False): + doc = frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "custom": 1, + "fields": [ + { + "label": "Some Field", + "fieldname": "some_fieldname", + "fieldtype": "Data", + "unique": unique, + "depends_on": depends_on, + } + ], + "permissions": [ + { + "role": "System Manager", + "read": 1, + } + ], + "name": name, + "autoname": "autoincrement" if autoincremented else "", + } + ) if fields: for f in fields: - doc.append('fields', f) + doc.append("fields", f) return doc diff --git a/frappe/core/doctype/doctype_action/doctype_action.py b/frappe/core/doctype/doctype_action/doctype_action.py index 807d1bf0b1..49cb21f99f 100644 --- a/frappe/core/doctype/doctype_action/doctype_action.py +++ b/frappe/core/doctype/doctype_action/doctype_action.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class DocTypeAction(Document): pass diff --git a/frappe/core/doctype/doctype_link/doctype_link.py b/frappe/core/doctype/doctype_link/doctype_link.py index ca2c4efa16..f534cd1780 100644 --- a/frappe/core/doctype/doctype_link/doctype_link.py +++ b/frappe/core/doctype/doctype_link/doctype_link.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class DocTypeLink(Document): pass diff --git a/frappe/core/doctype/doctype_state/doctype_state.py b/frappe/core/doctype/doctype_state/doctype_state.py index 3172834180..be888d2d85 100644 --- a/frappe/core/doctype/doctype_state/doctype_state.py +++ b/frappe/core/doctype/doctype_state/doctype_state.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class DocTypeState(Document): pass diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 5c445fd058..41aa3d7aff 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -3,10 +3,11 @@ # License: MIT. See LICENSE import frappe +from frappe import _ from frappe.model.document import Document -from frappe.utils.data import evaluate_filters from frappe.model.naming import parse_naming_series -from frappe import _ +from frappe.utils.data import evaluate_filters + class DocumentNamingRule(Document): def validate(self): @@ -17,23 +18,30 @@ class DocumentNamingRule(Document): docfields = [x.fieldname for x in frappe.get_meta(self.document_type).fields] for condition in self.conditions: if condition.field not in docfields: - frappe.throw(_("{0} is not a field of doctype {1}").format(frappe.bold(condition.field), frappe.bold(self.document_type))) + frappe.throw( + _("{0} is not a field of doctype {1}").format( + frappe.bold(condition.field), frappe.bold(self.document_type) + ) + ) def apply(self, doc): - ''' + """ Apply naming rules for the given document. Will set `name` if the rule is matched. - ''' + """ if self.conditions: - if not evaluate_filters(doc, [(self.document_type, d.field, d.condition, d.value) for d in self.conditions]): + if not evaluate_filters( + doc, [(self.document_type, d.field, d.condition, d.value) for d in self.conditions] + ): return - counter = frappe.db.get_value(self.doctype, self.name, 'counter', for_update=True) or 0 + counter = frappe.db.get_value(self.doctype, self.name, "counter", for_update=True) or 0 naming_series = parse_naming_series(self.prefix, doc=doc) - doc.name = naming_series + ('%0'+str(self.prefix_digits)+'d') % (counter + 1) - frappe.db.set_value(self.doctype, self.name, 'counter', counter + 1) + doc.name = naming_series + ("%0" + str(self.prefix_digits) + "d") % (counter + 1) + frappe.db.set_value(self.doctype, self.name, "counter", counter + 1) + @frappe.whitelist() def update_current(name, new_counter): - frappe.only_for('System Manager') - frappe.db.set_value('Document Naming Rule', name, 'counter', new_counter) + frappe.only_for("System Manager") + frappe.db.set_value("Document Naming Rule", name, "counter", new_counter) diff --git a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py index 50f1386758..459b17da8b 100644 --- a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py @@ -1,79 +1,68 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestDocumentNamingRule(unittest.TestCase): def test_naming_rule_by_series(self): - naming_rule = frappe.get_doc(dict( - doctype = 'Document Naming Rule', - document_type = 'ToDo', - prefix = 'test-todo-', - prefix_digits = 5 - )).insert() + naming_rule = frappe.get_doc( + dict(doctype="Document Naming Rule", document_type="ToDo", prefix="test-todo-", prefix_digits=5) + ).insert() - todo = frappe.get_doc(dict( - doctype = 'ToDo', - description = 'Is this my name ' + frappe.generate_hash() - )).insert() + todo = frappe.get_doc( + dict(doctype="ToDo", description="Is this my name " + frappe.generate_hash()) + ).insert() - self.assertEqual(todo.name, 'test-todo-00001') + self.assertEqual(todo.name, "test-todo-00001") naming_rule.delete() todo.delete() def test_naming_rule_by_condition(self): - naming_rule = frappe.get_doc(dict( - doctype = 'Document Naming Rule', - document_type = 'ToDo', - prefix = 'test-high-', - prefix_digits = 5, - priority = 10, - conditions = [dict( - field = 'priority', - condition = '=', - value = 'High' - )] - )).insert() + naming_rule = frappe.get_doc( + dict( + doctype="Document Naming Rule", + document_type="ToDo", + prefix="test-high-", + prefix_digits=5, + priority=10, + conditions=[dict(field="priority", condition="=", value="High")], + ) + ).insert() # another rule naming_rule_1 = frappe.copy_doc(naming_rule) - naming_rule_1.prefix = 'test-medium-' - naming_rule_1.conditions[0].value = 'Medium' + naming_rule_1.prefix = "test-medium-" + naming_rule_1.conditions[0].value = "Medium" naming_rule_1.insert() # default rule with low priority - should not get applied for rules # with higher priority naming_rule_2 = frappe.copy_doc(naming_rule) - naming_rule_2.prefix = 'test-low-' + naming_rule_2.prefix = "test-low-" naming_rule_2.priority = 0 naming_rule_2.conditions = [] naming_rule_2.insert() + todo = frappe.get_doc( + dict(doctype="ToDo", priority="High", description="Is this my name " + frappe.generate_hash()) + ).insert() - todo = frappe.get_doc(dict( - doctype = 'ToDo', - priority = 'High', - description = 'Is this my name ' + frappe.generate_hash() - )).insert() - - todo_1 = frappe.get_doc(dict( - doctype = 'ToDo', - priority = 'Medium', - description = 'Is this my name ' + frappe.generate_hash() - )).insert() + todo_1 = frappe.get_doc( + dict(doctype="ToDo", priority="Medium", description="Is this my name " + frappe.generate_hash()) + ).insert() - todo_2 = frappe.get_doc(dict( - doctype = 'ToDo', - priority = 'Low', - description = 'Is this my name ' + frappe.generate_hash() - )).insert() + todo_2 = frappe.get_doc( + dict(doctype="ToDo", priority="Low", description="Is this my name " + frappe.generate_hash()) + ).insert() try: - self.assertEqual(todo.name, 'test-high-00001') - self.assertEqual(todo_1.name, 'test-medium-00001') - self.assertEqual(todo_2.name, 'test-low-00001') + self.assertEqual(todo.name, "test-high-00001") + self.assertEqual(todo_1.name, "test-medium-00001") + self.assertEqual(todo_2.name, "test-low-00001") finally: naming_rule.delete() naming_rule_1.delete() diff --git a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py index 4706492cea..dc45798c34 100644 --- a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class DocumentNamingRuleCondition(Document): pass diff --git a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py index 3d0565234c..d88335758a 100644 --- a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDocumentNamingRuleCondition(unittest.TestCase): pass diff --git a/frappe/core/doctype/domain/domain.py b/frappe/core/doctype/domain/domain.py index ebd6e3ac9e..897f7ee655 100644 --- a/frappe/core/doctype/domain/domain.py +++ b/frappe/core/doctype/domain/domain.py @@ -3,16 +3,17 @@ # License: MIT. See LICENSE import frappe - -from frappe.model.document import Document from frappe.custom.doctype.custom_field.custom_field import create_custom_fields +from frappe.model.document import Document + class Domain(Document): - '''Domain documents are created automatically when DocTypes + """Domain documents are created automatically when DocTypes with "Restricted" domains are imported during - installation or migration''' + installation or migration""" + def setup_domain(self): - '''Setup domain icons, permissions, custom fields etc.''' + """Setup domain icons, permissions, custom fields etc.""" self.setup_data() self.setup_roles() self.setup_properties() @@ -31,20 +32,20 @@ class Domain(Document): frappe.get_attr(self.data.on_setup)() def remove_domain(self): - '''Unset domain settings''' + """Unset domain settings""" self.setup_data() if self.data.restricted_roles: for role_name in self.data.restricted_roles: - if frappe.db.exists('Role', role_name): - role = frappe.get_doc('Role', role_name) + if frappe.db.exists("Role", role_name): + role = frappe.get_doc("Role", role_name) role.disabled = 1 role.save() self.remove_custom_field() def remove_custom_field(self): - '''Remove custom_fields when disabling domain''' + """Remove custom_fields when disabling domain""" if self.data.custom_fields: for doctype in self.data.custom_fields: custom_fields = self.data.custom_fields[doctype] @@ -54,47 +55,48 @@ class Domain(Document): custom_fields = [custom_fields] for custom_field_detail in custom_fields: - custom_field_name = frappe.db.get_value('Custom Field', - dict(dt=doctype, fieldname=custom_field_detail.get('fieldname'))) + custom_field_name = frappe.db.get_value( + "Custom Field", dict(dt=doctype, fieldname=custom_field_detail.get("fieldname")) + ) if custom_field_name: - frappe.delete_doc('Custom Field', custom_field_name) + frappe.delete_doc("Custom Field", custom_field_name) def setup_roles(self): - '''Enable roles that are restricted to this domain''' + """Enable roles that are restricted to this domain""" if self.data.restricted_roles: user = frappe.get_doc("User", frappe.session.user) for role_name in self.data.restricted_roles: user.append("roles", {"role": role_name}) - if not frappe.db.get_value('Role', role_name): - frappe.get_doc(dict(doctype='Role', role_name=role_name)).insert() + if not frappe.db.get_value("Role", role_name): + frappe.get_doc(dict(doctype="Role", role_name=role_name)).insert() continue - role = frappe.get_doc('Role', role_name) + role = frappe.get_doc("Role", role_name) role.disabled = 0 role.save() user.save() def setup_data(self, domain=None): - '''Load domain info via hooks''' + """Load domain info via hooks""" self.data = frappe.get_domain_data(self.name) def get_domain_data(self, module): - return frappe.get_attr(frappe.get_hooks('domains')[self.name] + '.data') + return frappe.get_attr(frappe.get_hooks("domains")[self.name] + ".data") def set_default_portal_role(self): - '''Set default portal role based on domain''' - if self.data.get('default_portal_role'): - frappe.db.set_value('Portal Settings', None, 'default_role', - self.data.get('default_portal_role')) + """Set default portal role based on domain""" + if self.data.get("default_portal_role"): + frappe.db.set_value( + "Portal Settings", None, "default_role", self.data.get("default_portal_role") + ) def setup_properties(self): if self.data.properties: for args in self.data.properties: frappe.make_property_setter(args) - def set_values(self): - '''set values based on `data.set_value`''' + """set values based on `data.set_value`""" if self.data.set_value: for args in self.data.set_value: frappe.reload_doctype(args[0]) @@ -103,19 +105,27 @@ class Domain(Document): doc.save() def setup_sidebar_items(self): - '''Enable / disable sidebar items''' + """Enable / disable sidebar items""" if self.data.allow_sidebar_items: # disable all - frappe.db.sql('update `tabPortal Menu Item` set enabled=0') + frappe.db.sql("update `tabPortal Menu Item` set enabled=0") # enable - frappe.db.sql('''update `tabPortal Menu Item` set enabled=1 - where route in ({0})'''.format(', '.join('"{0}"'.format(d) for d in self.data.allow_sidebar_items))) + frappe.db.sql( + """update `tabPortal Menu Item` set enabled=1 + where route in ({0})""".format( + ", ".join('"{0}"'.format(d) for d in self.data.allow_sidebar_items) + ) + ) if self.data.remove_sidebar_items: # disable all - frappe.db.sql('update `tabPortal Menu Item` set enabled=1') + frappe.db.sql("update `tabPortal Menu Item` set enabled=1") # enable - frappe.db.sql('''update `tabPortal Menu Item` set enabled=0 - where route in ({0})'''.format(', '.join('"{0}"'.format(d) for d in self.data.remove_sidebar_items))) + frappe.db.sql( + """update `tabPortal Menu Item` set enabled=0 + where route in ({0})""".format( + ", ".join('"{0}"'.format(d) for d in self.data.remove_sidebar_items) + ) + ) diff --git a/frappe/core/doctype/domain/test_domain.py b/frappe/core/doctype/domain/test_domain.py index d7924ebc90..85f613a6bd 100644 --- a/frappe/core/doctype/domain/test_domain.py +++ b/frappe/core/doctype/domain/test_domain.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestDomain(unittest.TestCase): pass diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 276411c2ab..ba7f397c51 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -5,13 +5,14 @@ import frappe from frappe.model.document import Document + class DomainSettings(Document): def set_active_domains(self, domains): active_domains = [d.domain for d in self.active_domains] added = False for d in domains: if not d in active_domains: - self.append('active_domains', dict(domain=d)) + self.append("active_domains", dict(domain=d)) added = True if added: @@ -22,49 +23,52 @@ class DomainSettings(Document): # set the flag to update the the desktop icons of all domains if i >= 1: frappe.flags.keep_desktop_icons = True - domain = frappe.get_doc('Domain', d.domain) + domain = frappe.get_doc("Domain", d.domain) domain.setup_domain() self.restrict_roles_and_modules() frappe.clear_cache() def restrict_roles_and_modules(self): - '''Disable all restricted roles and set `restrict_to_domain` property in Module Def''' + """Disable all restricted roles and set `restrict_to_domain` property in Module Def""" active_domains = frappe.get_active_domains() - all_domains = list((frappe.get_hooks('domains') or {})) + all_domains = list((frappe.get_hooks("domains") or {})) def remove_role(role): frappe.db.delete("Has Role", {"role": role}) - frappe.set_value('Role', role, 'disabled', 1) + frappe.set_value("Role", role, "disabled", 1) for domain in all_domains: data = frappe.get_domain_data(domain) - if not frappe.db.get_value('Domain', domain): - frappe.get_doc(dict(doctype='Domain', domain=domain)).insert() - if 'modules' in data: - for module in data.get('modules'): - frappe.db.set_value('Module Def', module, 'restrict_to_domain', domain) - - if 'restricted_roles' in data: - for role in data['restricted_roles']: - if not frappe.db.get_value('Role', role): - frappe.get_doc(dict(doctype='Role', role_name=role)).insert() - frappe.db.set_value('Role', role, 'restrict_to_domain', domain) + if not frappe.db.get_value("Domain", domain): + frappe.get_doc(dict(doctype="Domain", domain=domain)).insert() + if "modules" in data: + for module in data.get("modules"): + frappe.db.set_value("Module Def", module, "restrict_to_domain", domain) + + if "restricted_roles" in data: + for role in data["restricted_roles"]: + if not frappe.db.get_value("Role", role): + frappe.get_doc(dict(doctype="Role", role_name=role)).insert() + frappe.db.set_value("Role", role, "restrict_to_domain", domain) if domain not in active_domains: remove_role(role) - if 'custom_fields' in data: + if "custom_fields" in data: if domain not in active_domains: inactive_domain = frappe.get_doc("Domain", domain) inactive_domain.setup_data() inactive_domain.remove_custom_field() + def get_active_domains(): - """ get the domains set in the Domain Settings as active domain """ + """get the domains set in the Domain Settings as active domain""" + def _get_active_domains(): - domains = frappe.get_all("Has Domain", filters={ "parent": "Domain Settings" }, - fields=["domain"], distinct=True) + domains = frappe.get_all( + "Has Domain", filters={"parent": "Domain Settings"}, fields=["domain"], distinct=True + ) active_domains = [row.get("domain") for row in domains] active_domains.append("") @@ -72,14 +76,16 @@ def get_active_domains(): return frappe.cache().get_value("active_domains", _get_active_domains) + def get_active_modules(): - """ get the active modules from Module Def""" + """get the active modules from Module Def""" + def _get_active_modules(): active_modules = [] active_domains = get_active_domains() - for m in frappe.get_all("Module Def", fields=['name', 'restrict_to_domain']): + for m in frappe.get_all("Module Def", fields=["name", "restrict_to_domain"]): if (not m.restrict_to_domain) or (m.restrict_to_domain in active_domains): active_modules.append(m.name) return active_modules - return frappe.cache().get_value('active_modules', _get_active_modules) + return frappe.cache().get_value("active_modules", _get_active_modules) diff --git a/frappe/core/doctype/dynamic_link/dynamic_link.py b/frappe/core/doctype/dynamic_link/dynamic_link.py index c0502824c6..e253147167 100644 --- a/frappe/core/doctype/dynamic_link/dynamic_link.py +++ b/frappe/core/doctype/dynamic_link/dynamic_link.py @@ -5,12 +5,15 @@ import frappe from frappe.model.document import Document + class DynamicLink(Document): pass + def on_doctype_update(): frappe.db.add_index("Dynamic Link", ["link_doctype", "link_name"]) + def deduplicate_dynamic_links(doc): links, duplicate = [], False for l in doc.links or []: @@ -23,4 +26,4 @@ def deduplicate_dynamic_links(doc): if duplicate: doc.links = [] for l in links: - doc.append('links', dict(link_doctype=l[0], link_name=l[1])) + doc.append("links", dict(link_doctype=l[0], link_name=l[1])) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 39c307520f..d93029179c 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -5,19 +5,24 @@ import frappe from frappe.model.document import Document + class ErrorLog(Document): def onload(self): if not self.seen: - self.db_set('seen', 1, update_modified=0) + self.db_set("seen", 1, update_modified=0) frappe.db.commit() + def set_old_logs_as_seen(): # set logs as seen - frappe.db.sql("""UPDATE `tabError Log` SET `seen`=1 - WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""") + frappe.db.sql( + """UPDATE `tabError Log` SET `seen`=1 + WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""" + ) + @frappe.whitelist() def clear_error_logs(): - '''Flush all Error Logs''' - frappe.only_for('System Manager') + """Flush all Error Logs""" + frappe.only_for("System Manager") frappe.db.truncate("Error Log") diff --git a/frappe/core/doctype/error_log/test_error_log.py b/frappe/core/doctype/error_log/test_error_log.py index 54a41cd4a9..e20ac92650 100644 --- a/frappe/core/doctype/error_log/test_error_log.py +++ b/frappe/core/doctype/error_log/test_error_log.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Error Log') + class TestErrorLog(unittest.TestCase): pass diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.py b/frappe/core/doctype/error_snapshot/error_snapshot.py index 85143b5aa6..82f189217f 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/error_snapshot.py @@ -5,12 +5,13 @@ import frappe from frappe.model.document import Document + class ErrorSnapshot(Document): no_feed_on_delete = True def onload(self): if not self.parent_error_snapshot: - self.db_set('seen', True, update_modified=False) + self.db_set("seen", True, update_modified=False) for relapsed in frappe.get_all("Error Snapshot", filters={"parent_error_snapshot": self.name}): frappe.db.set_value("Error Snapshot", relapsed.name, "seen", True, update_modified=False) @@ -18,13 +19,16 @@ class ErrorSnapshot(Document): frappe.local.flags.commit = True def validate(self): - parent = frappe.get_all("Error Snapshot", + parent = frappe.get_all( + "Error Snapshot", filters={"evalue": self.evalue, "parent_error_snapshot": ""}, - fields=["name", "relapses", "seen"], limit_page_length=1) + fields=["name", "relapses", "seen"], + limit_page_length=1, + ) if parent: parent = parent[0] - self.update({"parent_error_snapshot": parent['name']}) - frappe.db.set_value('Error Snapshot', parent['name'], 'relapses', parent["relapses"] + 1) + self.update({"parent_error_snapshot": parent["name"]}) + frappe.db.set_value("Error Snapshot", parent["name"], "relapses", parent["relapses"] + 1) if parent["seen"]: frappe.db.set_value("Error Snapshot", parent["name"], "seen", False) diff --git a/frappe/core/doctype/error_snapshot/test_error_snapshot.py b/frappe/core/doctype/error_snapshot/test_error_snapshot.py index 86928db9cc..40596b3d22 100644 --- a/frappe/core/doctype/error_snapshot/test_error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/test_error_snapshot.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Error Snapshot') + class TestErrorSnapshot(unittest.TestCase): pass diff --git a/frappe/core/doctype/feedback/feedback.py b/frappe/core/doctype/feedback/feedback.py index 3704ee66e0..c616787e4b 100644 --- a/frappe/core/doctype/feedback/feedback.py +++ b/frappe/core/doctype/feedback/feedback.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class Feedback(Document): pass diff --git a/frappe/core/doctype/feedback/test_feedback.py b/frappe/core/doctype/feedback/test_feedback.py index 66f644ccd3..e8e29e75ae 100644 --- a/frappe/core/doctype/feedback/test_feedback.py +++ b/frappe/core/doctype/feedback/test_feedback.py @@ -1,9 +1,11 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestFeedback(unittest.TestCase): def tearDown(self): frappe.form_dict.reference_doctype = None @@ -13,16 +15,17 @@ class TestFeedback(unittest.TestCase): def test_feedback_creation_updation(self): from frappe.website.doctype.blog_post.test_blog_post import make_test_blog + test_blog = make_test_blog() frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) from frappe.templates.includes.feedback.feedback import give_feedback - frappe.form_dict.reference_doctype = 'Blog Post' + frappe.form_dict.reference_doctype = "Blog Post" frappe.form_dict.reference_name = test_blog.name frappe.form_dict.like = True - frappe.local.request_ip = '127.0.0.1' + frappe.local.request_ip = "127.0.0.1" feedback = give_feedback() @@ -36,4 +39,4 @@ class TestFeedback(unittest.TestCase): frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - test_blog.delete() \ No newline at end of file + test_blog.delete() diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 50a7b31bca..d8b45cf043 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -16,20 +16,29 @@ import os import re import shutil import zipfile +from io import BytesIO from typing import TYPE_CHECKING, Tuple +from urllib.parse import quote, unquote import requests -from requests.exceptions import HTTPError, SSLError from PIL import Image, ImageFile, ImageOps -from io import BytesIO -from urllib.parse import quote, unquote +from requests.exceptions import HTTPError, SSLError import frappe from frappe import _, conf, safe_decode from frappe.model.document import Document -from frappe.utils import call_hook_method, cint, cstr, encode, get_files_path, get_hook_method, random_string, strip -from frappe.utils.image import strip_exif_data, optimize_image +from frappe.utils import ( + call_hook_method, + cint, + cstr, + encode, + get_files_path, + get_hook_method, + random_string, + strip, +) from frappe.utils.file_manager import safe_b64decode +from frappe.utils.image import optimize_image, strip_exif_data if TYPE_CHECKING: from PIL.ImageFile import ImageFile @@ -39,9 +48,11 @@ if TYPE_CHECKING: class MaxFileSizeReachedError(frappe.ValidationError): pass + class FolderNotEmpty(frappe.ValidationError): pass + exclude_from_linked_with = True ImageFile.LOAD_TRUNCATED_IMAGES = True @@ -53,7 +64,7 @@ class File(Document): frappe.local.rollback_observers.append(self) self.set_folder_name() if self.file_name: - self.file_name = re.sub(r'/', '', self.file_name) + self.file_name = re.sub(r"/", "", self.file_name) self.content = self.get("content", None) self.decode = self.get("decode", False) if self.content: @@ -76,21 +87,25 @@ class File(Document): def after_insert(self): if not self.is_folder: - self.add_comment_in_reference_doc('Attachment', - _('Added {0}').format("{file_name}{icon}".format(**{ - "icon": ' ' if self.is_private else "", - "file_url": quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name, - "file_name": self.file_name or self.file_url - }))) + self.add_comment_in_reference_doc( + "Attachment", + _("Added {0}").format( + "{file_name}{icon}".format( + **{ + "icon": ' ' if self.is_private else "", + "file_url": quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name, + "file_name": self.file_name or self.file_url, + } + ) + ), + ) def after_rename(self, olddn, newdn, merge=False): for successor in self.get_successor(): setup_folder_path(successor[0], self.name) def get_successor(self): - return frappe.db.get_values(doctype='File', - filters={'folder': self.name}, - fieldname='name') + return frappe.db.get_values(doctype="File", filters={"folder": self.name}, fieldname="name") def validate(self): if self.is_new(): @@ -117,10 +132,7 @@ class File(Document): # Probably an invalid web URL if not self.file_url.startswith(("/files/", "/private/files/")): - frappe.throw( - _("URL must start with http:// or https://"), - title=_('Invalid URL') - ) + frappe.throw(_("URL must start with http:// or https://"), title=_("Invalid URL")) # Ensure correct formatting and type self.file_url = unquote(self.file_url) @@ -130,25 +142,17 @@ class File(Document): base_path = os.path.realpath(get_files_path(is_private=self.is_private)) if not os.path.realpath(self.get_full_path()).startswith(base_path): - frappe.throw( - _("The File URL you've entered is incorrect"), - title=_('Invalid File URL') - ) + frappe.throw(_("The File URL you've entered is incorrect"), title=_("Invalid File URL")) def handle_is_private_changed(self): - if not frappe.db.exists( - 'File', { - 'name': self.name, - 'is_private': cint(not self.is_private) - } - ): + if not frappe.db.exists("File", {"name": self.name, "is_private": cint(not self.is_private)}): return old_file_url = self.file_url - file_name = self.file_url.split('/')[-1] - private_file_path = frappe.get_site_path('private', 'files', file_name) - public_file_path = frappe.get_site_path('public', 'files', file_name) + file_name = self.file_url.split("/")[-1] + private_file_path = frappe.get_site_path("private", "files", file_name) + public_file_path = frappe.get_site_path("public", "files", file_name) if self.is_private: shutil.move(public_file_path, private_file_path) @@ -167,15 +171,15 @@ class File(Document): ): return - frappe.db.set_value(self.attached_to_doctype, self.attached_to_name, - self.attached_to_field, self.file_url) + frappe.db.set_value( + self.attached_to_doctype, self.attached_to_name, self.attached_to_field, self.file_url + ) def fetch_attached_to_field(self, old_file_url): if self.attached_to_field: return True - reference_dict = frappe.get_doc( - self.attached_to_doctype, self.attached_to_name).as_dict() + reference_dict = frappe.get_doc(self.attached_to_doctype, self.attached_to_name).as_dict() for key, value in reference_dict.items(): if value == old_file_url: @@ -188,10 +192,16 @@ class File(Document): attachment_limit = cint(frappe.get_meta(self.attached_to_doctype).max_attachments) if attachment_limit: - current_attachment_count = len(frappe.get_all('File', filters={ - 'attached_to_doctype': self.attached_to_doctype, - 'attached_to_name': self.attached_to_name, - }, limit=attachment_limit + 1)) + current_attachment_count = len( + frappe.get_all( + "File", + filters={ + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_name, + }, + limit=attachment_limit + 1, + ) + ) if current_attachment_count >= attachment_limit: frappe.throw( @@ -199,7 +209,7 @@ class File(Document): frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name ), exc=frappe.exceptions.AttachmentLimitReached, - title=_('Attachment Limit Reached') + title=_("Attachment Limit Reached"), ) def set_folder_name(self): @@ -208,8 +218,7 @@ class File(Document): self.folder = frappe.db.get_value("File", {"is_attachments_folder": 1}) def validate_folder(self): - if not self.is_home_folder and not self.folder and \ - not self.flags.ignore_folder_validate: + if not self.is_home_folder and not self.folder and not self.flags.ignore_folder_validate: self.folder = "Home" def validate_file(self): @@ -218,7 +227,7 @@ class File(Document): """ full_path = self.get_full_path() - if full_path.startswith('http'): + if full_path.startswith("http"): return True if not os.path.exists(full_path): @@ -232,33 +241,32 @@ class File(Document): # check duplicate name # check duplicate assignment filters = { - 'content_hash': self.content_hash, - 'is_private': self.is_private, - 'name': ('!=', self.name) + "content_hash": self.content_hash, + "is_private": self.is_private, + "name": ("!=", self.name), } if self.attached_to_doctype and self.attached_to_name: - filters.update({ - 'attached_to_doctype': self.attached_to_doctype, - 'attached_to_name': self.attached_to_name - }) - duplicate_file = frappe.db.get_value('File', filters, ['name', 'file_url'], as_dict=1) + filters.update( + {"attached_to_doctype": self.attached_to_doctype, "attached_to_name": self.attached_to_name} + ) + duplicate_file = frappe.db.get_value("File", filters, ["name", "file_url"], as_dict=1) if duplicate_file: - duplicate_file_doc = frappe.get_cached_doc('File', duplicate_file.name) + duplicate_file_doc = frappe.get_cached_doc("File", duplicate_file.name) if duplicate_file_doc.exists_on_disk(): - # just use the url, to avoid uploading a duplicate - self.file_url = duplicate_file.file_url + # just use the url, to avoid uploading a duplicate + self.file_url = duplicate_file.file_url def set_file_name(self): if not self.file_name and self.file_url: - self.file_name = self.file_url.split('/')[-1] + self.file_name = self.file_url.split("/")[-1] else: - self.file_name = re.sub(r'/', '', self.file_name) + self.file_name = re.sub(r"/", "", self.file_name) def generate_content_hash(self): - if self.content_hash or not self.file_url or self.file_url.startswith('http'): + if self.content_hash or not self.file_url or self.file_url.startswith("http"): return - file_name = self.file_url.split('/')[-1] + file_name = self.file_url.split("/")[-1] try: file_path = get_files_path(file_name, is_private=self.is_private) with open(file_path, "rb") as f: @@ -272,9 +280,11 @@ class File(Document): self.check_folder_is_empty() self.call_delete_file() if not self.is_folder: - self.add_comment_in_reference_doc('Attachment Removed', _("Removed {0}").format(self.file_name)) + self.add_comment_in_reference_doc("Attachment Removed", _("Removed {0}").format(self.file_name)) - def make_thumbnail(self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False): + def make_thumbnail( + self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False + ): if self.file_url: try: if self.file_url.startswith(("/files", "/private/files")): @@ -282,7 +292,7 @@ class File(Document): else: image, filename, extn = get_web_image(self.file_url) except (HTTPError, SSLError, IOError, TypeError): - return + return size = width, height if crop: @@ -313,9 +323,14 @@ class File(Document): def call_delete_file(self): """If file not attached to any other record, delete it""" - if self.file_name and self.content_hash and (not frappe.db.count("File", - {"content_hash": self.content_hash, "name": ["!=", self.name]})): - self.delete_file_data_content() + if ( + self.file_name + and self.content_hash + and ( + not frappe.db.count("File", {"content_hash": self.content_hash, "name": ["!=", self.name]}) + ) + ): + self.delete_file_data_content() elif self.file_url: self.delete_file_data_content(only_thumbnail=True) @@ -332,7 +347,7 @@ class File(Document): self.on_trash() def unzip(self): - '''Unzip current file and replace it by its children''' + """Unzip current file and replace it by its children""" if not self.file_url.endswith(".zip"): frappe.throw(_("{0} is not a zip file").format(self.file_name)) @@ -341,16 +356,16 @@ class File(Document): files = [] with zipfile.ZipFile(zip_path) as z: for file in z.filelist: - if file.is_dir() or file.filename.startswith('__MACOSX/'): + if file.is_dir() or file.filename.startswith("__MACOSX/"): # skip directories and macos hidden directory continue filename = os.path.basename(file.filename) - if filename.startswith('.'): + if filename.startswith("."): # skip hidden files continue - file_doc = frappe.new_doc('File') + file_doc = frappe.new_doc("File") file_doc.content = z.read(file.filename) file_doc.file_name = filename file_doc.folder = self.folder @@ -360,28 +375,26 @@ class File(Document): file_doc.save() files.append(file_doc) - frappe.delete_doc('File', self.name) + frappe.delete_doc("File", self.name) return files - def exists_on_disk(self): exists = os.path.exists(self.get_full_path()) return exists - def get_content(self): """Returns [`file_name`, `content`] for given file name `fname`""" if self.is_folder: frappe.throw(_("Cannot get file contents of a Folder")) - if self.get('content'): + if self.get("content"): return self.content self.validate_url() file_path = self.get_full_path() # read the file - with io.open(encode(file_path), mode='rb') as f: + with io.open(encode(file_path), mode="rb") as f: content = f.read() try: # for plain text files @@ -419,7 +432,7 @@ class File(Document): file_path = get_files_path(is_private=self.is_private) if os.path.sep in self.file_name: - frappe.throw(_('File name cannot have {0}').format(os.path.sep)) + frappe.throw(_("File name cannot have {0}").format(os.path.sep)) # create directory (if not exists) frappe.create_folder(file_path) @@ -427,7 +440,7 @@ class File(Document): self.content = self.get_content() if isinstance(self.content, str): self.content = self.content.encode() - with open(os.path.join(file_path.encode('utf-8'), self.file_name.encode('utf-8')), 'wb+') as f: + with open(os.path.join(file_path.encode("utf-8"), self.file_name.encode("utf-8")), "wb+") as f: f.write(self.content) return get_files_path(self.file_name, is_private=self.is_private) @@ -452,7 +465,8 @@ class File(Document): self.file_size = self.check_max_file_size() if ( - self.content_type and self.content_type == "image/jpeg" + self.content_type + and self.content_type == "image/jpeg" and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images") ): self.content = strip_exif_data(self.content, self.content_type) @@ -463,16 +477,17 @@ class File(Document): # check if a file exists with the same content hash and is also in the same folder (public or private) if not ignore_existing_file_check: - duplicate_file = frappe.get_value("File", { - "content_hash": self.content_hash, - "is_private": self.is_private - }, - ["file_url", "name"], as_dict=True) + duplicate_file = frappe.get_value( + "File", + {"content_hash": self.content_hash, "is_private": self.is_private}, + ["file_url", "name"], + as_dict=True, + ) if duplicate_file: - file_doc = frappe.get_cached_doc('File', duplicate_file.name) + file_doc = frappe.get_cached_doc("File", duplicate_file.name) if file_doc.exists_on_disk(): - self.file_url = duplicate_file.file_url + self.file_url = duplicate_file.file_url file_exists = True if os.path.exists(encode(get_files_path(self.file_name, is_private=self.is_private))): @@ -480,12 +495,11 @@ class File(Document): if not file_exists: call_hook_method("before_write_file", file_size=self.file_size) - write_file_method = get_hook_method('write_file') + write_file_method = get_hook_method("write_file") if write_file_method: return write_file_method(self) return self.save_file_on_filesystem() - def save_file_on_filesystem(self): fpath = self.write_file() @@ -494,31 +508,27 @@ class File(Document): else: self.file_url = "/files/{0}".format(self.file_name) - return { - 'file_name': os.path.basename(fpath), - 'file_url': self.file_url - } + return {"file_name": os.path.basename(fpath), "file_url": self.file_url} def check_max_file_size(self): max_file_size = get_max_file_size() file_size = len(self.content) if file_size > max_file_size: - frappe.msgprint(_("File size exceeded the maximum allowed size of {0} MB").format( - max_file_size / 1048576), - raise_exception=MaxFileSizeReachedError) + frappe.msgprint( + _("File size exceeded the maximum allowed size of {0} MB").format(max_file_size / 1048576), + raise_exception=MaxFileSizeReachedError, + ) return file_size - def delete_file_data_content(self, only_thumbnail=False): - method = get_hook_method('delete_file_data_content') + method = get_hook_method("delete_file_data_content") if method: method(self, only_thumbnail=only_thumbnail) else: self.delete_file_from_filesystem(only_thumbnail=only_thumbnail) - def delete_file_from_filesystem(self, only_thumbnail=False): """Delete file, thumbnail from File document""" if only_thumbnail: @@ -528,10 +538,10 @@ class File(Document): delete_file(self.thumbnail_url) def is_downloadable(self): - return has_permission(self, 'read') + return has_permission(self, "read") def get_extension(self): - '''returns split filename and extension''' + """returns split filename and extension""" return os.path.splitext(self.file_name) def add_comment_in_reference_doc(self, comment_type, text): @@ -544,28 +554,28 @@ class File(Document): def set_is_private(self): if self.file_url: - self.is_private = cint(self.file_url.startswith('/private')) + self.is_private = cint(self.file_url.startswith("/private")) @frappe.whitelist() def optimize_file(self): if self.is_folder: - raise TypeError('Folders cannot be optimized') + raise TypeError("Folders cannot be optimized") content_type = mimetypes.guess_type(self.file_name)[0] - is_local_image = content_type.startswith('image/') and self.file_size > 0 - is_svg = content_type == 'image/svg+xml' + is_local_image = content_type.startswith("image/") and self.file_size > 0 + is_svg = content_type == "image/svg+xml" if not is_local_image: - raise NotImplementedError('Only local image files can be optimized') + raise NotImplementedError("Only local image files can be optimized") if is_svg: - raise TypeError('Optimization of SVG images is not supported') + raise TypeError("Optimization of SVG images is not supported") content = self.get_content() file_path = self.get_full_path() optimized_content = optimize_image(content, content_type) - with open(file_path, 'wb+') as f: + with open(file_path, "wb+") as f: f.write(optimized_content) self.file_size = len(optimized_content) @@ -594,26 +604,26 @@ class File(Document): def on_doctype_update(): frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"]) + def make_home_folder(): - home = frappe.get_doc({ - "doctype": "File", - "is_folder": 1, - "is_home_folder": 1, - "file_name": _("Home") - }).insert() - - frappe.get_doc({ - "doctype": "File", - "folder": home.name, - "is_folder": 1, - "is_attachments_folder": 1, - "file_name": _("Attachments") - }).insert() + home = frappe.get_doc( + {"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": _("Home")} + ).insert() + + frappe.get_doc( + { + "doctype": "File", + "folder": home.name, + "is_folder": 1, + "is_attachments_folder": 1, + "file_name": _("Attachments"), + } + ).insert() @frappe.whitelist() def create_new_folder(file_name, folder): - """ create new folder under current parent folder """ + """create new folder under current parent folder""" file = frappe.new_doc("File") file.file_name = file_name file.is_folder = 1 @@ -621,6 +631,7 @@ def create_new_folder(file_name, folder): file.insert(ignore_if_duplicate=True) return file + @frappe.whitelist() def move_file(file_list, new_parent, old_parent): @@ -651,8 +662,10 @@ def setup_folder_path(filename, new_parent): if file.is_folder: from frappe.model.rename_doc import rename_doc + rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) + def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str: mimetype = None @@ -666,8 +679,8 @@ def get_extension(filename, extn, content: bytes = None, response: "Response" = if extn: # remove '?' char and parameters from extn if present - if '?' in extn: - extn = extn.split('?', 1)[0] + if "?" in extn: + extn = extn.split("?", 1)[0] mimetype = mimetypes.guess_type(filename + "." + extn)[0] @@ -677,9 +690,10 @@ def get_extension(filename, extn, content: bytes = None, response: "Response" = return extn + def get_local_image(file_url): if file_url.startswith("/private"): - file_url_path = (file_url.lstrip("/"), ) + file_url_path = (file_url.lstrip("/"),) else: file_url_path = ("public", file_url.lstrip("/")) @@ -706,6 +720,7 @@ def get_local_image(file_url): return image, filename, extn + def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]: # download file_url = frappe.utils.get_url(file_url) @@ -745,10 +760,12 @@ def delete_file(path): """Delete file from `public folder`""" if path: if ".." in path.split("/"): - frappe.throw(_("It is risky to delete this file: {0}. Please contact your System Manager.").format(path)) + frappe.throw( + _("It is risky to delete this file: {0}. Please contact your System Manager.").format(path) + ) parts = os.path.split(path.strip("/")) - if parts[0]=="files": + if parts[0] == "files": path = frappe.utils.get_site_path("public", "files", parts[-1]) else: @@ -761,17 +778,17 @@ def delete_file(path): @frappe.whitelist() def get_max_file_size(): - return cint(conf.get('max_file_size')) or 10485760 + return cint(conf.get("max_file_size")) or 10485760 def has_permission(doc, ptype=None, user=None): has_access = False user = user or frappe.session.user - if ptype == 'create': - has_access = frappe.has_permission('File', 'create', user=user) + if ptype == "create": + has_access = frappe.has_permission("File", "create", user=user) - if not doc.is_private or doc.owner in [user, 'Guest'] or user == 'Administrator': + if not doc.is_private or doc.owner in [user, "Guest"] or user == "Administrator": has_access = True if doc.attached_to_doctype and doc.attached_to_name: @@ -781,15 +798,18 @@ def has_permission(doc, ptype=None, user=None): try: ref_doc = frappe.get_doc(attached_to_doctype, attached_to_name) - if ptype in ['write', 'create', 'delete']: - has_access = ref_doc.has_permission('write') + if ptype in ["write", "create", "delete"]: + has_access = ref_doc.has_permission("write") - if ptype == 'delete' and not has_access: - frappe.throw(_("Cannot delete file as it belongs to {0} {1} for which you do not have permissions").format( - doc.attached_to_doctype, doc.attached_to_name), - frappe.PermissionError) + if ptype == "delete" and not has_access: + frappe.throw( + _( + "Cannot delete file as it belongs to {0} {1} for which you do not have permissions" + ).format(doc.attached_to_doctype, doc.attached_to_name), + frappe.PermissionError, + ) else: - has_access = ref_doc.has_permission('read') + has_access = ref_doc.has_permission("read") except frappe.DoesNotExistError: # if parent doc is not created before file is created # we cannot check its permission so we will use file's permission @@ -800,34 +820,34 @@ def has_permission(doc, ptype=None, user=None): def remove_file_by_url(file_url, doctype=None, name=None): if doctype and name: - fid = frappe.db.get_value("File", { - "file_url": file_url, - "attached_to_doctype": doctype, - "attached_to_name": name}) + fid = frappe.db.get_value( + "File", {"file_url": file_url, "attached_to_doctype": doctype, "attached_to_name": name} + ) else: fid = frappe.db.get_value("File", {"file_url": file_url}) if fid: from frappe.utils.file_manager import remove_file + return remove_file(fid=fid) def get_content_hash(content): if isinstance(content, str): content = content.encode() - return hashlib.md5(content).hexdigest() #nosec + return hashlib.md5(content).hexdigest() # nosec def get_file_name(fname, optional_suffix): # convert to unicode fname = cstr(fname) - f = fname.rsplit('.', 1) + f = fname.rsplit(".", 1) if len(f) == 1: partial, extn = f[0], "" else: partial, extn = f[0], "." + f[1] - return '{partial}{suffix}{extn}'.format(partial=partial, extn=extn, suffix=optional_suffix) + return "{partial}{suffix}{extn}".format(partial=partial, extn=extn, suffix=optional_suffix) @frappe.whitelist() @@ -847,6 +867,7 @@ def download_file(file_url): frappe.local.response.filecontent = file_doc.get_content() frappe.local.response.type = "download" + def extract_images_from_doc(doc, fieldname): content = doc.get(fieldname) content = extract_images_from_html(doc, content) @@ -882,15 +903,17 @@ def extract_images_from_html(doc, content, is_private=False): doctype = doc.parenttype if doc.get("parent") else doc.doctype name = doc.get("parent") or doc.name - _file = frappe.get_doc({ - "doctype": "File", - "file_name": filename, - "attached_to_doctype": doctype, - "attached_to_name": name, - "content": content, - "decode": False, - "is_private": is_private - }) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": filename, + "attached_to_doctype": doctype, + "attached_to_name": name, + "content": content, + "decode": False, + "is_private": is_private, + } + ) _file.save(ignore_permissions=True) file_url = _file.file_url if not frappe.flags.has_dataurl: @@ -914,25 +937,25 @@ def get_random_filename(content_type=None): @frappe.whitelist() def unzip_file(name): - '''Unzip the given file and make file records for each of the extracted files''' - file_obj = frappe.get_doc('File', name) + """Unzip the given file and make file records for each of the extracted files""" + file_obj = frappe.get_doc("File", name) files = file_obj.unzip() return files @frappe.whitelist() def get_attached_images(doctype, names): - '''get list of image urls attached in form - returns {name: ['image.jpg', 'image.png']}''' + """get list of image urls attached in form + returns {name: ['image.jpg', 'image.png']}""" if isinstance(names, str): names = json.loads(names) - img_urls = frappe.db.get_list('File', filters={ - 'attached_to_doctype': doctype, - 'attached_to_name': ('in', names), - 'is_folder': 0 - }, fields=['file_url', 'attached_to_name as docname']) + img_urls = frappe.db.get_list( + "File", + filters={"attached_to_doctype": doctype, "attached_to_name": ("in", names), "is_folder": 0}, + fields=["file_url", "attached_to_name as docname"], + ) out = frappe._dict() for i in img_urls: @@ -947,41 +970,40 @@ def get_files_in_folder(folder, start=0, page_length=20): start = cint(start) page_length = cint(page_length) - attachment_folder = frappe.db.get_value('File', - 'Home/Attachments', - ['name', 'file_name', 'file_url', 'is_folder', 'modified'], - as_dict=1 + attachment_folder = frappe.db.get_value( + "File", "Home/Attachments", ["name", "file_name", "file_url", "is_folder", "modified"], as_dict=1 ) - files = frappe.db.get_list('File', - { 'folder': folder }, - ['name', 'file_name', 'file_url', 'is_folder', 'modified'], + files = frappe.db.get_list( + "File", + {"folder": folder}, + ["name", "file_name", "file_url", "is_folder", "modified"], start=start, - page_length=page_length + 1 + page_length=page_length + 1, ) - if folder == 'Home' and attachment_folder not in files: + if folder == "Home" and attachment_folder not in files: files.insert(0, attachment_folder) - return { - 'files': files[:page_length], - 'has_more': len(files) > page_length - } + return {"files": files[:page_length], "has_more": len(files) > page_length} + @frappe.whitelist() def get_files_by_search_text(text): if not text: return [] - text = '%' + cstr(text).lower() + '%' - return frappe.db.get_all('File', - fields=['name', 'file_name', 'file_url', 'is_folder', 'modified'], - filters={'is_folder': False}, - or_filters={'file_name': ('like', text), 'file_url': text, 'name': ('like', text)}, - order_by='modified desc', - limit=20 + text = "%" + cstr(text).lower() + "%" + return frappe.db.get_all( + "File", + fields=["name", "file_name", "file_url", "is_folder", "modified"], + filters={"is_folder": False}, + or_filters={"file_name": ("like", text), "file_url": text, "name": ("like", text)}, + order_by="modified desc", + limit=20, ) + def update_existing_file_docs(doc): # Update is private and file url of all file docs that point to the same file file_doctype = frappe.qb.DocType("File") @@ -993,30 +1015,32 @@ def update_existing_file_docs(doc): .where(file_doctype.name != doc.name) ).run() + def attach_files_to_document(doc, event): - """ Runs on on_update hook of all documents. + """Runs on on_update hook of all documents. Goes through every Attach and Attach Image field and attaches the file url to the document if it is not already attached. """ - attach_fields = doc.meta.get( - "fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]} - ) + attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) for df in attach_fields: # this method runs in on_update hook of all documents # we dont want the update to fail if file cannot be attached for some reason try: value = doc.get(df.fieldname) - if not (value or '').startswith(("/files", "/private/files")): + if not (value or "").startswith(("/files", "/private/files")): return - if frappe.db.exists("File", { - "file_url": value, - "attached_to_name": doc.name, - "attached_to_doctype": doc.doctype, - "attached_to_field": df.fieldname, - }): + if frappe.db.exists( + "File", + { + "file_url": value, + "attached_to_name": doc.name, + "attached_to_doctype": doc.doctype, + "attached_to_field": df.fieldname, + }, + ): return frappe.get_doc( diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index fb98a18d6e..b02bb581ab 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -2,21 +2,27 @@ # License: MIT. See LICENSE import base64 import json -import frappe import os import unittest +import frappe from frappe import _ -from frappe.core.doctype.file.file import File, get_attached_images, move_file, get_files_in_folder, unzip_file +from frappe.core.doctype.file.file import ( + File, + get_attached_images, + get_files_in_folder, + move_file, + unzip_file, +) from frappe.utils import get_files_path -test_content1 = 'Hello' -test_content2 = 'Hello World' +test_content1 = "Hello" +test_content2 = "Hello World" def make_test_doc(): - d = frappe.new_doc('ToDo') - d.description = 'Test' + d = frappe.new_doc("ToDo") + d.description = "Test" d.assigned_by = frappe.session.user d.save() return d.doctype, d.name @@ -26,12 +32,15 @@ class TestSimpleFile(unittest.TestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = test_content1 - _file = frappe.get_doc({ - "doctype": "File", - "file_name": "test1.txt", - "attached_to_doctype": self.attached_to_doctype, - "attached_to_name": self.attached_to_docname, - "content": self.test_content}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": "test1.txt", + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_docname, + "content": self.test_content, + } + ) _file.save() self.saved_file_url = _file.file_url @@ -44,14 +53,17 @@ class TestSimpleFile(unittest.TestCase): class TestBase64File(unittest.TestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() - self.test_content = base64.b64encode(test_content1.encode('utf-8')) - _file = frappe.get_doc({ - "doctype": "File", - "file_name": "test_base64.txt", - "attached_to_doctype": self.attached_to_doctype, - "attached_to_docname": self.attached_to_docname, - "content": self.test_content, - "decode": True}) + self.test_content = base64.b64encode(test_content1.encode("utf-8")) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": "test_base64.txt", + "attached_to_doctype": self.attached_to_doctype, + "attached_to_docname": self.attached_to_docname, + "content": self.test_content, + "decode": True, + } + ) _file.save() self.saved_file_url = _file.file_url @@ -66,24 +78,29 @@ class TestSameFileName(unittest.TestCase): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content1 = test_content1 self.test_content2 = test_content2 - _file1 = frappe.get_doc({ - "doctype": "File", - "file_name": "testing.txt", - "attached_to_doctype": self.attached_to_doctype, - "attached_to_name": self.attached_to_docname, - "content": self.test_content1}) + _file1 = frappe.get_doc( + { + "doctype": "File", + "file_name": "testing.txt", + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_docname, + "content": self.test_content1, + } + ) _file1.save() - _file2 = frappe.get_doc({ - "doctype": "File", - "file_name": "testing.txt", - "attached_to_doctype": self.attached_to_doctype, - "attached_to_name": self.attached_to_docname, - "content": self.test_content2}) + _file2 = frappe.get_doc( + { + "doctype": "File", + "file_name": "testing.txt", + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_docname, + "content": self.test_content2, + } + ) _file2.save() self.saved_file_url1 = _file1.file_url self.saved_file_url2 = _file2.file_url - _file = frappe.get_doc("File", {"file_url": self.saved_file_url1}) content1 = _file.get_content() self.assertEqual(content1, self.test_content1) @@ -92,18 +109,22 @@ class TestSameFileName(unittest.TestCase): self.assertEqual(content2, self.test_content2) def test_saved_content_private(self): - _file1 = frappe.get_doc({ - "doctype": "File", - "file_name": "testing-private.txt", - "content": test_content1, - "is_private": 1 - }).insert() - _file2 = frappe.get_doc({ - "doctype": "File", - "file_name": "testing-private.txt", - "content": test_content2, - "is_private": 1 - }).insert() + _file1 = frappe.get_doc( + { + "doctype": "File", + "file_name": "testing-private.txt", + "content": test_content1, + "is_private": 1, + } + ).insert() + _file2 = frappe.get_doc( + { + "doctype": "File", + "file_name": "testing-private.txt", + "content": test_content2, + "is_private": 1, + } + ).insert() _file = frappe.get_doc("File", {"file_url": _file1.file_url}) self.assertEqual(_file.get_content(), test_content1) @@ -118,59 +139,71 @@ class TestSameContent(unittest.TestCase): self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc() self.test_content1 = test_content1 self.test_content2 = test_content1 - self.orig_filename = 'hello.txt' - self.dup_filename = 'hello2.txt' - _file1 = frappe.get_doc({ - "doctype": "File", - "file_name": self.orig_filename, - "attached_to_doctype": self.attached_to_doctype1, - "attached_to_name": self.attached_to_docname1, - "content": self.test_content1}) + self.orig_filename = "hello.txt" + self.dup_filename = "hello2.txt" + _file1 = frappe.get_doc( + { + "doctype": "File", + "file_name": self.orig_filename, + "attached_to_doctype": self.attached_to_doctype1, + "attached_to_name": self.attached_to_docname1, + "content": self.test_content1, + } + ) _file1.save() - _file2 = frappe.get_doc({ - "doctype": "File", - "file_name": self.dup_filename, - "attached_to_doctype": self.attached_to_doctype2, - "attached_to_name": self.attached_to_docname2, - "content": self.test_content2}) + _file2 = frappe.get_doc( + { + "doctype": "File", + "file_name": self.dup_filename, + "attached_to_doctype": self.attached_to_doctype2, + "attached_to_name": self.attached_to_docname2, + "content": self.test_content2, + } + ) _file2.save() - def test_saved_content(self): self.assertFalse(os.path.exists(get_files_path(self.dup_filename))) def test_attachment_limit(self): doctype, docname = make_test_doc() from frappe.custom.doctype.property_setter.property_setter import make_property_setter - limit_property = make_property_setter('ToDo', None, 'max_attachments', 1, 'int', for_doctype=True) - file1 = frappe.get_doc({ - "doctype": "File", - "file_name": 'test-attachment', - "attached_to_doctype": doctype, - "attached_to_name": docname, - "content": 'test' - }) + + limit_property = make_property_setter( + "ToDo", None, "max_attachments", 1, "int", for_doctype=True + ) + file1 = frappe.get_doc( + { + "doctype": "File", + "file_name": "test-attachment", + "attached_to_doctype": doctype, + "attached_to_name": docname, + "content": "test", + } + ) file1.insert() - file2 = frappe.get_doc({ - "doctype": "File", - "file_name": 'test-attachment', - "attached_to_doctype": doctype, - "attached_to_name": docname, - "content": 'test2' - }) + file2 = frappe.get_doc( + { + "doctype": "File", + "file_name": "test-attachment", + "attached_to_doctype": doctype, + "attached_to_name": docname, + "content": "test2", + } + ) self.assertRaises(frappe.exceptions.AttachmentLimitReached, file2.insert) limit_property.delete() - frappe.clear_cache(doctype='ToDo') + frappe.clear_cache(doctype="ToDo") class TestFile(unittest.TestCase): def setUp(self): - frappe.set_user('Administrator') + frappe.set_user("Administrator") self.delete_test_data() self.upload_file() @@ -180,7 +213,6 @@ class TestFile(unittest.TestCase): except frappe.DoesNotExistError: pass - def delete_test_data(self): test_file_data = frappe.db.get_all( "File", @@ -192,34 +224,31 @@ class TestFile(unittest.TestCase): frappe.delete_doc("File", f) def upload_file(self): - _file = frappe.get_doc({ - "doctype": "File", - "file_name": "file_copy.txt", - "attached_to_name": "", - "attached_to_doctype": "", - "folder": self.get_folder("Test Folder 1", "Home").name, - "content": "Testing file copy example."}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": "file_copy.txt", + "attached_to_name": "", + "attached_to_doctype": "", + "folder": self.get_folder("Test Folder 1", "Home").name, + "content": "Testing file copy example.", + } + ) _file.save() self.saved_folder = _file.folder self.saved_name = _file.name self.saved_filename = get_files_path(_file.file_name) - def get_folder(self, folder_name, parent_folder="Home"): - return frappe.get_doc({ - "doctype": "File", - "file_name": _(folder_name), - "is_folder": 1, - "folder": _(parent_folder) - }).insert() - + return frappe.get_doc( + {"doctype": "File", "file_name": _(folder_name), "is_folder": 1, "folder": _(parent_folder)} + ).insert() def tests_after_upload(self): self.assertEqual(self.saved_folder, _("Home/Test Folder 1")) file_folder = frappe.db.get_value("File", self.saved_name, "folder") self.assertEqual(file_folder, _("Home/Test Folder 1")) - def test_file_copy(self): folder = self.get_folder("Test Folder 2", "Home") @@ -237,48 +266,47 @@ class TestFile(unittest.TestCase): result3 = self.get_folder("d3", "Home/d1/d2") self.assertEqual(result3.name, "Home/d1/d2/d3") result4 = self.get_folder("d4", "Home/d1/d2/d3") - _file = frappe.get_doc({ - "doctype": "File", - "file_name": "folder_copy.txt", - "attached_to_name": "", - "attached_to_doctype": "", - "folder": result4.name, - "content": "Testing folder copy example"}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": "folder_copy.txt", + "attached_to_name": "", + "attached_to_doctype": "", + "folder": result4.name, + "content": "Testing folder copy example", + } + ) _file.save() - def test_folder_copy(self): folder = self.get_folder("Test Folder 2", "Home") folder = self.get_folder("Test Folder 3", "Home/Test Folder 2") - _file = frappe.get_doc({ - "doctype": "File", - "file_name": "folder_copy.txt", - "attached_to_name": "", - "attached_to_doctype": "", - "folder": folder.name, - "content": "Testing folder copy example"}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": "folder_copy.txt", + "attached_to_name": "", + "attached_to_doctype": "", + "folder": folder.name, + "content": "Testing folder copy example", + } + ) _file.save() - move_file([{"name": folder.name}], 'Home/Test Folder 1', folder.folder) + move_file([{"name": folder.name}], "Home/Test Folder 1", folder.folder) - file = frappe.get_doc("File", {"file_name":"folder_copy.txt"}) - file_copy_txt = frappe.get_value("File", {"file_name":"file_copy.txt"}) + file = frappe.get_doc("File", {"file_name": "folder_copy.txt"}) + file_copy_txt = frappe.get_value("File", {"file_name": "file_copy.txt"}) if file_copy_txt: frappe.get_doc("File", file_copy_txt).delete() self.assertEqual(_("Home/Test Folder 1/Test Folder 3"), file.folder) - def test_default_folder(self): - d = frappe.get_doc({ - "doctype": "File", - "file_name": _("Test_Folder"), - "is_folder": 1 - }) + d = frappe.get_doc({"doctype": "File", "file_name": _("Test_Folder"), "is_folder": 1}) d.save() self.assertEqual(d.folder, "Home") - def test_on_delete(self): file = frappe.get_doc("File", {"file_name": "file_copy.txt"}) file.delete() @@ -286,13 +314,16 @@ class TestFile(unittest.TestCase): self.assertEqual(frappe.db.get_value("File", _("Home/Test Folder 1"), "file_size"), 0) folder = self.get_folder("Test Folder 3", "Home/Test Folder 1") - _file = frappe.get_doc({ - "doctype": "File", - "file_name": "folder_copy.txt", - "attached_to_name": "", - "attached_to_doctype": "", - "folder": folder.name, - "content": "Testing folder copy example"}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": "folder_copy.txt", + "attached_to_name": "", + "attached_to_doctype": "", + "folder": folder.name, + "content": "Testing folder copy example", + } + ) _file.save() folder = frappe.get_doc("File", "Home/Test Folder 1/Test Folder 3") @@ -302,21 +333,27 @@ class TestFile(unittest.TestCase): attached_to_doctype1, attached_to_docname1 = make_test_doc() attached_to_doctype2, attached_to_docname2 = make_test_doc() - file1 = frappe.get_doc({ - "doctype": "File", - "file_name": 'file1.txt', - "attached_to_doctype": attached_to_doctype1, - "attached_to_name": attached_to_docname1, - "is_private": 1, - "content": test_content1}).insert() - - file2 = frappe.get_doc({ - "doctype": "File", - "file_name": 'file2.txt', - "attached_to_doctype": attached_to_doctype2, - "attached_to_name": attached_to_docname2, - "is_private": 1, - "content": test_content1}).insert() + file1 = frappe.get_doc( + { + "doctype": "File", + "file_name": "file1.txt", + "attached_to_doctype": attached_to_doctype1, + "attached_to_name": attached_to_docname1, + "is_private": 1, + "content": test_content1, + } + ).insert() + + file2 = frappe.get_doc( + { + "doctype": "File", + "file_name": "file2.txt", + "attached_to_doctype": attached_to_doctype2, + "attached_to_name": attached_to_docname2, + "is_private": 1, + "content": test_content1, + } + ).insert() self.assertEqual(file1.is_private, file2.is_private, 1) self.assertEqual(file1.file_url, file2.file_url) @@ -325,45 +362,50 @@ class TestFile(unittest.TestCase): file1.is_private = 0 file1.save() - file2 = frappe.get_doc('File', file2.name) + file2 = frappe.get_doc("File", file2.name) self.assertEqual(file1.is_private, file2.is_private, 0) self.assertEqual(file1.file_url, file2.file_url) self.assertTrue(os.path.exists(file2.get_full_path())) def test_parent_directory_validation_in_file_url(self): - file1 = frappe.get_doc({ - "doctype": "File", - "file_name": 'parent_dir.txt', - "attached_to_doctype": "", - "attached_to_name": "", - "is_private": 1, - "content": test_content1}).insert() - - file1.file_url = '/private/files/../test.txt' + file1 = frappe.get_doc( + { + "doctype": "File", + "file_name": "parent_dir.txt", + "attached_to_doctype": "", + "attached_to_name": "", + "is_private": 1, + "content": test_content1, + } + ).insert() + + file1.file_url = "/private/files/../test.txt" self.assertRaises(frappe.exceptions.ValidationError, file1.save) # No validation to see if file exists file1.reload() - file1.file_url = '/private/files/parent_dir2.txt' + file1.file_url = "/private/files/parent_dir2.txt" file1.save() def test_file_url_validation(self): - test_file = frappe.get_doc({ - "doctype": "File", - "file_name": 'logo', - "file_url": 'https://frappe.io/files/frappe.png' - }) + test_file = frappe.get_doc( + {"doctype": "File", "file_name": "logo", "file_url": "https://frappe.io/files/frappe.png"} + ) self.assertIsNone(test_file.validate()) # bad path test_file.file_url = "/usr/bin/man" - self.assertRaisesRegex(frappe.exceptions.ValidationError, "URL must start with http:// or https://", test_file.validate) + self.assertRaisesRegex( + frappe.exceptions.ValidationError, "URL must start with http:// or https://", test_file.validate + ) test_file.file_url = None test_file.file_name = "/usr/bin/man" - self.assertRaisesRegex(frappe.exceptions.ValidationError, "There is some problem with the file url", test_file.validate) + self.assertRaisesRegex( + frappe.exceptions.ValidationError, "There is some problem with the file url", test_file.validate + ) test_file.file_url = None test_file.file_name = "_file" @@ -375,192 +417,220 @@ class TestFile(unittest.TestCase): def test_make_thumbnail(self): # test web image - test_file: File = frappe.get_doc({ - "doctype": "File", - "file_name": 'logo', - "file_url": frappe.utils.get_url('/_test/assets/image.jpg'), - }).insert(ignore_permissions=True) + test_file: File = frappe.get_doc( + { + "doctype": "File", + "file_name": "logo", + "file_url": frappe.utils.get_url("/_test/assets/image.jpg"), + } + ).insert(ignore_permissions=True) test_file.make_thumbnail() - self.assertEqual(test_file.thumbnail_url, '/files/image_small.jpg') + self.assertEqual(test_file.thumbnail_url, "/files/image_small.jpg") # test web image without extension - test_file = frappe.get_doc({ - "doctype": "File", - "file_name": 'logo', - "file_url": frappe.utils.get_url('/_test/assets/image'), - }).insert(ignore_permissions=True) + test_file = frappe.get_doc( + { + "doctype": "File", + "file_name": "logo", + "file_url": frappe.utils.get_url("/_test/assets/image"), + } + ).insert(ignore_permissions=True) test_file.make_thumbnail() self.assertTrue(test_file.thumbnail_url.endswith("_small.jpeg")) # test local image - test_file.db_set('thumbnail_url', None) + test_file.db_set("thumbnail_url", None) test_file.reload() test_file.file_url = "/files/image_small.jpg" test_file.make_thumbnail(suffix="xs", crop=True) - self.assertEqual(test_file.thumbnail_url, '/files/image_small_xs.jpg') + self.assertEqual(test_file.thumbnail_url, "/files/image_small_xs.jpg") frappe.clear_messages() - test_file.db_set('thumbnail_url', None) + test_file.db_set("thumbnail_url", None) test_file.reload() - test_file.file_url = frappe.utils.get_url('unknown.jpg') + test_file.file_url = frappe.utils.get_url("unknown.jpg") test_file.make_thumbnail(suffix="xs") - self.assertEqual(json.loads(frappe.message_log[0]).get("message"), f"File '{frappe.utils.get_url('unknown.jpg')}' not found") + self.assertEqual( + json.loads(frappe.message_log[0]).get("message"), + f"File '{frappe.utils.get_url('unknown.jpg')}' not found", + ) self.assertEqual(test_file.thumbnail_url, None) def test_file_unzip(self): - file_path = frappe.get_app_path('frappe', 'www/_test/assets/file.zip') - public_file_path = frappe.get_site_path('public', 'files') + file_path = frappe.get_app_path("frappe", "www/_test/assets/file.zip") + public_file_path = frappe.get_site_path("public", "files") try: import shutil + shutil.copy(file_path, public_file_path) except Exception: pass - test_file = frappe.get_doc({ - "doctype": "File", - "file_url": '/files/file.zip', - }).insert(ignore_permissions=True) + test_file = frappe.get_doc( + { + "doctype": "File", + "file_url": "/files/file.zip", + } + ).insert(ignore_permissions=True) - self.assertListEqual([file.file_name for file in unzip_file(test_file.name)], - ['css_asset.css', 'image.jpg', 'js_asset.min.js']) + self.assertListEqual( + [file.file_name for file in unzip_file(test_file.name)], + ["css_asset.css", "image.jpg", "js_asset.min.js"], + ) - test_file = frappe.get_doc({ - "doctype": "File", - "file_url": frappe.utils.get_url('/_test/assets/image.jpg'), - }).insert(ignore_permissions=True) - self.assertRaisesRegex(frappe.exceptions.ValidationError, 'not a zip file', test_file.unzip) + test_file = frappe.get_doc( + { + "doctype": "File", + "file_url": frappe.utils.get_url("/_test/assets/image.jpg"), + } + ).insert(ignore_permissions=True) + self.assertRaisesRegex(frappe.exceptions.ValidationError, "not a zip file", test_file.unzip) class TestAttachment(unittest.TestCase): - test_doctype = 'Test For Attachment' + test_doctype = "Test For Attachment" def setUp(self): - if frappe.db.exists('DocType', self.test_doctype): + if frappe.db.exists("DocType", self.test_doctype): return frappe.get_doc( - doctype='DocType', + doctype="DocType", name=self.test_doctype, - module='Custom', + module="Custom", custom=1, fields=[ - {'label': 'Title', 'fieldname': 'title', 'fieldtype': 'Data'}, - {'label': 'Attachment', 'fieldname': 'attachment', 'fieldtype': 'Attach'}, - ] + {"label": "Title", "fieldname": "title", "fieldtype": "Data"}, + {"label": "Attachment", "fieldname": "attachment", "fieldtype": "Attach"}, + ], ).insert() def tearDown(self): - frappe.delete_doc('DocType', self.test_doctype) + frappe.delete_doc("DocType", self.test_doctype) def test_file_attachment_on_update(self): - doc = frappe.get_doc( - doctype=self.test_doctype, - title='test for attachment on update' - ).insert() + doc = frappe.get_doc(doctype=self.test_doctype, title="test for attachment on update").insert() - file = frappe.get_doc({ - 'doctype': 'File', - 'file_name': 'test_attach.txt', - 'content': 'Test Content' - }) + file = frappe.get_doc( + {"doctype": "File", "file_name": "test_attach.txt", "content": "Test Content"} + ) file.save() doc.attachment = file.file_url doc.save() - exists = frappe.db.exists('File', { - 'file_name': 'test_attach.txt', - 'file_url': file.file_url, - 'attached_to_doctype': self.test_doctype, - 'attached_to_name': doc.name, - 'attached_to_field': 'attachment' - }) + exists = frappe.db.exists( + "File", + { + "file_name": "test_attach.txt", + "file_url": file.file_url, + "attached_to_doctype": self.test_doctype, + "attached_to_name": doc.name, + "attached_to_field": "attachment", + }, + ) self.assertTrue(exists) class TestAttachmentsAccess(unittest.TestCase): - def test_attachments_access(self): - frappe.set_user('test4@example.com') + frappe.set_user("test4@example.com") self.attached_to_doctype, self.attached_to_docname = make_test_doc() - frappe.get_doc({ - "doctype": "File", - "file_name": 'test_user.txt', - "attached_to_doctype": self.attached_to_doctype, - "attached_to_name": self.attached_to_docname, - "content": 'Testing User' - }).insert() - - frappe.get_doc({ - "doctype": "File", - "file_name": "test_user_home.txt", - "content": 'User Home', - }).insert() - - frappe.set_user('test@example.com') - - frappe.get_doc({ - "doctype": "File", - "file_name": 'test_system_manager.txt', - "attached_to_doctype": self.attached_to_doctype, - "attached_to_name": self.attached_to_docname, - "content": 'Testing System Manager' - }).insert() - - frappe.get_doc({ - "doctype": "File", - "file_name": "test_sm_home.txt", - "content": 'System Manager Home', - }).insert() - - system_manager_files = [file.file_name for file in get_files_in_folder('Home')['files']] - system_manager_attachments_files = [file.file_name for file in get_files_in_folder('Home/Attachments')['files']] - - frappe.set_user('test4@example.com') - user_files = [file.file_name for file in get_files_in_folder('Home')['files']] - user_attachments_files = [file.file_name for file in get_files_in_folder('Home/Attachments')['files']] - - self.assertIn('test_sm_home.txt', system_manager_files) - self.assertNotIn('test_sm_home.txt', user_files) - self.assertIn('test_user_home.txt', system_manager_files) - self.assertIn('test_user_home.txt', user_files) - - self.assertIn('test_system_manager.txt', system_manager_attachments_files) - self.assertNotIn('test_system_manager.txt', user_attachments_files) - self.assertIn('test_user.txt', system_manager_attachments_files) - self.assertIn('test_user.txt', user_attachments_files) - - frappe.set_user('Administrator') + frappe.get_doc( + { + "doctype": "File", + "file_name": "test_user.txt", + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_docname, + "content": "Testing User", + } + ).insert() + + frappe.get_doc( + { + "doctype": "File", + "file_name": "test_user_home.txt", + "content": "User Home", + } + ).insert() + + frappe.set_user("test@example.com") + + frappe.get_doc( + { + "doctype": "File", + "file_name": "test_system_manager.txt", + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_docname, + "content": "Testing System Manager", + } + ).insert() + + frappe.get_doc( + { + "doctype": "File", + "file_name": "test_sm_home.txt", + "content": "System Manager Home", + } + ).insert() + + system_manager_files = [file.file_name for file in get_files_in_folder("Home")["files"]] + system_manager_attachments_files = [ + file.file_name for file in get_files_in_folder("Home/Attachments")["files"] + ] + + frappe.set_user("test4@example.com") + user_files = [file.file_name for file in get_files_in_folder("Home")["files"]] + user_attachments_files = [ + file.file_name for file in get_files_in_folder("Home/Attachments")["files"] + ] + + self.assertIn("test_sm_home.txt", system_manager_files) + self.assertNotIn("test_sm_home.txt", user_files) + self.assertIn("test_user_home.txt", system_manager_files) + self.assertIn("test_user_home.txt", user_files) + + self.assertIn("test_system_manager.txt", system_manager_attachments_files) + self.assertNotIn("test_system_manager.txt", user_attachments_files) + self.assertIn("test_user.txt", system_manager_attachments_files) + self.assertIn("test_user.txt", user_attachments_files) + + frappe.set_user("Administrator") frappe.db.rollback() class TestFileUtils(unittest.TestCase): def test_extract_images_from_doc(self): # with filename in data URI - todo = frappe.get_doc({ - "doctype": "ToDo", - "description": 'Test ' - }).insert() + todo = frappe.get_doc( + { + "doctype": "ToDo", + "description": 'Test ', + } + ).insert() self.assertTrue(frappe.db.exists("File", {"attached_to_name": todo.name})) self.assertIn('', todo.description) - self.assertListEqual(get_attached_images('ToDo', [todo.name])[todo.name], ['/files/pix.png']) + self.assertListEqual(get_attached_images("ToDo", [todo.name])[todo.name], ["/files/pix.png"]) # without filename in data URI - todo = frappe.get_doc({ - "doctype": "ToDo", - "description": 'Test ' - }).insert() + todo = frappe.get_doc( + { + "doctype": "ToDo", + "description": 'Test ', + } + ).insert() filename = frappe.db.exists("File", {"attached_to_name": todo.name}) self.assertIn(f' Error Logs ') + "show_alert": True, + "message": _("You have unseen {0}").format( + ' Error Logs ' + ), } if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): - log_settings = frappe.get_cached_doc('Log Settings') + log_settings = frappe.get_cached_doc("Log Settings") if log_settings.users_to_notify: if user in [u.user for u in log_settings.users_to_notify]: @@ -51,4 +56,4 @@ def has_unseen_error_log(user): else: return _get_response(show_alert=False) else: - return _get_response() \ No newline at end of file + return _get_response() diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py index f398577665..1b78745103 100644 --- a/frappe/core/doctype/log_settings/test_log_settings.py +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -4,9 +4,9 @@ from datetime import datetime import frappe -from frappe.utils import now_datetime, add_to_date from frappe.core.doctype.log_settings.log_settings import run_log_clean_up from frappe.tests.utils import FrappeTestCase +from frappe.utils import add_to_date, now_datetime class TestLogSettings(FrappeTestCase): @@ -36,15 +36,9 @@ class TestLogSettings(FrappeTestCase): def test_delete_logs(self): # make sure test data is present - activity_log_count = frappe.db.count( - "Activity Log", {"creation": ("<=", self.datetime.past)} - ) - error_log_count = frappe.db.count( - "Error Log", {"creation": ("<=", self.datetime.past)} - ) - email_queue_count = frappe.db.count( - "Email Queue", {"creation": ("<=", self.datetime.past)} - ) + activity_log_count = frappe.db.count("Activity Log", {"creation": ("<=", self.datetime.past)}) + error_log_count = frappe.db.count("Error Log", {"creation": ("<=", self.datetime.past)}) + email_queue_count = frappe.db.count("Email Queue", {"creation": ("<=", self.datetime.past)}) self.assertNotEqual(activity_log_count, 0) self.assertNotEqual(error_log_count, 0) @@ -54,15 +48,9 @@ class TestLogSettings(FrappeTestCase): run_log_clean_up() # test if logs are deleted - activity_log_count = frappe.db.count( - "Activity Log", {"creation": ("<", self.datetime.past)} - ) - error_log_count = frappe.db.count( - "Error Log", {"creation": ("<", self.datetime.past)} - ) - email_queue_count = frappe.db.count( - "Email Queue", {"creation": ("<", self.datetime.past)} - ) + activity_log_count = frappe.db.count("Activity Log", {"creation": ("<", self.datetime.past)}) + error_log_count = frappe.db.count("Error Log", {"creation": ("<", self.datetime.past)}) + email_queue_count = frappe.db.count("Email Queue", {"creation": ("<", self.datetime.past)}) self.assertEqual(activity_log_count, 0) self.assertEqual(error_log_count, 0) diff --git a/frappe/core/doctype/module_def/__init__.py b/frappe/core/doctype/module_def/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/module_def/__init__.py +++ b/frappe/core/doctype/module_def/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py index 6b420430b8..8f80ffd4ee 100644 --- a/frappe/core/doctype/module_def/module_def.py +++ b/frappe/core/doctype/module_def/module_def.py @@ -1,14 +1,17 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, os, json +import json +import os +import frappe from frappe.model.document import Document + class ModuleDef(Document): def on_update(self): """If in `developer_mode`, create folder for module and - add in `modules.txt` of app if missing.""" + add in `modules.txt` of app if missing.""" frappe.clear_cache() if not self.custom and frappe.conf.get("developer_mode"): self.create_modules_folder() @@ -42,7 +45,7 @@ class ModuleDef(Document): def on_trash(self): """Delete module name from modules.txt""" - if not frappe.conf.get('developer_mode') or frappe.flags.in_uninstall or self.custom: + if not frappe.conf.get("developer_mode") or frappe.flags.in_uninstall or self.custom: return modules = None @@ -60,6 +63,7 @@ class ModuleDef(Document): frappe.clear_cache() frappe.setup_module_map() + @frappe.whitelist() def get_installed_apps(): - return json.dumps(frappe.get_installed_apps()) \ No newline at end of file + return json.dumps(frappe.get_installed_apps()) diff --git a/frappe/core/doctype/module_def/test_module_def.py b/frappe/core/doctype/module_def/test_module_def.py index 69a114d765..3d269e4734 100644 --- a/frappe/core/doctype/module_def/test_module_def.py +++ b/frappe/core/doctype/module_def/test_module_def.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Module Def') + class TestModuleDef(unittest.TestCase): pass diff --git a/frappe/core/doctype/module_profile/module_profile.py b/frappe/core/doctype/module_profile/module_profile.py index 930c3879b6..7c5f896ba8 100644 --- a/frappe/core/doctype/module_profile/module_profile.py +++ b/frappe/core/doctype/module_profile/module_profile.py @@ -4,8 +4,9 @@ from frappe.model.document import Document + class ModuleProfile(Document): def onload(self): from frappe.config import get_modules_from_all_apps - self.set_onload('all_modules', - [m.get("module_name") for m in get_modules_from_all_apps()]) + + self.set_onload("all_modules", [m.get("module_name") for m in get_modules_from_all_apps()]) diff --git a/frappe/core/doctype/module_profile/test_module_profile.py b/frappe/core/doctype/module_profile/test_module_profile.py index e676767db6..e15a70d93f 100644 --- a/frappe/core/doctype/module_profile/test_module_profile.py +++ b/frappe/core/doctype/module_profile/test_module_profile.py @@ -1,31 +1,31 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestModuleProfile(unittest.TestCase): def test_make_new_module_profile(self): - if not frappe.db.get_value('Module Profile', '_Test Module Profile'): - frappe.get_doc({ - 'doctype': 'Module Profile', - 'module_profile_name': '_Test Module Profile', - 'block_modules': [ - {'module': 'Accounts'} - ] - }).insert() + if not frappe.db.get_value("Module Profile", "_Test Module Profile"): + frappe.get_doc( + { + "doctype": "Module Profile", + "module_profile_name": "_Test Module Profile", + "block_modules": [{"module": "Accounts"}], + } + ).insert() # add to user and check - if not frappe.db.get_value('User', 'test-for-module_profile@example.com'): - new_user = frappe.get_doc({ - 'doctype': 'User', - 'email':'test-for-module_profile@example.com', - 'first_name':'Test User' - }).insert() + if not frappe.db.get_value("User", "test-for-module_profile@example.com"): + new_user = frappe.get_doc( + {"doctype": "User", "email": "test-for-module_profile@example.com", "first_name": "Test User"} + ).insert() else: - new_user = frappe.get_doc('User', 'test-for-module_profile@example.com') + new_user = frappe.get_doc("User", "test-for-module_profile@example.com") - new_user.module_profile = '_Test Module Profile' + new_user.module_profile = "_Test Module Profile" new_user.save() - self.assertEqual(new_user.block_modules[0].module, 'Accounts') + self.assertEqual(new_user.block_modules[0].module, "Accounts") diff --git a/frappe/core/doctype/navbar_item/navbar_item.py b/frappe/core/doctype/navbar_item/navbar_item.py index d4952a75f2..27aa339c93 100644 --- a/frappe/core/doctype/navbar_item/navbar_item.py +++ b/frappe/core/doctype/navbar_item/navbar_item.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class NavbarItem(Document): pass diff --git a/frappe/core/doctype/navbar_item/test_navbar_item.py b/frappe/core/doctype/navbar_item/test_navbar_item.py index bb4b2a837a..913e84704b 100644 --- a/frappe/core/doctype/navbar_item/test_navbar_item.py +++ b/frappe/core/doctype/navbar_item/test_navbar_item.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestNavbarItem(unittest.TestCase): pass diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py index c46d0081b6..6243107d63 100644 --- a/frappe/core/doctype/navbar_settings/navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/navbar_settings.py @@ -3,8 +3,9 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document + class NavbarSettings(Document): def validate(self): @@ -16,22 +17,28 @@ class NavbarSettings(Document): if not doc_before_save: return - before_save_items = [item for item in \ - doc_before_save.help_dropdown + doc_before_save.settings_dropdown if item.is_standard] + before_save_items = [ + item + for item in doc_before_save.help_dropdown + doc_before_save.settings_dropdown + if item.is_standard + ] - after_save_items = [item for item in \ - self.help_dropdown + self.settings_dropdown if item.is_standard] + after_save_items = [ + item for item in self.help_dropdown + self.settings_dropdown if item.is_standard + ] if not frappe.flags.in_patch and (len(before_save_items) > len(after_save_items)): frappe.throw(_("Please hide the standard navbar items instead of deleting them")) + def get_app_logo(): - app_logo = frappe.db.get_single_value('Navbar Settings', 'app_logo', cache=True) + app_logo = frappe.db.get_single_value("Navbar Settings", "app_logo", cache=True) if not app_logo: - app_logo = frappe.get_hooks('app_logo_url')[-1] + app_logo = frappe.get_hooks("app_logo_url")[-1] return app_logo + def get_navbar_settings(): - navbar_settings = frappe.get_single('Navbar Settings') + navbar_settings = frappe.get_single("Navbar Settings") return navbar_settings diff --git a/frappe/core/doctype/navbar_settings/test_navbar_settings.py b/frappe/core/doctype/navbar_settings/test_navbar_settings.py index 01497d9035..f63e361e8b 100644 --- a/frappe/core/doctype/navbar_settings/test_navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/test_navbar_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestNavbarSettings(unittest.TestCase): pass diff --git a/frappe/core/doctype/package/package.py b/frappe/core/doctype/package/package.py index aa9735c061..a32f1bc534 100644 --- a/frappe/core/doctype/package/package.py +++ b/frappe/core/doctype/package/package.py @@ -1,18 +1,21 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -import frappe import os + +import frappe from frappe.model.document import Document + class Package(Document): def validate(self): if not self.package_name: - self.package_name = self.name.lower().replace(' ', '-') + self.package_name = self.name.lower().replace(" ", "-") + @frappe.whitelist() def get_license_text(license_type): - with open(os.path.join(os.path.dirname(__file__), 'licenses', - license_type + '.md'), 'r') as textfile: + with open( + os.path.join(os.path.dirname(__file__), "licenses", license_type + ".md"), "r" + ) as textfile: return textfile.read() - diff --git a/frappe/core/doctype/package/test_package.py b/frappe/core/doctype/package/test_package.py index 3fb8d48274..0bd52a9e3c 100644 --- a/frappe/core/doctype/package/test_package.py +++ b/frappe/core/doctype/package/test_package.py @@ -1,11 +1,13 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # See license.txt -import frappe -import os import json +import os import unittest +import frappe + + class TestPackage(unittest.TestCase): def test_package_release(self): make_test_package() @@ -15,75 +17,94 @@ class TestPackage(unittest.TestCase): make_test_web_page() # make release - frappe.get_doc(dict( - doctype = 'Package Release', - package = 'Test Package', - publish = 1 - )).insert() - - self.assertTrue(os.path.exists(frappe.get_site_path('packages', 'test-package'))) - self.assertTrue(os.path.exists(frappe.get_site_path('packages', 'test-package', 'test_module_for_package'))) - self.assertTrue(os.path.exists(frappe.get_site_path('packages', 'test-package', 'test_module_for_package', 'doctype', 'test_doctype_for_package'))) - with open(frappe.get_site_path('packages', 'test-package', 'test_module_for_package', - 'doctype', 'test_doctype_for_package', 'test_doctype_for_package.json')) as f: + frappe.get_doc(dict(doctype="Package Release", package="Test Package", publish=1)).insert() + + self.assertTrue(os.path.exists(frappe.get_site_path("packages", "test-package"))) + self.assertTrue( + os.path.exists(frappe.get_site_path("packages", "test-package", "test_module_for_package")) + ) + self.assertTrue( + os.path.exists( + frappe.get_site_path( + "packages", "test-package", "test_module_for_package", "doctype", "test_doctype_for_package" + ) + ) + ) + with open( + frappe.get_site_path( + "packages", + "test-package", + "test_module_for_package", + "doctype", + "test_doctype_for_package", + "test_doctype_for_package.json", + ) + ) as f: doctype = json.loads(f.read()) - self.assertEqual(doctype['doctype'], 'DocType') - self.assertEqual(doctype['name'], 'Test DocType for Package') - self.assertEqual(doctype['fields'][0]['fieldname'], 'test_field') + self.assertEqual(doctype["doctype"], "DocType") + self.assertEqual(doctype["name"], "Test DocType for Package") + self.assertEqual(doctype["fields"][0]["fieldname"], "test_field") def make_test_package(): - if not frappe.db.exists('Package', 'Test Package'): - frappe.get_doc(dict( - doctype = 'Package', - name = 'Test Package', - package_name = 'test-package', - readme = '# Test Package' - )).insert() + if not frappe.db.exists("Package", "Test Package"): + frappe.get_doc( + dict( + doctype="Package", name="Test Package", package_name="test-package", readme="# Test Package" + ) + ).insert() + def make_test_module(): - if not frappe.db.exists('Module Def', 'Test Module for Package'): - frappe.get_doc(dict( - doctype = 'Module Def', - module_name = 'Test Module for Package', - custom = 1, - app_name = 'frappe', - package = 'Test Package' - )).insert() + if not frappe.db.exists("Module Def", "Test Module for Package"): + frappe.get_doc( + dict( + doctype="Module Def", + module_name="Test Module for Package", + custom=1, + app_name="frappe", + package="Test Package", + ) + ).insert() + def make_test_doctype(): - if not frappe.db.exists('DocType', 'Test DocType for Package'): - frappe.get_doc(dict( - doctype = 'DocType', - name = 'Test DocType for Package', - custom = 1, - module = 'Test Module for Package', - autoname = 'Prompt', - fields = [dict( - fieldname = 'test_field', - fieldtype = 'Data', - label = 'Test Field' - )] - )).insert() + if not frappe.db.exists("DocType", "Test DocType for Package"): + frappe.get_doc( + dict( + doctype="DocType", + name="Test DocType for Package", + custom=1, + module="Test Module for Package", + autoname="Prompt", + fields=[dict(fieldname="test_field", fieldtype="Data", label="Test Field")], + ) + ).insert() + def make_test_server_script(): - if not frappe.db.exists('Server Script', 'Test Script for Package'): - frappe.get_doc(dict( - doctype = 'Server Script', - name = 'Test Script for Package', - module = 'Test Module for Package', - script_type = 'DocType Event', - reference_doctype = 'Test DocType for Package', - doctype_event = 'Before Save', - script = 'frappe.msgprint("Test")' - )).insert() + if not frappe.db.exists("Server Script", "Test Script for Package"): + frappe.get_doc( + dict( + doctype="Server Script", + name="Test Script for Package", + module="Test Module for Package", + script_type="DocType Event", + reference_doctype="Test DocType for Package", + doctype_event="Before Save", + script='frappe.msgprint("Test")', + ) + ).insert() + def make_test_web_page(): - if not frappe.db.exists('Web Page', 'test-web-page-for-package'): - frappe.get_doc(dict( - doctype = "Web Page", - module = 'Test Module for Package', - main_section = "Some content", - published = 1, - title = "Test Web Page for Package" - )).insert() + if not frappe.db.exists("Web Page", "test-web-page-for-package"): + frappe.get_doc( + dict( + doctype="Web Page", + module="Test Module for Package", + main_section="Some content", + published=1, + title="Test Web Page for Package", + ) + ).insert() diff --git a/frappe/core/doctype/package_import/package_import.py b/frappe/core/doctype/package_import/package_import.py index f4a2d666dd..659017c498 100644 --- a/frappe/core/doctype/package_import/package_import.py +++ b/frappe/core/doctype/package_import/package_import.py @@ -1,14 +1,16 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt -import frappe -import os import json +import os import subprocess -from frappe.model.document import Document + +import frappe from frappe.desk.form.load import get_attachments +from frappe.model.document import Document from frappe.model.sync import get_doc_files -from frappe.modules.import_file import import_file_by_path, import_doc +from frappe.modules.import_file import import_doc, import_file_by_path + class PackageImport(Document): def validate(self): @@ -19,24 +21,30 @@ class PackageImport(Document): attachment = get_attachments(self.doctype, self.name) if not attachment: - frappe.throw(frappe._('Please attach the package')) + frappe.throw(frappe._("Please attach the package")) attachment = attachment[0] # get package_name from file (package_name-0.0.0.tar.gz) - package_name = attachment.file_name.split('.')[0].rsplit('-', 1)[0] - if not os.path.exists(frappe.get_site_path('packages')): - os.makedirs(frappe.get_site_path('packages')) + package_name = attachment.file_name.split(".")[0].rsplit("-", 1)[0] + if not os.path.exists(frappe.get_site_path("packages")): + os.makedirs(frappe.get_site_path("packages")) # extract - subprocess.check_output(['tar', 'xzf', - frappe.get_site_path(attachment.file_url.strip('/')), '-C', - frappe.get_site_path('packages')]) + subprocess.check_output( + [ + "tar", + "xzf", + frappe.get_site_path(attachment.file_url.strip("/")), + "-C", + frappe.get_site_path("packages"), + ] + ) - package_path = frappe.get_site_path('packages', package_name) + package_path = frappe.get_site_path("packages", package_name) # import Package - with open(os.path.join(package_path, package_name + '.json'), 'r') as packagefile: + with open(os.path.join(package_path, package_name + ".json"), "r") as packagefile: doc_dict = json.loads(packagefile.read()) frappe.flags.package = import_doc(doc_dict) @@ -51,8 +59,7 @@ class PackageImport(Document): # import files for file in files: - import_file_by_path(file, force=self.force, ignore_version=True, - for_sync=True) - log.append('Imported {}'.format(file)) + import_file_by_path(file, force=self.force, ignore_version=True, for_sync=True) + log.append("Imported {}".format(file)) - self.log = '\n'.join(log) + self.log = "\n".join(log) diff --git a/frappe/core/doctype/package_import/test_package_import.py b/frappe/core/doctype/package_import/test_package_import.py index 04628fed93..7e8008cc44 100644 --- a/frappe/core/doctype/package_import/test_package_import.py +++ b/frappe/core/doctype/package_import/test_package_import.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestPackageImport(unittest.TestCase): pass diff --git a/frappe/core/doctype/package_release/package_release.py b/frappe/core/doctype/package_release/package_release.py index d23ae917c4..05277dcf2e 100644 --- a/frappe/core/doctype/package_release/package_release.py +++ b/frappe/core/doctype/package_release/package_release.py @@ -1,11 +1,12 @@ # Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt +import os +import subprocess + import frappe from frappe.model.document import Document from frappe.modules.export_file import export_doc -import os -import subprocess from frappe.query_builder.functions import Max @@ -14,43 +15,55 @@ class PackageRelease(Document): # set the next patch release by default doctype = frappe.qb.DocType("Package Release") if not self.major: - self.major = frappe.qb.from_(doctype) \ - .where(doctype.package == self.package) \ - .select(Max(doctype.minor)).run()[0][0] or 0 + self.major = ( + frappe.qb.from_(doctype) + .where(doctype.package == self.package) + .select(Max(doctype.minor)) + .run()[0][0] + or 0 + ) if not self.minor: - self.minor = frappe.qb.from_(doctype) \ - .where(doctype.package == self.package) \ - .select(Max("minor")).run()[0][0] or 0 + self.minor = ( + frappe.qb.from_(doctype) + .where(doctype.package == self.package) + .select(Max("minor")) + .run()[0][0] + or 0 + ) if not self.patch: - value = frappe.qb.from_(doctype) \ - .where(doctype.package == self.package) \ - .select(Max("patch")).run()[0][0] or 0 + value = ( + frappe.qb.from_(doctype) + .where(doctype.package == self.package) + .select(Max("patch")) + .run()[0][0] + or 0 + ) self.patch = value + 1 def autoname(self): self.set_version() - self.name = '{}-{}.{}.{}'.format( - frappe.db.get_value('Package', self.package, 'package_name'), - self.major, self.minor, self.patch) + self.name = "{}-{}.{}.{}".format( + frappe.db.get_value("Package", self.package, "package_name"), self.major, self.minor, self.patch + ) def validate(self): if self.publish: self.export_files() def export_files(self): - '''Export all the documents in this package to site/packages folder''' - package = frappe.get_doc('Package', self.package) + """Export all the documents in this package to site/packages folder""" + package = frappe.get_doc("Package", self.package) self.export_modules() self.export_package_files(package) self.make_tarfile(package) def export_modules(self): - for m in frappe.db.get_all('Module Def', dict(package=self.package)): - module = frappe.get_doc('Module Def', m.name) + for m in frappe.db.get_all("Module Def", dict(package=self.package)): + module = frappe.get_doc("Module Def", m.name) for l in module.meta.links: - if l.link_doctype == 'Module Def': + if l.link_doctype == "Module Def": continue # all documents of the type in the module for d in frappe.get_all(l.link_doctype, dict(module=m.name)): @@ -58,35 +71,41 @@ class PackageRelease(Document): def export_package_files(self, package): # write readme - with open(frappe.get_site_path('packages', package.package_name, 'README.md'), 'w') as readme: + with open(frappe.get_site_path("packages", package.package_name, "README.md"), "w") as readme: readme.write(package.readme) # write license if package.license: - with open(frappe.get_site_path('packages', package.package_name, 'LICENSE.md'), 'w') as license: + with open(frappe.get_site_path("packages", package.package_name, "LICENSE.md"), "w") as license: license.write(package.license) # write package.json as `frappe_package.json` - with open(frappe.get_site_path('packages', package.package_name, package.package_name + '.json'), 'w') as packagefile: + with open( + frappe.get_site_path("packages", package.package_name, package.package_name + ".json"), "w" + ) as packagefile: packagefile.write(frappe.as_json(package.as_dict(no_nulls=True))) def make_tarfile(self, package): # make tarfile - filename = '{}.tar.gz'.format(self.name) - subprocess.check_output(['tar', 'czf', filename, package.package_name], - cwd=frappe.get_site_path('packages')) + filename = "{}.tar.gz".format(self.name) + subprocess.check_output( + ["tar", "czf", filename, package.package_name], cwd=frappe.get_site_path("packages") + ) # move file - subprocess.check_output(['mv', frappe.get_site_path('packages', filename), - frappe.get_site_path('public', 'files')]) + subprocess.check_output( + ["mv", frappe.get_site_path("packages", filename), frappe.get_site_path("public", "files")] + ) # make attachment - file = frappe.get_doc(dict( - doctype = 'File', - file_url = '/' + os.path.join('files', filename), - attached_to_doctype = self.doctype, - attached_to_name = self.name - )) + file = frappe.get_doc( + dict( + doctype="File", + file_url="/" + os.path.join("files", filename), + attached_to_doctype=self.doctype, + attached_to_name=self.name, + ) + ) file.flags.ignore_duplicate_entry_error = True file.insert() diff --git a/frappe/core/doctype/package_release/test_package_release.py b/frappe/core/doctype/package_release/test_package_release.py index 6a15e8625b..e7b680463d 100644 --- a/frappe/core/doctype/package_release/test_package_release.py +++ b/frappe/core/doctype/package_release/test_package_release.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestPackageRelease(unittest.TestCase): pass diff --git a/frappe/core/doctype/page/__init__.py b/frappe/core/doctype/page/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/page/__init__.py +++ b/frappe/core/doctype/page/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 894e180bb1..7185a25e01 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -1,72 +1,82 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import os -from frappe.model.document import Document + +import frappe +from frappe import _, conf, safe_decode from frappe.build import html_to_js_template -from frappe.model.utils import render_include -from frappe import conf, _, safe_decode +from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles from frappe.desk.form.meta import get_code_files_via_hooks, get_js from frappe.desk.utils import validate_route_conflict -from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles +from frappe.model.document import Document +from frappe.model.utils import render_include + class Page(Document): def autoname(self): """ - Creates a url friendly name for this page. - Will restrict the name to 30 characters, if there exists a similar name, - it will add name-1, name-2 etc. + Creates a url friendly name for this page. + Will restrict the name to 30 characters, if there exists a similar name, + it will add name-1, name-2 etc. """ from frappe.utils import cint - if (self.name and self.name.startswith('New Page')) or not self.name: - self.name = self.page_name.lower().replace('"','').replace("'",'').\ - replace(' ', '-')[:20] - if frappe.db.exists('Page',self.name): - cnt = frappe.db.sql("""select name from tabPage - where name like "%s-%%" order by name desc limit 1""" % self.name) + + if (self.name and self.name.startswith("New Page")) or not self.name: + self.name = self.page_name.lower().replace('"', "").replace("'", "").replace(" ", "-")[:20] + if frappe.db.exists("Page", self.name): + cnt = frappe.db.sql( + """select name from tabPage + where name like "%s-%%" order by name desc limit 1""" + % self.name + ) if cnt: - cnt = cint(cnt[0][0].split('-')[-1]) + 1 + cnt = cint(cnt[0][0].split("-")[-1]) + 1 else: cnt = 1 - self.name += '-' + str(cnt) + self.name += "-" + str(cnt) def validate(self): validate_route_conflict(self.doctype, self.name) - if self.is_new() and not getattr(conf,'developer_mode', 0): + if self.is_new() and not getattr(conf, "developer_mode", 0): frappe.throw(_("Not in Developer Mode")) - #setting ignore_permissions via update_setup_wizard_access (setup_wizard.py) - if frappe.session.user!="Administrator" and not self.flags.ignore_permissions: + # setting ignore_permissions via update_setup_wizard_access (setup_wizard.py) + if frappe.session.user != "Administrator" and not self.flags.ignore_permissions: frappe.throw(_("Only Administrator can edit")) # export def on_update(self): """ - Writes the .json for this page and if write_content is checked, - it will write out a .html file + Writes the .json for this page and if write_content is checked, + it will write out a .html file """ if self.flags.do_not_update_json: return from frappe.core.doctype.doctype.doctype import make_module_and_roles + make_module_and_roles(self, "roles") from frappe.modules.utils import export_module_json - path = export_module_json(self, self.standard=='Yes', self.module) + + path = export_module_json(self, self.standard == "Yes", self.module) if path: # js - if not os.path.exists(path + '.js'): - with open(path + '.js', 'w') as f: - f.write("""frappe.pages['%s'].on_page_load = function(wrapper) { + if not os.path.exists(path + ".js"): + with open(path + ".js", "w") as f: + f.write( + """frappe.pages['%s'].on_page_load = function(wrapper) { var page = frappe.ui.make_app_page({ parent: wrapper, title: '%s', single_column: true }); -}""" % (self.name, self.title)) +}""" + % (self.name, self.title) + ) def as_dict(self, no_nulls=False): d = super(Page, self).as_dict(no_nulls=no_nulls) @@ -75,16 +85,17 @@ class Page(Document): return d def on_trash(self): - delete_custom_role('page', self.name) + delete_custom_role("page", self.name) def is_permitted(self): """Returns true if Has Role is not set or the user is allowed.""" from frappe.utils import has_common - allowed = [d.role for d in frappe.get_all("Has Role", fields=["role"], - filters={"parent": self.name})] + allowed = [ + d.role for d in frappe.get_all("Has Role", fields=["role"], filters={"parent": self.name}) + ] - custom_roles = get_custom_allowed_roles('page', self.name) + custom_roles = get_custom_allowed_roles("page", self.name) allowed.extend(custom_roles) if not allowed: @@ -96,40 +107,42 @@ class Page(Document): return True def load_assets(self): - from frappe.modules import get_module_path, scrub import os - self.script = '' + + from frappe.modules import get_module_path, scrub + + self.script = "" page_name = scrub(self.name) - path = os.path.join(get_module_path(self.module), 'page', page_name) + path = os.path.join(get_module_path(self.module), "page", page_name) # script - fpath = os.path.join(path, page_name + '.js') + fpath = os.path.join(path, page_name + ".js") if os.path.exists(fpath): - with open(fpath, 'r') as f: + with open(fpath, "r") as f: self.script = render_include(f.read()) self.script += f"\n\n//# sourceURL={page_name}.js" # css - fpath = os.path.join(path, page_name + '.css') + fpath = os.path.join(path, page_name + ".css") if os.path.exists(fpath): - with open(fpath, 'r') as f: + with open(fpath, "r") as f: self.style = safe_decode(f.read()) # html as js template for fname in os.listdir(path): if fname.endswith(".html"): - with open(os.path.join(path, fname), 'r') as f: + with open(os.path.join(path, fname), "r") as f: template = f.read() if "" in template: context = frappe._dict({}) try: - out = frappe.get_attr("{app}.{module}.page.{page}.{page}.get_context".format( - app = frappe.local.module_app[scrub(self.module)], - module = scrub(self.module), - page = page_name - ))(context) + out = frappe.get_attr( + "{app}.{module}.page.{page}.{page}.get_context".format( + app=frappe.local.module_app[scrub(self.module)], module=scrub(self.module), page=page_name + ) + )(context) if out: context = out @@ -142,8 +155,9 @@ class Page(Document): # flag for not caching this page self._dynamic_page = True - if frappe.lang != 'en': + if frappe.lang != "en": from frappe.translate import get_lang_js + self.script += get_lang_js("page", self.name) for path in get_code_files_via_hooks("page_js", self.name): @@ -151,7 +165,8 @@ class Page(Document): if js: self.script += "\n\n" + js + def delete_custom_role(field, docname): - name = frappe.db.get_value('Custom Role', {field: docname}, "name") + name = frappe.db.get_value("Custom Role", {field: docname}, "name") if name: - frappe.delete_doc('Custom Role', name) + frappe.delete_doc("Custom Role", name) diff --git a/frappe/core/doctype/page/patches/drop_unused_pages.py b/frappe/core/doctype/page/patches/drop_unused_pages.py index 93b47cebcc..bdf6a81b10 100644 --- a/frappe/core/doctype/page/patches/drop_unused_pages.py +++ b/frappe/core/doctype/page/patches/drop_unused_pages.py @@ -1,5 +1,6 @@ import frappe + def execute(): - for name in ('desktop', 'space'): - frappe.delete_doc('Page', name) \ No newline at end of file + for name in ("desktop", "space"): + frappe.delete_doc("Page", name) diff --git a/frappe/core/doctype/page/test_page.py b/frappe/core/doctype/page/test_page.py index 7db32497a8..1eff54cad7 100644 --- a/frappe/core/doctype/page/test_page.py +++ b/frappe/core/doctype/page/test_page.py @@ -1,11 +1,19 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest -test_records = frappe.get_test_records('Page') +import frappe + +test_records = frappe.get_test_records("Page") + class TestPage(unittest.TestCase): def test_naming(self): - self.assertRaises(frappe.NameError, frappe.get_doc(dict(doctype='Page', page_name='DocType', module='Core')).insert) - self.assertRaises(frappe.NameError, frappe.get_doc(dict(doctype='Page', page_name='Settings', module='Core')).insert) + self.assertRaises( + frappe.NameError, + frappe.get_doc(dict(doctype="Page", page_name="DocType", module="Core")).insert, + ) + self.assertRaises( + frappe.NameError, + frappe.get_doc(dict(doctype="Page", page_name="Settings", module="Core")).insert, + ) diff --git a/frappe/core/doctype/patch_log/__init__.py b/frappe/core/doctype/patch_log/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/patch_log/__init__.py +++ b/frappe/core/doctype/patch_log/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/patch_log/patch_log.py b/frappe/core/doctype/patch_log/patch_log.py index 9a5da24e37..dbedbebdeb 100644 --- a/frappe/core/doctype/patch_log/patch_log.py +++ b/frappe/core/doctype/patch_log/patch_log.py @@ -4,8 +4,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class PatchLog(Document): - pass \ No newline at end of file + pass diff --git a/frappe/core/doctype/patch_log/test_patch_log.py b/frappe/core/doctype/patch_log/test_patch_log.py index df1ca16b22..521eaf5e41 100644 --- a/frappe/core/doctype/patch_log/test_patch_log.py +++ b/frappe/core/doctype/patch_log/test_patch_log.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Patch Log') + class TestPatchLog(unittest.TestCase): pass diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.py b/frappe/core/doctype/payment_gateway/payment_gateway.py index d0fa550ea1..c48fd340cd 100644 --- a/frappe/core/doctype/payment_gateway/payment_gateway.py +++ b/frappe/core/doctype/payment_gateway/payment_gateway.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class PaymentGateway(Document): - pass \ No newline at end of file + pass diff --git a/frappe/core/doctype/payment_gateway/test_payment_gateway.py b/frappe/core/doctype/payment_gateway/test_payment_gateway.py index e2ad081cfa..d40c7bbece 100644 --- a/frappe/core/doctype/payment_gateway/test_payment_gateway.py +++ b/frappe/core/doctype/payment_gateway/test_payment_gateway.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Payment Gateway') + class TestPaymentGateway(unittest.TestCase): pass diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index 2d1b026572..c3122fe52f 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -12,6 +12,7 @@ from frappe.model.document import Document from frappe.utils import gzip_compress, gzip_decompress from frappe.utils.background_jobs import enqueue + class PreparedReport(Document): def before_insert(self): self.status = "Queued" @@ -21,7 +22,6 @@ class PreparedReport(Document): enqueue(run_background, prepared_report=self.name, timeout=6000) - def run_background(prepared_report): instance = frappe.get_doc("Prepared Report", prepared_report) report = frappe.get_doc("Report", instance.ref_report_doctype) @@ -38,11 +38,7 @@ def run_background(prepared_report): if data: report.custom_columns = data["columns"] - result = generate_report_result( - report=report, - filters=instance.filters, - user=instance.owner - ) + result = generate_report_result(report=report, filters=instance.filters, user=instance.owner) create_json_gz_file(result["result"], "Prepared Report", instance.name) instance.status = "Completed" @@ -59,45 +55,50 @@ def run_background(prepared_report): frappe.publish_realtime( "report_generated", - { - "report_name": instance.report_name, - "name": instance.name - }, - user=frappe.session.user + {"report_name": instance.report_name, "name": instance.name}, + user=frappe.session.user, ) + @frappe.whitelist() def get_reports_in_queued_state(report_name, filters): - reports = frappe.get_all('Prepared Report', - filters = { - 'report_name': report_name, - 'filters': json.dumps(json.loads(filters)), - 'status': 'Queued' - }) + reports = frappe.get_all( + "Prepared Report", + filters={ + "report_name": report_name, + "filters": json.dumps(json.loads(filters)), + "status": "Queued", + }, + ) return reports + def delete_expired_prepared_reports(): - system_settings = frappe.get_single('System Settings') + system_settings = frappe.get_single("System Settings") enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion if enable_auto_deletion: expiry_period = system_settings.prepared_report_expiry_period - prepared_reports_to_delete = frappe.get_all('Prepared Report', - filters = { - 'creation': ['<', frappe.utils.add_days(frappe.utils.now(), -expiry_period)] - }) + prepared_reports_to_delete = frappe.get_all( + "Prepared Report", + filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]}, + ) batches = frappe.utils.create_batch(prepared_reports_to_delete, 100) for batch in batches: args = { - 'reports': batch, + "reports": batch, } enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args) + @frappe.whitelist() def delete_prepared_reports(reports): reports = frappe.parse_json(reports) for report in reports: - frappe.delete_doc('Prepared Report', report['name'], ignore_permissions=True, delete_permanently=True) + frappe.delete_doc( + "Prepared Report", report["name"], ignore_permissions=True, delete_permanently=True + ) + def create_json_gz_file(data, dt, dn): # Storing data in CSV file causes information loss @@ -109,14 +110,16 @@ def create_json_gz_file(data, dt, dn): compressed_content = gzip_compress(encoded_content) # Call save() file function to upload and attach the file - _file = frappe.get_doc({ - "doctype": "File", - "file_name": json_filename, - "attached_to_doctype": dt, - "attached_to_name": dn, - "content": compressed_content, - "is_private": 1 - }) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": json_filename, + "attached_to_doctype": dt, + "attached_to_name": dn, + "content": compressed_content, + "is_private": 1, + } + ) _file.save(ignore_permissions=True) @@ -130,11 +133,13 @@ def download_attachment(dn): def get_permission_query_condition(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user if user == "Administrator": return None from frappe.utils.user import UserPermissions + user = UserPermissions(user) if "System Manager" in user.roles: @@ -142,16 +147,19 @@ def get_permission_query_condition(user): reports = [frappe.db.escape(report) for report in user.get_all_reports().keys()] - return """`tabPrepared Report`.ref_report_doctype in ({reports})"""\ - .format(reports=','.join(reports)) + return """`tabPrepared Report`.ref_report_doctype in ({reports})""".format( + reports=",".join(reports) + ) def has_permission(doc, user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user if user == "Administrator": return True from frappe.utils.user import UserPermissions + user = UserPermissions(user) if "System Manager" in user.roles: diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py index 5b12990f64..86793cb802 100644 --- a/frappe/core/doctype/prepared_report/test_prepared_report.py +++ b/frappe/core/doctype/prepared_report/test_prepared_report.py @@ -1,32 +1,29 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -import unittest import json +import unittest + +import frappe class TestPreparedReport(unittest.TestCase): def setUp(self): - self.report = frappe.get_doc({ - "doctype": "Report", - "name": "Permitted Documents For User" - }) - self.filters = { - "user": "Administrator", - "doctype": "Role" - } - self.prepared_report_doc = frappe.get_doc({ - "doctype": "Prepared Report", - "report_name": self.report.name, - "filters": json.dumps(self.filters), - "ref_report_doctype": self.report.name - }).insert() + self.report = frappe.get_doc({"doctype": "Report", "name": "Permitted Documents For User"}) + self.filters = {"user": "Administrator", "doctype": "Role"} + self.prepared_report_doc = frappe.get_doc( + { + "doctype": "Prepared Report", + "report_name": self.report.name, + "filters": json.dumps(self.filters), + "ref_report_doctype": self.report.name, + } + ).insert() def tearDown(self): frappe.set_user("Administrator") self.prepared_report_doc.delete() def test_for_creation(self): - self.assertTrue('QUEUED' == self.prepared_report_doc.status.upper()) + self.assertTrue("QUEUED" == self.prepared_report_doc.status.upper()) self.assertTrue(self.prepared_report_doc.report_start_time) diff --git a/frappe/core/doctype/report/__init__.py b/frappe/core/doctype/report/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/report/__init__.py +++ b/frappe/core/doctype/report/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/report/boilerplate/controller.py b/frappe/core/doctype/report/boilerplate/controller.py index 72da0c7ce5..dd4339dd5f 100644 --- a/frappe/core/doctype/report/boilerplate/controller.py +++ b/frappe/core/doctype/report/boilerplate/controller.py @@ -3,6 +3,7 @@ # import frappe + def execute(filters=None): columns, data = [], [] return columns, data diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index bf82a3f684..9e6cc73f11 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -1,17 +1,19 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import datetime +import json + import frappe -import json, datetime -from frappe import _, scrub import frappe.desk.query_report -from frappe.utils import cint, cstr -from frappe.model.document import Document -from frappe.modules.export_file import export_to_files -from frappe.modules import make_boilerplate -from frappe.core.doctype.page.page import delete_custom_role +from frappe import _, scrub from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles +from frappe.core.doctype.page.page import delete_custom_role from frappe.desk.reportview import append_totals_row -from frappe.utils.safe_exec import safe_exec, check_safe_sql_query +from frappe.model.document import Document +from frappe.modules import make_boilerplate +from frappe.modules.export_file import export_to_files +from frappe.utils import cint, cstr +from frappe.utils.safe_exec import check_safe_sql_query, safe_exec class Report(Document): @@ -22,18 +24,20 @@ class Report(Document): if not self.is_standard: self.is_standard = "No" - if frappe.session.user=="Administrator" and getattr(frappe.local.conf, 'developer_mode',0)==1: + if ( + frappe.session.user == "Administrator" and getattr(frappe.local.conf, "developer_mode", 0) == 1 + ): self.is_standard = "Yes" if self.is_standard == "No": # allow only script manager to edit scripts - if self.report_type != 'Report Builder': - frappe.only_for('Script Manager', True) + if self.report_type != "Report Builder": + frappe.only_for("Script Manager", True) if frappe.db.get_value("Report", self.name, "is_standard") == "Yes": frappe.throw(_("Cannot edit a standard report. Please duplicate and create a new report")) - if self.is_standard == "Yes" and frappe.session.user!="Administrator": + if self.is_standard == "Yes" and frappe.session.user != "Administrator": frappe.throw(_("Only Administrator can save a standard report. Please rename and save.")) if self.report_type == "Report Builder": @@ -46,39 +50,45 @@ class Report(Document): self.export_doc() def on_trash(self): - if (self.is_standard == 'Yes' - and not cint(getattr(frappe.local.conf, 'developer_mode', 0)) - and not frappe.flags.in_patch): + if ( + self.is_standard == "Yes" + and not cint(getattr(frappe.local.conf, "developer_mode", 0)) + and not frappe.flags.in_patch + ): frappe.throw(_("You are not allowed to delete Standard Report")) - delete_custom_role('report', self.name) + delete_custom_role("report", self.name) self.delete_prepared_reports() def delete_prepared_reports(self): - prepared_reports = frappe.get_all("Prepared Report", filters={'ref_report_doctype': self.name}, pluck='name') + prepared_reports = frappe.get_all( + "Prepared Report", filters={"ref_report_doctype": self.name}, pluck="name" + ) for report in prepared_reports: - frappe.delete_doc("Prepared Report", report, ignore_missing=True, force=True, - delete_permanently=True) + frappe.delete_doc( + "Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True + ) def get_columns(self): return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns] @frappe.whitelist() def set_doctype_roles(self): - if not self.get('roles') and self.is_standard == 'No': + if not self.get("roles") and self.is_standard == "No": meta = frappe.get_meta(self.ref_doctype) if not meta.istable: - roles = [{'role': d.role} for d in meta.permissions if d.permlevel==0] - self.set('roles', roles) + roles = [{"role": d.role} for d in meta.permissions if d.permlevel == 0] + self.set("roles", roles) def is_permitted(self): """Returns true if Has Role is not set or the user is allowed.""" from frappe.utils import has_common - allowed = [d.role for d in frappe.get_all("Has Role", fields=["role"], - filters={"parent": self.name})] + allowed = [ + d.role for d in frappe.get_all("Has Role", fields=["role"], filters={"parent": self.name}) + ] - custom_roles = get_custom_allowed_roles('report', self.name) + custom_roles = get_custom_allowed_roles("report", self.name) allowed.extend(custom_roles) if not allowed: @@ -89,15 +99,16 @@ class Report(Document): def update_report_json(self): if not self.json: - self.json = '{}' + self.json = "{}" def export_doc(self): if frappe.flags.in_import: return - if self.is_standard == 'Yes' and (frappe.local.conf.get('developer_mode') or 0) == 1: - export_to_files(record_list=[['Report', self.name]], - record_module=self.module, create_init=True) + if self.is_standard == "Yes" and (frappe.local.conf.get("developer_mode") or 0) == 1: + export_to_files( + record_list=[["Report", self.name]], record_module=self.module, create_init=True + ) self.create_report_py() @@ -108,7 +119,7 @@ class Report(Document): def execute_query_report(self, filters): if not self.query: - frappe.throw(_("Must specify a Query to run"), title=_('Report Document Error')) + frappe.throw(_("Must specify a Query to run"), title=_("Report Document Error")) check_safe_sql_query(self.query) @@ -125,7 +136,7 @@ class Report(Document): start_time = datetime.datetime.now() # The JOB - if self.is_standard == 'Yes': + if self.is_standard == "Yes": res = self.execute_module(filters) else: res = self.execute_script(filters) @@ -133,9 +144,9 @@ class Report(Document): # automatically set as prepared execution_time = (datetime.datetime.now() - start_time).total_seconds() if execution_time > threshold and not self.prepared_report: - self.db_set('prepared_report', 1) + self.db_set("prepared_report", 1) - frappe.cache().hset('report_execution_time', self.name, execution_time) + frappe.cache().hset("report_execution_time", self.name, execution_time) return res @@ -147,15 +158,17 @@ class Report(Document): def execute_script(self, filters): # server script - loc = {"filters": frappe._dict(filters), 'data':None, 'result':None} + loc = {"filters": frappe._dict(filters), "data": None, "result": None} safe_exec(self.report_script, None, loc) - if loc['data']: - return loc['data'] + if loc["data"]: + return loc["data"] else: - return self.get_columns(), loc['result'] + return self.get_columns(), loc["result"] - def get_data(self, filters=None, limit=None, user=None, as_dict=False, ignore_prepared_report=False): - if self.report_type in ('Query Report', 'Script Report', 'Custom Report'): + def get_data( + self, filters=None, limit=None, user=None, as_dict=False, ignore_prepared_report=False + ): + if self.report_type in ("Query Report", "Script Report", "Custom Report"): columns, result = self.run_query_report(filters, user, ignore_prepared_report) else: columns, result = self.run_standard_report(filters, limit, user) @@ -167,10 +180,11 @@ class Report(Document): def run_query_report(self, filters, user, ignore_prepared_report=False): columns, result = [], [] - data = frappe.desk.query_report.run(self.name, - filters=filters, user=user, ignore_prepared_report=ignore_prepared_report) + data = frappe.desk.query_report.run( + self.name, filters=filters, user=user, ignore_prepared_report=ignore_prepared_report + ) - for d in data.get('columns'): + for d in data.get("columns"): if isinstance(d, dict): col = frappe._dict(d) if not col.fieldname: @@ -178,16 +192,18 @@ class Report(Document): columns.append(col) else: fieldtype, options = "Data", None - parts = d.split(':') + parts = d.split(":") if len(parts) > 1: if parts[1]: fieldtype, options = parts[1], None - if fieldtype and '/' in fieldtype: - fieldtype, options = fieldtype.split('/') + if fieldtype and "/" in fieldtype: + fieldtype, options = fieldtype.split("/") - columns.append(frappe._dict(label=parts[0], fieldtype=fieldtype, fieldname=parts[0], options=options)) + columns.append( + frappe._dict(label=parts[0], fieldtype=fieldtype, fieldname=parts[0], options=options) + ) - result += data.get('result') + result += data.get("result") return columns, result @@ -197,23 +213,27 @@ class Report(Document): result = [] order_by, group_by, group_by_args = self.get_standard_report_order_by(params) - _result = frappe.get_list(self.ref_doctype, - fields = [ - get_group_by_field(group_by_args, c[1]) if c[0] == '_aggregate_column' and group_by_args - else Report._format([c[1], c[0]]) for c in columns + _result = frappe.get_list( + self.ref_doctype, + fields=[ + get_group_by_field(group_by_args, c[1]) + if c[0] == "_aggregate_column" and group_by_args + else Report._format([c[1], c[0]]) + for c in columns ], - filters = self.get_standard_report_filters(params, filters), - order_by = order_by, - group_by = group_by, - as_list = True, - limit = limit, - user = user) + filters=self.get_standard_report_filters(params, filters), + order_by=order_by, + group_by=group_by, + as_list=True, + limit=limit, + user=user, + ) columns = self.build_standard_report_columns(columns, group_by_args) result = result + [list(d) for d in _result] - if params.get('add_totals_row'): + if params.get("add_totals_row"): result = append_totals_row(result) return columns, result @@ -221,17 +241,17 @@ class Report(Document): @staticmethod def _format(parts): # sort by is saved as DocType.fieldname, covert it to sql - return '`tab{0}`.`{1}`'.format(*parts) + return "`tab{0}`.`{1}`".format(*parts) def get_standard_report_columns(self, params): - if params.get('fields'): - columns = params.get('fields') - elif params.get('columns'): - columns = params.get('columns') - elif params.get('fields'): - columns = params.get('fields') + if params.get("fields"): + columns = params.get("fields") + elif params.get("columns"): + columns = params.get("columns") + elif params.get("fields"): + columns = params.get("fields") else: - columns = [['name', self.ref_doctype]] + columns = [["name", self.ref_doctype]] for df in frappe.get_meta(self.ref_doctype).fields: if df.in_list_view: columns.append([df.fieldname, self.ref_doctype]) @@ -239,11 +259,11 @@ class Report(Document): return columns def get_standard_report_filters(self, params, filters): - _filters = params.get('filters') or [] + _filters = params.get("filters") or [] if filters: for key, value in filters.items(): - condition, _value = '=', value + condition, _value = "=", value if isinstance(value, (list, tuple)): condition, _value = value _filters.append([key, condition, _value]) @@ -252,22 +272,27 @@ class Report(Document): def get_standard_report_order_by(self, params): group_by_args = None - if params.get('sort_by'): - order_by = Report._format(params.get('sort_by').split('.')) + ' ' + params.get('sort_order') + if params.get("sort_by"): + order_by = Report._format(params.get("sort_by").split(".")) + " " + params.get("sort_order") - elif params.get('order_by'): - order_by = params.get('order_by') + elif params.get("order_by"): + order_by = params.get("order_by") else: - order_by = Report._format([self.ref_doctype, 'modified']) + ' desc' + order_by = Report._format([self.ref_doctype, "modified"]) + " desc" - if params.get('sort_by_next'): - order_by += ', ' + Report._format(params.get('sort_by_next').split('.')) + ' ' + params.get('sort_order_next') + if params.get("sort_by_next"): + order_by += ( + ", " + + Report._format(params.get("sort_by_next").split(".")) + + " " + + params.get("sort_order_next") + ) group_by = None - if params.get('group_by'): - group_by_args = frappe._dict(params['group_by']) - group_by = group_by_args['group_by'] - order_by = '_aggregate_column desc' + if params.get("group_by"): + group_by_args = frappe._dict(params["group_by"]) + group_by = group_by_args["group_by"] + order_by = "_aggregate_column desc" return order_by, group_by, group_by_args @@ -280,7 +305,7 @@ class Report(Document): if meta.get_field(fieldname): field = meta.get_field(fieldname) else: - if fieldname == '_aggregate_column': + if fieldname == "_aggregate_column": label = get_group_by_column_label(group_by_args, meta) else: label = meta.get_label(fieldname) @@ -301,7 +326,7 @@ class Report(Document): if isinstance(row, (list, tuple)): _row = frappe._dict() for i, val in enumerate(row): - _row[columns[i].get('fieldname')] = val + _row[columns[i].get("fieldname")] = val elif isinstance(row, dict): # no need to convert from dict to dict _row = frappe._dict(row) @@ -311,42 +336,47 @@ class Report(Document): @frappe.whitelist() def toggle_disable(self, disable): - if not self.has_permission('write'): + if not self.has_permission("write"): frappe.throw(_("You are not allowed to edit the report.")) self.db_set("disabled", cint(disable)) + @frappe.whitelist() def is_prepared_report_disabled(report): - return frappe.db.get_value('Report', - report, 'disable_prepared_report') or 0 + return frappe.db.get_value("Report", report, "disable_prepared_report") or 0 + def get_report_module_dotted_path(module, report_name): - return frappe.local.module_app[scrub(module)] + "." + scrub(module) \ - + ".report." + scrub(report_name) + "." + scrub(report_name) + return ( + frappe.local.module_app[scrub(module)] + + "." + + scrub(module) + + ".report." + + scrub(report_name) + + "." + + scrub(report_name) + ) + def get_group_by_field(args, doctype): - if args['aggregate_function'] == 'count': - group_by_field = 'count(*) as _aggregate_column' + if args["aggregate_function"] == "count": + group_by_field = "count(*) as _aggregate_column" else: - group_by_field = '{0}({1}) as _aggregate_column'.format( - args.aggregate_function, - args.aggregate_on + group_by_field = "{0}({1}) as _aggregate_column".format( + args.aggregate_function, args.aggregate_on ) return group_by_field + def get_group_by_column_label(args, meta): - if args['aggregate_function'] == 'count': - label = 'Count' + if args["aggregate_function"] == "count": + label = "Count" else: - sql_fn_map = { - 'avg': 'Average', - 'sum': 'Sum' - } + sql_fn_map = {"avg": "Average", "sum": "Sum"} aggregate_on_label = meta.get_label(args.aggregate_on) - label = _('{function} of {fieldlabel}').format( - function=sql_fn_map[args.aggregate_function], - fieldlabel = aggregate_on_label + label = _("{function} of {fieldlabel}").format( + function=sql_fn_map[args.aggregate_function], fieldlabel=aggregate_on_label ) return label diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index e58d038993..7b17a5a8d5 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -1,50 +1,56 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json +import os import textwrap -import frappe, json, os -from frappe.desk.query_report import run, save_report, add_total_row -from frappe.desk.reportview import delete_report, save_report as _save_report -from frappe.custom.doctype.customize_form.customize_form import reset_customization +import frappe from frappe.core.doctype.user_permission.test_user_permission import create_user +from frappe.custom.doctype.customize_form.customize_form import reset_customization +from frappe.desk.query_report import add_total_row, run, save_report +from frappe.desk.reportview import delete_report +from frappe.desk.reportview import save_report as _save_report from frappe.tests.utils import FrappeTestCase -test_records = frappe.get_test_records('Report') -test_dependencies = ['User'] +test_records = frappe.get_test_records("Report") +test_dependencies = ["User"] + class TestReport(FrappeTestCase): def test_report_builder(self): - if frappe.db.exists('Report', 'User Activity Report'): - frappe.delete_doc('Report', 'User Activity Report') + if frappe.db.exists("Report", "User Activity Report"): + frappe.delete_doc("Report", "User Activity Report") - with open(os.path.join(os.path.dirname(__file__), 'user_activity_report.json'), 'r') as f: + with open(os.path.join(os.path.dirname(__file__), "user_activity_report.json"), "r") as f: frappe.get_doc(json.loads(f.read())).insert() - report = frappe.get_doc('Report', 'User Activity Report') + report = frappe.get_doc("Report", "User Activity Report") columns, data = report.get_data() - self.assertEqual(columns[0].get('label'), 'ID') - self.assertEqual(columns[1].get('label'), 'User Type') - self.assertTrue('Administrator' in [d[0] for d in data]) + self.assertEqual(columns[0].get("label"), "ID") + self.assertEqual(columns[1].get("label"), "User Type") + self.assertTrue("Administrator" in [d[0] for d in data]) def test_query_report(self): - report = frappe.get_doc('Report', 'Permitted Documents For User') - columns, data = report.get_data(filters={'user': 'Administrator', 'doctype': 'DocType'}) - self.assertEqual(columns[0].get('label'), 'Name') - self.assertEqual(columns[1].get('label'), 'Module') - self.assertTrue('User' in [d.get('name') for d in data]) + report = frappe.get_doc("Report", "Permitted Documents For User") + columns, data = report.get_data(filters={"user": "Administrator", "doctype": "DocType"}) + self.assertEqual(columns[0].get("label"), "Name") + self.assertEqual(columns[1].get("label"), "Module") + self.assertTrue("User" in [d.get("name") for d in data]) def test_save_or_delete_report(self): - '''Test for validations when editing / deleting report of type Report Builder''' + """Test for validations when editing / deleting report of type Report Builder""" try: - report = frappe.get_doc({ - 'doctype': 'Report', - 'ref_doctype': 'User', - 'report_name': 'Test Delete Report', - 'report_type': 'Report Builder', - 'is_standard': 'No', - }).insert() + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": "Test Delete Report", + "report_type": "Report Builder", + "is_standard": "No", + } + ).insert() # Check for PermissionError create_user("test_report_owner@example.com", "Website Manager") @@ -58,26 +64,30 @@ class TestReport(FrappeTestCase): frappe.ValidationError, "Only reports of type Report Builder can be deleted", delete_report, - report.name + report.name, ) # Check if creating and deleting works with proper validations frappe.set_user("test@example.com") report_name = _save_report( - 'Dummy Report', - 'User', - json.dumps([{ - 'fieldname': 'email', - 'fieldtype': 'Data', - 'label': 'Email', - 'insert_after_index': 0, - 'link_field': 'name', - 'doctype': 'User', - 'options': 'Email', - 'width': 100, - 'id':'email', - 'name': 'Email' - }]) + "Dummy Report", + "User", + json.dumps( + [ + { + "fieldname": "email", + "fieldtype": "Data", + "label": "Email", + "insert_after_index": 0, + "link_field": "name", + "doctype": "User", + "options": "Email", + "width": 100, + "id": "email", + "name": "Email", + } + ] + ), ) doc = frappe.get_doc("Report", report_name) @@ -87,116 +97,128 @@ class TestReport(FrappeTestCase): frappe.set_user("Administrator") frappe.db.rollback() - def test_custom_report(self): - reset_customization('User') + reset_customization("User") custom_report_name = save_report( - 'Permitted Documents For User', - 'Permitted Documents For User Custom', - json.dumps([{ - 'fieldname': 'email', - 'fieldtype': 'Data', - 'label': 'Email', - 'insert_after_index': 0, - 'link_field': 'name', - 'doctype': 'User', - 'options': 'Email', - 'width': 100, - 'id':'email', - 'name': 'Email' - }])) - custom_report = frappe.get_doc('Report', custom_report_name) + "Permitted Documents For User", + "Permitted Documents For User Custom", + json.dumps( + [ + { + "fieldname": "email", + "fieldtype": "Data", + "label": "Email", + "insert_after_index": 0, + "link_field": "name", + "doctype": "User", + "options": "Email", + "width": 100, + "id": "email", + "name": "Email", + } + ] + ), + ) + custom_report = frappe.get_doc("Report", custom_report_name) columns, result = custom_report.run_query_report( - filters={ - 'user': 'Administrator', - 'doctype': 'User' - }, user=frappe.session.user) + filters={"user": "Administrator", "doctype": "User"}, user=frappe.session.user + ) - self.assertListEqual(['email'], [column.get('fieldname') for column in columns]) - admin_dict = frappe.core.utils.find(result, lambda d: d['name'] == 'Administrator') - self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) + self.assertListEqual(["email"], [column.get("fieldname") for column in columns]) + admin_dict = frappe.core.utils.find(result, lambda d: d["name"] == "Administrator") + self.assertDictEqual( + {"name": "Administrator", "user_type": "System User", "email": "admin@example.com"}, admin_dict + ) def test_report_with_custom_column(self): - reset_customization('User') - response = run('Permitted Documents For User', - filters={'user': 'Administrator', 'doctype': 'User'}, - custom_columns=[{ - 'fieldname': 'email', - 'fieldtype': 'Data', - 'label': 'Email', - 'insert_after_index': 0, - 'link_field': 'name', - 'doctype': 'User', - 'options': 'Email', - 'width': 100, - 'id':'email', - 'name': 'Email' - }]) - result = response.get('result') - columns = response.get('columns') - self.assertListEqual(['name', 'email', 'user_type'], [column.get('fieldname') for column in columns]) - admin_dict = frappe.core.utils.find(result, lambda d: d['name'] == 'Administrator') - self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) + reset_customization("User") + response = run( + "Permitted Documents For User", + filters={"user": "Administrator", "doctype": "User"}, + custom_columns=[ + { + "fieldname": "email", + "fieldtype": "Data", + "label": "Email", + "insert_after_index": 0, + "link_field": "name", + "doctype": "User", + "options": "Email", + "width": 100, + "id": "email", + "name": "Email", + } + ], + ) + result = response.get("result") + columns = response.get("columns") + self.assertListEqual( + ["name", "email", "user_type"], [column.get("fieldname") for column in columns] + ) + admin_dict = frappe.core.utils.find(result, lambda d: d["name"] == "Administrator") + self.assertDictEqual( + {"name": "Administrator", "user_type": "System User", "email": "admin@example.com"}, admin_dict + ) def test_report_permissions(self): - frappe.set_user('test@example.com') - frappe.db.delete("Has Role", { - "parent": frappe.session.user, - "role": "Test Has Role" - }) + frappe.set_user("test@example.com") + frappe.db.delete("Has Role", {"parent": frappe.session.user, "role": "Test Has Role"}) frappe.db.commit() - if not frappe.db.exists('Role', 'Test Has Role'): - role = frappe.get_doc({ - 'doctype': 'Role', - 'role_name': 'Test Has Role' - }).insert(ignore_permissions=True) + if not frappe.db.exists("Role", "Test Has Role"): + role = frappe.get_doc({"doctype": "Role", "role_name": "Test Has Role"}).insert( + ignore_permissions=True + ) if not frappe.db.exists("Report", "Test Report"): - report = frappe.get_doc({ - 'doctype': 'Report', - 'ref_doctype': 'User', - 'report_name': 'Test Report', - 'report_type': 'Query Report', - 'is_standard': 'No', - 'roles': [ - {'role': 'Test Has Role'} - ] - }).insert(ignore_permissions=True) + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": "Test Report", + "report_type": "Query Report", + "is_standard": "No", + "roles": [{"role": "Test Has Role"}], + } + ).insert(ignore_permissions=True) else: - report = frappe.get_doc('Report', 'Test Report') + report = frappe.get_doc("Report", "Test Report") self.assertNotEqual(report.is_permitted(), True) - frappe.set_user('Administrator') + frappe.set_user("Administrator") # test for the `_format` method if report data doesn't have sort_by parameter def test_format_method(self): - if frappe.db.exists('Report', 'User Activity Report Without Sort'): - frappe.delete_doc('Report', 'User Activity Report Without Sort') - with open(os.path.join(os.path.dirname(__file__), 'user_activity_report_without_sort.json'), 'r') as f: + if frappe.db.exists("Report", "User Activity Report Without Sort"): + frappe.delete_doc("Report", "User Activity Report Without Sort") + with open( + os.path.join(os.path.dirname(__file__), "user_activity_report_without_sort.json"), "r" + ) as f: frappe.get_doc(json.loads(f.read())).insert() - report = frappe.get_doc('Report', 'User Activity Report Without Sort') + report = frappe.get_doc("Report", "User Activity Report Without Sort") columns, data = report.get_data() - self.assertEqual(columns[0].get('label'), 'ID') - self.assertEqual(columns[1].get('label'), 'User Type') - self.assertTrue('Administrator' in [d[0] for d in data]) - frappe.delete_doc('Report', 'User Activity Report Without Sort') + self.assertEqual(columns[0].get("label"), "ID") + self.assertEqual(columns[1].get("label"), "User Type") + self.assertTrue("Administrator" in [d[0] for d in data]) + frappe.delete_doc("Report", "User Activity Report Without Sort") def test_non_standard_script_report(self): - report_name = 'Test Non Standard Script Report' + report_name = "Test Non Standard Script Report" if not frappe.db.exists("Report", report_name): - report = frappe.get_doc({ - 'doctype': 'Report', - 'ref_doctype': 'User', - 'report_name': report_name, - 'report_type': 'Script Report', - 'is_standard': 'No', - }).insert(ignore_permissions=True) + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": report_name, + "report_type": "Script Report", + "is_standard": "No", + } + ).insert(ignore_permissions=True) else: - report = frappe.get_doc('Report', report_name) + report = frappe.get_doc("Report", report_name) - report.report_script = ''' + report.report_script = """ totals = {} for user in frappe.get_all('User', fields = ['name', 'user_type', 'creation']): if not user.user_type in totals: @@ -212,35 +234,37 @@ data = [ {"type":key, "value": value} for key, value in totals.items() ] ] -''' +""" report.save() data = report.get_data() # check columns - self.assertEqual(data[0][0]['label'], 'Type') + self.assertEqual(data[0][0]["label"], "Type") # check values - self.assertTrue('System User' in [d.get('type') for d in data[1]]) + self.assertTrue("System User" in [d.get("type") for d in data[1]]) def test_script_report_with_columns(self): - report_name = 'Test Script Report With Columns' + report_name = "Test Script Report With Columns" if frappe.db.exists("Report", report_name): - frappe.delete_doc('Report', report_name) - - report = frappe.get_doc({ - 'doctype': 'Report', - 'ref_doctype': 'User', - 'report_name': report_name, - 'report_type': 'Script Report', - 'is_standard': 'No', - 'columns': [ - dict(fieldname='type', label='Type', fieldtype='Data'), - dict(fieldname='value', label='Value', fieldtype='Int'), - ] - }).insert(ignore_permissions=True) - - report.report_script = ''' + frappe.delete_doc("Report", report_name) + + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": report_name, + "report_type": "Script Report", + "is_standard": "No", + "columns": [ + dict(fieldname="type", label="Type", fieldtype="Data"), + dict(fieldname="value", label="Value", fieldtype="Int"), + ], + } + ).insert(ignore_permissions=True) + + report.report_script = """ totals = {} for user in frappe.get_all('User', fields = ['name', 'user_type', 'creation']): if not user.user_type in totals: @@ -250,112 +274,88 @@ for user in frappe.get_all('User', fields = ['name', 'user_type', 'creation']): result = [ {"type":key, "value": value} for key, value in totals.items() ] -''' +""" report.save() data = report.get_data() # check columns - self.assertEqual(data[0][0]['label'], 'Type') + self.assertEqual(data[0][0]["label"], "Type") # check values - self.assertTrue('System User' in [d.get('type') for d in data[1]]) + self.assertTrue("System User" in [d.get("type") for d in data[1]]) def test_toggle_disabled(self): - """Make sure that authorization is respected. - """ + """Make sure that authorization is respected.""" # Assuming that there will be reports in the system. - reports = frappe.get_all(doctype='Report', limit=1) - report_name = reports[0]['name'] - doc = frappe.get_doc('Report', report_name) + reports = frappe.get_all(doctype="Report", limit=1) + report_name = reports[0]["name"] + doc = frappe.get_doc("Report", report_name) status = doc.disabled # User has write permission on reports and should pass through - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") doc.toggle_disable(not status) doc.reload() self.assertNotEqual(status, doc.disabled) # User has no write permission on reports, permission error is expected. - frappe.set_user('test1@example.com') - doc = frappe.get_doc('Report', report_name) + frappe.set_user("test1@example.com") + doc = frappe.get_doc("Report", report_name) with self.assertRaises(frappe.exceptions.ValidationError): doc.toggle_disable(1) # Set user back to administrator - frappe.set_user('Administrator') + frappe.set_user("Administrator") def test_add_total_row_for_tree_reports(self): - report_settings = { - 'tree': True, - 'parent_field': 'parent_value' - } + report_settings = {"tree": True, "parent_field": "parent_value"} columns = [ - { - "fieldname": "parent_column", - "label": "Parent Column", - "fieldtype": "Data", - "width": 10 - }, - { - "fieldname": "column_1", - "label": "Column 1", - "fieldtype": "Float", - "width": 10 - }, - { - "fieldname": "column_2", - "label": "Column 2", - "fieldtype": "Float", - "width": 10 - } + {"fieldname": "parent_column", "label": "Parent Column", "fieldtype": "Data", "width": 10}, + {"fieldname": "column_1", "label": "Column 1", "fieldtype": "Float", "width": 10}, + {"fieldname": "column_2", "label": "Column 2", "fieldtype": "Float", "width": 10}, ] result = [ - { - "parent_column": "Parent 1", - "column_1": 200, - "column_2": 150.50 - }, - { - "parent_column": "Child 1", - "column_1": 100, - "column_2": 75.25, - "parent_value": "Parent 1" - }, - { - "parent_column": "Child 2", - "column_1": 100, - "column_2": 75.25, - "parent_value": "Parent 1" - } + {"parent_column": "Parent 1", "column_1": 200, "column_2": 150.50}, + {"parent_column": "Child 1", "column_1": 100, "column_2": 75.25, "parent_value": "Parent 1"}, + {"parent_column": "Child 2", "column_1": 100, "column_2": 75.25, "parent_value": "Parent 1"}, ] - result = add_total_row(result, columns, meta=None, is_tree=report_settings['tree'], - parent_field=report_settings['parent_field']) + result = add_total_row( + result, + columns, + meta=None, + is_tree=report_settings["tree"], + parent_field=report_settings["parent_field"], + ) self.assertEqual(result[-1][0], "Total") self.assertEqual(result[-1][1], 200) self.assertEqual(result[-1][2], 150.50) def test_cte_in_query_report(self): - cte_query = textwrap.dedent(""" + cte_query = textwrap.dedent( + """ with enabled_users as ( select name from `tabUser` where enabled = 1 ) select * from enabled_users; - """) - - report = frappe.get_doc({ - "doctype": "Report", - "ref_doctype": "User", - "report_name": "Enabled Users List", - "report_type": "Query Report", - "is_standard": "No", - "query": cte_query, - }).insert() + """ + ) + + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": "Enabled Users List", + "report_type": "Query Report", + "is_standard": "No", + "query": cte_query, + } + ).insert() if frappe.db.db_type == "mariadb": col, rows = report.execute_query_report(filters={}) diff --git a/frappe/core/doctype/report_column/report_column.py b/frappe/core/doctype/report_column/report_column.py index 3b2c1e130b..c0984a5ca8 100644 --- a/frappe/core/doctype/report_column/report_column.py +++ b/frappe/core/doctype/report_column/report_column.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ReportColumn(Document): pass diff --git a/frappe/core/doctype/report_filter/report_filter.py b/frappe/core/doctype/report_filter/report_filter.py index b325985308..e35d7064d2 100644 --- a/frappe/core/doctype/report_filter/report_filter.py +++ b/frappe/core/doctype/report_filter/report_filter.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ReportFilter(Document): pass diff --git a/frappe/core/doctype/role/__init__.py b/frappe/core/doctype/role/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/doctype/role/__init__.py +++ b/frappe/core/doctype/role/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py b/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py index dc17526047..87de6ac79a 100644 --- a/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py +++ b/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py @@ -1,11 +1,13 @@ import frappe + from ..role import desk_properties + def execute(): - frappe.reload_doctype('user') - frappe.reload_doctype('role') - for role in frappe.get_all('Role', ['name', 'desk_access']): - role_doc = frappe.get_doc('Role', role.name) + frappe.reload_doctype("user") + frappe.reload_doctype("role") + for role in frappe.get_all("Role", ["name", "desk_access"]): + role_doc = frappe.get_doc("Role", role.name) for key in desk_properties: role_doc.set(key, role_doc.desk_access) role_doc.save() diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index f955c29462..7092004eaf 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -4,24 +4,27 @@ import frappe from frappe.model.document import Document -desk_properties = ("search_bar", "notifications", "list_sidebar", - "bulk_actions", "view_switcher", "form_sidebar", "timeline", "dashboard") - -STANDARD_ROLES = ( - "Administrator", - "System Manager", - "Script Manager", - "All", - "Guest" +desk_properties = ( + "search_bar", + "notifications", + "list_sidebar", + "bulk_actions", + "view_switcher", + "form_sidebar", + "timeline", + "dashboard", ) +STANDARD_ROLES = ("Administrator", "System Manager", "Script Manager", "All", "Guest") + + class Role(Document): def before_rename(self, old, new, merge=False): if old in STANDARD_ROLES: frappe.throw(frappe._("Standard roles cannot be renamed")) def after_insert(self): - frappe.cache().hdel('roles', 'Administrator') + frappe.cache().hdel("roles", "Administrator") def validate(self): if self.disabled: @@ -37,7 +40,7 @@ class Role(Document): def set_desk_properties(self): # set if desk_access is not allowed, unset all desk properties - if self.name == 'Guest': + if self.name == "Guest": self.desk_access = 0 if not self.desk_access: @@ -49,25 +52,29 @@ class Role(Document): frappe.clear_cache() def on_update(self): - '''update system user desk access if this has changed in this update''' - if frappe.flags.in_install: return - if self.has_value_changed('desk_access'): + """update system user desk access if this has changed in this update""" + if frappe.flags.in_install: + return + if self.has_value_changed("desk_access"): for user_name in get_users(self.name): - user = frappe.get_doc('User', user_name) + user = frappe.get_doc("User", user_name) user_type = user.user_type user.set_system_user() if user_type != user.user_type: user.save() -def get_info_based_on_role(role, field='email'): - ''' Get information of all users that have been assigned this role ''' - users = frappe.get_list("Has Role", filters={"role": role}, parent_doctype="User", - fields=["parent as user_name"]) + +def get_info_based_on_role(role, field="email"): + """Get information of all users that have been assigned this role""" + users = frappe.get_list( + "Has Role", filters={"role": role}, parent_doctype="User", fields=["parent as user_name"] + ) return get_user_info(users, field) -def get_user_info(users, field='email'): - ''' Fetch details about users for the specified field ''' + +def get_user_info(users, field="email"): + """Fetch details about users for the specified field""" info_list = [] for user in users: user_info, enabled = frappe.db.get_value("User", user.get("user_name"), [field, "enabled"]) @@ -75,18 +82,24 @@ def get_user_info(users, field='email'): info_list.append(user_info) return info_list + def get_users(role): - return [d.parent for d in frappe.get_all("Has Role", filters={"role": role, "parenttype": "User"}, - fields=["parent"])] + return [ + d.parent + for d in frappe.get_all( + "Has Role", filters={"role": role, "parenttype": "User"}, fields=["parent"] + ) + ] # searches for active employees @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def role_query(doctype, txt, searchfield, start, page_len, filters): - report_filters = [['Role', 'name', 'like', '%{}%'.format(txt)], ['Role', 'is_custom', '=', 0]] + report_filters = [["Role", "name", "like", "%{}%".format(txt)], ["Role", "is_custom", "=", 0]] if filters and isinstance(filters, list): report_filters.extend(filters) - return frappe.get_all('Role', limit_start=start, limit_page_length=page_len, - filters=report_filters, as_list=1) + return frappe.get_all( + "Role", limit_start=start, limit_page_length=page_len, filters=report_filters, as_list=1 + ) diff --git a/frappe/core/doctype/role/test_role.py b/frappe/core/doctype/role/test_role.py index 1671f9a9c8..a94796436d 100644 --- a/frappe/core/doctype/role/test_role.py +++ b/frappe/core/doctype/role/test_role.py @@ -1,9 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest -test_records = frappe.get_test_records('Role') +import frappe + +test_records = frappe.get_test_records("Role") + class TestUser(unittest.TestCase): def test_disable_role(self): @@ -23,26 +25,21 @@ class TestUser(unittest.TestCase): self.assertTrue("_Test Role 3" in frappe.get_roles("test@example.com")) def test_change_desk_access(self): - '''if we change desk acecss from role, remove from user''' - frappe.delete_doc_if_exists('User', 'test-user-for-desk-access@example.com') - frappe.delete_doc_if_exists('Role', 'desk-access-test') - user = frappe.get_doc(dict( - doctype='User', - email='test-user-for-desk-access@example.com', - first_name='test')).insert() - role = frappe.get_doc(dict( - doctype = 'Role', - role_name = 'desk-access-test', - desk_access = 0 - )).insert() + """if we change desk acecss from role, remove from user""" + frappe.delete_doc_if_exists("User", "test-user-for-desk-access@example.com") + frappe.delete_doc_if_exists("Role", "desk-access-test") + user = frappe.get_doc( + dict(doctype="User", email="test-user-for-desk-access@example.com", first_name="test") + ).insert() + role = frappe.get_doc(dict(doctype="Role", role_name="desk-access-test", desk_access=0)).insert() user.add_roles(role.name) user.save() - self.assertTrue(user.user_type=='Website User') + self.assertTrue(user.user_type == "Website User") role.desk_access = 1 role.save() user.reload() - self.assertTrue(user.user_type=='System User') + self.assertTrue(user.user_type == "System User") role.desk_access = 0 role.save() user.reload() - self.assertTrue(user.user_type=='Website User') + self.assertTrue(user.user_type == "Website User") diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py index cd9a6dc0fa..d89131b0d7 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py @@ -6,6 +6,7 @@ import frappe from frappe.core.doctype.report.report import is_prepared_report_disabled from frappe.model.document import Document + class RolePermissionforPageandReport(Document): @frappe.whitelist() def set_report_page_data(self): @@ -14,16 +15,16 @@ class RolePermissionforPageandReport(Document): def set_custom_roles(self): args = self.get_args() - self.set('roles', []) + self.set("roles", []) - name = frappe.db.get_value('Custom Role', args, "name") + name = frappe.db.get_value("Custom Role", args, "name") if name: - doc = frappe.get_doc('Custom Role', name) + doc = frappe.get_doc("Custom Role", name) roles = doc.roles else: roles = self.get_standard_roles() - self.set('roles', roles) + self.set("roles", roles) def check_prepared_report_disabled(self): if self.report: @@ -31,14 +32,14 @@ class RolePermissionforPageandReport(Document): def get_standard_roles(self): doctype = self.set_role_for - docname = self.page if self.set_role_for == 'Page' else self.report + docname = self.page if self.set_role_for == "Page" else self.report doc = frappe.get_doc(doctype, docname) return doc.roles @frappe.whitelist() def reset_roles(self): roles = self.get_standard_roles() - self.set('roles', roles) + self.set("roles", roles) self.update_custom_roles() self.update_disable_prepared_report() @@ -49,19 +50,16 @@ class RolePermissionforPageandReport(Document): def update_custom_roles(self): args = self.get_args() - name = frappe.db.get_value('Custom Role', args, "name") + name = frappe.db.get_value("Custom Role", args, "name") - args.update({ - 'doctype': 'Custom Role', - 'roles': self.get_roles() - }) + args.update({"doctype": "Custom Role", "roles": self.get_roles()}) if self.report: - args.update({'ref_doctype': frappe.db.get_value('Report', self.report, 'ref_doctype')}) + args.update({"ref_doctype": frappe.db.get_value("Report", self.report, "ref_doctype")}) if name: custom_role = frappe.get_doc("Custom Role", name) - custom_role.set('roles', self.get_roles()) + custom_role.set("roles", self.get_roles()) custom_role.save() else: frappe.get_doc(args).insert() @@ -69,25 +67,23 @@ class RolePermissionforPageandReport(Document): def update_disable_prepared_report(self): if self.report: # intentionally written update query in frappe.db.sql instead of frappe.db.set_value - frappe.db.sql(""" update `tabReport` set disable_prepared_report = %s - where name = %s""", (self.disable_prepared_report, self.report)) + frappe.db.sql( + """ update `tabReport` set disable_prepared_report = %s + where name = %s""", + (self.disable_prepared_report, self.report), + ) def get_args(self, row=None): - name = self.page if self.set_role_for == 'Page' else self.report - check_for_field = self.set_role_for.replace(" ","_").lower() + name = self.page if self.set_role_for == "Page" else self.report + check_for_field = self.set_role_for.replace(" ", "_").lower() - return { - check_for_field: name - } + return {check_for_field: name} def get_roles(self): roles = [] for data in self.roles: if data.role != "All": - roles.append({ - 'role': data.role, - 'parenttype': 'Custom Role' - }) + roles.append({"role": data.role, "parenttype": "Custom Role"}) return roles def update_status(self): diff --git a/frappe/core/doctype/role_profile/role_profile.py b/frappe/core/doctype/role_profile/role_profile.py index cb0a43d68f..ab4660d7c9 100644 --- a/frappe/core/doctype/role_profile/role_profile.py +++ b/frappe/core/doctype/role_profile/role_profile.py @@ -2,8 +2,9 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE -from frappe.model.document import Document import frappe +from frappe.model.document import Document + class RoleProfile(Document): def autoname(self): @@ -11,10 +12,10 @@ class RoleProfile(Document): self.name = self.role_profile def on_update(self): - """ Changes in role_profile reflected across all its user """ - users = frappe.get_all('User', filters={'role_profile_name': self.name}) + """Changes in role_profile reflected across all its user""" + users = frappe.get_all("User", filters={"role_profile_name": self.name}) roles = [role.role for role in self.roles] for d in users: - user = frappe.get_doc('User', d) - user.set('roles', []) + user = frappe.get_doc("User", d) + user.set("roles", []) user.add_roles(*roles) diff --git a/frappe/core/doctype/role_profile/test_role_profile.py b/frappe/core/doctype/role_profile/test_role_profile.py index b208a186de..19239a81cd 100644 --- a/frappe/core/doctype/role_profile/test_role_profile.py +++ b/frappe/core/doctype/role_profile/test_role_profile.py @@ -1,38 +1,42 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest -test_dependencies = ['Role'] +import frappe + +test_dependencies = ["Role"] + class TestRoleProfile(unittest.TestCase): def test_make_new_role_profile(self): - frappe.delete_doc_if_exists('Role Profile', 'Test 1', force=1) - new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert() + frappe.delete_doc_if_exists("Role Profile", "Test 1", force=1) + new_role_profile = frappe.get_doc(dict(doctype="Role Profile", role_profile="Test 1")).insert() - self.assertEqual(new_role_profile.role_profile, 'Test 1') + self.assertEqual(new_role_profile.role_profile, "Test 1") # add role - new_role_profile.append("roles", { - "role": '_Test Role 2' - }) + new_role_profile.append("roles", {"role": "_Test Role 2"}) new_role_profile.save() - self.assertEqual(new_role_profile.roles[0].role, '_Test Role 2') + self.assertEqual(new_role_profile.roles[0].role, "_Test Role 2") # user with a role profile random_user = frappe.mock("email") random_user_name = frappe.mock("name") - random_user = frappe.get_doc({ - "doctype": "User", - "email": random_user, - "enabled": 1, - "first_name": random_user_name, - "new_password": "Eastern_43A1W", - "role_profile_name": 'Test 1' - }).insert(ignore_permissions=True, ignore_if_duplicate=True) - self.assertListEqual([role.role for role in random_user.roles], [role.role for role in new_role_profile.roles]) + random_user = frappe.get_doc( + { + "doctype": "User", + "email": random_user, + "enabled": 1, + "first_name": random_user_name, + "new_password": "Eastern_43A1W", + "role_profile_name": "Test 1", + } + ).insert(ignore_permissions=True, ignore_if_duplicate=True) + self.assertListEqual( + [role.role for role in random_user.roles], [role.role for role in new_role_profile.roles] + ) # clear roles new_role_profile.roles = [] @@ -41,4 +45,4 @@ class TestRoleProfile(unittest.TestCase): # user roles with the role profile should also be updated random_user.reload() - self.assertListEqual(random_user.roles, []) \ No newline at end of file + self.assertListEqual(random_user.roles, []) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py index bd5c15bc31..bead463ba5 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ScheduledJobLog(Document): pass diff --git a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py index 9957f6c34c..3c99bb5cb8 100644 --- a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestScheduledJobLog(unittest.TestCase): pass diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 1a795bab82..9665a20843 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -32,19 +32,22 @@ class ScheduledJobType(Document): self.execute() else: if not self.is_job_in_queue(): - enqueue('frappe.core.doctype.scheduled_job_type.scheduled_job_type.run_scheduled_job', - queue = self.get_queue_name(), job_type=self.method) + enqueue( + "frappe.core.doctype.scheduled_job_type.scheduled_job_type.run_scheduled_job", + queue=self.get_queue_name(), + job_type=self.method, + ) return True return False - def is_event_due(self, current_time = None): - '''Return true if event is due based on time lapsed since last execution''' + def is_event_due(self, current_time=None): + """Return true if event is due based on time lapsed since last execution""" # if the next scheduled event is before NOW, then its due! return self.get_next_execution() <= (current_time or now_datetime()) def is_job_in_queue(self): - queued_jobs = get_jobs(site=frappe.local.site, key='job_type')[frappe.local.site] + queued_jobs = get_jobs(site=frappe.local.site, key="job_type")[frappe.local.site] return self.method in queued_jobs def get_next_execution(self): @@ -65,24 +68,25 @@ class ScheduledJobType(Document): if not self.cron_format: self.cron_format = CRON_MAP[self.frequency] - return croniter(self.cron_format, - get_datetime(self.last_execution or datetime(2000, 1, 1))).get_next(datetime) + return croniter( + self.cron_format, get_datetime(self.last_execution or datetime(2000, 1, 1)) + ).get_next(datetime) def execute(self): self.scheduler_log = None try: - self.log_status('Start') + self.log_status("Start") if self.server_script: script_name = frappe.db.get_value("Server Script", self.server_script) if script_name: - frappe.get_doc('Server Script', script_name).execute_scheduled_method() + frappe.get_doc("Server Script", script_name).execute_scheduled_method() else: frappe.get_attr(self.method)() frappe.db.commit() - self.log_status('Complete') + self.log_status("Complete") except Exception: frappe.db.rollback() - self.log_status('Failed') + self.log_status("Failed") def log_status(self, status): # log file @@ -92,21 +96,23 @@ class ScheduledJobType(Document): def update_scheduler_log(self, status): if not self.create_log: # self.get_next_execution will work properly iff self.last_execution is properly set - if self.frequency == "All" and status == 'Start': - self.db_set('last_execution', now_datetime(), update_modified=False) + if self.frequency == "All" and status == "Start": + self.db_set("last_execution", now_datetime(), update_modified=False) frappe.db.commit() return if not self.scheduler_log: - self.scheduler_log = frappe.get_doc(dict(doctype = 'Scheduled Job Log', scheduled_job_type=self.name)).insert(ignore_permissions=True) - self.scheduler_log.db_set('status', status) - if status == 'Failed': - self.scheduler_log.db_set('details', frappe.get_traceback()) - if status == 'Start': - self.db_set('last_execution', now_datetime(), update_modified=False) + self.scheduler_log = frappe.get_doc( + dict(doctype="Scheduled Job Log", scheduled_job_type=self.name) + ).insert(ignore_permissions=True) + self.scheduler_log.db_set("status", status) + if status == "Failed": + self.scheduler_log.db_set("details", frappe.get_traceback()) + if status == "Start": + self.db_set("last_execution", now_datetime(), update_modified=False) frappe.db.commit() def get_queue_name(self): - return 'long' if ('Long' in self.frequency) else 'default' + return "long" if ("Long" in self.frequency) else "default" def on_trash(self): frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": self.name}) @@ -187,9 +193,7 @@ def insert_single_event(frequency: str, event: str, cron_format: str = None): def clear_events(all_events: List): - for event in frappe.get_all( - "Scheduled Job Type", fields=["name", "method", "server_script"] - ): + for event in frappe.get_all("Scheduled Job Type", fields=["name", "method", "server_script"]): is_server_script = event.server_script is_defined_in_hooks = event.method in all_events diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index a11966c47e..3e63985692 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest -from frappe.utils import get_datetime +import frappe from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs +from frappe.utils import get_datetime + class TestScheduledJobType(unittest.TestCase): def setUp(self): @@ -15,51 +16,60 @@ class TestScheduledJobType(unittest.TestCase): frappe.db.commit() def test_sync_jobs(self): - all_job = frappe.get_doc('Scheduled Job Type', - dict(method='frappe.email.queue.flush')) - self.assertEqual(all_job.frequency, 'All') + all_job = frappe.get_doc("Scheduled Job Type", dict(method="frappe.email.queue.flush")) + self.assertEqual(all_job.frequency, "All") - daily_job = frappe.get_doc('Scheduled Job Type', - dict(method='frappe.email.queue.set_expiry_for_email_queue')) - self.assertEqual(daily_job.frequency, 'Daily') + daily_job = frappe.get_doc( + "Scheduled Job Type", dict(method="frappe.email.queue.set_expiry_for_email_queue") + ) + self.assertEqual(daily_job.frequency, "Daily") # check if cron jobs are synced - cron_job = frappe.get_doc('Scheduled Job Type', - dict(method='frappe.oauth.delete_oauth2_data')) - self.assertEqual(cron_job.frequency, 'Cron') - self.assertEqual(cron_job.cron_format, '0/15 * * * *') + cron_job = frappe.get_doc("Scheduled Job Type", dict(method="frappe.oauth.delete_oauth2_data")) + self.assertEqual(cron_job.frequency, "Cron") + self.assertEqual(cron_job.cron_format, "0/15 * * * *") # check if jobs are synced after change in hooks - updated_scheduler_events = { "hourly": ["frappe.email.queue.flush"] } + updated_scheduler_events = {"hourly": ["frappe.email.queue.flush"]} sync_jobs(updated_scheduler_events) - updated_scheduled_job = frappe.get_doc("Scheduled Job Type", {"method": "frappe.email.queue.flush"}) + updated_scheduled_job = frappe.get_doc( + "Scheduled Job Type", {"method": "frappe.email.queue.flush"} + ) self.assertEqual(updated_scheduled_job.frequency, "Hourly") def test_daily_job(self): - job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.queue.set_expiry_for_email_queue')) - job.db_set('last_execution', '2019-01-01 00:00:00') - self.assertTrue(job.is_event_due(get_datetime('2019-01-02 00:00:06'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:00:06'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-01 23:59:59'))) + job = frappe.get_doc( + "Scheduled Job Type", dict(method="frappe.email.queue.set_expiry_for_email_queue") + ) + job.db_set("last_execution", "2019-01-01 00:00:00") + self.assertTrue(job.is_event_due(get_datetime("2019-01-02 00:00:06"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-01 00:00:06"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-01 23:59:59"))) def test_weekly_job(self): - job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary')) - job.db_set('last_execution', '2019-01-01 00:00:00') - self.assertTrue(job.is_event_due(get_datetime('2019-01-06 00:00:01'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-02 00:00:06'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-05 23:59:59'))) + job = frappe.get_doc( + "Scheduled Job Type", + dict(method="frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary"), + ) + job.db_set("last_execution", "2019-01-01 00:00:00") + self.assertTrue(job.is_event_due(get_datetime("2019-01-06 00:00:01"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-02 00:00:06"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-05 23:59:59"))) def test_monthly_job(self): - job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.email.doctype.auto_email_report.auto_email_report.send_monthly')) - job.db_set('last_execution', '2019-01-01 00:00:00') - self.assertTrue(job.is_event_due(get_datetime('2019-02-01 00:00:01'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-15 00:00:06'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-31 23:59:59'))) + job = frappe.get_doc( + "Scheduled Job Type", + dict(method="frappe.email.doctype.auto_email_report.auto_email_report.send_monthly"), + ) + job.db_set("last_execution", "2019-01-01 00:00:00") + self.assertTrue(job.is_event_due(get_datetime("2019-02-01 00:00:01"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-15 00:00:06"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-31 23:59:59"))) def test_cron_job(self): # runs every 15 mins - job = frappe.get_doc('Scheduled Job Type', dict(method = 'frappe.oauth.delete_oauth2_data')) - job.db_set('last_execution', '2019-01-01 00:00:00') - self.assertTrue(job.is_event_due(get_datetime('2019-01-01 00:15:01'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:05:06'))) - self.assertFalse(job.is_event_due(get_datetime('2019-01-01 00:14:59'))) + job = frappe.get_doc("Scheduled Job Type", dict(method="frappe.oauth.delete_oauth2_data")) + job.db_set("last_execution", "2019-01-01 00:00:00") + self.assertTrue(job.is_event_due(get_datetime("2019-01-01 00:15:01"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-01 00:05:06"))) + self.assertFalse(job.is_event_due(get_datetime("2019-01-01 00:14:59"))) diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index 5b1aab1241..0e2eac16ba 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -7,9 +7,9 @@ from types import FunctionType, MethodType, ModuleType from typing import Dict, List import frappe -from frappe.model.document import Document -from frappe.utils.safe_exec import get_safe_globals, safe_exec, NamespaceDict from frappe import _ +from frappe.model.document import Document +from frappe.utils.safe_exec import NamespaceDict, get_safe_globals, safe_exec class ServerScript(Document): @@ -28,9 +28,7 @@ class ServerScript(Document): frappe.delete_doc("Scheduled Job Type", job.name) def get_code_fields(self): - return { - 'script': 'py' - } + return {"script": "py"} @property def scheduled_jobs(self) -> List[Dict[str, str]]: @@ -40,10 +38,8 @@ class ServerScript(Document): fields=["name", "stopped"], ) - def sync_scheduled_jobs(self): - """Sync Scheduled Job Type statuses if Server Script's disabled status is changed - """ + """Sync Scheduled Job Type statuses if Server Script's disabled status is changed""" if self.script_type != "Scheduler Event" or not self.has_value_changed("disabled"): return @@ -54,14 +50,12 @@ class ServerScript(Document): job.save() def sync_scheduler_events(self): - """Create or update Scheduled Job Type documents for Scheduler Event Server Scripts - """ + """Create or update Scheduled Job Type documents for Scheduler Event Server Scripts""" if not self.disabled and self.event_frequency and self.script_type == "Scheduler Event": setup_scheduler_events(script_name=self.name, frequency=self.event_frequency) def clear_scheduled_events(self): - """Deletes existing scheduled jobs by Server Script if self.event_frequency has changed - """ + """Deletes existing scheduled jobs by Server Script if self.event_frequency has changed""" if self.script_type == "Scheduler Event" and self.has_value_changed("event_frequency"): for scheduled_job in self.scheduled_jobs: frappe.delete_doc("Scheduled Job Type", scheduled_job.name) @@ -70,11 +64,11 @@ class ServerScript(Document): """Specific to API endpoint Server Scripts Raises: - frappe.DoesNotExistError: If self.script_type is not API - frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user + frappe.DoesNotExistError: If self.script_type is not API + frappe.PermissionError: If self.allow_guest is unset for API accessed by Guest user Returns: - dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals + dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals """ # wrong report type! if self.script_type != "API": @@ -92,7 +86,7 @@ class ServerScript(Document): """Specific to Document Event triggered Server Scripts Args: - doc (Document): Executes script with for a certain document's events + doc (Document): Executes script with for a certain document's events """ safe_exec(self.script, _locals={"doc": doc}, restrict_commit_rollback=True) @@ -100,7 +94,7 @@ class ServerScript(Document): """Specific to Scheduled Jobs via Server Scripts Raises: - frappe.DoesNotExistError: If script type is not a scheduler event + frappe.DoesNotExistError: If script type is not a scheduler event """ if self.script_type != "Scheduler Event": raise frappe.DoesNotExistError @@ -111,10 +105,10 @@ class ServerScript(Document): """Specific to Permission Query Server Scripts Args: - user (str): Takes user email to execute script and return list of conditions + user (str): Takes user email to execute script and return list of conditions Returns: - list: Returns list of conditions defined by rules in self.script + list: Returns list of conditions defined by rules in self.script """ locals = {"user": user, "conditions": ""} safe_exec(self.script, None, locals) @@ -127,21 +121,22 @@ class ServerScript(Document): that is used while executing a Server Script. Returns: - list: Returns list of autocompletion items. - For e.g., ["frappe.utils.cint", "frappe.db.get_all", ...] + list: Returns list of autocompletion items. + For e.g., ["frappe.utils.cint", "frappe.db.get_all", ...] """ + def get_keys(obj): out = [] for key in obj: - if key.startswith('_'): + if key.startswith("_"): continue value = obj[key] if isinstance(value, (NamespaceDict, dict)) and value: - if key == 'form_dict': - out.append(['form_dict', 7]) + if key == "form_dict": + out.append(["form_dict", 7]) continue for subkey, score in get_keys(value): - fullkey = f'{key}.{subkey}' + fullkey = f"{key}.{subkey}" out.append([fullkey, score]) else: if isinstance(value, type) and issubclass(value, Exception): @@ -159,11 +154,11 @@ class ServerScript(Document): out.append([key, score]) return out - items = frappe.cache().get_value('server_script_autocompletion_items') + items = frappe.cache().get_value("server_script_autocompletion_items") if not items: items = get_keys(get_safe_globals()) - items = [{'value': d[0], 'score': d[1]} for d in items] - frappe.cache().set_value('server_script_autocompletion_items', items) + items = [{"value": d[0], "score": d[1]} for d in items] + frappe.cache().set_value("server_script_autocompletion_items", items) return items @@ -172,8 +167,8 @@ def setup_scheduler_events(script_name, frequency): """Creates or Updates Scheduled Job Type documents based on the specified script name and frequency Args: - script_name (str): Name of the Server Script document - frequency (str): Event label compatible with the Frappe scheduler + script_name (str): Name of the Server Script document + frequency (str): Event label compatible with the Frappe scheduler """ method = frappe.scrub(f"{script_name}-{frequency}") scheduled_script = frappe.db.get_value("Scheduled Job Type", {"method": method}) @@ -199,6 +194,4 @@ def setup_scheduler_events(script_name, frequency): doc.frequency = frequency doc.save() - frappe.msgprint( - _("Scheduled execution for script {0} has updated").format(script_name) - ) + frappe.msgprint(_("Scheduled execution for script {0} has updated").format(script_name)) diff --git a/frappe/core/doctype/server_script/server_script_utils.py b/frappe/core/doctype/server_script/server_script_utils.py index b5f3ba7168..5300baa199 100644 --- a/frappe/core/doctype/server_script/server_script_utils.py +++ b/frappe/core/doctype/server_script/server_script_utils.py @@ -4,21 +4,22 @@ import frappe # to avoid circular imports EVENT_MAP = { - 'before_insert': 'Before Insert', - 'after_insert': 'After Insert', - 'before_validate': 'Before Validate', - 'validate': 'Before Save', - 'on_update': 'After Save', - 'before_submit': 'Before Submit', - 'on_submit': 'After Submit', - 'before_cancel': 'Before Cancel', - 'on_cancel': 'After Cancel', - 'on_trash': 'Before Delete', - 'after_delete': 'After Delete', - 'before_update_after_submit': 'Before Save (Submitted Document)', - 'on_update_after_submit': 'After Save (Submitted Document)' + "before_insert": "Before Insert", + "after_insert": "After Insert", + "before_validate": "Before Validate", + "validate": "Before Save", + "on_update": "After Save", + "before_submit": "Before Submit", + "on_submit": "After Submit", + "before_cancel": "Before Cancel", + "on_cancel": "After Cancel", + "on_trash": "Before Delete", + "after_delete": "After Delete", + "before_update_after_submit": "Before Save (Submitted Document)", + "on_update_after_submit": "After Save (Submitted Document)", } + def run_server_script_for_doc_event(doc, event): # run document event method if not event in EVENT_MAP: @@ -34,13 +35,14 @@ def run_server_script_for_doc_event(doc, event): if scripts: # run all scripts for this doctype + event for script_name in scripts: - frappe.get_doc('Server Script', script_name).execute_doc(doc) + frappe.get_doc("Server Script", script_name).execute_doc(doc) + def get_server_script_map(): # fetch cached server script methods # { # '[doctype]': { - # 'Before Insert': ['[server script 1]', '[server script 2]'] + # 'Before Insert': ['[server script 1]', '[server script 2]'] # }, # '_api': { # '[path]': '[server script]' @@ -49,25 +51,27 @@ def get_server_script_map(): # 'DocType': '[server script]' # } # } - if frappe.flags.in_patch and not frappe.db.table_exists('Server Script'): + if frappe.flags.in_patch and not frappe.db.table_exists("Server Script"): return {} - script_map = frappe.cache().get_value('server_script_map') + script_map = frappe.cache().get_value("server_script_map") if script_map is None: - script_map = { - 'permission_query': {} - } - enabled_server_scripts = frappe.get_all('Server Script', - fields=('name', 'reference_doctype', 'doctype_event','api_method', 'script_type'), - filters={'disabled': 0}) + script_map = {"permission_query": {}} + enabled_server_scripts = frappe.get_all( + "Server Script", + fields=("name", "reference_doctype", "doctype_event", "api_method", "script_type"), + filters={"disabled": 0}, + ) for script in enabled_server_scripts: - if script.script_type == 'DocType Event': - script_map.setdefault(script.reference_doctype, {}).setdefault(script.doctype_event, []).append(script.name) - elif script.script_type == 'Permission Query': - script_map['permission_query'][script.reference_doctype] = script.name + if script.script_type == "DocType Event": + script_map.setdefault(script.reference_doctype, {}).setdefault( + script.doctype_event, [] + ).append(script.name) + elif script.script_type == "Permission Query": + script_map["permission_query"][script.reference_doctype] = script.name else: - script_map.setdefault('_api', {})[script.api_method] = script.name + script_map.setdefault("_api", {})[script.api_method] = script.name - frappe.cache().set_value('server_script_map', script_map) + frappe.cache().set_value("server_script_map", script_map) return script_map diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index aa4507b858..2685367695 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -1,85 +1,90 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest + import requests + +import frappe from frappe.utils import get_site_url scripts = [ dict( - name='test_todo', - script_type = 'DocType Event', - doctype_event = 'Before Insert', - reference_doctype = 'ToDo', - script = ''' + name="test_todo", + script_type="DocType Event", + doctype_event="Before Insert", + reference_doctype="ToDo", + script=""" if "test" in doc.description: doc.status = 'Closed' -''' +""", ), dict( - name='test_todo_validate', - script_type = 'DocType Event', - doctype_event = 'Before Insert', - reference_doctype = 'ToDo', - script = ''' + name="test_todo_validate", + script_type="DocType Event", + doctype_event="Before Insert", + reference_doctype="ToDo", + script=""" if "validate" in doc.description: raise frappe.ValidationError -''' +""", ), dict( - name='test_api', - script_type = 'API', - api_method = 'test_server_script', - allow_guest = 1, - script = ''' + name="test_api", + script_type="API", + api_method="test_server_script", + allow_guest=1, + script=""" frappe.response['message'] = 'hello' -''' +""", ), dict( - name='test_return_value', - script_type = 'API', - api_method = 'test_return_value', - allow_guest = 1, - script = ''' + name="test_return_value", + script_type="API", + api_method="test_return_value", + allow_guest=1, + script=""" frappe.flags = 'hello' -''' +""", ), dict( - name='test_permission_query', - script_type = 'Permission Query', - reference_doctype = 'ToDo', - script = ''' + name="test_permission_query", + script_type="Permission Query", + reference_doctype="ToDo", + script=""" conditions = '1 = 1' -'''), - dict( - name='test_invalid_namespace_method', - script_type = 'DocType Event', - doctype_event = 'Before Insert', - reference_doctype = 'Note', - script = ''' +""", + ), + dict( + name="test_invalid_namespace_method", + script_type="DocType Event", + doctype_event="Before Insert", + reference_doctype="Note", + script=""" frappe.method_that_doesnt_exist("do some magic") -''' +""", ), dict( - name='test_todo_commit', - script_type = 'DocType Event', - doctype_event = 'Before Save', - reference_doctype = 'ToDo', - disabled = 1, - script = ''' + name="test_todo_commit", + script_type="DocType Event", + doctype_event="Before Save", + reference_doctype="ToDo", + disabled=1, + script=""" frappe.db.commit() -''' - ) +""", + ), ] + + class TestServerScript(unittest.TestCase): @classmethod def setUpClass(cls): frappe.db.commit() frappe.db.truncate("Server Script") - frappe.get_doc('User', 'Administrator').add_roles('Script Manager') + frappe.get_doc("User", "Administrator").add_roles("Script Manager") for script in scripts: - script_doc = frappe.get_doc(doctype ='Server Script') + script_doc = frappe.get_doc(doctype="Server Script") script_doc.update(script) script_doc.insert() @@ -89,19 +94,21 @@ class TestServerScript(unittest.TestCase): def tearDownClass(cls): frappe.db.commit() frappe.db.truncate("Server Script") - frappe.cache().delete_value('server_script_map') + frappe.cache().delete_value("server_script_map") def setUp(self): - frappe.cache().delete_value('server_script_map') + frappe.cache().delete_value("server_script_map") def test_doctype_event(self): - todo = frappe.get_doc(dict(doctype='ToDo', description='hello')).insert() - self.assertEqual(todo.status, 'Open') + todo = frappe.get_doc(dict(doctype="ToDo", description="hello")).insert() + self.assertEqual(todo.status, "Open") - todo = frappe.get_doc(dict(doctype='ToDo', description='test todo')).insert() - self.assertEqual(todo.status, 'Closed') + todo = frappe.get_doc(dict(doctype="ToDo", description="test todo")).insert() + self.assertEqual(todo.status, "Closed") - self.assertRaises(frappe.ValidationError, frappe.get_doc(dict(doctype='ToDo', description='validate me')).insert) + self.assertRaises( + frappe.ValidationError, frappe.get_doc(dict(doctype="ToDo", description="validate me")).insert + ) def test_api(self): response = requests.post(get_site_url(frappe.local.site) + "/api/method/test_server_script") @@ -109,14 +116,14 @@ class TestServerScript(unittest.TestCase): self.assertEqual("hello", response.json()["message"]) def test_api_return(self): - self.assertEqual(frappe.get_doc('Server Script', 'test_return_value').execute_method(), 'hello') + self.assertEqual(frappe.get_doc("Server Script", "test_return_value").execute_method(), "hello") def test_permission_query(self): if frappe.conf.db_type == "mariadb": - self.assertTrue('where (1 = 1)' in frappe.db.get_list('ToDo', run=False)) + self.assertTrue("where (1 = 1)" in frappe.db.get_list("ToDo", run=False)) else: - self.assertTrue('where (1 = \'1\')' in frappe.db.get_list('ToDo', run=False)) - self.assertTrue(isinstance(frappe.db.get_list('ToDo'), list)) + self.assertTrue("where (1 = '1')" in frappe.db.get_list("ToDo", run=False)) + self.assertTrue(isinstance(frappe.db.get_list("ToDo"), list)) def test_attribute_error(self): """Raise AttributeError if method not found in Namespace""" @@ -130,15 +137,18 @@ class TestServerScript(unittest.TestCase): with self.assertRaises(frappe.ValidationError) as se: frappe.get_doc(doctype="Server Script", **server_script).insert() - self.assertTrue("invalid python code" in str(se.exception).lower(), - msg="Python code validation not working") + self.assertTrue( + "invalid python code" in str(se.exception).lower(), msg="Python code validation not working" + ) def test_commit_in_doctype_event(self): - server_script = frappe.get_doc('Server Script', 'test_todo_commit') + server_script = frappe.get_doc("Server Script", "test_todo_commit") server_script.disabled = 0 server_script.save() - self.assertRaises(AttributeError, frappe.get_doc(dict(doctype='ToDo', description='test me')).insert) + self.assertRaises( + AttributeError, frappe.get_doc(dict(doctype="ToDo", description="test me")).insert + ) server_script.disabled = 1 server_script.save() @@ -148,15 +158,15 @@ class TestServerScript(unittest.TestCase): todo.insert() script = frappe.get_doc( - doctype='Server Script', - name='test_qb_restrictions', - script_type = 'API', - api_method = 'test_qb_restrictions', - allow_guest = 1, + doctype="Server Script", + name="test_qb_restrictions", + script_type="API", + api_method="test_qb_restrictions", + allow_guest=1, # whitelisted update - script = f''' + script=f""" frappe.db.set_value("ToDo", "{todo.name}", "description", "safe") -''' +""", ) script.insert() script.execute_method() diff --git a/frappe/core/doctype/session_default/session_default.py b/frappe/core/doctype/session_default/session_default.py index 9470a1bb38..df261f4a39 100644 --- a/frappe/core/doctype/session_default/session_default.py +++ b/frappe/core/doctype/session_default/session_default.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class SessionDefault(Document): pass diff --git a/frappe/core/doctype/session_default_settings/session_default_settings.py b/frappe/core/doctype/session_default_settings/session_default_settings.py index 52c917223e..4ac9b61553 100644 --- a/frappe/core/doctype/session_default_settings/session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/session_default_settings.py @@ -2,29 +2,35 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -import json from frappe.model.document import Document + class SessionDefaultSettings(Document): pass + @frappe.whitelist() def get_session_default_values(): - settings = frappe.get_single('Session Default Settings') + settings = frappe.get_single("Session Default Settings") fields = [] for default_values in settings.session_defaults: reference_doctype = frappe.scrub(default_values.ref_doctype) - fields.append({ - 'fieldname': reference_doctype, - 'fieldtype': 'Link', - 'options': default_values.ref_doctype, - 'label': _('Default {0}').format(_(default_values.ref_doctype)), - 'default': frappe.defaults.get_user_default(reference_doctype) - }) + fields.append( + { + "fieldname": reference_doctype, + "fieldtype": "Link", + "options": default_values.ref_doctype, + "label": _("Default {0}").format(_(default_values.ref_doctype)), + "default": frappe.defaults.get_user_default(reference_doctype), + } + ) return json.dumps(fields) + @frappe.whitelist() def set_session_default_values(default_values): default_values = frappe.parse_json(default_values) @@ -35,8 +41,9 @@ def set_session_default_values(default_values): return return "success" -#called on hook 'on_logout' to clear defaults for the session + +# called on hook 'on_logout' to clear defaults for the session def clear_session_defaults(): - settings = frappe.get_single('Session Default Settings').session_defaults + settings = frappe.get_single("Session Default Settings").session_defaults for entry in settings: frappe.defaults.clear_user_default(frappe.scrub(entry.ref_doctype)) diff --git a/frappe/core/doctype/session_default_settings/test_session_default_settings.py b/frappe/core/doctype/session_default_settings/test_session_default_settings.py index 7a7e971aed..f763f90a1d 100644 --- a/frappe/core/doctype/session_default_settings/test_session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/test_session_default_settings.py @@ -1,26 +1,33 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest -from frappe.core.doctype.session_default_settings.session_default_settings import set_session_default_values, clear_session_defaults + +import frappe +from frappe.core.doctype.session_default_settings.session_default_settings import ( + clear_session_defaults, + set_session_default_values, +) + class TestSessionDefaultSettings(unittest.TestCase): def test_set_session_default_settings(self): frappe.set_user("Administrator") settings = frappe.get_single("Session Default Settings") settings.session_defaults = [] - settings.append("session_defaults", { - "ref_doctype": "Role" - }) + settings.append("session_defaults", {"ref_doctype": "Role"}) settings.save() set_session_default_values({"role": "Website Manager"}) - todo = frappe.get_doc(dict(doctype="ToDo", description="test session defaults set", assigned_by="Administrator")).insert() + todo = frappe.get_doc( + dict(doctype="ToDo", description="test session defaults set", assigned_by="Administrator") + ).insert() self.assertEqual(todo.role, "Website Manager") def test_clear_session_defaults(self): clear_session_defaults() - todo = frappe.get_doc(dict(doctype="ToDo", description="test session defaults cleared", assigned_by="Administrator")).insert() + todo = frappe.get_doc( + dict(doctype="ToDo", description="test session defaults cleared", assigned_by="Administrator") + ).insert() self.assertNotEqual(todo.role, "Website Manager") diff --git a/frappe/core/doctype/sms_parameter/sms_parameter.py b/frappe/core/doctype/sms_parameter/sms_parameter.py index fb8466eac6..d67e905234 100644 --- a/frappe/core/doctype/sms_parameter/sms_parameter.py +++ b/frappe/core/doctype/sms_parameter/sms_parameter.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class SMSParameter(Document): - pass \ No newline at end of file + pass diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index f15ba7e4f6..e1da200ee5 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -3,15 +3,15 @@ # License: MIT. See LICENSE import frappe - -from frappe import _, throw, msgprint +from frappe import _, msgprint, throw +from frappe.model.document import Document from frappe.utils import nowdate -from frappe.model.document import Document class SMSSettings(Document): pass + def validate_receiver_nos(receiver_list): validated_receiver_list = [] for d in receiver_list: @@ -19,8 +19,8 @@ def validate_receiver_nos(receiver_list): break # remove invalid character - for x in [' ','-', '(', ')']: - d = d.replace(x, '') + for x in [" ", "-", "(", ")"]: + d = d.replace(x, "") validated_receiver_list.append(d) @@ -29,22 +29,28 @@ def validate_receiver_nos(receiver_list): return validated_receiver_list + @frappe.whitelist() def get_contact_number(contact_name, ref_doctype, ref_name): "returns mobile number of the contact" - number = frappe.db.sql("""select mobile_no, phone from tabContact + number = frappe.db.sql( + """select mobile_no, phone from tabContact where name=%s and exists( select name from `tabDynamic Link` where link_doctype=%s and link_name=%s ) - """, (contact_name, ref_doctype, ref_name)) + """, + (contact_name, ref_doctype, ref_name), + ) + + return number and (number[0][0] or number[0][1]) or "" - return number and (number[0][0] or number[0][1]) or '' @frappe.whitelist() -def send_sms(receiver_list, msg, sender_name = '', success_msg = True): +def send_sms(receiver_list, msg, sender_name="", success_msg=True): import json + if isinstance(receiver_list, str): receiver_list = json.loads(receiver_list) if not isinstance(receiver_list, list): @@ -53,29 +59,30 @@ def send_sms(receiver_list, msg, sender_name = '', success_msg = True): receiver_list = validate_receiver_nos(receiver_list) arg = { - 'receiver_list' : receiver_list, - 'message' : frappe.safe_decode(msg).encode('utf-8'), - 'success_msg' : success_msg + "receiver_list": receiver_list, + "message": frappe.safe_decode(msg).encode("utf-8"), + "success_msg": success_msg, } - if frappe.db.get_value('SMS Settings', None, 'sms_gateway_url'): + if frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): send_via_gateway(arg) else: msgprint(_("Please Update SMS Settings")) + def send_via_gateway(arg): - ss = frappe.get_doc('SMS Settings', 'SMS Settings') + ss = frappe.get_doc("SMS Settings", "SMS Settings") headers = get_headers(ss) use_json = headers.get("Content-Type") == "application/json" - message = frappe.safe_decode(arg.get('message')) + message = frappe.safe_decode(arg.get("message")) args = {ss.message_parameter: message} for d in ss.get("parameters"): if not d.header: args[d.parameter] = d.value success_list = [] - for d in arg.get('receiver_list'): + for d in arg.get("receiver_list"): args[ss.receiver_parameter] = d status = send_request(ss.sms_gateway_url, args, headers, ss.use_post, use_json) @@ -85,20 +92,22 @@ def send_via_gateway(arg): if len(success_list) > 0: args.update(arg) create_sms_log(args, success_list) - if arg.get('success_msg'): + if arg.get("success_msg"): frappe.msgprint(_("SMS sent to following numbers: {0}").format("\n" + "\n".join(success_list))) + def get_headers(sms_settings=None): if not sms_settings: - sms_settings = frappe.get_doc('SMS Settings', 'SMS Settings') + sms_settings = frappe.get_doc("SMS Settings", "SMS Settings") - headers={'Accept': "text/plain, text/html, */*"} + headers = {"Accept": "text/plain, text/html, */*"} for d in sms_settings.get("parameters"): if d.header == 1: headers.update({d.parameter: d.value}) return headers + def send_request(gateway_url, params, headers=None, use_post=False, use_json=False): import requests @@ -124,11 +133,11 @@ def send_request(gateway_url, params, headers=None, use_post=False, use_json=Fal # Create SMS Log # ========================================================= def create_sms_log(args, sent_to): - sl = frappe.new_doc('SMS Log') + sl = frappe.new_doc("SMS Log") sl.sent_on = nowdate() - sl.message = args['message'].decode('utf-8') - sl.no_of_requested_sms = len(args['receiver_list']) - sl.requested_numbers = "\n".join(args['receiver_list']) + sl.message = args["message"].decode("utf-8") + sl.no_of_requested_sms = len(args["receiver_list"]) + sl.requested_numbers = "\n".join(args["receiver_list"]) sl.no_of_sent_sms = len(sent_to) sl.sent_to = "\n".join(sent_to) sl.flags.ignore_permissions = True diff --git a/frappe/core/doctype/sms_settings/test_sms_settings.py b/frappe/core/doctype/sms_settings/test_sms_settings.py index b3be912f9e..a7ec761b82 100644 --- a/frappe/core/doctype/sms_settings/test_sms_settings.py +++ b/frappe/core/doctype/sms_settings/test_sms_settings.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestSMSSettings(unittest.TestCase): pass diff --git a/frappe/core/doctype/success_action/success_action.py b/frappe/core/doctype/success_action/success_action.py index afb3a87485..95a81ee0fb 100644 --- a/frappe/core/doctype/success_action/success_action.py +++ b/frappe/core/doctype/success_action/success_action.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class SuccessAction(Document): pass diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index 1ae8e9e79e..3d01015087 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -3,17 +3,18 @@ import frappe from frappe import _ -from frappe.model.document import Document from frappe.model import no_value_fields +from frappe.model.document import Document from frappe.translate import set_default_language +from frappe.twofactor import toggle_two_factor_auth from frappe.utils import cint, today from frappe.utils.momentjs import get_all_timezones -from frappe.twofactor import toggle_two_factor_auth + class SystemSettings(Document): def validate(self): enable_password_policy = cint(self.enable_password_policy) and True or False - minimum_password_score = cint(getattr(self, 'minimum_password_score', 0)) or 0 + minimum_password_score = cint(getattr(self, "minimum_password_score", 0)) or 0 if enable_password_policy and minimum_password_score <= 0: frappe.throw(_("Please select Minimum Password Score")) elif not enable_password_policy: @@ -22,21 +23,24 @@ class SystemSettings(Document): for key in ("session_expiry", "session_expiry_mobile"): if self.get(key): parts = self.get(key).split(":") - if len(parts)!=2 or not (cint(parts[0]) or cint(parts[1])): + if len(parts) != 2 or not (cint(parts[0]) or cint(parts[1])): frappe.throw(_("Session Expiry must be in format {0}").format("hh:mm")) if self.enable_two_factor_auth: - if self.two_factor_method=='SMS': - if not frappe.db.get_value('SMS Settings', None, 'sms_gateway_url'): - frappe.throw(_('Please setup SMS before setting it as an authentication method, via SMS Settings')) - toggle_two_factor_auth(True, roles=['All']) + if self.two_factor_method == "SMS": + if not frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + frappe.throw( + _("Please setup SMS before setting it as an authentication method, via SMS Settings") + ) + toggle_two_factor_auth(True, roles=["All"]) else: self.bypass_2fa_for_retricted_ip_users = 0 self.bypass_restrict_ip_check_if_2fa_enabled = 0 frappe.flags.update_last_reset_password_date = False - if (self.force_user_to_reset_password and - not cint(frappe.db.get_single_value("System Settings", "force_user_to_reset_password"))): + if self.force_user_to_reset_password and not cint( + frappe.db.get_single_value("System Settings", "force_user_to_reset_password") + ): frappe.flags.update_last_reset_password_date = True def on_update(self): @@ -47,19 +51,24 @@ class SystemSettings(Document): if self.language: set_default_language(self.language) - frappe.cache().delete_value('system_settings') - frappe.cache().delete_value('time_zone') + frappe.cache().delete_value("system_settings") + frappe.cache().delete_value("time_zone") frappe.local.system_settings = {} if frappe.flags.update_last_reset_password_date: update_last_reset_password_date() + def update_last_reset_password_date(): - frappe.db.sql(""" UPDATE `tabUser` + frappe.db.sql( + """ UPDATE `tabUser` SET last_password_reset_date = %s WHERE - last_password_reset_date is null""", today()) + last_password_reset_date is null""", + today(), + ) + @frappe.whitelist() def load(): @@ -73,7 +82,4 @@ def load(): if df.fieldtype in ("Select", "Data"): defaults[df.fieldname] = all_defaults.get(df.fieldname) - return { - "timezones": get_all_timezones(), - "defaults": defaults - } + return {"timezones": get_all_timezones(), "defaults": defaults} diff --git a/frappe/core/doctype/system_settings/test_system_settings.py b/frappe/core/doctype/system_settings/test_system_settings.py index f95e26b793..955f4193f0 100644 --- a/frappe/core/doctype/system_settings/test_system_settings.py +++ b/frappe/core/doctype/system_settings/test_system_settings.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestSystemSettings(unittest.TestCase): pass diff --git a/frappe/core/doctype/test/test.py b/frappe/core/doctype/test/test.py index ab6fcb6de4..8f3cf7111d 100644 --- a/frappe/core/doctype/test/test.py +++ b/frappe/core/doctype/test/test.py @@ -2,13 +2,13 @@ # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + # import frappe from frappe.model.document import Document -import json class test(Document): - def db_insert(self): d = self.get_valid_dict(convert_dates_to_str=True) with open("data_file.json", "w+") as read_file: @@ -42,4 +42,3 @@ class test(Document): # return [] with open("data_file.json", "r") as read_file: return [json.load(read_file)] - diff --git a/frappe/core/doctype/test/test_test.py b/frappe/core/doctype/test/test_test.py index d8508b8651..e4ee3de5dd 100644 --- a/frappe/core/doctype/test/test_test.py +++ b/frappe/core/doctype/test/test_test.py @@ -4,5 +4,6 @@ # import frappe import unittest + class Testtest(unittest.TestCase): pass diff --git a/frappe/core/doctype/transaction_log/test_transaction_log.py b/frappe/core/doctype/transaction_log/test_transaction_log.py index c332a82f65..a4bb066eea 100644 --- a/frappe/core/doctype/transaction_log/test_transaction_log.py +++ b/frappe/core/doctype/transaction_log/test_transaction_log.py @@ -1,40 +1,47 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -import unittest import hashlib +import unittest + +import frappe test_records = [] -class TestTransactionLog(unittest.TestCase): + +class TestTransactionLog(unittest.TestCase): def test_validate_chaining(self): - frappe.get_doc({ - "doctype": "Transaction Log", - "reference_doctype": "Test Doctype", - "document_name": "Test Document 1", - "data": "first_data" - }).insert(ignore_permissions=True) - - second_log = frappe.get_doc({ - "doctype": "Transaction Log", - "reference_doctype": "Test Doctype", - "document_name": "Test Document 2", - "data": "second_data" - }).insert(ignore_permissions=True) - - third_log = frappe.get_doc({ - "doctype": "Transaction Log", - "reference_doctype": "Test Doctype", - "document_name": "Test Document 3", - "data": "third_data" - }).insert(ignore_permissions=True) + frappe.get_doc( + { + "doctype": "Transaction Log", + "reference_doctype": "Test Doctype", + "document_name": "Test Document 1", + "data": "first_data", + } + ).insert(ignore_permissions=True) + + second_log = frappe.get_doc( + { + "doctype": "Transaction Log", + "reference_doctype": "Test Doctype", + "document_name": "Test Document 2", + "data": "second_data", + } + ).insert(ignore_permissions=True) + third_log = frappe.get_doc( + { + "doctype": "Transaction Log", + "reference_doctype": "Test Doctype", + "document_name": "Test Document 3", + "data": "third_data", + } + ).insert(ignore_permissions=True) sha = hashlib.sha256() sha.update( - frappe.safe_encode(str(third_log.transaction_hash)) + - frappe.safe_encode(str(second_log.chaining_hash)) + frappe.safe_encode(str(third_log.transaction_hash)) + + frappe.safe_encode(str(second_log.chaining_hash)) ) self.assertEqual(sha.hexdigest(), third_log.chaining_hash) diff --git a/frappe/core/doctype/transaction_log/transaction_log.py b/frappe/core/doctype/transaction_log/transaction_log.py index 0a480f6660..d0c0342f6f 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.py +++ b/frappe/core/doctype/transaction_log/transaction_log.py @@ -16,7 +16,9 @@ class TransactionLog(Document): self.row_index = index self.timestamp = now_datetime() if index != 1: - prev_hash = frappe.get_all("Transaction Log", filters={"row_index":str(index-1)}, pluck="chaining_hash", limit=1) + prev_hash = frappe.get_all( + "Transaction Log", filters={"row_index": str(index - 1)}, pluck="chaining_hash", limit=1 + ) if prev_hash: self.previous_hash = prev_hash[0] else: @@ -38,25 +40,26 @@ class TransactionLog(Document): def hash_chain(self): sha = hashlib.sha256() - sha.update(frappe.safe_encode(str(self.transaction_hash)) + frappe.safe_encode(str(self.previous_hash))) + sha.update( + frappe.safe_encode(str(self.transaction_hash)) + frappe.safe_encode(str(self.previous_hash)) + ) return sha.hexdigest() def get_current_index(): series = DocType("Series") current = ( - frappe.qb.from_(series) - .where(series.name == "TRANSACTLOG") - .for_update() - .select("current") + frappe.qb.from_(series).where(series.name == "TRANSACTLOG").for_update().select("current") ).run() if current and current[0][0] is not None: current = current[0][0] - frappe.db.sql("""UPDATE `tabSeries` + frappe.db.sql( + """UPDATE `tabSeries` SET `current` = `current` + 1 - where `name` = 'TRANSACTLOG'""") + where `name` = 'TRANSACTLOG'""" + ) current = cint(current) + 1 else: frappe.db.sql("INSERT INTO `tabSeries` (name, current) VALUES ('TRANSACTLOG', 1)") diff --git a/frappe/core/doctype/translation/test_translation.py b/frappe/core/doctype/translation/test_translation.py index 982d9bf976..df5ae3767a 100644 --- a/frappe/core/doctype/translation/test_translation.py +++ b/frappe/core/doctype/translation/test_translation.py @@ -1,58 +1,59 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe from frappe import _ + class TestTranslation(unittest.TestCase): def setUp(self): frappe.db.delete("Translation") def tearDown(self): - frappe.local.lang = 'en' - frappe.local.lang_full_dict=None + frappe.local.lang = "en" + frappe.local.lang_full_dict = None def test_doctype(self): translation_data = get_translation_data() for key, val in translation_data.items(): frappe.local.lang = key - frappe.local.lang_full_dict=None + frappe.local.lang_full_dict = None translation = create_translation(key, val) self.assertEqual(_(val[0]), val[1]) - frappe.delete_doc('Translation', translation.name) - frappe.local.lang_full_dict=None + frappe.delete_doc("Translation", translation.name) + frappe.local.lang_full_dict = None self.assertEqual(_(val[0]), val[0]) def test_parent_language(self): data = [ - ['es', ['Test Data', 'datos de prueba']], - ['es', ['Test Spanish', 'prueba de español']], - ['es-MX', ['Test Data', 'pruebas de datos']] + ["es", ["Test Data", "datos de prueba"]], + ["es", ["Test Spanish", "prueba de español"]], + ["es-MX", ["Test Data", "pruebas de datos"]], ] for key, val in data: create_translation(key, val) - frappe.local.lang = 'es' + frappe.local.lang = "es" - frappe.local.lang_full_dict=None + frappe.local.lang_full_dict = None self.assertTrue(_(data[0][0]), data[0][1]) - frappe.local.lang_full_dict=None + frappe.local.lang_full_dict = None self.assertTrue(_(data[1][0]), data[1][1]) - frappe.local.lang = 'es-MX' + frappe.local.lang = "es-MX" # different translation for es-MX - frappe.local.lang_full_dict=None + frappe.local.lang_full_dict = None self.assertTrue(_(data[2][0]), data[2][1]) # from spanish (general) - frappe.local.lang_full_dict=None + frappe.local.lang_full_dict = None self.assertTrue(_(data[1][0]), data[1][1]) def test_html_content_data_translation(self): @@ -73,7 +74,7 @@ class TestTranslation(unittest.TestCase): los procesadores Intel Core i5 e i7 de quinta generación con Intel HD Graphics 6000 son capaces de hacerlo. """ - create_translation('es', [source, target]) + create_translation("es", [source, target]) source = """ Test Data""" html_translated_data = """ testituloksia """ - return {'hr': ['Test data', 'Testdaten'], - 'ms': ['Test Data','ujian Data'], - 'et': ['Test Data', 'testandmed'], - 'es': ['Test Data', 'datos de prueba'], - 'en': ['Quotation', 'Tax Invoice'], - 'fi': [html_source_data, html_translated_data]} + return { + "hr": ["Test data", "Testdaten"], + "ms": ["Test Data", "ujian Data"], + "et": ["Test Data", "testandmed"], + "es": ["Test Data", "datos de prueba"], + "en": ["Quotation", "Tax Invoice"], + "fi": [html_source_data, html_translated_data], + } + def create_translation(key, val): - translation = frappe.new_doc('Translation') + translation = frappe.new_doc("Translation") translation.language = key translation.source_text = val[0] translation.translated_text = val[1] diff --git a/frappe/core/doctype/translation/translation.py b/frappe/core/doctype/translation/translation.py index a01552903c..90ea4d1523 100644 --- a/frappe/core/doctype/translation/translation.py +++ b/frappe/core/doctype/translation/translation.py @@ -2,11 +2,13 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe from frappe.model.document import Document -from frappe.utils import strip_html_tags, is_html from frappe.translate import get_translator_url -import json +from frappe.utils import is_html, strip_html_tags + class Translation(Document): def validate(self): @@ -28,6 +30,7 @@ class Translation(Document): def get_contribution_status(self): pass + @frappe.whitelist() def create_translations(translation_map, language): from frappe.frappeclient import FrappeClient @@ -37,44 +40,54 @@ def create_translations(translation_map, language): # first create / update local user translations for source_id, translation_dict in translation_map.items(): translation_dict = frappe._dict(translation_dict) - existing_doc_name = frappe.db.get_all('Translation', { - 'source_text': translation_dict.source_text, - 'context': translation_dict.context or '', - 'language': language, - }) + existing_doc_name = frappe.db.get_all( + "Translation", + { + "source_text": translation_dict.source_text, + "context": translation_dict.context or "", + "language": language, + }, + ) translation_map_to_send[source_id] = translation_dict if existing_doc_name: - frappe.db.set_value('Translation', existing_doc_name[0].name, { - 'translated_text': translation_dict.translated_text, - 'contributed': 1, - 'contribution_status': 'Pending' - }) + frappe.db.set_value( + "Translation", + existing_doc_name[0].name, + { + "translated_text": translation_dict.translated_text, + "contributed": 1, + "contribution_status": "Pending", + }, + ) translation_map_to_send[source_id].name = existing_doc_name[0].name else: - doc = frappe.get_doc({ - 'doctype': 'Translation', - 'source_text': translation_dict.source_text, - 'contributed': 1, - 'contribution_status': 'Pending', - 'translated_text': translation_dict.translated_text, - 'context': translation_dict.context, - 'language': language - }) + doc = frappe.get_doc( + { + "doctype": "Translation", + "source_text": translation_dict.source_text, + "contributed": 1, + "contribution_status": "Pending", + "translated_text": translation_dict.translated_text, + "context": translation_dict.context, + "language": language, + } + ) doc.insert() translation_map_to_send[source_id].name = doc.name params = { - 'language': language, - 'contributor_email': frappe.session.user, - 'contributor_name': frappe.utils.get_fullname(frappe.session.user), - 'translation_map': json.dumps(translation_map_to_send) + "language": language, + "contributor_email": frappe.session.user, + "contributor_name": frappe.utils.get_fullname(frappe.session.user), + "translation_map": json.dumps(translation_map_to_send), } translator = FrappeClient(get_translator_url()) - added_translations = translator.post_api('translator.api.add_translations', params=params) + added_translations = translator.post_api("translator.api.add_translations", params=params) for local_docname, remote_docname in added_translations.items(): - frappe.db.set_value('Translation', local_docname, 'contribution_docname', remote_docname) + frappe.db.set_value("Translation", local_docname, "contribution_docname", remote_docname) + def clear_user_translation_cache(lang): - frappe.cache().hdel('lang_user_translations', lang) + frappe.cache().hdel("lang_user_translations", lang) diff --git a/frappe/core/doctype/user/test_user.py b/frappe/core/doctype/user/test_user.py index 3e6e1ec7e2..16a46e356c 100644 --- a/frappe/core/doctype/user/test_user.py +++ b/frappe/core/doctype/user/test_user.py @@ -6,14 +6,21 @@ from unittest.mock import patch import frappe import frappe.exceptions -from frappe.core.doctype.user.user import (extract_mentions, reset_password, - sign_up, test_password_strength, update_password, verify_password) +from frappe.core.doctype.user.user import ( + extract_mentions, + reset_password, + sign_up, + test_password_strength, + update_password, + verify_password, +) from frappe.frappeclient import FrappeClient from frappe.model.delete_doc import delete_doc from frappe.utils import get_url user_module = frappe.core.doctype.user.user -test_records = frappe.get_test_records('User') +test_records = frappe.get_test_records("User") + class TestUser(unittest.TestCase): def tearDown(self): @@ -21,41 +28,41 @@ class TestUser(unittest.TestCase): frappe.db.set_value("System Settings", "System Settings", "enable_password_policy", 0) frappe.db.set_value("System Settings", "System Settings", "minimum_password_score", "") frappe.db.set_value("System Settings", "System Settings", "password_reset_limit", 3) - frappe.set_user('Administrator') + frappe.set_user("Administrator") def test_user_type(self): - new_user = frappe.get_doc(dict(doctype='User', email='test-for-type@example.com', - first_name='Tester')).insert(ignore_if_duplicate=True) - self.assertEqual(new_user.user_type, 'Website User') + new_user = frappe.get_doc( + dict(doctype="User", email="test-for-type@example.com", first_name="Tester") + ).insert(ignore_if_duplicate=True) + self.assertEqual(new_user.user_type, "Website User") # social login userid for frappe self.assertTrue(new_user.social_logins[0].userid) self.assertEqual(new_user.social_logins[0].provider, "frappe") # role with desk access - new_user.add_roles('_Test Role 2') + new_user.add_roles("_Test Role 2") new_user.save() - self.assertEqual(new_user.user_type, 'System User') + self.assertEqual(new_user.user_type, "System User") # clear role new_user.roles = [] new_user.save() - self.assertEqual(new_user.user_type, 'Website User') + self.assertEqual(new_user.user_type, "Website User") # role without desk access - new_user.add_roles('_Test Role 4') + new_user.add_roles("_Test Role 4") new_user.save() - self.assertEqual(new_user.user_type, 'Website User') + self.assertEqual(new_user.user_type, "Website User") delete_contact(new_user.name) - frappe.delete_doc('User', new_user.name) - + frappe.delete_doc("User", new_user.name) def test_delete(self): frappe.get_doc("User", "test@example.com").add_roles("_Test Role 2") self.assertRaises(frappe.LinkExistsError, delete_doc, "Role", "_Test Role 2") frappe.db.delete("Has Role", {"role": "_Test Role 2"}) - delete_doc("Role","_Test Role 2") + delete_doc("Role", "_Test Role 2") if frappe.db.exists("User", "_test@example.com"): delete_contact("_test@example.com") @@ -70,33 +77,43 @@ class TestUser(unittest.TestCase): delete_contact("_test@example.com") delete_doc("User", "_test@example.com") - self.assertTrue(not frappe.db.sql("""select * from `tabToDo` where allocated_to=%s""", - ("_test@example.com",))) + self.assertTrue( + not frappe.db.sql("""select * from `tabToDo` where allocated_to=%s""", ("_test@example.com",)) + ) from frappe.core.doctype.role.test_role import test_records as role_records + frappe.copy_doc(role_records[1]).insert() def test_get_value(self): self.assertEqual(frappe.db.get_value("User", "test@example.com"), "test@example.com") - self.assertEqual(frappe.db.get_value("User", {"email":"test@example.com"}), "test@example.com") - self.assertEqual(frappe.db.get_value("User", {"email":"test@example.com"}, "email"), "test@example.com") - self.assertEqual(frappe.db.get_value("User", {"email":"test@example.com"}, ["first_name", "email"]), - ("_Test", "test@example.com")) - self.assertEqual(frappe.db.get_value("User", - {"email":"test@example.com", "first_name": "_Test"}, - ["first_name", "email"]), - ("_Test", "test@example.com")) - - test_user = frappe.db.sql("select * from tabUser where name='test@example.com'", - as_dict=True)[0] - self.assertEqual(frappe.db.get_value("User", {"email":"test@example.com"}, "*", as_dict=True), - test_user) + self.assertEqual(frappe.db.get_value("User", {"email": "test@example.com"}), "test@example.com") + self.assertEqual( + frappe.db.get_value("User", {"email": "test@example.com"}, "email"), "test@example.com" + ) + self.assertEqual( + frappe.db.get_value("User", {"email": "test@example.com"}, ["first_name", "email"]), + ("_Test", "test@example.com"), + ) + self.assertEqual( + frappe.db.get_value( + "User", {"email": "test@example.com", "first_name": "_Test"}, ["first_name", "email"] + ), + ("_Test", "test@example.com"), + ) + + test_user = frappe.db.sql("select * from tabUser where name='test@example.com'", as_dict=True)[0] + self.assertEqual( + frappe.db.get_value("User", {"email": "test@example.com"}, "*", as_dict=True), test_user + ) self.assertEqual(frappe.db.get_value("User", "xxxtest@example.com"), None) frappe.db.set_value("Website Settings", "Website Settings", "_test", "_test_val") self.assertEqual(frappe.db.get_value("Website Settings", None, "_test"), "_test_val") - self.assertEqual(frappe.db.get_value("Website Settings", "Website Settings", "_test"), "_test_val") + self.assertEqual( + frappe.db.get_value("Website Settings", "Website Settings", "_test"), "_test_val" + ) def test_high_permlevel_validations(self): user = frappe.get_meta("User") @@ -111,7 +128,7 @@ class TestUser(unittest.TestCase): me.add_roles("System Manager") # system manager is not added (it is reset) - self.assertFalse('System Manager' in [d.role for d in me.roles]) + self.assertFalse("System Manager" in [d.role for d in me.roles]) frappe.set_user("Administrator") @@ -122,27 +139,30 @@ class TestUser(unittest.TestCase): self.assertTrue("System Manager" in [d.role for d in me.get("roles")]) def test_delete_user(self): - new_user = frappe.get_doc(dict(doctype='User', email='test-for-delete@example.com', - first_name='Tester Delete User')).insert(ignore_if_duplicate=True) - self.assertEqual(new_user.user_type, 'Website User') + new_user = frappe.get_doc( + dict(doctype="User", email="test-for-delete@example.com", first_name="Tester Delete User") + ).insert(ignore_if_duplicate=True) + self.assertEqual(new_user.user_type, "Website User") # role with desk access - new_user.add_roles('_Test Role 2') + new_user.add_roles("_Test Role 2") new_user.save() - self.assertEqual(new_user.user_type, 'System User') - - comm = frappe.get_doc({ - "doctype":"Communication", - "subject": "To check user able to delete even if linked with communication", - "content": "To check user able to delete even if linked with communication", - "sent_or_received": "Sent", - "user": new_user.name - }) + self.assertEqual(new_user.user_type, "System User") + + comm = frappe.get_doc( + { + "doctype": "Communication", + "subject": "To check user able to delete even if linked with communication", + "content": "To check user able to delete even if linked with communication", + "sent_or_received": "Sent", + "user": new_user.name, + } + ) comm.insert(ignore_permissions=True) delete_contact(new_user.name) - frappe.delete_doc('User', new_user.name) - self.assertFalse(frappe.db.exists('User', new_user.name)) + frappe.delete_doc("User", new_user.name) + self.assertFalse(frappe.db.exists("User", new_user.name)) def test_password_strength(self): # Test Password without Password Strength Policy @@ -158,12 +178,11 @@ class TestUser(unittest.TestCase): # Score 1; should now fail result = test_password_strength("bee2ve") - self.assertEqual(result['feedback']['password_policy_validation_passed'], False) + self.assertEqual(result["feedback"]["password_policy_validation_passed"], False) # Score 4; should pass result = test_password_strength("Eastern_43A1W") - self.assertEqual(result['feedback']['password_policy_validation_passed'], True) - + self.assertEqual(result["feedback"]["password_policy_validation_passed"], True) # test password strength while saving user with new password user = frappe.get_doc("User", "test@example.com") @@ -176,14 +195,14 @@ class TestUser(unittest.TestCase): frappe.flags.in_test = True def test_comment_mentions(self): - comment = ''' + comment = """ @Test - ''' + """ self.assertEqual(extract_mentions(comment)[0], "test.comment@example.com") - comment = ''' + comment = """
Testing comment, @@ -191,9 +210,9 @@ class TestUser(unittest.TestCase): please check
- ''' + """ self.assertEqual(extract_mentions(comment)[0], "test.comment@example.com") - comment = ''' + comment = """
Testing comment for @@ -205,24 +224,22 @@ class TestUser(unittest.TestCase): please check
- ''' + """ self.assertEqual(extract_mentions(comment)[0], "test_user@example.com") self.assertEqual(extract_mentions(comment)[1], "test.again@example1.com") frappe.delete_doc("User Group", "Team") - doc = frappe.get_doc({ - 'doctype': 'User Group', - 'name': 'Team', - 'user_group_members': [{ - 'user': 'test@example.com' - }, { - 'user': 'test1@example.com' - }] - }) + doc = frappe.get_doc( + { + "doctype": "User Group", + "name": "Team", + "user_group_members": [{"user": "test@example.com"}, {"user": "test1@example.com"}], + } + ) doc.insert() - comment = ''' + comment = """
Testing comment for @@ -233,8 +250,8 @@ class TestUser(unittest.TestCase): please check
- ''' - self.assertListEqual(extract_mentions(comment), ['test@example.com', 'test1@example.com']) + """ + self.assertListEqual(extract_mentions(comment), ["test@example.com", "test1@example.com"]) def test_rate_limiting_for_reset_password(self): # Allow only one reset request for a day @@ -242,7 +259,7 @@ class TestUser(unittest.TestCase): frappe.db.commit() url = get_url() - data={'cmd': 'frappe.core.doctype.user.user.reset_password', 'user': 'test@test.com'} + data = {"cmd": "frappe.core.doctype.user.user.reset_password", "user": "test@test.com"} # Clear rate limit tracker to start fresh key = f"rl:{data['cmd']}:{data['user']}" @@ -257,56 +274,73 @@ class TestUser(unittest.TestCase): def test_user_rename(self): old_name = "test_user_rename@example.com" new_name = "test_user_rename_new@example.com" - user = frappe.get_doc({ - "doctype": "User", - "email": old_name, - "enabled": 1, - "first_name": "_Test", - "new_password": "Eastern_43A1W", - "roles": [ - { - "doctype": "Has Role", - "parentfield": "roles", - "role": "System Manager" - }] - }).insert(ignore_permissions=True, ignore_if_duplicate=True) - - frappe.rename_doc('User', user.name, new_name) + user = frappe.get_doc( + { + "doctype": "User", + "email": old_name, + "enabled": 1, + "first_name": "_Test", + "new_password": "Eastern_43A1W", + "roles": [{"doctype": "Has Role", "parentfield": "roles", "role": "System Manager"}], + } + ).insert(ignore_permissions=True, ignore_if_duplicate=True) + + frappe.rename_doc("User", user.name, new_name) self.assertTrue(frappe.db.exists("Notification Settings", new_name)) frappe.delete_doc("User", new_name) def test_signup(self): import frappe.website.utils - random_user = frappe.mock('email') - random_user_name = frappe.mock('name') + + random_user = frappe.mock("email") + random_user_name = frappe.mock("name") # disabled signup with patch.object(user_module, "is_signup_disabled", return_value=True): - self.assertRaisesRegex(frappe.exceptions.ValidationError, "Sign Up is disabled", - sign_up, random_user, random_user_name, "/signup") + self.assertRaisesRegex( + frappe.exceptions.ValidationError, + "Sign Up is disabled", + sign_up, + random_user, + random_user_name, + "/signup", + ) - self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (1, "Please check your email for verification")) - self.assertEqual(frappe.cache().hget('redirect_after_login', random_user), "/welcome") + self.assertTupleEqual( + sign_up(random_user, random_user_name, "/welcome"), + (1, "Please check your email for verification"), + ) + self.assertEqual(frappe.cache().hget("redirect_after_login", random_user), "/welcome") # re-register - self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (0, "Already Registered")) + self.assertTupleEqual( + sign_up(random_user, random_user_name, "/welcome"), (0, "Already Registered") + ) # disabled user user = frappe.get_doc("User", random_user) user.enabled = 0 user.save() - self.assertTupleEqual(sign_up(random_user, random_user_name, "/welcome"), (0, "Registered but disabled")) + self.assertTupleEqual( + sign_up(random_user, random_user_name, "/welcome"), (0, "Registered but disabled") + ) # throttle user creation with patch.object(user_module.frappe.db, "get_creation_count", return_value=301): - self.assertRaisesRegex(frappe.exceptions.ValidationError, "Throttled", - sign_up, frappe.mock('email'), random_user_name, "/signup") - + self.assertRaisesRegex( + frappe.exceptions.ValidationError, + "Throttled", + sign_up, + frappe.mock("email"), + random_user_name, + "/signup", + ) def test_reset_password(self): from frappe.auth import CookieManager, LoginManager from frappe.utils import set_request + old_password = "Eastern_43A1W" new_password = "easy_password" @@ -318,7 +352,10 @@ class TestUser(unittest.TestCase): test_user = frappe.get_doc("User", "testpassword@example.com") test_user.reset_password() self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/app") - self.assertEqual(update_password(new_password, key="wrong_key"), "The Link specified has either been used before or Invalid") + self.assertEqual( + update_password(new_password, key="wrong_key"), + "The Link specified has either been used before or Invalid", + ) # password verification should fail with old password self.assertRaises(frappe.exceptions.AuthenticationError, verify_password, old_password) @@ -327,19 +364,26 @@ class TestUser(unittest.TestCase): # reset password update_password(old_password, old_password=new_password) - self.assertRaisesRegex(frappe.exceptions.ValidationError, "Invalid key type", update_password, "test", 1, ['like', '%']) + self.assertRaisesRegex( + frappe.exceptions.ValidationError, "Invalid key type", update_password, "test", 1, ["like", "%"] + ) password_strength_response = { - "feedback": { - "password_policy_validation_passed": False, - "suggestions": ["Fix password"] - } + "feedback": {"password_policy_validation_passed": False, "suggestions": ["Fix password"]} } # password strength failure test - with patch.object(user_module, "test_password_strength", return_value=password_strength_response): - self.assertRaisesRegex(frappe.exceptions.ValidationError, "Fix password", update_password, new_password, 0, test_user.reset_password_key) - + with patch.object( + user_module, "test_password_strength", return_value=password_strength_response + ): + self.assertRaisesRegex( + frappe.exceptions.ValidationError, + "Fix password", + update_password, + new_password, + 0, + test_user.reset_password_key, + ) # test redirect URL for website users frappe.set_user("test2@example.com") @@ -348,7 +392,7 @@ class TestUser(unittest.TestCase): update_password(old_password, old_password=new_password) # test API endpoint - with patch.object(user_module.frappe, 'sendmail') as sendmail: + with patch.object(user_module.frappe, "sendmail") as sendmail: frappe.clear_messages() test_user = frappe.get_doc("User", "test2@example.com") self.assertEqual(reset_password(user="test2@example.com"), None) @@ -357,7 +401,7 @@ class TestUser(unittest.TestCase): update_password(old_password, old_password=new_password) self.assertEqual( json.loads(frappe.message_log[0]).get("message"), - "Password reset instructions have been sent to your email" + "Password reset instructions have been sent to your email", ) sendmail.assert_called_once() @@ -370,11 +414,14 @@ class TestUser(unittest.TestCase): def test_user_onload_modules(self): from frappe.config import get_modules_from_all_apps from frappe.desk.form.load import getdoc + frappe.response.docs = [] getdoc("User", "Administrator") doc = frappe.response.docs[0] - self.assertListEqual(doc.get("__onload").get('all_modules', []), - [m.get("module_name") for m in get_modules_from_all_apps()]) + self.assertListEqual( + doc.get("__onload").get("all_modules", []), + [m.get("module_name") for m in get_modules_from_all_apps()], + ) def delete_contact(user): diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 1ad977547c..c90cbf1fce 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -1,32 +1,46 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE from bs4 import BeautifulSoup + import frappe -import frappe.share import frappe.defaults import frappe.permissions -from frappe.model.document import Document -from frappe.utils import (cint, flt, has_gravatar, escape_html, format_datetime, - now_datetime, get_formatted_email, today, get_time_zone) -from frappe import throw, msgprint, _ -from frappe.utils.password import update_password as _update_password, check_password, get_password_reset_limit +import frappe.share +from frappe import _, msgprint, throw +from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype +from frappe.desk.doctype.notification_settings.notification_settings import ( + create_notification_settings, + toggle_notifications, +) from frappe.desk.notifications import clear_notifications -from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings, toggle_notifications +from frappe.model.document import Document +from frappe.query_builder import DocType +from frappe.rate_limiter import rate_limit +from frappe.utils import ( + cint, + escape_html, + flt, + format_datetime, + get_formatted_email, + get_time_zone, + has_gravatar, + now_datetime, + today, +) +from frappe.utils.password import check_password, get_password_reset_limit +from frappe.utils.password import update_password as _update_password from frappe.utils.user import get_system_managers from frappe.website.utils import is_signup_disabled -from frappe.rate_limiter import rate_limit -from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype -from frappe.query_builder import DocType - STANDARD_USERS = frappe.STANDARD_USERS + class User(Document): __new_password = None def __setup__(self): # because it is handled separately - self.flags.ignore_save_passwords = ['new_password'] + self.flags.ignore_save_passwords = ["new_password"] def autoname(self): """set name as Email Address""" @@ -38,8 +52,8 @@ class User(Document): def onload(self): from frappe.config import get_modules_from_all_apps - self.set_onload('all_modules', - [m.get("module_name") for m in get_modules_from_all_apps()]) + + self.set_onload("all_modules", [m.get("module_name") for m in get_modules_from_all_apps()]) def before_insert(self): self.flags.in_insert = True @@ -47,8 +61,8 @@ class User(Document): def after_insert(self): create_notification_settings(self.name) - frappe.cache().delete_key('users_for_mentions') - frappe.cache().delete_key('enabled_users') + frappe.cache().delete_key("users_for_mentions") + frappe.cache().delete_key("enabled_users") def validate(self): # clear new password @@ -79,23 +93,23 @@ class User(Document): if self.language == "Loading...": self.language = None - if (self.name not in ["Administrator", "Guest"]) and (not self.get_social_login_userid("frappe")): + if (self.name not in ["Administrator", "Guest"]) and ( + not self.get_social_login_userid("frappe") + ): self.set_social_login_userid("frappe", frappe.generate_hash(length=39)) def validate_roles(self): if self.role_profile_name: - role_profile = frappe.get_doc('Role Profile', self.role_profile_name) - self.set('roles', []) + role_profile = frappe.get_doc("Role Profile", self.role_profile_name) + self.set("roles", []) self.append_roles(*[role.role for role in role_profile.roles]) def validate_allowed_modules(self): if self.module_profile: - module_profile = frappe.get_doc('Module Profile', self.module_profile) - self.set('block_modules', []) - for d in module_profile.get('block_modules'): - self.append('block_modules', { - 'module': d.module - }) + module_profile = frappe.get_doc("Module Profile", self.module_profile) + self.set("block_modules", []) + for d in module_profile.get("block_modules"): + self.append("block_modules", {"module": d.module}) def validate_user_image(self): if self.user_image and len(self.user_image) > 2000: @@ -106,26 +120,23 @@ class User(Document): self.share_with_self() clear_notifications(user=self.name) frappe.clear_cache(user=self.name) - now=frappe.flags.in_test or frappe.flags.in_install + now = frappe.flags.in_test or frappe.flags.in_install self.send_password_notification(self.__new_password) frappe.enqueue( - 'frappe.core.doctype.user.user.create_contact', - user=self, - ignore_mandatory=True, - now=now + "frappe.core.doctype.user.user.create_contact", user=self, ignore_mandatory=True, now=now ) - if self.name not in ('Administrator', 'Guest') and not self.user_image: - frappe.enqueue('frappe.core.doctype.user.user.update_gravatar', name=self.name, now=now) + if self.name not in ("Administrator", "Guest") and not self.user_image: + frappe.enqueue("frappe.core.doctype.user.user.update_gravatar", name=self.name, now=now) # Set user selected timezone if self.time_zone: frappe.defaults.set_default("time_zone", self.time_zone, self.name) - if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'): - frappe.cache().delete_key('users_for_mentions') + if self.has_value_changed("allow_in_mentions") or self.has_value_changed("user_type"): + frappe.cache().delete_key("users_for_mentions") - if self.has_value_changed('enabled'): - frappe.cache().delete_key('enabled_users') + if self.has_value_changed("enabled"): + frappe.cache().delete_key("enabled_users") def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" @@ -151,65 +162,60 @@ class User(Document): def add_system_manager_role(self): # if adding system manager, do nothing - if not cint(self.enabled) or ("System Manager" in [user_role.role for user_role in - self.get("roles")]): + if not cint(self.enabled) or ( + "System Manager" in [user_role.role for user_role in self.get("roles")] + ): return - if (self.name not in STANDARD_USERS and self.user_type == "System User" and not self.get_other_system_managers() - and cint(frappe.db.get_single_value('System Settings', 'setup_complete'))): + if ( + self.name not in STANDARD_USERS + and self.user_type == "System User" + and not self.get_other_system_managers() + and cint(frappe.db.get_single_value("System Settings", "setup_complete")) + ): msgprint(_("Adding System Manager to this User as there must be atleast one System Manager")) - self.append("roles", { - "doctype": "Has Role", - "role": "System Manager" - }) + self.append("roles", {"doctype": "Has Role", "role": "System Manager"}) - if self.name == 'Administrator': + if self.name == "Administrator": # Administrator should always have System Manager Role - self.extend("roles", [ - { - "doctype": "Has Role", - "role": "System Manager" - }, - { - "doctype": "Has Role", - "role": "Administrator" - } - ]) + self.extend( + "roles", + [ + {"doctype": "Has Role", "role": "System Manager"}, + {"doctype": "Has Role", "role": "Administrator"}, + ], + ) def email_new_password(self, new_password=None): if new_password and not self.flags.in_insert: _update_password(user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions) def set_system_user(self): - '''For the standard users like admin and guest, the user type is fixed.''' - user_type_mapper = { - 'Administrator': 'System User', - 'Guest': 'Website User' - } + """For the standard users like admin and guest, the user type is fixed.""" + user_type_mapper = {"Administrator": "System User", "Guest": "Website User"} - if self.user_type and not frappe.get_cached_value('User Type', self.user_type, 'is_standard'): + if self.user_type and not frappe.get_cached_value("User Type", self.user_type, "is_standard"): if user_type_mapper.get(self.name): self.user_type = user_type_mapper.get(self.name) else: self.set_roles_and_modules_based_on_user_type() else: - '''Set as System User if any of the given roles has desk_access''' - self.user_type = 'System User' if self.has_desk_access() else 'Website User' + """Set as System User if any of the given roles has desk_access""" + self.user_type = "System User" if self.has_desk_access() else "Website User" def set_roles_and_modules_based_on_user_type(self): - user_type_doc = frappe.get_cached_doc('User Type', self.user_type) + user_type_doc = frappe.get_cached_doc("User Type", self.user_type) if user_type_doc.role: self.roles = [] # Check whether User has linked with the 'Apply User Permission On' doctype or not if user_linked_with_permission_on_doctype(user_type_doc, self.name): - self.append('roles', { - 'role': user_type_doc.role - }) + self.append("roles", {"role": user_type_doc.role}) - frappe.msgprint(_('Role has been set as per the user type {0}') - .format(self.user_type), alert=True) + frappe.msgprint( + _("Role has been set as per the user type {0}").format(self.user_type), alert=True + ) user_type_doc.update_modules_in_user(self) @@ -219,20 +225,24 @@ class User(Document): return False role_table = DocType("Role") - return frappe.db.count(role_table, ((role_table.desk_access == 1) & (role_table.name.isin([d.role for d in self.roles])))) + return frappe.db.count( + role_table, + ((role_table.desk_access == 1) & (role_table.name.isin([d.role for d in self.roles]))), + ) def share_with_self(self): - frappe.share.add(self.doctype, self.name, self.name, write=1, share=1, - flags={"ignore_share_permission": True}) + frappe.share.add( + self.doctype, self.name, self.name, write=1, share=1, flags={"ignore_share_permission": True} + ) def validate_share(self, docshare): pass # if docshare.user == self.name: - # if self.user_type=="System User": - # if docshare.share != 1: - # frappe.throw(_("Sorry! User should have complete access to their own record.")) - # else: - # frappe.throw(_("Sorry! Sharing with Website User is prohibited.")) + # if self.user_type=="System User": + # if docshare.share != 1: + # frappe.throw(_("Sorry! User should have complete access to their own record.")) + # else: + # frappe.throw(_("Sorry! Sharing with Website User is prohibited.")) def send_password_notification(self, new_password): try: @@ -240,13 +250,14 @@ class User(Document): if self.name not in STANDARD_USERS: if new_password: # new password given, no email required - _update_password(user=self.name, pwd=new_password, - logout_all_sessions=self.logout_all_sessions) + _update_password( + user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions + ) if not self.flags.no_welcome_mail and cint(self.send_welcome_email): self.send_welcome_mail_to_user() self.flags.email_sent = 1 - if frappe.session.user != 'Guest': + if frappe.session.user != "Guest": msgprint(_("Welcome email sent")) return else: @@ -261,14 +272,14 @@ class User(Document): pass def reset_password(self, send_email=False, password_expired=False): - from frappe.utils import random_string, get_url + from frappe.utils import get_url, random_string key = random_string(32) self.db_set("reset_password_key", key) url = "/update-password?key=" + key if password_expired: - url = "/update-password?key=" + key + '&password_expired=true' + url = "/update-password?key=" + key + "&password_expired=true" link = get_url(url) if send_email: @@ -283,7 +294,7 @@ class User(Document): frappe.qb.from_(user_doctype) .from_(user_role_doctype) .select(user_doctype.name) - .where(user_role_doctype.role == 'System Manager') + .where(user_role_doctype.role == "System Manager") .where(user_doctype.docstatus < 2) .where(user_doctype.enabled == 1) .where(user_role_doctype.parent == user_doctype.name) @@ -294,57 +305,68 @@ class User(Document): def get_fullname(self): """get first_name space last_name""" - return (self.first_name or '') + \ - (self.first_name and " " or '') + (self.last_name or '') + return (self.first_name or "") + (self.first_name and " " or "") + (self.last_name or "") def password_reset_mail(self, link): - self.send_login_mail(_("Password Reset"), - "password_reset", {"link": link}, now=True) + self.send_login_mail(_("Password Reset"), "password_reset", {"link": link}, now=True) def send_welcome_mail_to_user(self): from frappe.utils import get_url + link = self.reset_password() subject = None method = frappe.get_hooks("welcome_email") if method: subject = frappe.get_attr(method[-1])() if not subject: - site_name = frappe.db.get_default('site_name') or frappe.get_conf().get("site_name") + site_name = frappe.db.get_default("site_name") or frappe.get_conf().get("site_name") if site_name: subject = _("Welcome to {0}").format(site_name) else: subject = _("Complete Registration") - self.send_login_mail(subject, "new_user", - dict( - link=link, - site_url=get_url(), - )) + self.send_login_mail( + subject, + "new_user", + dict( + link=link, + site_url=get_url(), + ), + ) def send_login_mail(self, subject, template, add_args, now=None): """send mail with login details""" - from frappe.utils.user import get_user_fullname from frappe.utils import get_url + from frappe.utils.user import get_user_fullname - created_by = get_user_fullname(frappe.session['user']) + created_by = get_user_fullname(frappe.session["user"]) if created_by == "Guest": created_by = "Administrator" args = { - 'first_name': self.first_name or self.last_name or "user", - 'user': self.name, - 'title': subject, - 'login_url': get_url(), - 'created_by': created_by + "first_name": self.first_name or self.last_name or "user", + "user": self.name, + "title": subject, + "login_url": get_url(), + "created_by": created_by, } args.update(add_args) - sender = frappe.session.user not in STANDARD_USERS and get_formatted_email(frappe.session.user) or None + sender = ( + frappe.session.user not in STANDARD_USERS and get_formatted_email(frappe.session.user) or None + ) - frappe.sendmail(recipients=self.email, sender=sender, subject=subject, - template=template, args=args, header=[subject, "green"], - delayed=(not now) if now is not None else self.flags.delay_emails, retry=3) + frappe.sendmail( + recipients=self.email, + sender=sender, + subject=subject, + template=template, + args=args, + header=[subject, "green"], + delayed=(not now) if now is not None else self.flags.delay_emails, + retry=3, + ) def a_system_manager_should_exist(self): if not self.get_other_system_managers(): @@ -389,18 +411,15 @@ class User(Document): ) # unlink contact table = DocType("Contact") - frappe.qb.update(table).where( - table.user == self.name - ).set(table.user, None).run() + frappe.qb.update(table).where(table.user == self.name).set(table.user, None).run() # delete notification settings frappe.delete_doc("Notification Settings", self.name, ignore_permissions=True) - if self.get('allow_in_mentions'): - frappe.cache().delete_key('users_for_mentions') - - frappe.cache().delete_key('enabled_users') + if self.get("allow_in_mentions"): + frappe.cache().delete_key("users_for_mentions") + frappe.cache().delete_key("enabled_users") def before_rename(self, old_name, new_name, merge=False): frappe.clear_cache(user=old_name) @@ -415,6 +434,7 @@ class User(Document): def validate_email_type(self, email): from frappe.utils import validate_email_address + validate_email_address(email.strip(), True) def after_rename(self, old_name, new_name, merge=False): @@ -423,13 +443,16 @@ class User(Document): desc = frappe.db.get_table_columns_description(tab) has_fields = [] for d in desc: - if d.get('name') in ['owner', 'modified_by']: - has_fields.append(d.get('name')) + if d.get("name") in ["owner", "modified_by"]: + has_fields.append(d.get("name")) for field in has_fields: - frappe.db.sql("""UPDATE `%s` + frappe.db.sql( + """UPDATE `%s` SET `%s` = %s - WHERE `%s` = %s""" % - (tab, field, '%s', field, '%s'), (new_name, old_name)) + WHERE `%s` = %s""" + % (tab, field, "%s", field, "%s"), + (new_name, old_name), + ) if frappe.db.exists("Notification Settings", old_name): frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False) @@ -463,10 +486,10 @@ class User(Document): self.set("roles", list(set(d for d in self.get("roles") if d.role == "Guest"))) def remove_disabled_roles(self): - disabled_roles = [d.name for d in frappe.get_all("Role", filters={"disabled":1})] - for role in list(self.get('roles')): + disabled_roles = [d.name for d in frappe.get_all("Role", filters={"disabled": 1})] + for role in list(self.get("roles")): if role.role in disabled_roles: - self.get('roles').remove(role) + self.get("roles").remove(role) def ensure_unique_roles(self): exists = [] @@ -487,23 +510,23 @@ class User(Document): self.username = self.username.strip(" @") if self.username_exists(): - if self.user_type == 'System User': + if self.user_type == "System User": frappe.msgprint(_("Username {0} already exists").format(self.username)) self.suggest_username() self.username = "" def password_strength_test(self): - """ test password strength """ + """test password strength""" if self.flags.ignore_password_policy: return if self.__new_password: user_data = (self.first_name, self.middle_name, self.last_name, self.email, self.birth_date) - result = test_password_strength(self.__new_password, '', None, user_data) + result = test_password_strength(self.__new_password, "", None, user_data) feedback = result.get("feedback", None) - if feedback and not feedback.get('password_policy_validation_passed', False): + if feedback and not feedback.get("password_policy_validation_passed", False): handle_password_test_fail(result) def suggest_username(self): @@ -518,7 +541,9 @@ class User(Document): if not username: # @firstname_last_name - username = _check_suggestion(frappe.scrub("{0} {1}".format(self.first_name, self.last_name or ""))) + username = _check_suggestion( + frappe.scrub("{0} {1}".format(self.first_name, self.last_name or "")) + ) if username: frappe.msgprint(_("Suggested Username: {0}").format(username)) @@ -526,16 +551,18 @@ class User(Document): return username def username_exists(self, username=None): - return frappe.db.get_value("User", {"username": username or self.username, "name": ("!=", self.name)}) + return frappe.db.get_value( + "User", {"username": username or self.username, "name": ("!=", self.name)} + ) def get_blocked_modules(self): """Returns list of modules blocked for that user""" return [d.module for d in self.block_modules] if self.block_modules else [] def validate_user_email_inbox(self): - """ check if same email account added in User Emails twice """ + """check if same email account added in User Emails twice""" - email_accounts = [ user_email.email_account for user_email in self.user_emails ] + email_accounts = [user_email.email_account for user_email in self.user_emails] if len(email_accounts) != len(set(email_accounts)): frappe.throw(_("Email Account added multiple times")) @@ -548,10 +575,7 @@ class User(Document): return None def set_social_login_userid(self, provider, userid, username=None): - social_logins = { - "provider": provider, - "userid": userid - } + social_logins = {"provider": provider, "userid": userid} if username: social_logins["username"] = username @@ -574,8 +598,12 @@ class User(Document): 3. If allow_login_using_user_name is set, you can use username while finding the user. """ - login_with_mobile = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number")) - login_with_username = cint(frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name")) + login_with_mobile = cint( + frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number") + ) + login_with_username = cint( + frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name") + ) or_filters = [{"name": user_name}] if login_with_mobile: @@ -583,17 +611,17 @@ class User(Document): if login_with_username: or_filters.append({"username": user_name}) - users = frappe.db.get_all('User', fields=['name', 'enabled'], or_filters=or_filters, limit=1) + users = frappe.db.get_all("User", fields=["name", "enabled"], or_filters=or_filters, limit=1) if not users: return user = users[0] - user['is_authenticated'] = True + user["is_authenticated"] = True if validate_password: try: - check_password(user['name'], password, delete_tracker_cache=False) + check_password(user["name"], password, delete_tracker_cache=False) except frappe.AuthenticationError: - user['is_authenticated'] = False + user["is_authenticated"] = False return user @@ -601,68 +629,74 @@ class User(Document): if not self.time_zone: self.time_zone = get_time_zone() + @frappe.whitelist() def get_timezones(): import pytz - return { - "timezones": pytz.all_timezones - } + + return {"timezones": pytz.all_timezones} + @frappe.whitelist() def get_all_roles(arg=None): """return all roles""" active_domains = frappe.get_active_domains() - roles = frappe.get_all("Role", filters={ - "name": ("not in", "Administrator,Guest,All"), - "disabled": 0 - }, or_filters={ - "ifnull(restrict_to_domain, '')": "", - "restrict_to_domain": ("in", active_domains) - }, order_by="name") + roles = frappe.get_all( + "Role", + filters={"name": ("not in", "Administrator,Guest,All"), "disabled": 0}, + or_filters={"ifnull(restrict_to_domain, '')": "", "restrict_to_domain": ("in", active_domains)}, + order_by="name", + ) + + return [role.get("name") for role in roles] - return [ role.get("name") for role in roles ] @frappe.whitelist() def get_roles(arg=None): """get roles for a user""" - return frappe.get_roles(frappe.form_dict['uid']) + return frappe.get_roles(frappe.form_dict["uid"]) + @frappe.whitelist() def get_perm_info(role): """get permission info""" from frappe.permissions import get_all_perms + return get_all_perms(role) + @frappe.whitelist(allow_guest=True) def update_password(new_password, logout_all_sessions=0, key=None, old_password=None): - #validate key to avoid key input like ['like', '%'], '', ['in', ['']] + # validate key to avoid key input like ['like', '%'], '', ['in', ['']] if key and not isinstance(key, str): - frappe.throw(_('Invalid key type')) + frappe.throw(_("Invalid key type")) result = test_password_strength(new_password, key, old_password) feedback = result.get("feedback", None) - if feedback and not feedback.get('password_policy_validation_passed', False): + if feedback and not feedback.get("password_policy_validation_passed", False): handle_password_test_fail(result) res = _get_user_for_update_password(key, old_password) - if res.get('message'): + if res.get("message"): frappe.local.response.http_status_code = 410 - return res['message'] + return res["message"] else: - user = res['user'] + user = res["user"] - logout_all_sessions = cint(logout_all_sessions) or frappe.db.get_single_value("System Settings", "logout_on_password_reset") + logout_all_sessions = cint(logout_all_sessions) or frappe.db.get_single_value( + "System Settings", "logout_on_password_reset" + ) _update_password(user, new_password, logout_all_sessions=cint(logout_all_sessions)) user_doc, redirect_url = reset_user_data(user) # get redirect url from cache - redirect_to = frappe.cache().hget('redirect_after_login', user) + redirect_to = frappe.cache().hget("redirect_after_login", user) if redirect_to: redirect_url = redirect_to - frappe.cache().hdel('redirect_after_login', user) + frappe.cache().hdel("redirect_after_login", user) frappe.local.login_manager.login_as(user) @@ -674,12 +708,17 @@ def update_password(new_password, logout_all_sessions=0, key=None, old_password= else: return redirect_url if redirect_url else "/" + @frappe.whitelist(allow_guest=True) def test_password_strength(new_password, key=None, old_password=None, user_data=None): from frappe.utils.password_strength import test_password_strength as _test_password_strength - password_policy = frappe.db.get_value("System Settings", None, - ["enable_password_policy", "minimum_password_score"], as_dict=True) or {} + password_policy = ( + frappe.db.get_value( + "System Settings", None, ["enable_password_policy", "minimum_password_score"], as_dict=True + ) + or {} + ) enable_password_policy = cint(password_policy.get("enable_password_policy", 0)) minimum_password_score = cint(password_policy.get("minimum_password_score", 0)) @@ -688,41 +727,54 @@ def test_password_strength(new_password, key=None, old_password=None, user_data= return {} if not user_data: - user_data = frappe.db.get_value('User', frappe.session.user, - ['first_name', 'middle_name', 'last_name', 'email', 'birth_date']) + user_data = frappe.db.get_value( + "User", frappe.session.user, ["first_name", "middle_name", "last_name", "email", "birth_date"] + ) if new_password: result = _test_password_strength(new_password, user_inputs=user_data) password_policy_validation_passed = False # score should be greater than 0 and minimum_password_score - if result.get('score') and result.get('score') >= minimum_password_score: + if result.get("score") and result.get("score") >= minimum_password_score: password_policy_validation_passed = True - result['feedback']['password_policy_validation_passed'] = password_policy_validation_passed + result["feedback"]["password_policy_validation_passed"] = password_policy_validation_passed return result -#for login + +# for login @frappe.whitelist() def has_email_account(email): return frappe.get_list("Email Account", filters={"email_id": email}) + @frappe.whitelist(allow_guest=False) def get_email_awaiting(user): - waiting = frappe.get_all("User Email", fields=["email_account", "email_id"], filters={"awaiting_password": 1, "parent": user}) + waiting = frappe.get_all( + "User Email", + fields=["email_account", "email_id"], + filters={"awaiting_password": 1, "parent": user}, + ) if waiting: return waiting else: user_email_table = DocType("User Email") - frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where(user_email_table.parent == user).run() + frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where( + user_email_table.parent == user + ).run() return False + def ask_pass_update(): # update the sys defaults as to awaiting users from frappe.utils import set_default - password_list = frappe.get_all("User Email", filters={"awaiting_password": True}, pluck="parent", distinct=True) - set_default("email_user_password", u','.join(password_list)) + password_list = frappe.get_all( + "User Email", filters={"awaiting_password": True}, pluck="parent", distinct=True + ) + set_default("email_user_password", ",".join(password_list)) + def _get_user_for_update_password(key, old_password): # verify old password @@ -740,19 +792,22 @@ def _get_user_for_update_password(key, old_password): return result + def reset_user_data(user): user_doc = frappe.get_doc("User", user) redirect_url = user_doc.redirect_url - user_doc.reset_password_key = '' - user_doc.redirect_url = '' + user_doc.reset_password_key = "" + user_doc.redirect_url = "" user_doc.save(ignore_permissions=True) return user_doc, redirect_url + @frappe.whitelist() def verify_password(password): frappe.local.login_manager.check_password(frappe.session.user, password) + @frappe.whitelist(allow_guest=True) def sign_up(email, full_name, redirect_to): if is_signup_disabled(): @@ -765,20 +820,27 @@ def sign_up(email, full_name, redirect_to): else: return 0, _("Registered but disabled") else: - if frappe.db.get_creation_count('User', 60) > 300: - frappe.respond_as_web_page(_('Temporarily Disabled'), - _('Too many users signed up recently, so the registration is disabled. Please try back in an hour'), - http_status_code=429) + if frappe.db.get_creation_count("User", 60) > 300: + frappe.respond_as_web_page( + _("Temporarily Disabled"), + _( + "Too many users signed up recently, so the registration is disabled. Please try back in an hour" + ), + http_status_code=429, + ) from frappe.utils import random_string - user = frappe.get_doc({ - "doctype":"User", - "email": email, - "first_name": escape_html(full_name), - "enabled": 1, - "new_password": random_string(10), - "user_type": "Website User" - }) + + user = frappe.get_doc( + { + "doctype": "User", + "email": email, + "first_name": escape_html(full_name), + "enabled": 1, + "new_password": random_string(10), + "user_type": "Website User", + } + ) user.flags.ignore_permissions = True user.flags.ignore_password_policy = True user.insert() @@ -789,49 +851,53 @@ def sign_up(email, full_name, redirect_to): user.add_roles(default_role) if redirect_to: - frappe.cache().hset('redirect_after_login', user.name, redirect_to) + frappe.cache().hset("redirect_after_login", user.name, redirect_to) if user.flags.email_sent: return 1, _("Please check your email for verification") else: return 2, _("Please ask your administrator to verify your sign-up") + @frappe.whitelist(allow_guest=True) -@rate_limit(limit=get_password_reset_limit, seconds = 24*60*60, methods=['POST']) +@rate_limit(limit=get_password_reset_limit, seconds=24 * 60 * 60, methods=["POST"]) def reset_password(user): - if user=="Administrator": - return 'not allowed' + if user == "Administrator": + return "not allowed" try: user = frappe.get_doc("User", user) if not user.enabled: - return 'disabled' + return "disabled" user.validate_reset_password() user.reset_password(send_email=True) return frappe.msgprint( msg=_("Password reset instructions have been sent to your email"), - title=_("Password Email Sent") + title=_("Password Email Sent"), ) except frappe.DoesNotExistError: - frappe.local.response['http_status_code'] = 400 + frappe.local.response["http_status_code"] = 400 frappe.clear_messages() - return 'not found' + return "not found" + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def user_query(doctype, txt, searchfield, start, page_len, filters): - from frappe.desk.reportview import get_match_cond, get_filters_cond - conditions=[] + from frappe.desk.reportview import get_filters_cond, get_match_cond + + conditions = [] user_type_condition = "and user_type != 'Website User'" - if filters and filters.get('ignore_user_type'): - user_type_condition = '' - filters.pop('ignore_user_type') + if filters and filters.get("ignore_user_type"): + user_type_condition = "" + filters.pop("ignore_user_type") txt = "%{}%".format(txt) - return frappe.db.sql("""SELECT `name`, CONCAT_WS(' ', first_name, middle_name, last_name) + return frappe.db.sql( + """SELECT `name`, CONCAT_WS(' ', first_name, middle_name, last_name) FROM `tabUser` WHERE `enabled`=1 {user_type_condition} @@ -847,22 +913,31 @@ def user_query(doctype, txt, searchfield, start, page_len, filters): NAME asc LIMIT %(page_len)s OFFSET %(start)s """.format( - user_type_condition = user_type_condition, + user_type_condition=user_type_condition, standard_users=", ".join(frappe.db.escape(u) for u in STANDARD_USERS), key=searchfield, fcond=get_filters_cond(doctype, filters, conditions), - mcond=get_match_cond(doctype) + mcond=get_match_cond(doctype), ), - dict(start=start, page_len=page_len, txt=txt) + dict(start=start, page_len=page_len, txt=txt), ) + def get_total_users(): """Returns total no. of system users""" - return flt(frappe.db.sql('''SELECT SUM(`simultaneous_sessions`) + return flt( + frappe.db.sql( + """SELECT SUM(`simultaneous_sessions`) FROM `tabUser` WHERE `enabled` = 1 AND `user_type` = 'System User' - AND `name` NOT IN ({})'''.format(", ".join(["%s"]*len(STANDARD_USERS))), STANDARD_USERS)[0][0]) + AND `name` NOT IN ({})""".format( + ", ".join(["%s"] * len(STANDARD_USERS)) + ), + STANDARD_USERS, + )[0][0] + ) + def get_system_users(exclude_users=None, limit=None): if not exclude_users: @@ -870,126 +945,160 @@ def get_system_users(exclude_users=None, limit=None): elif not isinstance(exclude_users, (list, tuple)): exclude_users = [exclude_users] - limit_cond = '' + limit_cond = "" if limit: - limit_cond = 'limit {0}'.format(limit) + limit_cond = "limit {0}".format(limit) exclude_users += list(STANDARD_USERS) - system_users = frappe.db.sql_list("""select name from `tabUser` + system_users = frappe.db.sql_list( + """select name from `tabUser` where enabled=1 and user_type != 'Website User' - and name not in ({}) {}""".format(", ".join(["%s"]*len(exclude_users)), limit_cond), - exclude_users) + and name not in ({}) {}""".format( + ", ".join(["%s"] * len(exclude_users)), limit_cond + ), + exclude_users, + ) return system_users + def get_active_users(): """Returns No. of system users who logged in, in the last 3 days""" - return frappe.db.sql("""select count(*) from `tabUser` + return frappe.db.sql( + """select count(*) from `tabUser` where enabled = 1 and user_type != 'Website User' and name not in ({}) - and hour(timediff(now(), last_active)) < 72""".format(", ".join(["%s"]*len(STANDARD_USERS))), STANDARD_USERS)[0][0] + and hour(timediff(now(), last_active)) < 72""".format( + ", ".join(["%s"] * len(STANDARD_USERS)) + ), + STANDARD_USERS, + )[0][0] + def get_website_users(): """Returns total no. of website users""" return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"}) + def get_active_website_users(): """Returns No. of website users who logged in, in the last 3 days""" - return frappe.db.sql("""select count(*) from `tabUser` + return frappe.db.sql( + """select count(*) from `tabUser` where enabled = 1 and user_type = 'Website User' - and hour(timediff(now(), last_active)) < 72""")[0][0] + and hour(timediff(now(), last_active)) < 72""" + )[0][0] + def get_permission_query_conditions(user): - if user=="Administrator": + if user == "Administrator": return "" else: return """(`tabUser`.name not in ({standard_users}))""".format( - standard_users = ", ".join(frappe.db.escape(user) for user in STANDARD_USERS)) + standard_users=", ".join(frappe.db.escape(user) for user in STANDARD_USERS) + ) + def has_permission(doc, user): if (user != "Administrator") and (doc.name in STANDARD_USERS): # dont allow non Administrator user to view / edit Administrator user return False + def notify_admin_access_to_system_manager(login_manager=None): - if (login_manager + if ( + login_manager and login_manager.user == "Administrator" - and frappe.local.conf.notify_admin_access_to_system_manager): + and frappe.local.conf.notify_admin_access_to_system_manager + ): site = '{0}'.format(frappe.local.request.host_url) - date_and_time = '{0}'.format(format_datetime(now_datetime(), format_string="medium")) + date_and_time = "{0}".format(format_datetime(now_datetime(), format_string="medium")) ip_address = frappe.local.request_ip - access_message = _('Administrator accessed {0} on {1} via IP Address {2}.').format( - site, date_and_time, ip_address) + access_message = _("Administrator accessed {0} on {1} via IP Address {2}.").format( + site, date_and_time, ip_address + ) frappe.sendmail( recipients=get_system_managers(), subject=_("Administrator Logged In"), template="administrator_logged_in", - args={'access_message': access_message}, - header=['Access Notification', 'orange'] + args={"access_message": access_message}, + header=["Access Notification", "orange"], ) + def extract_mentions(txt): """Find all instances of @mentions in the html.""" - soup = BeautifulSoup(txt, 'html.parser') + soup = BeautifulSoup(txt, "html.parser") emails = [] - for mention in soup.find_all(class_='mention'): - if mention.get('data-is-group') == 'true': + for mention in soup.find_all(class_="mention"): + if mention.get("data-is-group") == "true": try: - user_group = frappe.get_cached_doc('User Group', mention['data-id']) + user_group = frappe.get_cached_doc("User Group", mention["data-id"]) emails += [d.user for d in user_group.user_group_members] except frappe.DoesNotExistError: pass continue - email = mention['data-id'] + email = mention["data-id"] emails.append(email) return emails + def handle_password_test_fail(result): - suggestions = result['feedback']['suggestions'][0] if result['feedback']['suggestions'] else '' - warning = result['feedback']['warning'] if 'warning' in result['feedback'] else '' - suggestions += "
" + _("Hint: Include symbols, numbers and capital letters in the password") + '
' - frappe.throw(' '.join([_('Invalid Password:'), warning, suggestions])) + suggestions = result["feedback"]["suggestions"][0] if result["feedback"]["suggestions"] else "" + warning = result["feedback"]["warning"] if "warning" in result["feedback"] else "" + suggestions += ( + "
" + _("Hint: Include symbols, numbers and capital letters in the password") + "
" + ) + frappe.throw(" ".join([_("Invalid Password:"), warning, suggestions])) + def update_gravatar(name): gravatar = has_gravatar(name) if gravatar: - frappe.db.set_value('User', name, 'user_image', gravatar) + frappe.db.set_value("User", name, "user_image", gravatar) + def throttle_user_creation(): if frappe.flags.in_import: return - if frappe.db.get_creation_count('User', 60) > frappe.local.conf.get("throttle_user_limit", 60): - frappe.throw(_('Throttled')) + if frappe.db.get_creation_count("User", 60) > frappe.local.conf.get("throttle_user_limit", 60): + frappe.throw(_("Throttled")) + @frappe.whitelist() def get_role_profile(role_profile): - roles = frappe.get_doc('Role Profile', {'role_profile': role_profile}) + roles = frappe.get_doc("Role Profile", {"role_profile": role_profile}) return roles.roles + @frappe.whitelist() def get_module_profile(module_profile): - module_profile = frappe.get_doc('Module Profile', {'module_profile_name': module_profile}) - return module_profile.get('block_modules') + module_profile = frappe.get_doc("Module Profile", {"module_profile_name": module_profile}) + return module_profile.get("block_modules") + def create_contact(user, ignore_links=False, ignore_mandatory=False): from frappe.contacts.doctype.contact.contact import get_contact_name - if user.name in ["Administrator", "Guest"]: return + + if user.name in ["Administrator", "Guest"]: + return contact_name = get_contact_name(user.email) if not contact_name: - contact = frappe.get_doc({ - "doctype": "Contact", - "first_name": user.first_name, - "last_name": user.last_name, - "user": user.name, - "gender": user.gender, - }) + contact = frappe.get_doc( + { + "doctype": "Contact", + "first_name": user.first_name, + "last_name": user.last_name, + "user": user.name, + "gender": user.gender, + } + ) if user.email: contact.add_email(user.email, is_primary=True) @@ -999,7 +1108,9 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False): if user.mobile_no: contact.add_phone(user.mobile_no, is_primary_mobile_no=True) - contact.insert(ignore_permissions=True, ignore_links=ignore_links, ignore_mandatory=ignore_mandatory) + contact.insert( + ignore_permissions=True, ignore_links=ignore_links, ignore_mandatory=ignore_mandatory + ) else: contact = frappe.get_doc("Contact", contact_name) contact.first_name = user.first_name @@ -1013,21 +1124,24 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False): user.phone, is_primary_phone=not any( new_contact.is_primary_phone == 1 for new_contact in contact.phone_nos - ) + ), ) # Add mobile number if mobile does not exists in contact - if user.mobile_no and not any(new_contact.phone == user.mobile_no for new_contact in contact.phone_nos): + if user.mobile_no and not any( + new_contact.phone == user.mobile_no for new_contact in contact.phone_nos + ): # Set primary mobile if there is no primary mobile number contact.add_phone( user.mobile_no, is_primary_mobile_no=not any( new_contact.is_primary_mobile_no == 1 for new_contact in contact.phone_nos - ) + ), ) contact.save(ignore_permissions=True) + @frappe.whitelist() def generate_keys(user): """ @@ -1053,6 +1167,7 @@ def switch_theme(theme): if theme in ["Dark", "Light", "Automatic"]: frappe.db.set_value("User", frappe.session.user, "desk_theme", theme) + def get_enabled_users(): def _get_enabled_users(): enabled_users = frappe.get_all("User", filters={"enabled": "1"}, pluck="name") diff --git a/frappe/core/doctype/user_document_type/user_document_type.py b/frappe/core/doctype/user_document_type/user_document_type.py index a14d735e6a..1e7eab4bd4 100644 --- a/frappe/core/doctype/user_document_type/user_document_type.py +++ b/frappe/core/doctype/user_document_type/user_document_type.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class UserDocumentType(Document): pass diff --git a/frappe/core/doctype/user_email/user_email.py b/frappe/core/doctype/user_email/user_email.py index daad083577..4e125f5308 100644 --- a/frappe/core/doctype/user_email/user_email.py +++ b/frappe/core/doctype/user_email/user_email.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class UserEmail(Document): pass diff --git a/frappe/core/doctype/user_group/test_user_group.py b/frappe/core/doctype/user_group/test_user_group.py index b5d642ae9c..546d4fd54c 100644 --- a/frappe/core/doctype/user_group/test_user_group.py +++ b/frappe/core/doctype/user_group/test_user_group.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestUserGroup(unittest.TestCase): pass diff --git a/frappe/core/doctype/user_group/user_group.py b/frappe/core/doctype/user_group/user_group.py index 05ff71e353..a59117426f 100644 --- a/frappe/core/doctype/user_group/user_group.py +++ b/frappe/core/doctype/user_group/user_group.py @@ -2,13 +2,15 @@ # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE +import frappe + # import frappe from frappe.model.document import Document -import frappe + class UserGroup(Document): def after_insert(self): - frappe.cache().delete_key('user_groups') + frappe.cache().delete_key("user_groups") def on_trash(self): - frappe.cache().delete_key('user_groups') + frappe.cache().delete_key("user_groups") diff --git a/frappe/core/doctype/user_group_member/test_user_group_member.py b/frappe/core/doctype/user_group_member/test_user_group_member.py index 6d4650a3d0..f1bdc41cff 100644 --- a/frappe/core/doctype/user_group_member/test_user_group_member.py +++ b/frappe/core/doctype/user_group_member/test_user_group_member.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestUserGroupMember(unittest.TestCase): pass diff --git a/frappe/core/doctype/user_group_member/user_group_member.py b/frappe/core/doctype/user_group_member/user_group_member.py index 69718d8d91..6b948d797f 100644 --- a/frappe/core/doctype/user_group_member/user_group_member.py +++ b/frappe/core/doctype/user_group_member/user_group_member.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class UserGroupMember(Document): pass diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index d4a9d68fd5..b963da2f49 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -1,12 +1,16 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # See LICENSE -from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable -from frappe.permissions import has_user_permission +import unittest + +import frappe from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.core.doctype.user_permission.user_permission import ( + add_user_permissions, + remove_applicable, +) +from frappe.permissions import has_user_permission from frappe.website.doctype.blog_post.test_blog_post import make_test_blog -import frappe -import unittest class TestUserPermission(unittest.TestCase): def setUp(self): @@ -15,64 +19,62 @@ class TestUserPermission(unittest.TestCase): "test_user_perm1@example.com", "nested_doc_user@example.com", ) - frappe.db.delete("User Permission", { - "user": ("in", test_users) - }) + frappe.db.delete("User Permission", {"user": ("in", test_users)}) frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") def test_default_user_permission_validation(self): - user = create_user('test_default_permission@example.com') - param = get_params(user, 'User', user.name, is_default=1) + user = create_user("test_default_permission@example.com") + param = get_params(user, "User", user.name, is_default=1) add_user_permissions(param) - #create a duplicate entry with default - perm_user = create_user('test_user_perm@example.com') - param = get_params(user, 'User', perm_user.name, is_default=1) + # create a duplicate entry with default + perm_user = create_user("test_user_perm@example.com") + param = get_params(user, "User", perm_user.name, is_default=1) self.assertRaises(frappe.ValidationError, add_user_permissions, param) def test_default_user_permission_corectness(self): - user = create_user('test_default_corectness_permission_1@example.com') - param = get_params(user, 'User', user.name, is_default=1, hide_descendants= 1) + user = create_user("test_default_corectness_permission_1@example.com") + param = get_params(user, "User", user.name, is_default=1, hide_descendants=1) add_user_permissions(param) - #create a duplicate entry with default - perm_user = create_user('test_default_corectness2@example.com') + # create a duplicate entry with default + perm_user = create_user("test_default_corectness2@example.com") test_blog = make_test_blog() - param = get_params(perm_user, 'Blog Post', test_blog.name, is_default=1, hide_descendants= 1) + param = get_params(perm_user, "Blog Post", test_blog.name, is_default=1, hide_descendants=1) add_user_permissions(param) - frappe.db.delete('User Permission', filters={'for_value': test_blog.name}) - frappe.delete_doc('Blog Post', test_blog.name) + frappe.db.delete("User Permission", filters={"for_value": test_blog.name}) + frappe.delete_doc("Blog Post", test_blog.name) def test_default_user_permission(self): - frappe.set_user('Administrator') - user = create_user('test_user_perm1@example.com', 'Website Manager') - for category in ['general', 'public']: - if not frappe.db.exists('Blog Category', category): - frappe.get_doc({'doctype': 'Blog Category', 'title': category}).insert() + frappe.set_user("Administrator") + user = create_user("test_user_perm1@example.com", "Website Manager") + for category in ["general", "public"]: + if not frappe.db.exists("Blog Category", category): + frappe.get_doc({"doctype": "Blog Category", "title": category}).insert() - param = get_params(user, 'Blog Category', 'general', is_default=1) + param = get_params(user, "Blog Category", "general", is_default=1) add_user_permissions(param) - param = get_params(user, 'Blog Category', 'public') + param = get_params(user, "Blog Category", "public") add_user_permissions(param) - frappe.set_user('test_user_perm1@example.com') + frappe.set_user("test_user_perm1@example.com") doc = frappe.new_doc("Blog Post") - self.assertEqual(doc.blog_category, 'general') - frappe.set_user('Administrator') + self.assertEqual(doc.blog_category, "general") + frappe.set_user("Administrator") def test_apply_to_all(self): - ''' Create User permission for User having access to all applicable Doctypes''' - user = create_user('test_bulk_creation_update@example.com') - param = get_params(user, 'User', user.name) + """Create User permission for User having access to all applicable Doctypes""" + user = create_user("test_bulk_creation_update@example.com") + param = get_params(user, "User", user.name) is_created = add_user_permissions(param) self.assertEqual(is_created, 1) def test_for_apply_to_all_on_update_from_apply_all(self): - user = create_user('test_bulk_creation_update@example.com') - param = get_params(user, 'User', user.name) + user = create_user("test_bulk_creation_update@example.com") + param = get_params(user, "User", user.name) # Initially create User Permission document with apply_to_all checked is_created = add_user_permissions(param) @@ -84,12 +86,12 @@ class TestUserPermission(unittest.TestCase): self.assertEqual(is_created, 0) def test_for_applicable_on_update_from_apply_to_all(self): - ''' Update User Permission from all to some applicable Doctypes''' - user = create_user('test_bulk_creation_update@example.com') - param = get_params(user,'User', user.name, applicable = ["Comment", "Contact"]) + """Update User Permission from all to some applicable Doctypes""" + user = create_user("test_bulk_creation_update@example.com") + param = get_params(user, "User", user.name, applicable=["Comment", "Contact"]) # Initially create User Permission document with apply_to_all checked - is_created = add_user_permissions(get_params(user, 'User', user.name)) + is_created = add_user_permissions(get_params(user, "User", user.name)) self.assertEqual(is_created, 1) @@ -97,8 +99,12 @@ class TestUserPermission(unittest.TestCase): frappe.db.commit() removed_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user)) - is_created_applicable_first = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Comment")) - is_created_applicable_second = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Contact")) + is_created_applicable_first = frappe.db.exists( + "User Permission", get_exists_param(user, applicable="Comment") + ) + is_created_applicable_second = frappe.db.exists( + "User Permission", get_exists_param(user, applicable="Contact") + ) # Check that apply_to_all is removed self.assertIsNone(removed_apply_to_all) @@ -109,19 +115,25 @@ class TestUserPermission(unittest.TestCase): self.assertEqual(is_created, 1) def test_for_apply_to_all_on_update_from_applicable(self): - ''' Update User Permission from some to all applicable Doctypes''' - user = create_user('test_bulk_creation_update@example.com') - param = get_params(user, 'User', user.name) + """Update User Permission from some to all applicable Doctypes""" + user = create_user("test_bulk_creation_update@example.com") + param = get_params(user, "User", user.name) # create User permissions that with applicable - is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Comment", "Contact"])) + is_created = add_user_permissions( + get_params(user, "User", user.name, applicable=["Comment", "Contact"]) + ) self.assertEqual(is_created, 1) is_created = add_user_permissions(param) is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user)) - removed_applicable_first = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Comment")) - removed_applicable_second = frappe.db.exists("User Permission", get_exists_param(user, applicable = "Contact")) + removed_applicable_first = frappe.db.exists( + "User Permission", get_exists_param(user, applicable="Comment") + ) + removed_applicable_second = frappe.db.exists( + "User Permission", get_exists_param(user, applicable="Contact") + ) # To check that a User permission with apply_to_all exists self.assertIsNotNone(is_created_apply_to_all) @@ -137,14 +149,11 @@ class TestUserPermission(unittest.TestCase): user = create_user("nested_doc_user@example.com", "Blogger") if not frappe.db.exists("DocType", "Person"): - doc = new_doctype("Person", - fields=[ - { - "label": "Person Name", - "fieldname": "person_name", - "fieldtype": "Data" - } - ], unique=0) + doc = new_doctype( + "Person", + fields=[{"label": "Person Name", "fieldname": "person_name", "fieldtype": "Data"}], + unique=0, + ) doc.is_tree = 1 doc.insert() @@ -153,7 +162,12 @@ class TestUserPermission(unittest.TestCase): ).insert() child_record = frappe.get_doc( - {"doctype": "Person", "person_name": "Child", "is_group": 0, "parent_person": parent_record.name} + { + "doctype": "Person", + "person_name": "Child", + "is_group": 0, + "parent_person": parent_record.name, + } ).insert() add_user_permissions(get_params(user, "Person", parent_record.name)) @@ -162,7 +176,9 @@ class TestUserPermission(unittest.TestCase): self.assertTrue(has_user_permission(frappe.get_doc("Person", parent_record.name), user.name)) self.assertTrue(has_user_permission(frappe.get_doc("Person", child_record.name), user.name)) - frappe.db.set_value("User Permission", {"allow": "Person", "for_value": parent_record.name}, "hide_descendants", 1) + frappe.db.set_value( + "User Permission", {"allow": "Person", "for_value": parent_record.name}, "hide_descendants", 1 + ) frappe.cache().delete_value("user_permissions") # check if adding perm on a group record with hide_descendants enabled, @@ -172,21 +188,24 @@ class TestUserPermission(unittest.TestCase): def test_user_perm_on_new_doc_with_field_default(self): """Test User Perm impact on frappe.new_doc. with *field* default value""" - frappe.set_user('Administrator') + frappe.set_user("Administrator") user = create_user("new_doc_test@example.com", "Blogger") # make a doctype "Doc A" with 'doctype' link field and default value ToDo if not frappe.db.exists("DocType", "Doc A"): - doc = new_doctype("Doc A", + doc = new_doctype( + "Doc A", fields=[ { "label": "DocType", "fieldname": "doc", "fieldtype": "Link", "options": "DocType", - "default": "ToDo" + "default": "ToDo", } - ], unique=0) + ], + unique=0, + ) doc.insert() # make User Perm on DocType 'ToDo' in Assignment Rule (unrelated doctype) @@ -199,20 +218,23 @@ class TestUserPermission(unittest.TestCase): # it should not have impact on Doc A self.assertEqual(new_doc.doc, "ToDo") - frappe.set_user('Administrator') + frappe.set_user("Administrator") remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo") def test_user_perm_on_new_doc_with_user_default(self): """Test User Perm impact on frappe.new_doc. with *user* default value""" - from frappe.core.doctype.session_default_settings.session_default_settings import (clear_session_defaults, - set_session_default_values) + from frappe.core.doctype.session_default_settings.session_default_settings import ( + clear_session_defaults, + set_session_default_values, + ) - frappe.set_user('Administrator') + frappe.set_user("Administrator") user = create_user("user_default_test@example.com", "Blogger") # make a doctype "Doc A" with 'doctype' link field if not frappe.db.exists("DocType", "Doc A"): - doc = new_doctype("Doc A", + doc = new_doctype( + "Doc A", fields=[ { "label": "DocType", @@ -220,15 +242,15 @@ class TestUserPermission(unittest.TestCase): "fieldtype": "Link", "options": "DocType", } - ], unique=0) + ], + unique=0, + ) doc.insert() # create a 'DocType' session default field if not frappe.db.exists("Session Default", {"ref_doctype": "DocType"}): - settings = frappe.get_single('Session Default Settings') - settings.append("session_defaults", { - "ref_doctype": "DocType" - }) + settings = frappe.get_single("Session Default Settings") + settings.append("session_defaults", {"ref_doctype": "DocType"}) settings.save() # make User Perm on DocType 'ToDo' in Assignment Rule (unrelated doctype) @@ -244,43 +266,46 @@ class TestUserPermission(unittest.TestCase): # it should not have impact on Doc A self.assertEqual(new_doc.doc, "ToDo") - frappe.set_user('Administrator') + frappe.set_user("Administrator") clear_session_defaults() remove_applicable(["Assignment Rule"], "user_default_test@example.com", "DocType", "ToDo") + def create_user(email, *roles): - ''' create user with role system manager ''' - if frappe.db.exists('User', email): - return frappe.get_doc('User', email) + """create user with role system manager""" + if frappe.db.exists("User", email): + return frappe.get_doc("User", email) - user = frappe.new_doc('User') + user = frappe.new_doc("User") user.email = email user.first_name = email.split("@")[0] if not roles: - roles = ('System Manager',) + roles = ("System Manager",) user.add_roles(*roles) return user + def get_params(user, doctype, docname, is_default=0, hide_descendants=0, applicable=None): - ''' Return param to insert ''' + """Return param to insert""" param = { "user": user.name, - "doctype":doctype, - "docname":docname, + "doctype": doctype, + "docname": docname, "is_default": is_default, "apply_to_all_doctypes": 1, "applicable_doctypes": [], - "hide_descendants": hide_descendants + "hide_descendants": hide_descendants, } if applicable: param.update({"apply_to_all_doctypes": 0}) param.update({"applicable_doctypes": applicable}) return param -def get_exists_param(user, applicable = None): - ''' param to check existing Document ''' + +def get_exists_param(user, applicable=None): + """param to check existing Document""" param = { "user": user.name, "allow": "User", diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index fb658481b2..e43a288744 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -1,13 +1,16 @@ # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe, json -from frappe.model.document import Document -from frappe.permissions import (get_valid_perms, update_permission_property) +import json + +import frappe from frappe import _ -from frappe.utils import cstr from frappe.core.utils import find from frappe.desk.form.linked_with import get_linked_doctypes +from frappe.model.document import Document +from frappe.permissions import get_valid_perms, update_permission_property +from frappe.utils import cstr + class UserPermission(Document): def validate(self): @@ -15,50 +18,55 @@ class UserPermission(Document): self.validate_default_permission() def on_update(self): - frappe.cache().hdel('user_permissions', self.user) - frappe.publish_realtime('update_user_permissions') + frappe.cache().hdel("user_permissions", self.user) + frappe.publish_realtime("update_user_permissions") - def on_trash(self): # pylint: disable=no-self-use - frappe.cache().hdel('user_permissions', self.user) - frappe.publish_realtime('update_user_permissions') + def on_trash(self): # pylint: disable=no-self-use + frappe.cache().hdel("user_permissions", self.user) + frappe.publish_realtime("update_user_permissions") def validate_user_permission(self): - ''' checks for duplicate user permission records''' - - duplicate_exists = frappe.db.get_all(self.doctype, filters={ - 'allow': self.allow, - 'for_value': self.for_value, - 'user': self.user, - 'applicable_for': cstr(self.applicable_for), - 'apply_to_all_doctypes': self.apply_to_all_doctypes, - 'name': ['!=', self.name] - }, limit=1) + """checks for duplicate user permission records""" + + duplicate_exists = frappe.db.get_all( + self.doctype, + filters={ + "allow": self.allow, + "for_value": self.for_value, + "user": self.user, + "applicable_for": cstr(self.applicable_for), + "apply_to_all_doctypes": self.apply_to_all_doctypes, + "name": ["!=", self.name], + }, + limit=1, + ) if duplicate_exists: frappe.throw(_("User permission already exists"), frappe.DuplicateEntryError) def validate_default_permission(self): - ''' validate user permission overlap for default value of a particular doctype ''' + """validate user permission overlap for default value of a particular doctype""" overlap_exists = [] if self.is_default: - overlap_exists = frappe.get_all(self.doctype, filters={ - 'allow': self.allow, - 'user': self.user, - 'is_default': 1, - 'name': ['!=', self.name] - }, or_filters={ - 'applicable_for': cstr(self.applicable_for), - 'apply_to_all_doctypes': 1, - }, limit=1) + overlap_exists = frappe.get_all( + self.doctype, + filters={"allow": self.allow, "user": self.user, "is_default": 1, "name": ["!=", self.name]}, + or_filters={ + "applicable_for": cstr(self.applicable_for), + "apply_to_all_doctypes": 1, + }, + limit=1, + ) if overlap_exists: ref_link = frappe.get_desk_link(self.doctype, overlap_exists[0].name) frappe.throw(_("{0} has already assigned default value for {1}.").format(ref_link, self.allow)) + @frappe.whitelist() def get_user_permissions(user=None): - '''Get all users permissions for the user as a dict of doctype''' + """Get all users permissions for the user as a dict of doctype""" # if this is called from client-side, # user can access only his/her user permissions - if frappe.request and frappe.local.form_dict.cmd == 'get_user_permissions': + if frappe.request and frappe.local.form_dict.cmd == "get_user_permissions": user = frappe.session.user if not user: @@ -81,16 +89,18 @@ def get_user_permissions(user=None): if not out.get(perm.allow): out[perm.allow] = [] - out[perm.allow].append(frappe._dict({ - 'doc': doc_name, - 'applicable_for': perm.get('applicable_for'), - 'is_default': is_default - })) + out[perm.allow].append( + frappe._dict( + {"doc": doc_name, "applicable_for": perm.get("applicable_for"), "is_default": is_default} + ) + ) try: - for perm in frappe.get_all('User Permission', - fields=['allow', 'for_value', 'applicable_for', 'is_default', 'hide_descendants'], - filters=dict(user=user)): + for perm in frappe.get_all( + "User Permission", + fields=["allow", "for_value", "applicable_for", "is_default", "hide_descendants"], + filters=dict(user=user), + ): meta = frappe.get_meta(perm.allow) add_doc_to_perm(perm, perm.for_value, perm.is_default) @@ -109,14 +119,20 @@ def get_user_permissions(user=None): return out + def user_permission_exists(user, allow, for_value, applicable_for=None): - '''Checks if similar user permission already exists''' + """Checks if similar user permission already exists""" user_permissions = get_user_permissions(user).get(allow, []) - if not user_permissions: return None - has_same_user_permission = find(user_permissions, lambda perm:perm["doc"] == for_value and perm.get('applicable_for') == applicable_for) + if not user_permissions: + return None + has_same_user_permission = find( + user_permissions, + lambda perm: perm["doc"] == for_value and perm.get("applicable_for") == applicable_for, + ) return has_same_user_permission + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters): @@ -142,34 +158,44 @@ def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, return return_list + def get_permitted_documents(doctype): - ''' Returns permitted documents from the given doctype for the session user ''' + """Returns permitted documents from the given doctype for the session user""" # sort permissions in a way to make the first permission in the list to be default - user_perm_list = sorted(get_user_permissions().get(doctype, []), key=lambda x: x.get('is_default'), reverse=True) + user_perm_list = sorted( + get_user_permissions().get(doctype, []), key=lambda x: x.get("is_default"), reverse=True + ) + + return [d.get("doc") for d in user_perm_list if d.get("doc")] - return [d.get('doc') for d in user_perm_list \ - if d.get('doc')] @frappe.whitelist() def check_applicable_doc_perm(user, doctype, docname): - frappe.only_for('System Manager') + frappe.only_for("System Manager") applicable = [] - doc_exists = frappe.get_all('User Permission', - fields=['name'], - filters={"user": user, + doc_exists = frappe.get_all( + "User Permission", + fields=["name"], + filters={ + "user": user, "allow": doctype, "for_value": docname, - "apply_to_all_doctypes":1, - }, limit=1) + "apply_to_all_doctypes": 1, + }, + limit=1, + ) if doc_exists: applicable = get_linked_doctypes(doctype).keys() else: - data = frappe.get_all('User Permission', - fields=['applicable_for'], - filters={"user": user, + data = frappe.get_all( + "User Permission", + fields=["applicable_for"], + filters={ + "user": user, "allow": doctype, - "for_value":docname, - }) + "for_value": docname, + }, + ) for permission in data: applicable.append(permission.applicable_for) return applicable @@ -181,46 +207,74 @@ def clear_user_permissions(user, for_doctype): total = frappe.db.count("User Permission", {"user": user, "allow": for_doctype}) if total: - frappe.db.delete("User Permission", { - "allow": for_doctype, - "user": user, - }) + frappe.db.delete( + "User Permission", + { + "allow": for_doctype, + "user": user, + }, + ) frappe.clear_cache() return total + @frappe.whitelist() def add_user_permissions(data): - ''' Add and update the user permissions ''' - frappe.only_for('System Manager') + """Add and update the user permissions""" + frappe.only_for("System Manager") if isinstance(data, str): data = json.loads(data) data = frappe._dict(data) # get all doctypes on whom this permission is applied perm_applied_docs = check_applicable_doc_perm(data.user, data.doctype, data.docname) - exists = frappe.db.exists("User Permission", { - "user": data.user, - "allow": data.doctype, - "for_value": data.docname, - "apply_to_all_doctypes": 1 - }) + exists = frappe.db.exists( + "User Permission", + { + "user": data.user, + "allow": data.doctype, + "for_value": data.docname, + "apply_to_all_doctypes": 1, + }, + ) if data.apply_to_all_doctypes == 1 and not exists: remove_applicable(perm_applied_docs, data.user, data.doctype, data.docname) - insert_user_perm(data.user, data.doctype, data.docname, data.is_default, data.hide_descendants, apply_to_all=1) + insert_user_perm( + data.user, data.doctype, data.docname, data.is_default, data.hide_descendants, apply_to_all=1 + ) return 1 elif len(data.applicable_doctypes) > 0 and data.apply_to_all_doctypes != 1: remove_apply_to_all(data.user, data.doctype, data.docname) - update_applicable(perm_applied_docs, data.applicable_doctypes, data.user, data.doctype, data.docname) - for applicable in data.applicable_doctypes : + update_applicable( + perm_applied_docs, data.applicable_doctypes, data.user, data.doctype, data.docname + ) + for applicable in data.applicable_doctypes: if applicable not in perm_applied_docs: - insert_user_perm(data.user, data.doctype, data.docname, data.is_default, data.hide_descendants, applicable=applicable) + insert_user_perm( + data.user, + data.doctype, + data.docname, + data.is_default, + data.hide_descendants, + applicable=applicable, + ) elif exists: - insert_user_perm(data.user, data.doctype, data.docname, data.is_default, data.hide_descendants, applicable=applicable) + insert_user_perm( + data.user, + data.doctype, + data.docname, + data.is_default, + data.hide_descendants, + applicable=applicable, + ) return 1 return 0 -def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, apply_to_all=None, applicable=None): + +def insert_user_perm( + user, doctype, docname, is_default=0, hide_descendants=0, apply_to_all=None, applicable=None +): user_perm = frappe.new_doc("User Permission") user_perm.user = user user_perm.allow = doctype @@ -234,29 +288,41 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a user_perm.apply_to_all_doctypes = 1 user_perm.insert() + def remove_applicable(perm_applied_docs, user, doctype, docname): for applicable_for in perm_applied_docs: - frappe.db.delete("User Permission", { - "applicable_for": applicable_for, + frappe.db.delete( + "User Permission", + { + "applicable_for": applicable_for, + "for_value": docname, + "allow": doctype, + "user": user, + }, + ) + + +def remove_apply_to_all(user, doctype, docname): + frappe.db.delete( + "User Permission", + { + "apply_to_all_doctypes": 1, "for_value": docname, "allow": doctype, "user": user, - }) + }, + ) -def remove_apply_to_all(user, doctype, docname): - frappe.db.delete("User Permission", { - "apply_to_all_doctypes": 1, - "for_value": docname, - "allow": doctype, - "user": user, - }) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: if applied not in to_apply: - frappe.db.delete("User Permission", { - "applicable_for": applied, - "for_value": docname, - "allow": doctype, - "user": user, - }) + frappe.db.delete( + "User Permission", + { + "applicable_for": applied, + "for_value": docname, + "allow": doctype, + "user": user, + }, + ) diff --git a/frappe/core/doctype/user_select_document_type/user_select_document_type.py b/frappe/core/doctype/user_select_document_type/user_select_document_type.py index 18a21931e5..07b8123a13 100644 --- a/frappe/core/doctype/user_select_document_type/user_select_document_type.py +++ b/frappe/core/doctype/user_select_document_type/user_select_document_type.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class UserSelectDocumentType(Document): pass diff --git a/frappe/core/doctype/user_social_login/user_social_login.py b/frappe/core/doctype/user_social_login/user_social_login.py index 80c0c89383..d12b5823d1 100644 --- a/frappe/core/doctype/user_social_login/user_social_login.py +++ b/frappe/core/doctype/user_social_login/user_social_login.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class UserSocialLogin(Document): pass diff --git a/frappe/core/doctype/user_type/test_user_type.py b/frappe/core/doctype/user_type/test_user_type.py index 6807f8fc9e..53999ed3df 100644 --- a/frappe/core/doctype/user_type/test_user_type.py +++ b/frappe/core/doctype/user_type/test_user_type.py @@ -1,22 +1,25 @@ # -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe from frappe.installer import update_site_config + class TestUserType(unittest.TestCase): def setUp(self): create_role() def test_add_select_perm_doctypes(self): - user_type = create_user_type('Test User Type') + user_type = create_user_type("Test User Type") # select perms added for all link fields - doc = frappe.get_meta('Contact') + doc = frappe.get_meta("Contact") link_fields = doc.get_link_fields() - select_doctypes = frappe.get_all('User Select Document Type', {'parent': user_type.name}, pluck='document_type') + select_doctypes = frappe.get_all( + "User Select Document Type", {"parent": user_type.name}, pluck="document_type" + ) for entry in link_fields: self.assertTrue(entry.options in select_doctypes) @@ -35,34 +38,29 @@ class TestUserType(unittest.TestCase): def create_user_type(user_type): - if frappe.db.exists('User Type', user_type): - frappe.delete_doc('User Type', user_type) + if frappe.db.exists("User Type", user_type): + frappe.delete_doc("User Type", user_type) user_type_limit = {frappe.scrub(user_type): 1} - update_site_config('user_type_doctype_limit', user_type_limit) - - doc = frappe.get_doc({ - 'doctype': 'User Type', - 'name': user_type, - 'role': '_Test User Type', - 'user_id_field': 'user', - 'apply_user_permission_on': 'User' - }) - - doc.append('user_doctypes', { - 'document_type': 'Contact', - 'read': 1, - 'write': 1 - }) + update_site_config("user_type_doctype_limit", user_type_limit) + + doc = frappe.get_doc( + { + "doctype": "User Type", + "name": user_type, + "role": "_Test User Type", + "user_id_field": "user", + "apply_user_permission_on": "User", + } + ) + + doc.append("user_doctypes", {"document_type": "Contact", "read": 1, "write": 1}) return doc.insert() def create_role(): - if not frappe.db.exists('Role', '_Test User Type'): - frappe.get_doc({ - 'doctype': 'Role', - 'role_name': '_Test User Type', - 'desk_access': 1, - 'is_custom': 1 - }).insert() \ No newline at end of file + if not frappe.db.exists("Role", "_Test User Type"): + frappe.get_doc( + {"doctype": "Role", "role_name": "_Test User Type", "desk_access": 1, "is_custom": 1} + ).insert() diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index c0dfd2e597..7efb66f569 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -4,10 +4,11 @@ import frappe from frappe import _ -from frappe.utils import get_link_to_form from frappe.config import get_modules_from_app -from frappe.permissions import add_permission, add_user_permission from frappe.model.document import Document +from frappe.permissions import add_permission, add_user_permission +from frappe.utils import get_link_to_form + class UserType(Document): def validate(self): @@ -29,14 +30,14 @@ class UserType(Document): def on_trash(self): if self.is_standard: - frappe.throw(_('Standard user type {0} can not be deleted.') - .format(frappe.bold(self.name))) + frappe.throw(_("Standard user type {0} can not be deleted.").format(frappe.bold(self.name))) def set_modules(self): if not self.user_doctypes: return - modules = frappe.get_all("DocType", + modules = frappe.get_all( + "DocType", filters={"name": ("in", [d.document_type for d in self.user_doctypes])}, distinct=True, pluck="module", @@ -47,64 +48,77 @@ class UserType(Document): self.append("user_type_modules", {"module": module}) def validate_document_type_limit(self): - limit = frappe.conf.get('user_type_doctype_limit', {}).get(frappe.scrub(self.name)) + limit = frappe.conf.get("user_type_doctype_limit", {}).get(frappe.scrub(self.name)) - if not limit and frappe.session.user != 'Administrator': - frappe.throw(_('User does not have permission to create the new {0}') - .format(frappe.bold(_('User Type'))), title=_('Permission Error')) + if not limit and frappe.session.user != "Administrator": + frappe.throw( + _("User does not have permission to create the new {0}").format(frappe.bold(_("User Type"))), + title=_("Permission Error"), + ) if not limit: - frappe.throw(_('The limit has not set for the user type {0} in the site config file.') - .format(frappe.bold(self.name)), title=_('Set Limit')) + frappe.throw( + _("The limit has not set for the user type {0} in the site config file.").format( + frappe.bold(self.name) + ), + title=_("Set Limit"), + ) if self.user_doctypes and len(self.user_doctypes) > limit: - frappe.throw(_('The total number of user document types limit has been crossed.'), - title=_('User Document Types Limit Exceeded')) + frappe.throw( + _("The total number of user document types limit has been crossed."), + title=_("User Document Types Limit Exceeded"), + ) custom_doctypes = [row.document_type for row in self.user_doctypes if row.is_custom] if custom_doctypes and len(custom_doctypes) > 3: - frappe.throw(_('You can only set the 3 custom doctypes in the Document Types table.'), - title=_('Custom Document Types Limit Exceeded')) + frappe.throw( + _("You can only set the 3 custom doctypes in the Document Types table."), + title=_("Custom Document Types Limit Exceeded"), + ) def validate_role(self): if not self.role: - frappe.throw(_("The field {0} is mandatory") - .format(frappe.bold(_('Role')))) + frappe.throw(_("The field {0} is mandatory").format(frappe.bold(_("Role")))) - if not frappe.db.get_value('Role', self.role, 'is_custom'): - frappe.throw(_("The role {0} should be a custom role.") - .format(frappe.bold(get_link_to_form('Role', self.role)))) + if not frappe.db.get_value("Role", self.role, "is_custom"): + frappe.throw( + _("The role {0} should be a custom role.").format( + frappe.bold(get_link_to_form("Role", self.role)) + ) + ) def update_users(self): - for row in frappe.get_all('User', filters = {'user_type': self.name}): - user = frappe.get_cached_doc('User', row.name) + for row in frappe.get_all("User", filters={"user_type": self.name}): + user = frappe.get_cached_doc("User", row.name) self.update_roles_in_user(user) self.update_modules_in_user(user) user.update_children() def update_roles_in_user(self, user): - user.set('roles', []) - user.append('roles', { - 'role': self.role - }) + user.set("roles", []) + user.append("roles", {"role": self.role}) def update_modules_in_user(self, user): - block_modules = frappe.get_all('Module Def', fields = ['name as module'], - filters={'name': ['not in', [d.module for d in self.user_type_modules]]}) + block_modules = frappe.get_all( + "Module Def", + fields=["name as module"], + filters={"name": ["not in", [d.module for d in self.user_type_modules]]}, + ) if block_modules: - user.set('block_modules', block_modules) + user.set("block_modules", block_modules) def add_role_permissions_for_user_doctypes(self): - perms = ['read', 'write', 'create', 'submit', 'cancel', 'amend', 'delete'] + perms = ["read", "write", "create", "submit", "cancel", "amend", "delete"] for row in self.user_doctypes: docperm = add_role_permissions(row.document_type, self.role) - values = {perm:row.get(perm) or 0 for perm in perms} - for perm in ['print', 'email', 'share']: + values = {perm: row.get(perm) or 0 for perm in perms} + for perm in ["print", "email", "share"]: values[perm] = 1 - frappe.db.set_value('Custom DocPerm', docperm, values) + frappe.db.set_value("Custom DocPerm", docperm, values) def add_select_perm_doctypes(self): if frappe.flags.ignore_select_perm: @@ -127,9 +141,7 @@ class UserType(Document): if select_doctypes: select_doctypes = set(select_doctypes) for select_doctype in select_doctypes: - self.append('select_doctypes', { - 'document_type': select_doctype - }) + self.append("select_doctypes", {"document_type": select_doctype}) def prepare_select_perm_doctypes(self, doc, user_doctypes, select_doctypes): for field in doc.get_link_fields(): @@ -137,108 +149,149 @@ class UserType(Document): select_doctypes.append(field.options) def add_role_permissions_for_select_doctypes(self): - for doctype in ['select_doctypes', 'custom_select_doctypes']: + for doctype in ["select_doctypes", "custom_select_doctypes"]: for row in self.get(doctype): docperm = add_role_permissions(row.document_type, self.role) - frappe.db.set_value('Custom DocPerm', docperm, - {'select': 1, 'read': 0, 'create': 0, 'write': 0}) + frappe.db.set_value( + "Custom DocPerm", docperm, {"select": 1, "read": 0, "create": 0, "write": 0} + ) def add_role_permissions_for_file(self): - docperm = add_role_permissions('File', self.role) - frappe.db.set_value('Custom DocPerm', docperm, - {'read': 1, 'create': 1, 'write': 1}) + docperm = add_role_permissions("File", self.role) + frappe.db.set_value("Custom DocPerm", docperm, {"read": 1, "create": 1, "write": 1}) def remove_permission_for_deleted_doctypes(self): doctypes = [d.document_type for d in self.user_doctypes] # Do not remove the doc permission for the file doctype - doctypes.append('File') + doctypes.append("File") - for doctype in ['select_doctypes', 'custom_select_doctypes']: + for doctype in ["select_doctypes", "custom_select_doctypes"]: for dt in self.get(doctype): doctypes.append(dt.document_type) - for perm in frappe.get_all('Custom DocPerm', - filters = {'role': self.role, 'parent': ['not in', doctypes]}): - frappe.delete_doc('Custom DocPerm', perm.name) + for perm in frappe.get_all( + "Custom DocPerm", filters={"role": self.role, "parent": ["not in", doctypes]} + ): + frappe.delete_doc("Custom DocPerm", perm.name) + def add_role_permissions(doctype, role): - name = frappe.get_value('Custom DocPerm', dict(parent=doctype, - role=role, permlevel=0)) + name = frappe.get_value("Custom DocPerm", dict(parent=doctype, role=role, permlevel=0)) if not name: name = add_permission(doctype, role, 0) return name + def get_non_standard_user_type_details(): - user_types = frappe.get_all('User Type', - fields=['apply_user_permission_on', 'name', 'user_id_field'], - filters={'is_standard': 0}) + user_types = frappe.get_all( + "User Type", + fields=["apply_user_permission_on", "name", "user_id_field"], + filters={"is_standard": 0}, + ) if user_types: user_type_details = {d.name: [d.apply_user_permission_on, d.user_id_field] for d in user_types} - frappe.cache().set_value('non_standard_user_types', user_type_details) + frappe.cache().set_value("non_standard_user_types", user_type_details) return user_type_details + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters): - modules = [d.get('module_name') for d in get_modules_from_app('frappe')] - - filters = [['DocField', 'options', '=', 'User'], ['DocType', 'is_submittable', '=', 0], - ['DocType', 'issingle', '=', 0], ['DocType', 'module', 'not in', modules], - ['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]] - - doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters, - order_by='`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1) - - custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)], - ['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']] - - custom_doctypes = frappe.get_all('Custom Field', fields = ['dt as name'], - filters= custom_dt_filters, as_list=1) + modules = [d.get("module_name") for d in get_modules_from_app("frappe")] + + filters = [ + ["DocField", "options", "=", "User"], + ["DocType", "is_submittable", "=", 0], + ["DocType", "issingle", "=", 0], + ["DocType", "module", "not in", modules], + ["DocType", "read_only", "=", 0], + ["DocType", "name", "like", "%{0}%".format(txt)], + ] + + doctypes = frappe.get_all( + "DocType", + fields=["`tabDocType`.`name`"], + filters=filters, + order_by="`tabDocType`.`idx` desc", + limit_start=start, + limit_page_length=page_len, + as_list=1, + ) + + custom_dt_filters = [ + ["Custom Field", "dt", "like", "%{0}%".format(txt)], + ["Custom Field", "options", "=", "User"], + ["Custom Field", "fieldtype", "=", "Link"], + ] + + custom_doctypes = frappe.get_all( + "Custom Field", fields=["dt as name"], filters=custom_dt_filters, as_list=1 + ) return doctypes + custom_doctypes + @frappe.whitelist() def get_user_id(parent): - data = frappe.get_all('DocField', fields = ['label', 'fieldname as value'], - filters= {'options': 'User', 'fieldtype': 'Link', 'parent': parent}) or [] - - data.extend(frappe.get_all('Custom Field', fields = ['label', 'fieldname as value'], - filters= {'options': 'User', 'fieldtype': 'Link', 'dt': parent})) + data = ( + frappe.get_all( + "DocField", + fields=["label", "fieldname as value"], + filters={"options": "User", "fieldtype": "Link", "parent": parent}, + ) + or [] + ) + + data.extend( + frappe.get_all( + "Custom Field", + fields=["label", "fieldname as value"], + filters={"options": "User", "fieldtype": "Link", "dt": parent}, + ) + ) return data + def user_linked_with_permission_on_doctype(doc, user): if not doc.apply_user_permission_on: return True if not doc.user_id_field: - frappe.throw(_('User Id Field is mandatory in the user type {0}') - .format(frappe.bold(doc.name))) + frappe.throw(_("User Id Field is mandatory in the user type {0}").format(frappe.bold(doc.name))) - if frappe.db.get_value(doc.apply_user_permission_on, - {doc.user_id_field: user}, 'name'): + if frappe.db.get_value(doc.apply_user_permission_on, {doc.user_id_field: user}, "name"): return True else: label = frappe.get_meta(doc.apply_user_permission_on).get_field(doc.user_id_field).label - frappe.msgprint(_("To set the role {0} in the user {1}, kindly set the {2} field as {3} in one of the {4} record.") - .format(frappe.bold(doc.role), frappe.bold(user), frappe.bold(label), - frappe.bold(user), frappe.bold(doc.apply_user_permission_on))) + frappe.msgprint( + _( + "To set the role {0} in the user {1}, kindly set the {2} field as {3} in one of the {4} record." + ).format( + frappe.bold(doc.role), + frappe.bold(user), + frappe.bold(label), + frappe.bold(user), + frappe.bold(doc.apply_user_permission_on), + ) + ) return False + def apply_permissions_for_non_standard_user_type(doc, method=None): - '''Create user permission for the non standard user type''' - if not frappe.db.table_exists('User Type'): + """Create user permission for the non standard user type""" + if not frappe.db.table_exists("User Type"): return - user_types = frappe.cache().get_value('non_standard_user_types') + user_types = frappe.cache().get_value("non_standard_user_types") if not user_types: user_types = get_non_standard_user_type_details() @@ -247,23 +300,28 @@ def apply_permissions_for_non_standard_user_type(doc, method=None): return for user_type, data in user_types.items(): - if (not doc.get(data[1]) or doc.doctype != data[0]): + if not doc.get(data[1]) or doc.doctype != data[0]: continue - if frappe.get_cached_value('User', doc.get(data[1]), 'user_type') != user_type: + if frappe.get_cached_value("User", doc.get(data[1]), "user_type") != user_type: return - if (doc.get(data[1]) and (not doc._doc_before_save or doc.get(data[1]) != doc._doc_before_save.get(data[1]) - or not frappe.db.get_value('User Permission', - {'user': doc.get(data[1]), 'allow': data[0], 'for_value': doc.name}, 'name'))): + if doc.get(data[1]) and ( + not doc._doc_before_save + or doc.get(data[1]) != doc._doc_before_save.get(data[1]) + or not frappe.db.get_value( + "User Permission", {"user": doc.get(data[1]), "allow": data[0], "for_value": doc.name}, "name" + ) + ): - perm_data = frappe.db.get_value('User Permission', - {'allow': doc.doctype, 'for_value': doc.name}, ['name', 'user']) + perm_data = frappe.db.get_value( + "User Permission", {"allow": doc.doctype, "for_value": doc.name}, ["name", "user"] + ) if not perm_data: - user_doc = frappe.get_cached_doc('User', doc.get(data[1])) + user_doc = frappe.get_cached_doc("User", doc.get(data[1])) user_doc.set_roles_and_modules_based_on_user_type() user_doc.update_children() add_user_permission(doc.doctype, doc.name, doc.get(data[1])) else: - frappe.db.set_value('User Permission', perm_data[0], 'user', doc.get(data[1])) + frappe.db.set_value("User Permission", perm_data[0], "user", doc.get(data[1])) diff --git a/frappe/core/doctype/user_type/user_type_dashboard.py b/frappe/core/doctype/user_type/user_type_dashboard.py index 6cdd2f82a5..65217cb66b 100644 --- a/frappe/core/doctype/user_type/user_type_dashboard.py +++ b/frappe/core/doctype/user_type/user_type_dashboard.py @@ -1,13 +1,5 @@ - from frappe import _ + def get_data(): - return { - 'fieldname': 'user_type', - 'transactions': [ - { - 'label': _('Reference'), - 'items': ['User'] - } - ] - } \ No newline at end of file + return {"fieldname": "user_type", "transactions": [{"label": _("Reference"), "items": ["User"]}]} diff --git a/frappe/core/doctype/user_type_module/user_type_module.py b/frappe/core/doctype/user_type_module/user_type_module.py index d25479f869..83662bfcaf 100644 --- a/frappe/core/doctype/user_type_module/user_type_module.py +++ b/frappe/core/doctype/user_type_module/user_type_module.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class UserTypeModule(Document): pass diff --git a/frappe/core/doctype/version/test_version.py b/frappe/core/doctype/version/test_version.py index 608dc9f0ab..c35430b17b 100644 --- a/frappe/core/doctype/version/test_version.py +++ b/frappe/core/doctype/version/test_version.py @@ -1,39 +1,45 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import copy +import unittest + import frappe -import unittest, copy -from frappe.test_runner import make_test_objects from frappe.core.doctype.version.version import get_diff +from frappe.test_runner import make_test_objects + class TestVersion(unittest.TestCase): def test_get_diff(self): - frappe.set_user('Administrator') - test_records = make_test_objects('Event', reset = True) + frappe.set_user("Administrator") + test_records = make_test_objects("Event", reset=True) old_doc = frappe.get_doc("Event", test_records[0]) new_doc = copy.deepcopy(old_doc) old_doc.color = None - new_doc.color = '#fafafa' + new_doc.color = "#fafafa" - diff = get_diff(old_doc, new_doc)['changed'] + diff = get_diff(old_doc, new_doc)["changed"] - self.assertEqual(get_fieldnames(diff)[0], 'color') + self.assertEqual(get_fieldnames(diff)[0], "color") self.assertTrue(get_old_values(diff)[0] is None) - self.assertEqual(get_new_values(diff)[0], '#fafafa') + self.assertEqual(get_new_values(diff)[0], "#fafafa") new_doc.starts_on = "2017-07-20" - diff = get_diff(old_doc, new_doc)['changed'] + diff = get_diff(old_doc, new_doc)["changed"] + + self.assertEqual(get_fieldnames(diff)[1], "starts_on") + self.assertEqual(get_old_values(diff)[1], "01-01-2014 00:00:00") + self.assertEqual(get_new_values(diff)[1], "07-20-2017 00:00:00") - self.assertEqual(get_fieldnames(diff)[1], 'starts_on') - self.assertEqual(get_old_values(diff)[1], '01-01-2014 00:00:00') - self.assertEqual(get_new_values(diff)[1], '07-20-2017 00:00:00') def get_fieldnames(change_array): return [d[0] for d in change_array] + def get_old_values(change_array): return [d[1] for d in change_array] + def get_new_values(change_array): return [d[2] for d in change_array] diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index fcb558650a..540f8c7a02 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -1,14 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json +import json -from frappe.model.document import Document +import frappe from frappe.model import no_value_fields, table_fields +from frappe.model.document import Document + class Version(Document): def set_diff(self, old, new): - '''Set the data property with the diff of the docs if present''' + """Set the data property with the diff of the docs if present""" diff = get_diff(old, new) if diff: self.ref_doctype = new.doctype @@ -21,9 +23,9 @@ class Version(Document): def for_insert(self, doc): updater_reference = doc.flags.updater_reference data = { - 'creation': doc.creation, - 'updater_reference': updater_reference, - 'created_by': doc.owner + "creation": doc.creation, + "updater_reference": updater_reference, + "created_by": doc.owner, } self.ref_doctype = doc.doctype self.docname = doc.name @@ -34,20 +36,20 @@ class Version(Document): def get_diff(old, new, for_child=False): - '''Get diff between 2 document objects + """Get diff between 2 document objects If there is a change, then returns a dict like: - { - "changed" : [[fieldname1, old, new], [fieldname2, old, new]], - "added" : [[table_fieldname1, {dict}], ], - "removed" : [[table_fieldname1, {dict}], ], - "row_changed": [[table_fieldname1, row_name1, row_index, - [[child_fieldname1, old, new], - [child_fieldname2, old, new]], ] - ], + { + "changed" : [[fieldname1, old, new], [fieldname2, old, new]], + "added" : [[table_fieldname1, {dict}], ], + "removed" : [[table_fieldname1, {dict}], ], + "row_changed": [[table_fieldname1, row_name1, row_index, + [[child_fieldname1, old, new], + [child_fieldname2, old, new]], ] + ], - }''' + }""" if not new: return None @@ -57,8 +59,14 @@ def get_diff(old, new, for_child=False): data_import = new.flags.via_data_import updater_reference = new.flags.updater_reference - out = frappe._dict(changed = [], added = [], removed = [], - row_changed = [], data_import=data_import, updater_reference=updater_reference) + out = frappe._dict( + changed=[], + added=[], + removed=[], + row_changed=[], + data_import=data_import, + updater_reference=updater_reference, + ) for df in new.meta.fields: if df.fieldtype in no_value_fields and df.fieldtype not in table_fields: @@ -88,7 +96,7 @@ def get_diff(old, new, for_child=False): if not d.name in new_row_by_name: out.removed.append([df.fieldname, d.as_dict()]) - elif (old_value != new_value): + elif old_value != new_value: if df.fieldtype not in blacklisted_fields: old_value = old.get_formatted(df.fieldname) if old_value else old_value new_value = new.get_formatted(df.fieldname) if new_value else new_value @@ -98,7 +106,7 @@ def get_diff(old, new, for_child=False): # docstatus if not for_child and old.docstatus != new.docstatus: - out.changed.append(['docstatus', old.docstatus, new.docstatus]) + out.changed.append(["docstatus", old.docstatus, new.docstatus]) if any((out.changed, out.added, out.removed, out.row_changed)): return out @@ -106,5 +114,6 @@ def get_diff(old, new, for_child=False): else: return None + def on_doctype_update(): frappe.db.add_index("Version", ["ref_doctype", "docname"]) diff --git a/frappe/core/doctype/view_log/test_view_log.py b/frappe/core/doctype/view_log/test_view_log.py index efa9538fbf..04a17cc526 100644 --- a/frappe/core/doctype/view_log/test_view_log.py +++ b/frappe/core/doctype/view_log/test_view_log.py @@ -1,35 +1,36 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestViewLog(unittest.TestCase): def tearDown(self): - frappe.set_user('Administrator') + frappe.set_user("Administrator") def test_if_user_is_added(self): - ev = frappe.get_doc({ - 'doctype': 'Event', - 'subject': 'test event for view logs', - 'starts_on': '2018-06-04 14:11:00', - 'event_type': 'Public' - }).insert() + ev = frappe.get_doc( + { + "doctype": "Event", + "subject": "test event for view logs", + "starts_on": "2018-06-04 14:11:00", + "event_type": "Public", + } + ).insert() - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") from frappe.desk.form.load import getdoc # load the form - getdoc('Event', ev.name) + getdoc("Event", ev.name) a = frappe.get_value( doctype="View Log", - filters={ - "reference_doctype": "Event", - "reference_name": ev.name - }, - fieldname=['viewed_by'] + filters={"reference_doctype": "Event", "reference_name": ev.name}, + fieldname=["viewed_by"], ) - self.assertEqual('test@example.com', a) - self.assertNotEqual('test1@example.com', a) \ No newline at end of file + self.assertEqual("test@example.com", a) + self.assertNotEqual("test1@example.com", a) diff --git a/frappe/core/doctype/view_log/view_log.py b/frappe/core/doctype/view_log/view_log.py index fbbd6e1154..4a3ba0e83c 100644 --- a/frappe/core/doctype/view_log/view_log.py +++ b/frappe/core/doctype/view_log/view_log.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class ViewLog(Document): pass diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index 5f41f217f0..a10f8ec5ae 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -14,28 +14,35 @@ def get_notification_config(): "ToDo": "frappe.core.notifications.get_things_todo", "Event": "frappe.core.notifications.get_todays_events", "Error Snapshot": {"seen": 0, "parent_error_snapshot": None}, - "Workflow Action": {"status": 'Open'} + "Workflow Action": {"status": "Open"}, }, } + def get_things_todo(as_list=False): """Returns a count of incomplete todos""" - data = frappe.get_list("ToDo", + data = frappe.get_list( + "ToDo", fields=["name", "description"] if as_list else "count(*)", filters=[["ToDo", "status", "=", "Open"]], - or_filters=[["ToDo", "allocated_to", "=", frappe.session.user], - ["ToDo", "assigned_by", "=", frappe.session.user]], - as_list=True) + or_filters=[ + ["ToDo", "allocated_to", "=", frappe.session.user], + ["ToDo", "assigned_by", "=", frappe.session.user], + ], + as_list=True, + ) if as_list: return data else: return data[0][0] + def get_todays_events(as_list=False): """Returns a count of todays events in calendar""" from frappe.desk.doctype.event.event import get_events from frappe.utils import nowdate + today = nowdate() events = get_events(today, today) return events if as_list else len(events) diff --git a/frappe/core/page/__init__.py b/frappe/core/page/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/page/__init__.py +++ b/frappe/core/page/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py index 960444c349..d3f5e3d32f 100644 --- a/frappe/core/page/background_jobs/background_jobs.py +++ b/frappe/core/page/background_jobs/background_jobs.py @@ -15,59 +15,49 @@ from frappe.utils.scheduler import is_scheduler_inactive if TYPE_CHECKING: from rq.job import Job -JOB_COLORS = { - 'queued': 'orange', - 'failed': 'red', - 'started': 'blue', - 'finished': 'green' -} +JOB_COLORS = {"queued": "orange", "failed": "red", "started": "blue", "finished": "green"} @frappe.whitelist() def get_info(view=None, queue_timeout=None, job_status=None) -> List[Dict]: jobs = [] - def add_job(job: 'Job', name: str) -> None: + def add_job(job: "Job", name: str) -> None: if job_status != "all" and job.get_status() != job_status: return - if queue_timeout != "all" and not name.endswith(f':{queue_timeout}'): + if queue_timeout != "all" and not name.endswith(f":{queue_timeout}"): return - if job.kwargs.get('site') == frappe.local.site: + if job.kwargs.get("site") == frappe.local.site: job_info = { - 'job_name': job.kwargs.get('kwargs', {}).get('playbook_method') - or job.kwargs.get('kwargs', {}).get('job_type') - or str(job.kwargs.get('job_name')), - 'status': job.get_status(), - 'queue': name, - 'creation': convert_utc_to_user_timezone(job.created_at), - 'color': JOB_COLORS[job.get_status()] + "job_name": job.kwargs.get("kwargs", {}).get("playbook_method") + or job.kwargs.get("kwargs", {}).get("job_type") + or str(job.kwargs.get("job_name")), + "status": job.get_status(), + "queue": name, + "creation": convert_utc_to_user_timezone(job.created_at), + "color": JOB_COLORS[job.get_status()], } if job.exc_info: - job_info['exc_info'] = job.exc_info + job_info["exc_info"] = job.exc_info jobs.append(job_info) - if view == 'Jobs': + if view == "Jobs": queues = get_queues() for queue in queues: for job in queue.jobs: add_job(job, queue.name) - elif view == 'Workers': + elif view == "Workers": workers = get_workers() for worker in workers: current_job = worker.get_current_job() - if current_job and current_job.kwargs.get('site') == frappe.local.site: + if current_job and current_job.kwargs.get("site") == frappe.local.site: add_job(current_job, job.origin) else: - jobs.append({ - 'queue': worker.name, - 'job_name': 'idle', - 'status': '', - 'creation': '' - }) + jobs.append({"queue": worker.name, "job_name": "idle", "status": "", "creation": ""}) return jobs @@ -85,5 +75,5 @@ def remove_failed_jobs(): @frappe.whitelist() def get_scheduler_status(): if is_scheduler_inactive(): - return {'status': 'inactive'} - return {'status': 'active'} + return {"status": "inactive"} + return {"status": "active"} diff --git a/frappe/core/page/permission_manager/__init__.py b/frappe/core/page/permission_manager/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/page/permission_manager/__init__.py +++ b/frappe/core/page/permission_manager/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 08642c599e..ad12e0fd4c 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -2,18 +2,27 @@ # License: MIT. See LICENSE import frappe -from frappe import _ import frappe.defaults +from frappe import _ +from frappe.core.doctype.doctype.doctype import ( + clear_permissions_cache, + validate_permissions_for_doctype, +) from frappe.modules.import_file import get_file_path, read_doc_from_file +from frappe.permissions import ( + add_permission, + get_all_perms, + get_linked_doctypes, + reset_perms, + setup_custom_perms, + update_permission_property, +) from frappe.translate import send_translations -from frappe.core.doctype.doctype.doctype import (clear_permissions_cache, - validate_permissions_for_doctype) -from frappe.permissions import (reset_perms, get_linked_doctypes, get_all_perms, - setup_custom_perms, add_permission, update_permission_property) from frappe.utils.user import get_users_with_role as _get_user_with_role not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"] + @frappe.whitelist() def get_roles_and_doctypes(): frappe.only_for("System Manager") @@ -21,38 +30,43 @@ def get_roles_and_doctypes(): active_domains = frappe.get_active_domains() - doctypes = frappe.get_all("DocType", filters={ - "istable": 0, - "name": ("not in", ",".join(not_allowed_in_permission_manager)), - }, or_filters={ - "ifnull(restrict_to_domain, '')": "", - "restrict_to_domain": ("in", active_domains) - }, fields=["name"]) - - restricted_roles = ['Administrator'] - if frappe.session.user != 'Administrator': - custom_user_type_roles = frappe.get_all('User Type', filters = {'is_standard': 0}, fields=['role']) + doctypes = frappe.get_all( + "DocType", + filters={ + "istable": 0, + "name": ("not in", ",".join(not_allowed_in_permission_manager)), + }, + or_filters={"ifnull(restrict_to_domain, '')": "", "restrict_to_domain": ("in", active_domains)}, + fields=["name"], + ) + + restricted_roles = ["Administrator"] + if frappe.session.user != "Administrator": + custom_user_type_roles = frappe.get_all("User Type", filters={"is_standard": 0}, fields=["role"]) for row in custom_user_type_roles: restricted_roles.append(row.role) - restricted_roles.append('All') + restricted_roles.append("All") - roles = frappe.get_all("Role", filters={ - "name": ("not in", restricted_roles), - "disabled": 0, - }, or_filters={ - "ifnull(restrict_to_domain, '')": "", - "restrict_to_domain": ("in", active_domains) - }, fields=["name"]) + roles = frappe.get_all( + "Role", + filters={ + "name": ("not in", restricted_roles), + "disabled": 0, + }, + or_filters={"ifnull(restrict_to_domain, '')": "", "restrict_to_domain": ("in", active_domains)}, + fields=["name"], + ) - doctypes_list = [ {"label":_(d.get("name")), "value":d.get("name")} for d in doctypes] - roles_list = [ {"label":_(d.get("name")), "value":d.get("name")} for d in roles] + doctypes_list = [{"label": _(d.get("name")), "value": d.get("name")} for d in doctypes] + roles_list = [{"label": _(d.get("name")), "value": d.get("name")} for d in roles] return { - "doctypes": sorted(doctypes_list, key=lambda d: d['label']), - "roles": sorted(roles_list, key=lambda d: d['label']) + "doctypes": sorted(doctypes_list, key=lambda d: d["label"]), + "roles": sorted(roles_list, key=lambda d: d["label"]), } + @frappe.whitelist() def get_permissions(doctype=None, role=None): frappe.only_for("System Manager") @@ -61,14 +75,14 @@ def get_permissions(doctype=None, role=None): if doctype: out = [p for p in out if p.parent == doctype] else: - filters=dict(parent = doctype) - if frappe.session.user != 'Administrator': - custom_roles = frappe.get_all('Role', filters={'is_custom': 1}) - filters['role'] = ['not in', [row.name for row in custom_roles]] + filters = dict(parent=doctype) + if frappe.session.user != "Administrator": + custom_roles = frappe.get_all("Role", filters={"is_custom": 1}) + filters["role"] = ["not in", [row.name for row in custom_roles]] - out = frappe.get_all('Custom DocPerm', fields='*', filters=filters, order_by="permlevel") + out = frappe.get_all("Custom DocPerm", fields="*", filters=filters, order_by="permlevel") if not out: - out = frappe.get_all('DocPerm', fields='*', filters=filters, order_by="permlevel") + out = frappe.get_all("DocPerm", fields="*", filters=filters, order_by="permlevel") linked_doctypes = {} for d in out: @@ -82,28 +96,31 @@ def get_permissions(doctype=None, role=None): return out + @frappe.whitelist() def add(parent, role, permlevel): frappe.only_for("System Manager") add_permission(parent, role, permlevel) + @frappe.whitelist() def update(doctype, role, permlevel, ptype, value=None): """Update role permission params Args: - doctype (str): Name of the DocType to update params for - role (str): Role to be updated for, eg "Website Manager". - permlevel (int): perm level the provided rule applies to - ptype (str): permission type, example "read", "delete", etc. - value (None, optional): value for ptype, None indicates False + doctype (str): Name of the DocType to update params for + role (str): Role to be updated for, eg "Website Manager". + permlevel (int): perm level the provided rule applies to + ptype (str): permission type, example "read", "delete", etc. + value (None, optional): value for ptype, None indicates False Returns: - str: Refresh flag is permission is updated successfully + str: Refresh flag is permission is updated successfully """ frappe.only_for("System Manager") out = update_permission_property(doctype, role, permlevel, ptype, value) - return 'refresh' if out else None + return "refresh" if out else None + @frappe.whitelist() def remove(doctype, role, permlevel): @@ -112,28 +129,31 @@ def remove(doctype, role, permlevel): frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel}) - if not frappe.get_all('Custom DocPerm', {"parent": doctype}): - frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) + if not frappe.get_all("Custom DocPerm", {"parent": doctype}): + frappe.throw(_("There must be atleast one permission rule."), title=_("Cannot Remove")) validate_permissions_for_doctype(doctype, for_remove=True, alert=True) + @frappe.whitelist() def reset(doctype): frappe.only_for("System Manager") reset_perms(doctype) clear_permissions_cache(doctype) + @frappe.whitelist() def get_users_with_role(role): frappe.only_for("System Manager") return _get_user_with_role(role) + @frappe.whitelist() def get_standard_permissions(doctype): frappe.only_for("System Manager") meta = frappe.get_meta(doctype) if meta.custom: - doc = frappe.get_doc('DocType', doctype) + doc = frappe.get_doc("DocType", doctype) return [p.as_dict() for p in doc.permissions] else: # also used to setup permissions via patch diff --git a/frappe/core/report/__init__.py b/frappe/core/report/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/core/report/__init__.py +++ b/frappe/core/report/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 535d354250..32d8bbe18f 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -2,21 +2,27 @@ # License: MIT. See LICENSE import frappe -from frappe import _, throw import frappe.utils.user -from frappe.permissions import check_admin_or_system_manager, rights +from frappe import _, throw from frappe.model import data_fieldtypes +from frappe.permissions import check_admin_or_system_manager, rights + def execute(filters=None): - user, doctype, show_permissions = filters.get("user"), filters.get("doctype"), filters.get("show_permissions") + user, doctype, show_permissions = ( + filters.get("user"), + filters.get("doctype"), + filters.get("show_permissions"), + ) - if not validate(user, doctype): return [], [] + if not validate(user, doctype): + return [], [] columns, fields = get_columns_and_fields(doctype) data = frappe.get_list(doctype, fields=fields, as_list=True, user=user) if show_permissions: - columns = columns + [frappe.unscrub(right) + ':Check:80' for right in rights] + columns = columns + [frappe.unscrub(right) + ":Check:80" for right in rights] data = list(data) for i, doc in enumerate(data): permission = frappe.permissions.get_doc_permissions(frappe.get_doc(doctype, doc[0]), user) @@ -24,29 +30,36 @@ def execute(filters=None): return columns, data + def validate(user, doctype): # check if current user is System Manager check_admin_or_system_manager() return user and doctype + def get_columns_and_fields(doctype): columns = ["Name:Link/{}:200".format(doctype)] fields = ["`name`"] for df in frappe.get_meta(doctype).fields: if df.in_list_view and df.fieldtype in data_fieldtypes: fields.append("`{0}`".format(df.fieldname)) - fieldtype = "Link/{}".format(df.options) if df.fieldtype=="Link" else df.fieldtype - columns.append("{label}:{fieldtype}:{width}".format(label=df.label, fieldtype=fieldtype, width=df.width or 100)) + fieldtype = "Link/{}".format(df.options) if df.fieldtype == "Link" else df.fieldtype + columns.append( + "{label}:{fieldtype}:{width}".format( + label=df.label, fieldtype=fieldtype, width=df.width or 100 + ) + ) return columns, fields + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def query_doctypes(doctype, txt, searchfield, start, page_len, filters): user = filters.get("user") user_perms = frappe.utils.user.UserPermissions(user) user_perms.build_permissions() - can_read = user_perms.can_read # Does not include child tables + can_read = user_perms.can_read # Does not include child tables single_doctypes = [d[0] for d in frappe.db.get_values("DocType", {"issingle": 1})] diff --git a/frappe/core/report/transaction_log_report/transaction_log_report.py b/frappe/core/report/transaction_log_report/transaction_log_report.py index e9c68cb0c7..51a01ffc57 100644 --- a/frappe/core/report/transaction_log_report/transaction_log_report.py +++ b/frappe/core/report/transaction_log_report/transaction_log_report.py @@ -1,16 +1,19 @@ # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import hashlib + +import frappe from frappe import _ from frappe.utils import format_datetime + def execute(filters=None): columns, data = get_columns(filters), get_data(filters) return columns, data + def get_data(filters=None): result = [] logs = frappe.get_all("Transaction Log", fields=["*"], order_by="creation desc") @@ -26,14 +29,35 @@ def get_data(filters=None): if not previous_hash: integrity = False else: - integrity = check_data_integrity(l.chaining_hash, l.transaction_hash, l.previous_hash, previous_hash[0][0]) + integrity = check_data_integrity( + l.chaining_hash, l.transaction_hash, l.previous_hash, previous_hash[0][0] + ) - result.append([_(str(integrity)), _(l.reference_doctype), l.document_name, l.owner, l.modified_by, format_datetime(l.timestamp, "YYYYMMDDHHmmss")]) + result.append( + [ + _(str(integrity)), + _(l.reference_doctype), + l.document_name, + l.owner, + l.modified_by, + format_datetime(l.timestamp, "YYYYMMDDHHmmss"), + ] + ) else: - result.append([_("First Transaction"), _(l.reference_doctype), l.document_name, l.owner, l.modified_by, format_datetime(l.timestamp, "YYYYMMDDHHmmss")]) + result.append( + [ + _("First Transaction"), + _(l.reference_doctype), + l.document_name, + l.owner, + l.modified_by, + format_datetime(l.timestamp, "YYYYMMDDHHmmss"), + ] + ) return result + def check_data_integrity(chaining_hash, transaction_hash, registered_previous_hash, previous_hash): if registered_previous_hash != previous_hash: return False @@ -45,6 +69,7 @@ def check_data_integrity(chaining_hash, transaction_hash, registered_previous_ha else: return True + def calculate_chain(transaction_hash, previous_hash): sha = hashlib.sha256() sha.update(str(transaction_hash) + str(previous_hash)) @@ -57,37 +82,17 @@ def get_columns(filters=None): "label": _("Chain Integrity"), "fieldname": "chain_integrity", "fieldtype": "Data", - "width": 150 + "width": 150, }, { "label": _("Reference Doctype"), "fieldname": "reference_doctype", "fieldtype": "Data", - "width": 150 - }, - { - "label": _("Reference Name"), - "fieldname": "reference_name", - "fieldtype": "Data", - "width": 150 - }, - { - "label": _("Owner"), - "fieldname": "owner", - "fieldtype": "Data", - "width": 100 + "width": 150, }, - { - "label": _("Modified By"), - "fieldname": "modified_by", - "fieldtype": "Data", - "width": 100 - }, - { - "label": _("Timestamp"), - "fieldname": "timestamp", - "fieldtype": "Data", - "width": 100 - } + {"label": _("Reference Name"), "fieldname": "reference_name", "fieldtype": "Data", "width": 150}, + {"label": _("Owner"), "fieldname": "owner", "fieldtype": "Data", "width": 100}, + {"label": _("Modified By"), "fieldname": "modified_by", "fieldtype": "Data", "width": 100}, + {"label": _("Timestamp"), "fieldname": "timestamp", "fieldtype": "Data", "width": 100}, ] return columns diff --git a/frappe/core/utils.py b/frappe/core/utils.py index d4690cae89..8581f30f89 100644 --- a/frappe/core/utils.py +++ b/frappe/core/utils.py @@ -13,6 +13,7 @@ def get_parent_doc(doc): doc.parent_doc = None return doc.parent_doc + def set_timeline_doc(doc): """Set timeline_doctype and timeline_name""" parent_doc = get_parent_doc(doc) @@ -33,47 +34,50 @@ def set_timeline_doc(doc): else: return + def find(list_of_dict, match_function): - '''Returns a dict in a list of dicts on matching the conditions - provided in match function + """Returns a dict in a list of dicts on matching the conditions + provided in match function Usage: - list_of_dict = [{'name': 'Suraj'}, {'name': 'Aditya'}] + list_of_dict = [{'name': 'Suraj'}, {'name': 'Aditya'}] - required_dict = find(list_of_dict, lambda d: d['name'] == 'Aditya') - ''' + required_dict = find(list_of_dict, lambda d: d['name'] == 'Aditya') + """ for entry in list_of_dict: if match_function(entry): return entry return None + def find_all(list_of_dict, match_function): - '''Returns all matching dicts in a list of dicts. - Uses matching function to filter out the dicts + """Returns all matching dicts in a list of dicts. + Uses matching function to filter out the dicts Usage: - colored_shapes = [ - {'color': 'red', 'shape': 'square'}, - {'color': 'red', 'shape': 'circle'}, - {'color': 'blue', 'shape': 'triangle'} - ] - - red_shapes = find_all(colored_shapes, lambda d: d['color'] == 'red') - ''' + colored_shapes = [ + {'color': 'red', 'shape': 'square'}, + {'color': 'red', 'shape': 'circle'}, + {'color': 'blue', 'shape': 'triangle'} + ] + + red_shapes = find_all(colored_shapes, lambda d: d['color'] == 'red') + """ found = [] for entry in list_of_dict: if match_function(entry): found.append(entry) return found + def ljust_list(_list, length, fill_word=None): """ Similar to ljust but for list. Usage: - $ ljust_list([1, 2, 3], 5) - > [1, 2, 3, None, None] + $ ljust_list([1, 2, 3], 5) + > [1, 2, 3, None, None] """ # make a copy to avoid mutation of passed list _list = list(_list) diff --git a/frappe/core/web_form/edit_profile/edit_profile.py b/frappe/core/web_form/edit_profile/edit_profile.py index e1ada61927..80b7b87352 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.py +++ b/frappe/core/web_form/edit_profile/edit_profile.py @@ -1,5 +1,6 @@ import frappe + def get_context(context): # do your magic here pass diff --git a/frappe/coverage.py b/frappe/coverage.py index 5f89800deb..ffa3576818 100644 --- a/frappe/coverage.py +++ b/frappe/coverage.py @@ -10,18 +10,18 @@ STANDARD_INCLUSIONS = ["*.py"] STANDARD_EXCLUSIONS = [ - '*.js', - '*.xml', - '*.pyc', - '*.css', - '*.less', - '*.scss', - '*.vue', - '*.html', - '*/test_*', - '*/node_modules/*', - '*/doctype/*/*_dashboard.py', - '*/patches/*', + "*.js", + "*.xml", + "*.pyc", + "*.css", + "*.less", + "*.scss", + "*.vue", + "*.html", + "*/test_*", + "*/node_modules/*", + "*/doctype/*/*_dashboard.py", + "*/patches/*", ] FRAPPE_EXCLUSIONS = [ @@ -35,22 +35,25 @@ FRAPPE_EXCLUSIONS = [ "*/patches/*", ] -class CodeCoverage(): + +class CodeCoverage: def __init__(self, with_coverage, app): self.with_coverage = with_coverage - self.app = app or 'frappe' + self.app = app or "frappe" def __enter__(self): if self.with_coverage: import os + from coverage import Coverage + from frappe.utils import get_bench_path # Generate coverage report only for app that is being tested - source_path = os.path.join(get_bench_path(), 'apps', self.app) + source_path = os.path.join(get_bench_path(), "apps", self.app) omit = STANDARD_EXCLUSIONS[:] - if self.app == 'frappe': + if self.app == "frappe": omit.extend(FRAPPE_EXCLUSIONS) self.coverage = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) @@ -60,4 +63,4 @@ class CodeCoverage(): if self.with_coverage: self.coverage.stop() self.coverage.save() - self.coverage.xml_report() \ No newline at end of file + self.coverage.xml_report() diff --git a/frappe/custom/doctype/client_script/__init__.py b/frappe/custom/doctype/client_script/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/custom/doctype/client_script/__init__.py +++ b/frappe/custom/doctype/client_script/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/custom/doctype/client_script/client_script.py b/frappe/custom/doctype/client_script/client_script.py index fd6bc9accd..3039e0a4a5 100644 --- a/frappe/custom/doctype/client_script/client_script.py +++ b/frappe/custom/doctype/client_script/client_script.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import frappe - from frappe import _ from frappe.model.document import Document @@ -14,9 +13,7 @@ class ClientScript(Document): if not self.is_new(): return - exists = frappe.db.exists( - "Client Script", {"dt": self.dt, "view": self.view} - ) + exists = frappe.db.exists("Client Script", {"dt": self.dt, "view": self.view}) if exists: frappe.throw( _("Client Script for {0} {1} already exists").format(frappe.bold(self.dt), self.view), diff --git a/frappe/custom/doctype/client_script/test_client_script.py b/frappe/custom/doctype/client_script/test_client_script.py index 4887956001..2538fdf515 100644 --- a/frappe/custom/doctype/client_script/test_client_script.py +++ b/frappe/custom/doctype/client_script/test_client_script.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Client Script') + class TestClientScript(unittest.TestCase): pass diff --git a/frappe/custom/doctype/custom_field/__init__.py b/frappe/custom/doctype/custom_field/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/custom/doctype/custom_field/__init__.py +++ b/frappe/custom/doctype/custom_field/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index fdd7bf3134..10ee4a503f 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -1,14 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import json -from frappe.utils import cstr + +import frappe from frappe import _ -from frappe.model.document import Document -from frappe.model.docfield import supports_translation from frappe.model import core_doctypes_list +from frappe.model.docfield import supports_translation +from frappe.model.document import Document from frappe.query_builder.functions import IfNull +from frappe.utils import cstr + class CustomField(Document): def autoname(self): @@ -25,8 +27,9 @@ class CustomField(Document): frappe.throw(_("Label is mandatory")) # remove special characters from fieldname - self.fieldname = "".join(filter(lambda x: x.isdigit() or x.isalpha() or '_', - cstr(label).replace(' ','_'))) + self.fieldname = "".join( + filter(lambda x: x.isdigit() or x.isalpha() or "_", cstr(label).replace(" ", "_")) + ) # fieldnames should be lowercase self.fieldname = self.fieldname.lower() @@ -36,8 +39,8 @@ class CustomField(Document): def validate(self): # these imports have been added to avoid cyclical import, should fix in future - from frappe.custom.doctype.customize_form.customize_form import CustomizeForm from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts + from frappe.custom.doctype.customize_form.customize_form import CustomizeForm # don't always get meta to improve performance # setting idx is just an improvement, not a requirement @@ -47,8 +50,9 @@ class CustomField(Document): if self.is_new() and self.fieldname in fieldnames: frappe.throw( - _("A field with the name {0} already exists in {1}") - .format(frappe.bold(self.fieldname), self.dt) + _("A field with the name {0} already exists in {1}").format( + frappe.bold(self.fieldname), self.dt + ) ) if self.insert_after == "append": @@ -64,14 +68,13 @@ class CustomField(Document): and not CustomizeForm.allow_fieldtype_change(old_fieldtype, self.fieldtype) ): frappe.throw( - _("Fieldtype cannot be changed from {0} to {1}") - .format(old_fieldtype, self.fieldtype) + _("Fieldtype cannot be changed from {0} to {1}").format(old_fieldtype, self.fieldtype) ) if not self.fieldname: frappe.throw(_("Fieldname not set for Custom Field")) - if self.get('translatable', 0) and not supports_translation(self.fieldtype): + if self.get("translatable", 0) and not supports_translation(self.fieldtype): self.translatable = 0 check_fieldname_conflicts(self) @@ -83,33 +86,39 @@ class CustomField(Document): if not self.flags.ignore_validate: # validate field from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype + validate_fields_for_doctype(self.dt) # update the schema - if not frappe.db.get_value('DocType', self.dt, 'issingle') and not frappe.flags.in_setup_wizard: + if not frappe.db.get_value("DocType", self.dt, "issingle") and not frappe.flags.in_setup_wizard: frappe.db.updatedb(self.dt) def on_trash(self): - #check if Admin owned field - if self.owner == 'Administrator' and frappe.session.user != 'Administrator': - frappe.throw(_("Custom Field {0} is created by the Administrator and can only be deleted through the Administrator account.").format( - frappe.bold(self.label))) + # check if Admin owned field + if self.owner == "Administrator" and frappe.session.user != "Administrator": + frappe.throw( + _( + "Custom Field {0} is created by the Administrator and can only be deleted through the Administrator account." + ).format(frappe.bold(self.label)) + ) # delete property setter entries - frappe.db.delete("Property Setter", { - "doc_type": self.dt, - "field_name": self.fieldname - }) + frappe.db.delete("Property Setter", {"doc_type": self.dt, "field_name": self.fieldname}) frappe.clear_cache(doctype=self.dt) def validate_insert_after(self, meta): if not meta.get_field(self.insert_after): - frappe.throw(_("Insert After field '{0}' mentioned in Custom Field '{1}', with label '{2}', does not exist") - .format(self.insert_after, self.name, self.label), frappe.DoesNotExistError) + frappe.throw( + _( + "Insert After field '{0}' mentioned in Custom Field '{1}', with label '{2}', does not exist" + ).format(self.insert_after, self.name, self.label), + frappe.DoesNotExistError, + ) if self.fieldname == self.insert_after: frappe.throw(_("Insert After cannot be set as {0}").format(meta.get_label(self.insert_after))) + @frappe.whitelist() def get_fields_label(doctype=None): meta = frappe.get_meta(doctype) @@ -120,36 +129,44 @@ def get_fields_label(doctype=None): if meta.custom: return frappe.msgprint(_("Custom Fields can only be added to a standard DocType.")) - return [{"value": df.fieldname or "", "label": _(df.label or "")} - for df in frappe.get_meta(doctype).get("fields")] + return [ + {"value": df.fieldname or "", "label": _(df.label or "")} + for df in frappe.get_meta(doctype).get("fields") + ] + def create_custom_field_if_values_exist(doctype, df): df = frappe._dict(df) - if df.fieldname in frappe.db.get_table_columns(doctype) and \ - frappe.db.count(dt=doctype, filters=IfNull(df.fieldname, "") != ""): + if df.fieldname in frappe.db.get_table_columns(doctype) and frappe.db.count( + dt=doctype, filters=IfNull(df.fieldname, "") != "" + ): create_custom_field(doctype, df) + def create_custom_field(doctype, df, ignore_validate=False, is_system_generated=True): df = frappe._dict(df) if not df.fieldname and df.label: df.fieldname = frappe.scrub(df.label) if not frappe.db.get_value("Custom Field", {"dt": doctype, "fieldname": df.fieldname}): - custom_field = frappe.get_doc({ - "doctype":"Custom Field", - "dt": doctype, - "permlevel": 0, - "fieldtype": 'Data', - "hidden": 0, - "is_system_generated": is_system_generated - }) + custom_field = frappe.get_doc( + { + "doctype": "Custom Field", + "dt": doctype, + "permlevel": 0, + "fieldtype": "Data", + "hidden": 0, + "is_system_generated": is_system_generated, + } + ) custom_field.update(df) custom_field.flags.ignore_validate = ignore_validate custom_field.insert() -def create_custom_fields(custom_fields, ignore_validate = False, update=True): - '''Add / update multiple custom fields - :param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`''' +def create_custom_fields(custom_fields, ignore_validate=False, update=True): + """Add / update multiple custom fields + + :param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`""" if not ignore_validate and frappe.flags.in_setup_wizard: ignore_validate = True @@ -182,7 +199,6 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True): frappe.db.updatedb(doctype) - @frappe.whitelist() def add_custom_field(doctype, df): df = json.loads(df) diff --git a/frappe/custom/doctype/custom_field/test_custom_field.py b/frappe/custom/doctype/custom_field/test_custom_field.py index ad3cf27eea..519ea7f2b4 100644 --- a/frappe/custom/doctype/custom_field/test_custom_field.py +++ b/frappe/custom/doctype/custom_field/test_custom_field.py @@ -3,9 +3,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + test_records = frappe.get_test_records("Custom Field") @@ -36,12 +37,6 @@ class TestCustomField(unittest.TestCase): frappe.db.commit() - self.assertTrue( - frappe.db.exists("Custom Field", "Address-_test_custom_field_1") - ) - self.assertTrue( - frappe.db.exists("Custom Field", "Address-_test_custom_field_2") - ) - self.assertTrue( - frappe.db.exists("Custom Field", "Contact-_test_custom_field_2") - ) + self.assertTrue(frappe.db.exists("Custom Field", "Address-_test_custom_field_1")) + self.assertTrue(frappe.db.exists("Custom Field", "Address-_test_custom_field_2")) + self.assertTrue(frappe.db.exists("Custom Field", "Contact-_test_custom_field_2")) diff --git a/frappe/custom/doctype/customize_form/__init__.py b/frappe/custom/doctype/customize_form/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/custom/doctype/customize_form/__init__.py +++ b/frappe/custom/doctype/customize_form/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 5ec5cae121..b4ccb21167 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -6,17 +6,21 @@ Thus providing a better UI from user perspective """ import json + import frappe import frappe.translate from frappe import _ -from frappe.utils import cint -from frappe.model.document import Document -from frappe.model import no_value_fields, core_doctypes_list -from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype, check_email_append_to +from frappe.core.doctype.doctype.doctype import ( + check_email_append_to, + validate_fields_for_doctype, + validate_series, +) from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.custom.doctype.property_setter.property_setter import delete_property_setter +from frappe.model import core_doctypes_list, no_value_fields from frappe.model.docfield import supports_translation -from frappe.core.doctype.doctype.doctype import validate_series +from frappe.model.document import Document +from frappe.utils import cint class CustomizeForm(Document): @@ -39,16 +43,16 @@ class CustomizeForm(Document): # load custom translation translation = self.get_name_translation() - self.label = translation.translated_text if translation else '' + self.label = translation.translated_text if translation else "" self.create_auto_repeat_custom_field_if_required(meta) # NOTE doc (self) is sent to clientside by run_method def validate_doctype(self, meta): - ''' + """ Check if the doctype is allowed to be customized. - ''' + """ if self.doc_type in core_doctypes_list: frappe.throw(_("Core DocTypes cannot be customized.")) @@ -59,9 +63,9 @@ class CustomizeForm(Document): frappe.throw(_("Only standard DocTypes are allowed to be customized from Customize Form.")) def load_properties(self, meta): - ''' + """ Load the customize object (this) with the metadata properties - ''' + """ # doctype properties for prop in doctype_properties: self.set(prop, meta.get(prop)) @@ -71,20 +75,20 @@ class CustomizeForm(Document): "fieldname": d.fieldname, "is_custom_field": d.get("is_custom_field"), "is_system_generated": d.get("is_system_generated"), - "name": d.name + "name": d.name, } for prop in docfield_properties: new_d[prop] = d.get(prop) self.append("fields", new_d) - for fieldname in ('links', 'actions', 'states'): + for fieldname in ("links", "actions", "states"): for d in meta.get(fieldname): self.append(fieldname, d) def create_auto_repeat_custom_field_if_required(self, meta): - ''' + """ Create auto repeat custom field if it's not already present - ''' + """ if self.allow_auto_repeat: all_fields = [df.fieldname for df in meta.fields] @@ -92,45 +96,51 @@ class CustomizeForm(Document): return insert_after = self.fields[len(self.fields) - 1].fieldname - create_custom_field(self.doc_type, dict( - fieldname='auto_repeat', - label='Auto Repeat', - fieldtype='Link', - options='Auto Repeat', - insert_after=insert_after, - read_only=1, no_copy=1, print_hide=1 - )) - + create_custom_field( + self.doc_type, + dict( + fieldname="auto_repeat", + label="Auto Repeat", + fieldtype="Link", + options="Auto Repeat", + insert_after=insert_after, + read_only=1, + no_copy=1, + print_hide=1, + ), + ) def get_name_translation(self): - '''Get translation object if exists of current doctype name in the default language''' - return frappe.get_value('Translation', { - 'source_text': self.doc_type, - 'language': frappe.local.lang or 'en' - }, ['name', 'translated_text'], as_dict=True) + """Get translation object if exists of current doctype name in the default language""" + return frappe.get_value( + "Translation", + {"source_text": self.doc_type, "language": frappe.local.lang or "en"}, + ["name", "translated_text"], + as_dict=True, + ) def set_name_translation(self): - '''Create, update custom translation for this doctype''' + """Create, update custom translation for this doctype""" current = self.get_name_translation() if not self.label: if current: # clear translation - frappe.delete_doc('Translation', current.name) + frappe.delete_doc("Translation", current.name) return if not current: frappe.get_doc( { - "doctype": 'Translation', + "doctype": "Translation", "source_text": self.doc_type, "translated_text": self.label, - "language_code": frappe.local.lang or 'en' + "language_code": frappe.local.lang or "en", } ).insert() return if self.label != current.translated_text: - frappe.db.set_value('Translation', current.name, 'translated_text', self.label) + frappe.db.set_value("Translation", current.name, "translated_text", self.label) frappe.translate.clear_cache() def clear_existing_doc(self): @@ -161,14 +171,15 @@ class CustomizeForm(Document): if self.flags.update_db: frappe.db.updatedb(self.doc_type) - if not hasattr(self, 'hide_success') or not self.hide_success: + if not hasattr(self, "hide_success") or not self.hide_success: frappe.msgprint(_("{0} updated").format(_(self.doc_type)), alert=True) frappe.clear_cache(doctype=self.doc_type) self.fetch_to_customize() if self.flags.rebuild_doctype_for_global_search: - frappe.enqueue('frappe.utils.global_search.rebuild_for_doctype', - now=True, doctype=self.doc_type) + frappe.enqueue( + "frappe.utils.global_search.rebuild_for_doctype", now=True, doctype=self.doc_type + ) def set_property_setters(self): meta = frappe.get_meta(self.doc_type) @@ -193,12 +204,11 @@ class CustomizeForm(Document): def set_property_setters_for_docfield(self, meta, df, meta_df): for prop, prop_type in docfield_properties.items(): - if prop != "idx" and (df.get(prop) or '') != (meta_df[0].get(prop) or ''): + if prop != "idx" and (df.get(prop) or "") != (meta_df[0].get(prop) or ""): if not self.allow_property_change(prop, meta_df, df): continue - self.make_property_setter(prop, df.get(prop), prop_type, - fieldname=df.fieldname) + self.make_property_setter(prop, df.get(prop), prop_type, fieldname=df.fieldname) def allow_property_change(self, prop, meta_df, df): if prop == "fieldtype": @@ -209,42 +219,61 @@ class CustomizeForm(Document): new_value_length = cint(df.get(prop)) if new_value_length and (old_value_length > new_value_length): - self.check_length_for_fieldtypes.append({'df': df, 'old_value': meta_df[0].get(prop)}) + self.check_length_for_fieldtypes.append({"df": df, "old_value": meta_df[0].get(prop)}) self.validate_fieldtype_length() else: self.flags.update_db = True elif prop == "allow_on_submit" and df.get(prop): - if not frappe.db.get_value("DocField", - {"parent": self.doc_type, "fieldname": df.fieldname}, "allow_on_submit"): - frappe.msgprint(_("Row {0}: Not allowed to enable Allow on Submit for standard fields")\ - .format(df.idx)) + if not frappe.db.get_value( + "DocField", {"parent": self.doc_type, "fieldname": df.fieldname}, "allow_on_submit" + ): + frappe.msgprint( + _("Row {0}: Not allowed to enable Allow on Submit for standard fields").format(df.idx) + ) return False - elif prop == "reqd" and \ - ((frappe.db.get_value("DocField", - {"parent":self.doc_type,"fieldname":df.fieldname}, "reqd") == 1) \ - and (df.get(prop) == 0)): - frappe.msgprint(_("Row {0}: Not allowed to disable Mandatory for standard fields")\ - .format(df.idx)) + elif prop == "reqd" and ( + ( + frappe.db.get_value("DocField", {"parent": self.doc_type, "fieldname": df.fieldname}, "reqd") + == 1 + ) + and (df.get(prop) == 0) + ): + frappe.msgprint( + _("Row {0}: Not allowed to disable Mandatory for standard fields").format(df.idx) + ) return False - elif prop == "in_list_view" and df.get(prop) \ - and df.fieldtype!="Attach Image" and df.fieldtype in no_value_fields: - frappe.msgprint(_("'In List View' not allowed for type {0} in row {1}") - .format(df.fieldtype, df.idx)) - return False + elif ( + prop == "in_list_view" + and df.get(prop) + and df.fieldtype != "Attach Image" + and df.fieldtype in no_value_fields + ): + frappe.msgprint( + _("'In List View' not allowed for type {0} in row {1}").format(df.fieldtype, df.idx) + ) + return False - elif prop == "precision" and cint(df.get("precision")) > 6 \ - and cint(df.get("precision")) > cint(meta_df[0].get("precision")): + elif ( + prop == "precision" + and cint(df.get("precision")) > 6 + and cint(df.get("precision")) > cint(meta_df[0].get("precision")) + ): self.flags.update_db = True elif prop == "unique": self.flags.update_db = True - elif (prop == "read_only" and cint(df.get("read_only"))==0 - and frappe.db.get_value("DocField", {"parent": self.doc_type, - "fieldname": df.fieldname}, "read_only")==1): + elif ( + prop == "read_only" + and cint(df.get("read_only")) == 0 + and frappe.db.get_value( + "DocField", {"parent": self.doc_type, "fieldname": df.fieldname}, "read_only" + ) + == 1 + ): # if docfield has read_only checked and user is trying to make it editable, don't allow it frappe.msgprint(_("You cannot unset 'Read Only' for field {0}").format(df.label)) return False @@ -253,25 +282,24 @@ class CustomizeForm(Document): frappe.msgprint(_("You can't set 'Options' for field {0}").format(df.label)) return False - elif prop == 'translatable' and not supports_translation(df.get('fieldtype')): + elif prop == "translatable" and not supports_translation(df.get("fieldtype")): frappe.msgprint(_("You can't set 'Translatable' for field {0}").format(df.label)) return False - elif (prop == 'in_global_search' and - df.in_global_search != meta_df[0].get("in_global_search")): + elif prop == "in_global_search" and df.in_global_search != meta_df[0].get("in_global_search"): self.flags.rebuild_doctype_for_global_search = True return True def set_property_setters_for_actions_and_links(self, meta): - ''' + """ Apply property setters or create custom records for DocType Action and DocType Link - ''' + """ for doctype, fieldname, field_map in ( - ('DocType Link', 'links', doctype_link_properties), - ('DocType Action', 'actions', doctype_action_properties), - ('DocType State', 'states', doctype_state_properties), - ): + ("DocType Link", "links", doctype_link_properties), + ("DocType Action", "actions", doctype_action_properties), + ("DocType State", "states", doctype_state_properties), + ): has_custom = False items = [] for i, d in enumerate(self.get(fieldname) or []): @@ -281,8 +309,7 @@ class CustomizeForm(Document): original = frappe.get_doc(doctype, d.name) for prop, prop_type in field_map.items(): if d.get(prop) != original.get(prop): - self.make_property_setter(prop, d.get(prop), prop_type, - apply_on=doctype, row_name=d.name) + self.make_property_setter(prop, d.get(prop), prop_type, apply_on=doctype, row_name=d.name) items.append(d.name) else: # custom - just insert/update @@ -296,34 +323,32 @@ class CustomizeForm(Document): self.clear_removed_items(doctype, items) def update_order_property_setter(self, has_custom, fieldname): - ''' + """ We need to maintain the order of the link/actions if the user has shuffled them. So we create a new property (ex `links_order`) to keep a list of items. - ''' - property_name = '{}_order'.format(fieldname) + """ + property_name = "{}_order".format(fieldname) if has_custom: # save the order of the actions and links - self.make_property_setter(property_name, - json.dumps([d.name for d in self.get(fieldname)]), 'Small Text') + self.make_property_setter( + property_name, json.dumps([d.name for d in self.get(fieldname)]), "Small Text" + ) else: - frappe.db.delete('Property Setter', dict(property=property_name, - doc_type=self.doc_type)) - + frappe.db.delete("Property Setter", dict(property=property_name, doc_type=self.doc_type)) def clear_removed_items(self, doctype, items): - ''' + """ Clear rows that do not appear in `items`. These have been removed by the user. - ''' + """ if items: - frappe.db.delete(doctype, dict(parent=self.doc_type, custom=1, - name=('not in', items))) + frappe.db.delete(doctype, dict(parent=self.doc_type, custom=1, name=("not in", items))) else: frappe.db.delete(doctype, dict(parent=self.doc_type, custom=1)) def update_custom_fields(self): for i, df in enumerate(self.get("fields")): if df.get("is_custom_field"): - if not frappe.db.exists('Custom Field', {'dt': self.doc_type, 'fieldname': df.fieldname}): + if not frappe.db.exists("Custom Field", {"dt": self.doc_type, "fieldname": df.fieldname}): self.add_custom_field(df, i) self.flags.update_db = True else: @@ -339,8 +364,8 @@ class CustomizeForm(Document): for prop in docfield_properties: d.set(prop, df.get(prop)) - if i!=0: - d.insert_after = self.fields[i-1].fieldname + if i != 0: + d.insert_after = self.fields[i - 1].fieldname d.idx = i d.insert() @@ -364,8 +389,8 @@ class CustomizeForm(Document): changed = True # check and update `insert_after` property - if i!=0: - insert_after = self.fields[i-1].fieldname + if i != 0: + insert_after = self.fields[i - 1].fieldname if custom_field.insert_after != insert_after: custom_field.insert_after = insert_after custom_field.idx = i @@ -374,46 +399,51 @@ class CustomizeForm(Document): if changed: custom_field.db_update() self.flags.update_db = True - #custom_field.save() + # custom_field.save() def delete_custom_fields(self): meta = frappe.get_meta(self.doc_type) - fields_to_remove = ( - {df.fieldname for df in meta.get("fields")} - {df.fieldname for df in self.get("fields")} - ) + fields_to_remove = {df.fieldname for df in meta.get("fields")} - { + df.fieldname for df in self.get("fields") + } for fieldname in fields_to_remove: df = meta.get("fields", {"fieldname": fieldname})[0] if df.get("is_custom_field"): frappe.delete_doc("Custom Field", df.name) - def make_property_setter(self, prop, value, property_type, fieldname=None, - apply_on=None, row_name = None): + def make_property_setter( + self, prop, value, property_type, fieldname=None, apply_on=None, row_name=None + ): delete_property_setter(self.doc_type, prop, fieldname, row_name) property_value = self.get_existing_property_value(prop, fieldname) - if property_value==value: + if property_value == value: return if not apply_on: apply_on = "DocField" if fieldname else "DocType" # create a new property setter - frappe.make_property_setter({ - "doctype": self.doc_type, - "doctype_or_field": apply_on, - "fieldname": fieldname, - "row_name": row_name, - "property": prop, - "value": value, - "property_type": property_type - }, is_system_generated=False) + frappe.make_property_setter( + { + "doctype": self.doc_type, + "doctype_or_field": apply_on, + "fieldname": fieldname, + "row_name": row_name, + "property": prop, + "value": value, + "property_type": property_type, + }, + is_system_generated=False, + ) def get_existing_property_value(self, property_name, fieldname=None): # check if there is any need to make property setter! if fieldname: - property_value = frappe.db.get_value("DocField", {"parent": self.doc_type, - "fieldname": fieldname}, property_name) + property_value = frappe.db.get_value( + "DocField", {"parent": self.doc_type, "fieldname": fieldname}, property_name + ) else: if frappe.db.has_column("DocType", property_name): property_value = frappe.db.get_value("DocType", self.doc_type, property_name) @@ -434,41 +464,47 @@ class CustomizeForm(Document): # Ignore fieldtype check validation if new field type has unspecified maxlength # Changes like DATA to TEXT, where new_value_lenth equals 0 will not be validated if new_value_length and (old_value_length > new_value_length): - self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value}) + self.check_length_for_fieldtypes.append({"df": df, "old_value": old_value}) self.validate_fieldtype_length() else: self.flags.update_db = True else: - frappe.throw(_("Fieldtype cannot be changed from {0} to {1} in row {2}").format(old_value, new_value, df.idx)) + frappe.throw( + _("Fieldtype cannot be changed from {0} to {1} in row {2}").format( + old_value, new_value, df.idx + ) + ) def validate_fieldtype_length(self): for field in self.check_length_for_fieldtypes: - df = field.get('df') + df = field.get("df") max_length = cint(frappe.db.type_map.get(df.fieldtype)[1]) fieldname = df.fieldname - docs = frappe.db.sql(''' + docs = frappe.db.sql( + """ SELECT name, {fieldname}, LENGTH({fieldname}) AS len FROM `tab{doctype}` WHERE LENGTH({fieldname}) > {max_length} - '''.format( - fieldname=fieldname, - doctype=self.doc_type, - max_length=max_length - ), as_dict=True) + """.format( + fieldname=fieldname, doctype=self.doc_type, max_length=max_length + ), + as_dict=True, + ) links = [] label = df.label for doc in docs: links.append(frappe.utils.get_link_to_form(self.doc_type, doc.name)) - links_str = ', '.join(links) + links_str = ", ".join(links) if docs: - frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') - .format( - frappe.bold(label), - links_str, - frappe.bold(max_length) - ), title=_('Data Too Long'), is_minimizable=len(docs) > 1) + frappe.throw( + _( + "Value for field {0} is too long in {1}. Length should be lesser than {2} characters" + ).format(frappe.bold(label), links_str, frappe.bold(max_length)), + title=_("Data Too Long"), + is_minimizable=len(docs) > 1, + ) self.flags.update_db = True @@ -482,136 +518,139 @@ class CustomizeForm(Document): @classmethod def allow_fieldtype_change(self, old_type: str, new_type: str) -> bool: - """ allow type change, if both old_type and new_type are in same field group. + """allow type change, if both old_type and new_type are in same field group. field groups are defined in ALLOWED_FIELDTYPE_CHANGE variables. """ in_field_group = lambda group: (old_type in group) and (new_type in group) return any(map(in_field_group, ALLOWED_FIELDTYPE_CHANGE)) + def reset_customization(doctype): - setters = frappe.get_all("Property Setter", filters={ - 'doc_type': doctype, - 'field_name': ['!=', 'naming_series'], - 'property': ['!=', 'options'], - 'is_system_generated': False - }, pluck='name') + setters = frappe.get_all( + "Property Setter", + filters={ + "doc_type": doctype, + "field_name": ["!=", "naming_series"], + "property": ["!=", "options"], + "is_system_generated": False, + }, + pluck="name", + ) for setter in setters: frappe.delete_doc("Property Setter", setter) - custom_fields = frappe.get_all("Custom Field", filters={ - 'dt': doctype, - 'is_system_generated': False - }, pluck='name') + custom_fields = frappe.get_all( + "Custom Field", filters={"dt": doctype, "is_system_generated": False}, pluck="name" + ) for field in custom_fields: frappe.delete_doc("Custom Field", field) frappe.clear_cache(doctype=doctype) + doctype_properties = { - 'search_fields': 'Data', - 'title_field': 'Data', - 'image_field': 'Data', - 'sort_field': 'Data', - 'sort_order': 'Data', - 'default_print_format': 'Data', - 'allow_copy': 'Check', - 'istable': 'Check', - 'quick_entry': 'Check', - 'editable_grid': 'Check', - 'max_attachments': 'Int', - 'track_changes': 'Check', - 'track_views': 'Check', - 'allow_auto_repeat': 'Check', - 'allow_import': 'Check', - 'show_preview_popup': 'Check', - 'default_email_template': 'Data', - 'email_append_to': 'Check', - 'subject_field': 'Data', - 'sender_field': 'Data', - 'autoname': 'Data', - 'show_title_field_in_link': 'Check' + "search_fields": "Data", + "title_field": "Data", + "image_field": "Data", + "sort_field": "Data", + "sort_order": "Data", + "default_print_format": "Data", + "allow_copy": "Check", + "istable": "Check", + "quick_entry": "Check", + "editable_grid": "Check", + "max_attachments": "Int", + "track_changes": "Check", + "track_views": "Check", + "allow_auto_repeat": "Check", + "allow_import": "Check", + "show_preview_popup": "Check", + "default_email_template": "Data", + "email_append_to": "Check", + "subject_field": "Data", + "sender_field": "Data", + "autoname": "Data", + "show_title_field_in_link": "Check", } docfield_properties = { - 'idx': 'Int', - 'label': 'Data', - 'fieldtype': 'Select', - 'options': 'Text', - 'fetch_from': 'Small Text', - 'fetch_if_empty': 'Check', - 'show_dashboard': 'Check', - 'permlevel': 'Int', - 'width': 'Data', - 'print_width': 'Data', - 'non_negative': 'Check', - 'reqd': 'Check', - 'unique': 'Check', - 'ignore_user_permissions': 'Check', - 'in_list_view': 'Check', - 'in_standard_filter': 'Check', - 'in_global_search': 'Check', - 'in_preview': 'Check', - 'bold': 'Check', - 'no_copy': 'Check', - 'hidden': 'Check', - 'collapsible': 'Check', - 'collapsible_depends_on': 'Data', - 'print_hide': 'Check', - 'print_hide_if_no_value': 'Check', - 'report_hide': 'Check', - 'allow_on_submit': 'Check', - 'translatable': 'Check', - 'mandatory_depends_on': 'Data', - 'read_only_depends_on': 'Data', - 'depends_on': 'Data', - 'description': 'Text', - 'default': 'Text', - 'precision': 'Select', - 'read_only': 'Check', - 'length': 'Int', - 'columns': 'Int', - 'remember_last_selected_value': 'Check', - 'allow_bulk_edit': 'Check', - 'auto_repeat': 'Link', - 'allow_in_quick_entry': 'Check', - 'hide_border': 'Check', - 'hide_days': 'Check', - 'hide_seconds': 'Check', - 'is_virtual': 'Check', + "idx": "Int", + "label": "Data", + "fieldtype": "Select", + "options": "Text", + "fetch_from": "Small Text", + "fetch_if_empty": "Check", + "show_dashboard": "Check", + "permlevel": "Int", + "width": "Data", + "print_width": "Data", + "non_negative": "Check", + "reqd": "Check", + "unique": "Check", + "ignore_user_permissions": "Check", + "in_list_view": "Check", + "in_standard_filter": "Check", + "in_global_search": "Check", + "in_preview": "Check", + "bold": "Check", + "no_copy": "Check", + "hidden": "Check", + "collapsible": "Check", + "collapsible_depends_on": "Data", + "print_hide": "Check", + "print_hide_if_no_value": "Check", + "report_hide": "Check", + "allow_on_submit": "Check", + "translatable": "Check", + "mandatory_depends_on": "Data", + "read_only_depends_on": "Data", + "depends_on": "Data", + "description": "Text", + "default": "Text", + "precision": "Select", + "read_only": "Check", + "length": "Int", + "columns": "Int", + "remember_last_selected_value": "Check", + "allow_bulk_edit": "Check", + "auto_repeat": "Link", + "allow_in_quick_entry": "Check", + "hide_border": "Check", + "hide_days": "Check", + "hide_seconds": "Check", + "is_virtual": "Check", } doctype_link_properties = { - 'link_doctype': 'Link', - 'link_fieldname': 'Data', - 'group': 'Data', - 'hidden': 'Check' + "link_doctype": "Link", + "link_fieldname": "Data", + "group": "Data", + "hidden": "Check", } doctype_action_properties = { - 'label': 'Link', - 'action_type': 'Select', - 'action': 'Small Text', - 'group': 'Data', - 'hidden': 'Check' + "label": "Link", + "action_type": "Select", + "action": "Small Text", + "group": "Data", + "hidden": "Check", } -doctype_state_properties = { - 'title': 'Data', - 'color': 'Select' -} +doctype_state_properties = {"title": "Data", "color": "Select"} ALLOWED_FIELDTYPE_CHANGE = ( - ('Currency', 'Float', 'Percent'), - ('Small Text', 'Data'), - ('Text', 'Data'), - ('Text', 'Text Editor', 'Code', 'Signature', 'HTML Editor'), - ('Data', 'Select'), - ('Text', 'Small Text'), - ('Text', 'Data', 'Barcode'), - ('Code', 'Geolocation'), - ('Table', 'Table MultiSelect')) - -ALLOWED_OPTIONS_CHANGE = ('Read Only', 'HTML', 'Data') + ("Currency", "Float", "Percent"), + ("Small Text", "Data"), + ("Text", "Data"), + ("Text", "Text Editor", "Code", "Signature", "HTML Editor"), + ("Data", "Select"), + ("Text", "Small Text"), + ("Text", "Data", "Barcode"), + ("Code", "Geolocation"), + ("Table", "Table MultiSelect"), +) + +ALLOWED_OPTIONS_CHANGE = ("Read Only", "HTML", "Data") diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index 2cae69ca21..5a1f629beb 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -1,30 +1,37 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest, json -from frappe.test_runner import make_test_records_for_doctype +import json +import unittest + +import frappe from frappe.core.doctype.doctype.doctype import InvalidFieldNameError from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.test_runner import make_test_records_for_doctype test_dependencies = ["Custom Field", "Property Setter"] + + class TestCustomizeForm(unittest.TestCase): def insert_custom_field(self): frappe.delete_doc_if_exists("Custom Field", "Event-test_custom_field") - frappe.get_doc({ - "doctype": "Custom Field", - "dt": "Event", - "label": "Test Custom Field", - "description": "A Custom Field for Testing", - "fieldtype": "Select", - "in_list_view": 1, - "options": "\nCustom 1\nCustom 2\nCustom 3", - "default": "Custom 3", - "insert_after": frappe.get_meta('Event').fields[-1].fieldname - }).insert() + frappe.get_doc( + { + "doctype": "Custom Field", + "dt": "Event", + "label": "Test Custom Field", + "description": "A Custom Field for Testing", + "fieldtype": "Select", + "in_list_view": 1, + "options": "\nCustom 1\nCustom 2\nCustom 3", + "default": "Custom 3", + "insert_after": frappe.get_meta("Event").fields[-1].fieldname, + } + ).insert() def setUp(self): self.insert_custom_field() - frappe.db.delete('Property Setter', dict(doc_type='Event')) + frappe.db.delete("Property Setter", dict(doc_type="Event")) frappe.db.commit() frappe.clear_cache(doctype="Event") @@ -52,8 +59,7 @@ class TestCustomizeForm(unittest.TestCase): d = self.get_customize_form("Event") self.assertEqual(d.doc_type, "Event") - self.assertEqual(len(d.get("fields")), - len(frappe.get_doc("DocType", d.doc_type).fields) + 1) + self.assertEqual(len(d.get("fields")), len(frappe.get_doc("DocType", d.doc_type).fields) + 1) self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field") self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1) @@ -61,35 +67,65 @@ class TestCustomizeForm(unittest.TestCase): def test_save_customization_property(self): d = self.get_customize_form("Event") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Event", "property": "allow_copy"}, "value"), None) + self.assertEqual( + frappe.db.get_value( + "Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value" + ), + None, + ) d.allow_copy = 1 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Event", "property": "allow_copy"}, "value"), '1') + self.assertEqual( + frappe.db.get_value( + "Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value" + ), + "1", + ) d.allow_copy = 0 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Event", "property": "allow_copy"}, "value"), None) + self.assertEqual( + frappe.db.get_value( + "Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value" + ), + None, + ) def test_save_customization_field_property(self): d = self.get_customize_form("Event") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None) + self.assertEqual( + frappe.db.get_value( + "Property Setter", + {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, + "value", + ), + None, + ) repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0] repeat_this_event_field.reqd = 1 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1') + self.assertEqual( + frappe.db.get_value( + "Property Setter", + {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, + "value", + ), + "1", + ) repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0] repeat_this_event_field.reqd = 0 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None) + self.assertEqual( + frappe.db.get_value( + "Property Setter", + {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, + "value", + ), + None, + ) def test_save_customization_custom_field_property(self): d = self.get_customize_form("Event") @@ -109,26 +145,36 @@ class TestCustomizeForm(unittest.TestCase): self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "no_copy"), 0) - def test_save_customization_new_field(self): d = self.get_customize_form("Event") last_fieldname = d.fields[-1].fieldname - d.append("fields", { - "label": "Test Add Custom Field Via Customize Form", - "fieldtype": "Data", - "is_custom_field": 1 - }) + d.append( + "fields", + { + "label": "Test Add Custom Field Via Customize Form", + "fieldtype": "Data", + "is_custom_field": 1, + }, + ) d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Custom Field", - "Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data") - - self.assertEqual(frappe.db.get_value("Custom Field", - "Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname) + self.assertEqual( + frappe.db.get_value( + "Custom Field", "Event-test_add_custom_field_via_customize_form", "fieldtype" + ), + "Data", + ) + + self.assertEqual( + frappe.db.get_value( + "Custom Field", "Event-test_add_custom_field_via_customize_form", "insert_after" + ), + last_fieldname, + ) frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form") - self.assertEqual(frappe.db.get_value("Custom Field", - "Event-test_add_custom_field_via_customize_form"), None) - + self.assertEqual( + frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form"), None + ) def test_save_customization_remove_field(self): d = self.get_customize_form("Event") @@ -144,7 +190,7 @@ class TestCustomizeForm(unittest.TestCase): def test_reset_to_defaults(self): d = frappe.get_doc("Customize Form") d.doc_type = "Event" - d.run_method('reset_to_defaults') + d.run_method("reset_to_defaults") self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0) @@ -191,7 +237,7 @@ class TestCustomizeForm(unittest.TestCase): d.run_method("save_customization") def test_core_doctype_customization(self): - self.assertRaises(frappe.ValidationError, self.get_customize_form, 'User') + self.assertRaises(frappe.ValidationError, self.get_customize_form, "User") def test_save_customization_length_field_property(self): # Using Notification Log doctype as it doesn't have any other custom fields @@ -201,41 +247,49 @@ class TestCustomizeForm(unittest.TestCase): document_name.length = 255 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Property Setter", - {"doc_type": "Notification Log", "property": "length", "field_name": "document_name"}, "value"), '255') + self.assertEqual( + frappe.db.get_value( + "Property Setter", + {"doc_type": "Notification Log", "property": "length", "field_name": "document_name"}, + "value", + ), + "255", + ) self.assertTrue(d.flags.update_db) - length = frappe.db.sql("""SELECT character_maximum_length + length = frappe.db.sql( + """SELECT character_maximum_length FROM information_schema.columns WHERE table_name = 'tabNotification Log' - AND column_name = 'document_name'""")[0][0] + AND column_name = 'document_name'""" + )[0][0] self.assertEqual(length, 255) def test_custom_link(self): try: # create a dummy doctype linked to Event - testdt_name = 'Test Link for Event' - testdt = new_doctype(testdt_name, fields=[ - dict(fieldtype='Link', fieldname='event', options='Event') - ]).insert() + testdt_name = "Test Link for Event" + testdt = new_doctype( + testdt_name, fields=[dict(fieldtype="Link", fieldname="event", options="Event")] + ).insert() - testdt_name1 = 'Test Link for Event 1' - testdt1 = new_doctype(testdt_name1, fields=[ - dict(fieldtype='Link', fieldname='event', options='Event') - ]).insert() + testdt_name1 = "Test Link for Event 1" + testdt1 = new_doctype( + testdt_name1, fields=[dict(fieldtype="Link", fieldname="event", options="Event")] + ).insert() # add a custom link d = self.get_customize_form("Event") - d.append('links', dict(link_doctype=testdt_name, link_fieldname='event', group='Tests')) - d.append('links', dict(link_doctype=testdt_name1, link_fieldname='event', group='Tests')) + d.append("links", dict(link_doctype=testdt_name, link_fieldname="event", group="Tests")) + d.append("links", dict(link_doctype=testdt_name1, link_fieldname="event", group="Tests")) d.run_method("save_customization") frappe.clear_cache() - event = frappe.get_meta('Event') + event = frappe.get_meta("Event") # check links exist self.assertTrue([d.name for d in event.links if d.link_doctype == testdt_name]) @@ -251,7 +305,7 @@ class TestCustomizeForm(unittest.TestCase): d.run_method("save_customization") frappe.clear_cache() - event = frappe.get_meta('Event') + event = frappe.get_meta("Event") self.assertFalse([d.name for d in (event.links or []) if d.link_doctype == testdt_name]) finally: testdt.delete() @@ -262,17 +316,26 @@ class TestCustomizeForm(unittest.TestCase): frappe.clear_cache() d = self.get_customize_form("User Group") - d.append('links', dict(link_doctype='User Group Member', parent_doctype='User Group', - link_fieldname='user', table_fieldname='user_group_members', group='Tests', custom=1)) + d.append( + "links", + dict( + link_doctype="User Group Member", + parent_doctype="User Group", + link_fieldname="user", + table_fieldname="user_group_members", + group="Tests", + custom=1, + ), + ) d.run_method("save_customization") frappe.clear_cache() - user_group = frappe.get_meta('User Group') + user_group = frappe.get_meta("User Group") # check links exist - self.assertTrue([d.name for d in user_group.links if d.link_doctype == 'User Group Member']) - self.assertTrue([d.name for d in user_group.links if d.parent_doctype == 'User Group']) + self.assertTrue([d.name for d in user_group.links if d.link_doctype == "User Group Member"]) + self.assertTrue([d.name for d in user_group.links if d.parent_doctype == "User Group"]) # remove the link d = self.get_customize_form("User Group") @@ -280,22 +343,24 @@ class TestCustomizeForm(unittest.TestCase): d.run_method("save_customization") frappe.clear_cache() - user_group = frappe.get_meta('Event') - self.assertFalse([d.name for d in (user_group.links or []) if d.link_doctype == 'User Group Member']) + user_group = frappe.get_meta("Event") + self.assertFalse( + [d.name for d in (user_group.links or []) if d.link_doctype == "User Group Member"] + ) def test_custom_action(self): - test_route = '/app/List/DocType' + test_route = "/app/List/DocType" # create a dummy action (route) d = self.get_customize_form("Event") - d.append('actions', dict(label='Test Action', action_type='Route', action=test_route)) + d.append("actions", dict(label="Test Action", action_type="Route", action=test_route)) d.run_method("save_customization") frappe.clear_cache() - event = frappe.get_meta('Event') + event = frappe.get_meta("Event") # check if added to meta - action = [d for d in event.actions if d.label=='Test Action'] + action = [d for d in event.actions if d.label == "Test Action"] self.assertEqual(len(action), 1) self.assertEqual(action[0].action, test_route) @@ -305,9 +370,9 @@ class TestCustomizeForm(unittest.TestCase): d.run_method("save_customization") frappe.clear_cache() - event = frappe.get_meta('Event') + event = frappe.get_meta("Event") - action = [d for d in event.actions if d.label=='Test Action'] + action = [d for d in event.actions if d.label == "Test Action"] self.assertEqual(len(action), 0) def test_custom_label(self): diff --git a/frappe/custom/doctype/customize_form_field/__init__.py b/frappe/custom/doctype/customize_form_field/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/custom/doctype/customize_form_field/__init__.py +++ b/frappe/custom/doctype/customize_form_field/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.py b/frappe/custom/doctype/customize_form_field/customize_form_field.py index 67563cf048..0e030ce812 100644 --- a/frappe/custom/doctype/customize_form_field/customize_form_field.py +++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class CustomizeFormField(Document): - pass \ No newline at end of file + pass diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.py b/frappe/custom/doctype/doctype_layout/doctype_layout.py index fa285ddb62..f5d37d6f60 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.py @@ -2,9 +2,9 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +from frappe.desk.utils import slug from frappe.model.document import Document -from frappe.desk.utils import slug class DocTypeLayout(Document): def validate(self): diff --git a/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py b/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py index 4e44743b48..59cdfffb21 100644 --- a/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py @@ -1,13 +1,18 @@ import frappe + def execute(): - for web_form_name in frappe.db.get_all('Web Form', pluck='name'): - web_form = frappe.get_doc('Web Form', web_form_name) - doctype_layout = frappe.get_doc(dict( - doctype = 'DocType Layout', - document_type = web_form.doc_type, - name = web_form.title, - route = web_form.route, - fields = [dict(fieldname = d.fieldname, label=d.label) for d in web_form.web_form_fields if d.fieldname] - )).insert() - print(doctype_layout.name) \ No newline at end of file + for web_form_name in frappe.db.get_all("Web Form", pluck="name"): + web_form = frappe.get_doc("Web Form", web_form_name) + doctype_layout = frappe.get_doc( + dict( + doctype="DocType Layout", + document_type=web_form.doc_type, + name=web_form.title, + route=web_form.route, + fields=[ + dict(fieldname=d.fieldname, label=d.label) for d in web_form.web_form_fields if d.fieldname + ], + ) + ).insert() + print(doctype_layout.name) diff --git a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py index a63dd7ee16..1373b4a53a 100644 --- a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDocTypeLayout(unittest.TestCase): pass diff --git a/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py b/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py index 3f8487b659..66fc111d32 100644 --- a/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py +++ b/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class DocTypeLayoutField(Document): pass diff --git a/frappe/custom/doctype/property_setter/__init__.py b/frappe/custom/doctype/property_setter/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/custom/doctype/property_setter/__init__.py +++ b/frappe/custom/doctype/property_setter/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/custom/doctype/property_setter/property_setter.py b/frappe/custom/doctype/property_setter/property_setter.py index a86cf5efd6..3034904381 100644 --- a/frappe/custom/doctype/property_setter/property_setter.py +++ b/frappe/custom/doctype/property_setter/property_setter.py @@ -3,17 +3,15 @@ import frappe from frappe import _ - from frappe.model.document import Document -not_allowed_fieldtype_change = ['naming_series'] +not_allowed_fieldtype_change = ["naming_series"] + class PropertySetter(Document): def autoname(self): - self.name = '{doctype}-{field}-{property}'.format( - doctype = self.doc_type, - field = self.field_name or self.row_name or 'main', - property = self.property + self.name = "{doctype}-{field}-{property}".format( + doctype=self.doc_type, field=self.field_name or self.row_name or "main", property=self.property ) def validate(self): @@ -21,16 +19,11 @@ class PropertySetter(Document): if self.is_new(): delete_property_setter(self.doc_type, self.property, self.field_name, self.row_name) - frappe.clear_cache(doctype = self.doc_type) + frappe.clear_cache(doctype=self.doc_type) def validate_fieldtype_change(self): - if ( - self.property == 'fieldtype' - and self.field_name in not_allowed_fieldtype_change - ): - frappe.throw( - _("Field type cannot be changed for {0}").format(self.field_name) - ) + if self.property == "fieldtype" and self.field_name in not_allowed_fieldtype_change: + frappe.throw(_("Field type cannot be changed for {0}").format(self.field_name)) def on_update(self): if frappe.flags.in_patch: @@ -38,21 +31,31 @@ class PropertySetter(Document): if not self.flags.ignore_validate and self.flags.validate_fields_for_doctype: from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype + validate_fields_for_doctype(self.doc_type) -def make_property_setter(doctype, fieldname, property, value, property_type, for_doctype = False, - validate_fields_for_doctype=True): +def make_property_setter( + doctype, + fieldname, + property, + value, + property_type, + for_doctype=False, + validate_fields_for_doctype=True, +): # WARNING: Ignores Permissions - property_setter = frappe.get_doc({ - "doctype":"Property Setter", - "doctype_or_field": for_doctype and "DocType" or "DocField", - "doc_type": doctype, - "field_name": fieldname, - "property": property, - "value": value, - "property_type": property_type - }) + property_setter = frappe.get_doc( + { + "doctype": "Property Setter", + "doctype_or_field": for_doctype and "DocType" or "DocField", + "doc_type": doctype, + "field_name": fieldname, + "property": property, + "value": value, + "property_type": property_type, + } + ) property_setter.flags.ignore_permissions = True property_setter.flags.validate_fields_for_doctype = validate_fields_for_doctype property_setter.insert() @@ -63,8 +66,8 @@ def delete_property_setter(doc_type, property, field_name=None, row_name=None): """delete other property setters on this, if this is new""" filters = dict(doc_type=doc_type, property=property) if field_name: - filters['field_name'] = field_name + filters["field_name"] = field_name if row_name: filters["row_name"] = row_name - frappe.db.delete('Property Setter', filters) + frappe.db.delete("Property Setter", filters) diff --git a/frappe/custom/doctype/property_setter/test_property_setter.py b/frappe/custom/doctype/property_setter/test_property_setter.py index 1bbbe59a0f..5b877ab18c 100644 --- a/frappe/custom/doctype/property_setter/test_property_setter.py +++ b/frappe/custom/doctype/property_setter/test_property_setter.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Property Setter') + class TestPropertySetter(unittest.TestCase): pass diff --git a/frappe/custom/doctype/test_rename_new/test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_rename_new.py index fc4ab97cfe..e79cb60bbe 100644 --- a/frappe/custom/doctype/test_rename_new/test_rename_new.py +++ b/frappe/custom/doctype/test_rename_new/test_rename_new.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class Testrenamenew(Document): pass diff --git a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_test_rename_new.py index 03202669ed..513a9286a3 100644 --- a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py +++ b/frappe/custom/doctype/test_rename_new/test_test_rename_new.py @@ -4,5 +4,6 @@ # import frappe import unittest + class Testrenamenew(unittest.TestCase): pass diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py index 5eca7cfac5..7d2b320c59 100644 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py @@ -1,6 +1,8 @@ from abc import ABCMeta, abstractmethod + from frappe.utils.password import get_decrypted_password + class BaseConnection(metaclass=ABCMeta): @abstractmethod def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): @@ -19,4 +21,4 @@ class BaseConnection(metaclass=ABCMeta): pass def get_password(self): - return get_decrypted_password('Data Migration Connector', self.connector.name) \ No newline at end of file + return get_decrypted_password("Data Migration Connector", self.connector.name) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py index 473a15c2dc..8228529562 100644 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py @@ -1,14 +1,16 @@ - import frappe from frappe.frappeclient import FrappeClient + from .base import BaseConnection + class FrappeConnection(BaseConnection): def __init__(self, connector): self.connector = connector - self.connection = FrappeClient(self.connector.hostname, - self.connector.username, self.get_password()) - self.name_field = 'name' + self.connection = FrappeClient( + self.connector.hostname, self.connector.username, self.get_password() + ) + self.name_field = "name" def insert(self, doctype, doc): doc = frappe._dict(doc) @@ -25,5 +27,6 @@ class FrappeConnection(BaseConnection): return self.connection.delete(doctype, migration_id) def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): - return self.connection.get_list(doctype, fields=fields, filters=filters, - limit_start=start, limit_page_length=page_length) + return self.connection.get_list( + doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length + ) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py index 2e4e4d45b3..9db7fc2445 100644 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py @@ -2,23 +2,27 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe, os -from frappe.model.document import Document +import os + +import frappe from frappe import _ +from frappe.model.document import Document from frappe.modules.export_file import create_init_py + from .connectors.base import BaseConnection from .connectors.frappe_connection import FrappeConnection + class DataMigrationConnector(Document): def validate(self): if not (self.python_module or self.connector_type): - frappe.throw(_('Enter python module or select connector type')) + frappe.throw(_("Enter python module or select connector type")) if self.python_module: try: get_connection_class(self.python_module) except: - frappe.throw(frappe._('Invalid module path')) + frappe.throw(frappe._("Invalid module path")) def get_connection(self): if self.python_module: @@ -29,37 +33,40 @@ class DataMigrationConnector(Document): return self.connection + @frappe.whitelist() def create_new_connection(module, connection_name): - if not frappe.conf.get('developer_mode'): - frappe.msgprint(_('Please enable developer mode to create new connection')) + if not frappe.conf.get("developer_mode"): + frappe.msgprint(_("Please enable developer mode to create new connection")) return # create folder module_path = frappe.get_module_path(module) - connectors_folder = os.path.join(module_path, 'connectors') + connectors_folder = os.path.join(module_path, "connectors") frappe.create_folder(connectors_folder) # create init py - create_init_py(module_path, 'connectors', '') + create_init_py(module_path, "connectors", "") - connection_class = connection_name.replace(' ', '') - file_name = frappe.scrub(connection_name) + '.py' - file_path = os.path.join(module_path, 'connectors', file_name) + connection_class = connection_name.replace(" ", "") + file_name = frappe.scrub(connection_name) + ".py" + file_path = os.path.join(module_path, "connectors", file_name) # create boilerplate file - with open(file_path, 'w') as f: + with open(file_path, "w") as f: f.write(connection_boilerplate.format(connection_class=connection_class)) # get python module string from file_path - app_name = frappe.db.get_value('Module Def', module, 'app_name') - python_module = os.path.relpath( - file_path, '../apps/{0}'.format(app_name)).replace(os.path.sep, '.')[:-3] + app_name = frappe.db.get_value("Module Def", module, "app_name") + python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( + os.path.sep, "." + )[:-3] return python_module + def get_connection_class(python_module): - filename = python_module.rsplit('.', 1)[-1] - classname = frappe.unscrub(filename).replace(' ', '') + filename = python_module.rsplit(".", 1)[-1] + classname = frappe.unscrub(filename).replace(" ", "") module = frappe.get_module(python_module) raise_error = False @@ -75,6 +82,7 @@ def get_connection_class(python_module): return _class + connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection class {connection_class}(BaseConnection): diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py index ffc96c8266..c4090796ab 100644 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestDataMigrationConnector(unittest.TestCase): pass diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py index 46d33eaca9..49af65e99b 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py @@ -6,6 +6,7 @@ import frappe from frappe.model.document import Document from frappe.utils.safe_exec import get_safe_globals + class DataMigrationMapping(Document): def get_filters(self): if self.condition: @@ -14,25 +15,25 @@ class DataMigrationMapping(Document): def get_fields(self): fields = [] for f in self.fields: - if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith('eval:')): + if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): fields.append(f.local_fieldname) if frappe.db.has_column(self.local_doctype, self.migration_id_field): fields.append(self.migration_id_field) - if 'name' not in fields: - fields.append('name') + if "name" not in fields: + fields.append("name") return fields def get_mapped_record(self, doc): - '''Build a mapped record using information from the fields table''' + """Build a mapped record using information from the fields table""" mapped = frappe._dict() - key_fieldname = 'remote_fieldname' - value_fieldname = 'local_fieldname' + key_fieldname = "remote_fieldname" + value_fieldname = "local_fieldname" - if self.mapping_type == 'Pull': + if self.mapping_type == "Pull": key_fieldname, value_fieldname = value_fieldname, key_fieldname for field_map in self.fields: @@ -44,25 +45,28 @@ class DataMigrationMapping(Document): else: # child table mapping mapping_name = field_map.child_table_mapping - value = get_mapped_child_records(mapping_name, - doc.get(get_source_value(field_map, value_fieldname))) + value = get_mapped_child_records( + mapping_name, doc.get(get_source_value(field_map, value_fieldname)) + ) mapped[key] = value return mapped + def get_mapped_child_records(mapping_name, child_docs): mapped_child_docs = [] - mapping = frappe.get_doc('Data Migration Mapping', mapping_name) + mapping = frappe.get_doc("Data Migration Mapping", mapping_name) for child_doc in child_docs: mapped_child_docs.append(mapping.get_mapped_record(child_doc)) return mapped_child_docs + def get_value_from_fieldname(field_map, fieldname_field, doc): field_name = get_source_value(field_map, fieldname_field) - if field_name.startswith('eval:'): + if field_name.startswith("eval:"): value = frappe.safe_eval(field_name[5:], get_safe_globals()) elif field_name[0] in ('"', "'"): value = field_name[1:-1] @@ -70,8 +74,9 @@ def get_value_from_fieldname(field_map, fieldname_field, doc): value = get_source_value(doc, field_name) return value + def get_source_value(source, key): - '''Get value from source (object or dict) based on key''' + """Get value from source (object or dict) based on key""" if isinstance(source, dict): return source.get(key) else: diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py index b1040aaa58..30d2a6bcfe 100644 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestDataMigrationMapping(unittest.TestCase): pass diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py index ce46f60f67..abd6348a26 100644 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class DataMigrationMappingDetail(Document): pass diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py index d13912b431..4118e8e7fe 100644 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py @@ -2,10 +2,10 @@ # License: MIT. See LICENSE import frappe -from frappe.modules import get_module_path, scrub_dt_dn -from frappe.modules.export_file import export_to_files, create_init_py from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.model.document import Document +from frappe.modules import get_module_path, scrub_dt_dn +from frappe.modules.export_file import create_init_py, export_to_files def get_mapping_module(module, mapping_name): @@ -14,9 +14,7 @@ def get_mapping_module(module, mapping_name): module = frappe.scrub(module) try: - return frappe.get_module( - f"{app_name}.{module}.data_migration_mapping.{mapping_name}" - ) + return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") except ImportError: return None @@ -29,52 +27,52 @@ class DataMigrationPlan(Document): if frappe.flags.in_import or frappe.flags.in_test: return - if frappe.local.conf.get('developer_mode'): - record_list =[['Data Migration Plan', self.name]] + if frappe.local.conf.get("developer_mode"): + record_list = [["Data Migration Plan", self.name]] for m in self.mappings: - record_list.append(['Data Migration Mapping', m.mapping]) + record_list.append(["Data Migration Mapping", m.mapping]) export_to_files(record_list=record_list, record_module=self.module) for m in self.mappings: - dt, dn = scrub_dt_dn('Data Migration Mapping', m.mapping) + dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) create_init_py(get_module_path(self.module), dt, dn) def make_custom_fields_for_mappings(self): frappe.flags.ignore_in_install = True - label = self.name + ' ID' + label = self.name + " ID" fieldname = frappe.scrub(label) df = { - 'label': label, - 'fieldname': fieldname, - 'fieldtype': 'Data', - 'hidden': 1, - 'read_only': 1, - 'unique': 1, - 'no_copy': 1 + "label": label, + "fieldname": fieldname, + "fieldtype": "Data", + "hidden": 1, + "read_only": 1, + "unique": 1, + "no_copy": 1, } for m in self.mappings: - mapping = frappe.get_doc('Data Migration Mapping', m.mapping) + mapping = frappe.get_doc("Data Migration Mapping", m.mapping) create_custom_field(mapping.local_doctype, df) mapping.migration_id_field = fieldname mapping.save() # Create custom field in Deleted Document - create_custom_field('Deleted Document', df) + create_custom_field("Deleted Document", df) frappe.flags.ignore_in_install = False def pre_process_doc(self, mapping_name, doc): module = get_mapping_module(self.module, mapping_name) - if module and hasattr(module, 'pre_process'): + if module and hasattr(module, "pre_process"): return module.pre_process(doc) return doc def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): module = get_mapping_module(self.module, mapping_name) - if module and hasattr(module, 'post_process'): + if module and hasattr(module, "post_process"): return module.post_process(local_doc=local_doc, remote_doc=remote_doc) diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py index 649f7db903..ef3bfa3a70 100644 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestDataMigrationPlan(unittest.TestCase): pass diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py index 7939a68d97..0650f4b2c7 100644 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class DataMigrationPlanMapping(Document): pass diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py index deb14baf27..c734cb105b 100644 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py @@ -2,11 +2,17 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe, json, math -from frappe.model.document import Document +import json +import math + +import frappe from frappe import _ +from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( + get_source_value, +) +from frappe.model.document import Document from frappe.utils import cstr -from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import get_source_value + class DataMigrationRun(Document): @frappe.whitelist() @@ -21,44 +27,47 @@ class DataMigrationRun(Document): next_mapping_name = self.get_next_mapping_name() if next_mapping_name: next_mapping = self.get_mapping(next_mapping_name) - self.db_set(dict( - current_mapping = next_mapping.name, - current_mapping_start = 0, - current_mapping_delete_start = 0, - current_mapping_action = 'Insert' - ), notify=True, commit=True) - frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test) + self.db_set( + dict( + current_mapping=next_mapping.name, + current_mapping_start=0, + current_mapping_delete_start=0, + current_mapping_action="Insert", + ), + notify=True, + commit=True, + ) + frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) else: self.complete() def enqueue_next_page(self): mapping = self.get_mapping(self.current_mapping) percent_complete = self.percent_complete + (100.0 / self.total_pages) - fields = dict( - percent_complete = percent_complete - ) - if self.current_mapping_action == 'Insert': + fields = dict(percent_complete=percent_complete) + if self.current_mapping_action == "Insert": start = self.current_mapping_start + mapping.page_length - fields['current_mapping_start'] = start - elif self.current_mapping_action == 'Delete': + fields["current_mapping_start"] = start + elif self.current_mapping_action == "Delete": delete_start = self.current_mapping_delete_start + mapping.page_length - fields['current_mapping_delete_start'] = delete_start + fields["current_mapping_delete_start"] = delete_start self.db_set(fields, notify=True, commit=True) - if(percent_complete < 100): - frappe.publish_realtime(self.trigger_name, - {"progress_percent": percent_complete}, user=frappe.session.user) + if percent_complete < 100: + frappe.publish_realtime( + self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user + ) - frappe.enqueue_doc(self.doctype, self.name, 'run_current_mapping', now=frappe.flags.in_test) + frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) def run_current_mapping(self): try: mapping = self.get_mapping(self.current_mapping) - if mapping.mapping_type == 'Push': + if mapping.mapping_type == "Push": done = self.push() - elif mapping.mapping_type == 'Pull': + elif mapping.mapping_type == "Pull": done = self.pull() if done: @@ -67,96 +76,106 @@ class DataMigrationRun(Document): self.enqueue_next_page() except Exception as e: - self.db_set('status', 'Error', notify=True, commit=True) - print('Data Migration Run failed') + self.db_set("status", "Error", notify=True, commit=True) + print("Data Migration Run failed") print(frappe.get_traceback()) - self.execute_postprocess('Error') + self.execute_postprocess("Error") raise e def get_last_modified_condition(self): - last_run_timestamp = frappe.db.get_value('Data Migration Run', dict( - data_migration_plan=self.data_migration_plan, - data_migration_connector=self.data_migration_connector, - name=('!=', self.name) - ), 'modified') + last_run_timestamp = frappe.db.get_value( + "Data Migration Run", + dict( + data_migration_plan=self.data_migration_plan, + data_migration_connector=self.data_migration_connector, + name=("!=", self.name), + ), + "modified", + ) if last_run_timestamp: - condition = dict(modified=('>', last_run_timestamp)) + condition = dict(modified=(">", last_run_timestamp)) else: condition = {} return condition def begin(self): plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] - self.mappings = [frappe.get_doc( - 'Data Migration Mapping', m.mapping) for m in plan_active_mappings] + self.mappings = [ + frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings + ] total_pages = 0 for m in [mapping for mapping in self.mappings]: - if m.mapping_type == 'Push': + if m.mapping_type == "Push": count = float(self.get_count(m)) page_count = math.ceil(count / m.page_length) total_pages += page_count - if m.mapping_type == 'Pull': + if m.mapping_type == "Pull": total_pages += 10 - self.db_set(dict( - status = 'Started', - current_mapping = None, - current_mapping_start = 0, - current_mapping_delete_start = 0, - percent_complete = 0, - current_mapping_action = 'Insert', - total_pages = total_pages - ), notify=True, commit=True) + self.db_set( + dict( + status="Started", + current_mapping=None, + current_mapping_start=0, + current_mapping_delete_start=0, + percent_complete=0, + current_mapping_action="Insert", + total_pages=total_pages, + ), + notify=True, + commit=True, + ) def complete(self): fields = dict() - push_failed = self.get_log('push_failed', []) - pull_failed = self.get_log('pull_failed', []) + push_failed = self.get_log("push_failed", []) + pull_failed = self.get_log("pull_failed", []) - status = 'Partial Success' + status = "Partial Success" if not push_failed and not pull_failed: - status = 'Success' - fields['percent_complete'] = 100 + status = "Success" + fields["percent_complete"] = 100 - fields['status'] = status + fields["status"] = status self.db_set(fields, notify=True, commit=True) self.execute_postprocess(status) - frappe.publish_realtime(self.trigger_name, - {"progress_percent": 100}, user=frappe.session.user) + frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) def execute_postprocess(self, status): # Execute post process postprocess_method_path = self.get_plan().postprocess_method if postprocess_method_path: - frappe.get_attr(postprocess_method_path)({ - "status": status, - "stats": { - "push_insert": self.push_insert, - "push_update": self.push_update, - "push_delete": self.push_delete, - "pull_insert": self.pull_insert, - "pull_update": self.pull_update + frappe.get_attr(postprocess_method_path)( + { + "status": status, + "stats": { + "push_insert": self.push_insert, + "push_update": self.push_update, + "push_delete": self.push_delete, + "pull_insert": self.pull_insert, + "pull_update": self.pull_update, + }, } - }) + ) def get_plan(self): - if not hasattr(self, 'plan'): - self.plan = frappe.get_doc('Data Migration Plan', self.data_migration_plan) + if not hasattr(self, "plan"): + self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) return self.plan def get_mapping(self, mapping_name): - if hasattr(self, 'mappings'): + if hasattr(self, "mappings"): for m in self.mappings: if m.name == mapping_name: return m - return frappe.get_doc('Data Migration Mapping', mapping_name) + return frappe.get_doc("Data Migration Mapping", mapping_name) def get_next_mapping_name(self): mappings = [m for m in self.get_plan().mappings if m.enabled] @@ -168,9 +187,9 @@ class DataMigrationRun(Document): # last return None if d.mapping == self.current_mapping: - return mappings[i+1].mapping + return mappings[i + 1].mapping - raise frappe.ValidationError('Mapping Broken') + raise frappe.ValidationError("Mapping Broken") def get_data(self, filters): mapping = self.get_mapping(self.current_mapping) @@ -178,79 +197,85 @@ class DataMigrationRun(Document): start = self.current_mapping_start data = [] - doclist = frappe.get_all(mapping.local_doctype, - filters=filters, or_filters=or_filters, - start=start, page_length=mapping.page_length) + doclist = frappe.get_all( + mapping.local_doctype, + filters=filters, + or_filters=or_filters, + start=start, + page_length=mapping.page_length, + ) for d in doclist: - doc = frappe.get_doc(mapping.local_doctype, d['name']) + doc = frappe.get_doc(mapping.local_doctype, d["name"]) data.append(doc) return data def get_new_local_data(self): - '''Fetch newly inserted local data using `frappe.get_all`. Used during Push''' + """Fetch newly inserted local data using `frappe.get_all`. Used during Push""" mapping = self.get_mapping(self.current_mapping) filters = mapping.get_filters() or {} # new docs dont have migration field set - filters.update({ - mapping.migration_id_field: '' - }) + filters.update({mapping.migration_id_field: ""}) return self.get_data(filters) def get_updated_local_data(self): - '''Fetch local updated data using `frappe.get_all`. Used during Push''' + """Fetch local updated data using `frappe.get_all`. Used during Push""" mapping = self.get_mapping(self.current_mapping) filters = mapping.get_filters() or {} # existing docs must have migration field set - filters.update({ - mapping.migration_id_field: ('!=', '') - }) + filters.update({mapping.migration_id_field: ("!=", "")}) return self.get_data(filters) def get_deleted_local_data(self): - '''Fetch local deleted data using `frappe.get_all`. Used during Push''' + """Fetch local deleted data using `frappe.get_all`. Used during Push""" mapping = self.get_mapping(self.current_mapping) filters = self.get_last_modified_condition() - filters.update({ - "deleted_doctype": mapping.local_doctype - }) + filters.update({"deleted_doctype": mapping.local_doctype}) - data = frappe.get_all('Deleted Document', fields=['name', 'data'], - filters=filters) + data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) _data = [] for d in data: doc = json.loads(d.data) if doc.get(mapping.migration_id_field): - doc['_deleted_document_name'] = d["name"] + doc["_deleted_document_name"] = d["name"] _data.append(doc) return _data def get_remote_data(self): - '''Fetch data from remote using `connection.get`. Used during Pull''' + """Fetch data from remote using `connection.get`. Used during Pull""" mapping = self.get_mapping(self.current_mapping) start = self.current_mapping_start filters = mapping.get_filters() or {} connection = self.get_connection() - return connection.get(mapping.remote_objectname, - fields=["*"], filters=filters, start=start, - page_length=mapping.page_length) + return connection.get( + mapping.remote_objectname, + fields=["*"], + filters=filters, + start=start, + page_length=mapping.page_length, + ) def get_count(self, mapping): filters = mapping.get_filters() or {} or_filters = self.get_or_filters(mapping) - to_insert = frappe.get_all(mapping.local_doctype, ['count(name) as total'], - filters=filters, or_filters=or_filters)[0].total + to_insert = frappe.get_all( + mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters + )[0].total - to_delete = frappe.get_all('Deleted Document', ['count(name) as total'], - filters={'deleted_doctype': mapping.local_doctype}, or_filters=or_filters)[0].total + to_delete = frappe.get_all( + "Deleted Document", + ["count(name) as total"], + filters={"deleted_doctype": mapping.local_doctype}, + or_filters=or_filters, + )[0].total return to_insert + to_delete @@ -259,36 +284,35 @@ class DataMigrationRun(Document): # docs whose migration_id_field is not set # failed in the previous run, include those too - or_filters.update({ - mapping.migration_id_field: ('=', '') - }) + or_filters.update({mapping.migration_id_field: ("=", "")}) return or_filters def get_connection(self): - if not hasattr(self, 'connection'): - self.connection = frappe.get_doc('Data Migration Connector', - self.data_migration_connector).get_connection() + if not hasattr(self, "connection"): + self.connection = frappe.get_doc( + "Data Migration Connector", self.data_migration_connector + ).get_connection() return self.connection def push(self): - self.db_set('current_mapping_type', 'Push') + self.db_set("current_mapping_type", "Push") done = True - if self.current_mapping_action == 'Insert': + if self.current_mapping_action == "Insert": done = self._push_insert() - elif self.current_mapping_action == 'Update': + elif self.current_mapping_action == "Update": done = self._push_update() - elif self.current_mapping_action == 'Delete': + elif self.current_mapping_action == "Delete": done = self._push_delete() return done def _push_insert(self): - '''Inserts new local docs on remote''' + """Inserts new local docs on remote""" mapping = self.get_mapping(self.current_mapping) connection = self.get_connection() data = self.get_new_local_data() @@ -300,31 +324,31 @@ class DataMigrationRun(Document): try: response_doc = connection.insert(mapping.remote_objectname, doc) - frappe.db.set_value(mapping.local_doctype, d.name, - mapping.migration_id_field, response_doc[connection.name_field], - update_modified=False) + frappe.db.set_value( + mapping.local_doctype, + d.name, + mapping.migration_id_field, + response_doc[connection.name_field], + update_modified=False, + ) frappe.db.commit() - self.update_log('push_insert', 1) + self.update_log("push_insert", 1) # post process after insert self.post_process_doc(local_doc=d, remote_doc=response_doc) except Exception as e: - self.update_log('push_failed', {d.name: cstr(e)}) + self.update_log("push_failed", {d.name: cstr(e)}) # update page_start - self.db_set('current_mapping_start', - self.current_mapping_start + mapping.page_length) + self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) if len(data) < mapping.page_length: # done, no more new data to insert - self.db_set({ - 'current_mapping_action': 'Update', - 'current_mapping_start': 0 - }) + self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) # not done with this mapping return False def _push_update(self): - '''Updates local modified docs on remote''' + """Updates local modified docs on remote""" mapping = self.get_mapping(self.current_mapping) connection = self.get_connection() data = self.get_updated_local_data() @@ -336,27 +360,23 @@ class DataMigrationRun(Document): doc = mapping.get_mapped_record(doc) try: response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) - self.update_log('push_update', 1) + self.update_log("push_update", 1) # post process after update self.post_process_doc(local_doc=d, remote_doc=response_doc) except Exception as e: - self.update_log('push_failed', {d.name: cstr(e)}) + self.update_log("push_failed", {d.name: cstr(e)}) # update page_start - self.db_set('current_mapping_start', - self.current_mapping_start + mapping.page_length) + self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) if len(data) < mapping.page_length: # done, no more data to update - self.db_set({ - 'current_mapping_action': 'Delete', - 'current_mapping_start': 0 - }) + self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) # not done with this mapping return False def _push_delete(self): - '''Deletes docs deleted from local on remote''' + """Deletes docs deleted from local on remote""" mapping = self.get_mapping(self.current_mapping) connection = self.get_connection() data = self.get_deleted_local_data() @@ -368,15 +388,14 @@ class DataMigrationRun(Document): self.pre_process_doc(d) try: response_doc = connection.delete(mapping.remote_objectname, migration_id_value) - self.update_log('push_delete', 1) + self.update_log("push_delete", 1) # post process only when action is success self.post_process_doc(local_doc=d, remote_doc=response_doc) except Exception as e: - self.update_log('push_failed', {d.name: cstr(e)}) + self.update_log("push_failed", {d.name: cstr(e)}) # update page_start - self.db_set('current_mapping_start', - self.current_mapping_start + mapping.page_length) + self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) if len(data) < mapping.page_length: # done, no more new data to delete @@ -384,7 +403,7 @@ class DataMigrationRun(Document): return True def pull(self): - self.db_set('current_mapping_type', 'Pull') + self.db_set("current_mapping_type", "Pull") connection = self.get_connection() mapping = self.get_mapping(self.current_mapping) @@ -401,21 +420,25 @@ class DataMigrationRun(Document): # insert new local doc local_doc = insert_local_doc(mapping, doc) - self.update_log('pull_insert', 1) + self.update_log("pull_insert", 1) # set migration id - frappe.db.set_value(mapping.local_doctype, local_doc.name, - mapping.migration_id_field, migration_id_value, - update_modified=False) + frappe.db.set_value( + mapping.local_doctype, + local_doc.name, + mapping.migration_id_field, + migration_id_value, + update_modified=False, + ) frappe.db.commit() else: # update doc local_doc = update_local_doc(mapping, doc, migration_id_value) - self.update_log('pull_update', 1) + self.update_log("pull_update", 1) # post process doc after success self.post_process_doc(remote_doc=d, local_doc=local_doc) except Exception as e: # failed, append to log - self.update_log('pull_failed', {migration_id_value: cstr(e)}) + self.update_log("pull_failed", {migration_id_value: cstr(e)}) if len(data) < mapping.page_length: # last page, done with pull @@ -432,16 +455,16 @@ class DataMigrationRun(Document): return doc def set_log(self, key, value): - value = json.dumps(value) if '_failed' in key else value + value = json.dumps(value) if "_failed" in key else value self.db_set(key, value) def update_log(self, key, value=None): - ''' + """ Helper for updating logs, push_failed and pull_failed are stored as json, other keys are stored as int - ''' - if '_failed' in key: + """ + if "_failed" in key: # json self.set_log(key, self.get_log(key, []) + [value]) else: @@ -450,11 +473,13 @@ class DataMigrationRun(Document): def get_log(self, key, default=None): value = self.db_get(key) - if '_failed' in key: - if not value: value = json.dumps(default) + if "_failed" in key: + if not value: + value = json.dumps(default) value = json.loads(value) return value or default + def insert_local_doc(mapping, doc): try: # insert new doc @@ -463,26 +488,27 @@ def insert_local_doc(mapping, doc): doc = frappe.get_doc(doc).insert() return doc except Exception: - print('Data Migration Run failed: Error in Pull insert') + print("Data Migration Run failed: Error in Pull insert") print(frappe.get_traceback()) return None + def update_local_doc(mapping, remote_doc, migration_id_value): try: # migration id value is set in migration_id_field in mapping.local_doctype - docname = frappe.db.get_value(mapping.local_doctype, - filters={ mapping.migration_id_field: migration_id_value }) + docname = frappe.db.get_value( + mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} + ) doc = frappe.get_doc(mapping.local_doctype, docname) doc.update(remote_doc) doc.save() return doc except Exception: - print('Data Migration Run failed: Error in Pull update') + print("Data Migration Run failed: Error in Pull update") print(frappe.get_traceback()) return None + def local_doc_exists(mapping, migration_id_value): - return frappe.db.exists(mapping.local_doctype, { - mapping.migration_id_field: migration_id_value - }) + return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py index 485f86a7f9..0357b1e0f5 100644 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py @@ -1,113 +1,128 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + +import frappe + class TestDataMigrationRun(unittest.TestCase): def test_run(self): create_plan() - description = 'data migration todo' - new_todo = frappe.get_doc({ - 'doctype': 'ToDo', - 'description': description - }).insert() - - event_subject = 'data migration event' - frappe.get_doc(dict( - doctype='Event', - subject=event_subject, - repeat_on='Monthly', - starts_on=frappe.utils.now_datetime() - )).insert() - - run = frappe.get_doc({ - 'doctype': 'Data Migration Run', - 'data_migration_plan': 'ToDo Sync', - 'data_migration_connector': 'Local Connector' - }).insert() + description = "data migration todo" + new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() + + event_subject = "data migration event" + frappe.get_doc( + dict( + doctype="Event", + subject=event_subject, + repeat_on="Monthly", + starts_on=frappe.utils.now_datetime(), + ) + ).insert() + + run = frappe.get_doc( + { + "doctype": "Data Migration Run", + "data_migration_plan": "ToDo Sync", + "data_migration_connector": "Local Connector", + } + ).insert() run.run() - self.assertEqual(run.db_get('status'), 'Success') + self.assertEqual(run.db_get("status"), "Success") - self.assertEqual(run.db_get('push_insert'), 1) - self.assertEqual(run.db_get('pull_insert'), 1) + self.assertEqual(run.db_get("push_insert"), 1) + self.assertEqual(run.db_get("pull_insert"), 1) - todo = frappe.get_doc('ToDo', new_todo.name) + todo = frappe.get_doc("ToDo", new_todo.name) self.assertTrue(todo.todo_sync_id) # Pushed Event - event = frappe.get_doc('Event', todo.todo_sync_id) + event = frappe.get_doc("Event", todo.todo_sync_id) self.assertEqual(event.subject, description) # Pulled ToDo - created_todo = frappe.get_doc('ToDo', {'description': event_subject}) + created_todo = frappe.get_doc("ToDo", {"description": event_subject}) self.assertEqual(created_todo.description, event_subject) - todo_list = frappe.get_list('ToDo', filters={'description': 'data migration todo'}, fields=['name']) + todo_list = frappe.get_list( + "ToDo", filters={"description": "data migration todo"}, fields=["name"] + ) todo_name = todo_list[0].name - todo = frappe.get_doc('ToDo', todo_name) - todo.description = 'data migration todo updated' + todo = frappe.get_doc("ToDo", todo_name) + todo.description = "data migration todo updated" todo.save() - run = frappe.get_doc({ - 'doctype': 'Data Migration Run', - 'data_migration_plan': 'ToDo Sync', - 'data_migration_connector': 'Local Connector' - }).insert() + run = frappe.get_doc( + { + "doctype": "Data Migration Run", + "data_migration_plan": "ToDo Sync", + "data_migration_connector": "Local Connector", + } + ).insert() run.run() # Update - self.assertEqual(run.db_get('status'), 'Success') - self.assertEqual(run.db_get('pull_update'), 1) + self.assertEqual(run.db_get("status"), "Success") + self.assertEqual(run.db_get("pull_update"), 1) + def create_plan(): - frappe.get_doc({ - 'doctype': 'Data Migration Mapping', - 'mapping_name': 'Todo to Event', - 'remote_objectname': 'Event', - 'remote_primary_key': 'name', - 'mapping_type': 'Push', - 'local_doctype': 'ToDo', - 'fields': [ - { 'remote_fieldname': 'subject', 'local_fieldname': 'description' }, - { 'remote_fieldname': 'starts_on', 'local_fieldname': 'eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())' } - ], - 'condition': '{"description": "data migration todo" }' - }).insert(ignore_if_duplicate=True) - - frappe.get_doc({ - 'doctype': 'Data Migration Mapping', - 'mapping_name': 'Event to ToDo', - 'remote_objectname': 'Event', - 'remote_primary_key': 'name', - 'local_doctype': 'ToDo', - 'local_primary_key': 'name', - 'mapping_type': 'Pull', - 'condition': '{"subject": "data migration event" }', - 'fields': [ - { 'remote_fieldname': 'subject', 'local_fieldname': 'description' } - ] - }).insert(ignore_if_duplicate=True) - - frappe.get_doc({ - 'doctype': 'Data Migration Plan', - 'plan_name': 'ToDo Sync', - 'module': 'Core', - 'mappings': [ - { 'mapping': 'Todo to Event' }, - { 'mapping': 'Event to ToDo' } - ] - }).insert(ignore_if_duplicate=True) - - frappe.get_doc({ - 'doctype': 'Data Migration Connector', - 'connector_name': 'Local Connector', - 'connector_type': 'Frappe', - # connect to same host. - 'hostname': frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), - 'username': 'Administrator', - 'password': frappe.conf.get("admin_password") or 'admin' - }).insert(ignore_if_duplicate=True) + frappe.get_doc( + { + "doctype": "Data Migration Mapping", + "mapping_name": "Todo to Event", + "remote_objectname": "Event", + "remote_primary_key": "name", + "mapping_type": "Push", + "local_doctype": "ToDo", + "fields": [ + {"remote_fieldname": "subject", "local_fieldname": "description"}, + { + "remote_fieldname": "starts_on", + "local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", + }, + ], + "condition": '{"description": "data migration todo" }', + } + ).insert(ignore_if_duplicate=True) + + frappe.get_doc( + { + "doctype": "Data Migration Mapping", + "mapping_name": "Event to ToDo", + "remote_objectname": "Event", + "remote_primary_key": "name", + "local_doctype": "ToDo", + "local_primary_key": "name", + "mapping_type": "Pull", + "condition": '{"subject": "data migration event" }', + "fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], + } + ).insert(ignore_if_duplicate=True) + + frappe.get_doc( + { + "doctype": "Data Migration Plan", + "plan_name": "ToDo Sync", + "module": "Core", + "mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], + } + ).insert(ignore_if_duplicate=True) + + frappe.get_doc( + { + "doctype": "Data Migration Connector", + "connector_name": "Local Connector", + "connector_type": "Frappe", + # connect to same host. + "hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), + "username": "Administrator", + "password": frappe.conf.get("admin_password") or "admin", + } + ).insert(ignore_if_duplicate=True) diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index 5db0537ed7..7de3fabf01 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -6,38 +6,60 @@ from frappe.database.database import savepoint + def setup_database(force, source_sql=None, verbose=None, no_mariadb_socket=False): import frappe - if frappe.conf.db_type == 'postgres': + + if frappe.conf.db_type == "postgres": import frappe.database.postgres.setup_db + return frappe.database.postgres.setup_db.setup_database(force, source_sql, verbose) else: import frappe.database.mariadb.setup_db - return frappe.database.mariadb.setup_db.setup_database(force, source_sql, verbose, no_mariadb_socket=no_mariadb_socket) + + return frappe.database.mariadb.setup_db.setup_database( + force, source_sql, verbose, no_mariadb_socket=no_mariadb_socket + ) + def drop_user_and_database(db_name, root_login=None, root_password=None): import frappe - if frappe.conf.db_type == 'postgres': + + if frappe.conf.db_type == "postgres": import frappe.database.postgres.setup_db - return frappe.database.postgres.setup_db.drop_user_and_database(db_name, root_login, root_password) + + return frappe.database.postgres.setup_db.drop_user_and_database( + db_name, root_login, root_password + ) else: import frappe.database.mariadb.setup_db - return frappe.database.mariadb.setup_db.drop_user_and_database(db_name, root_login, root_password) + + return frappe.database.mariadb.setup_db.drop_user_and_database( + db_name, root_login, root_password + ) + def get_db(host=None, user=None, password=None, port=None): import frappe - if frappe.conf.db_type == 'postgres': + + if frappe.conf.db_type == "postgres": import frappe.database.postgres.database + return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port) else: import frappe.database.mariadb.database + return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port) + def setup_help_database(help_db_name): import frappe - if frappe.conf.db_type == 'postgres': + + if frappe.conf.db_type == "postgres": import frappe.database.postgres.setup_db + return frappe.database.postgres.setup_db.setup_help_database(help_db_name) else: import frappe.database.mariadb.setup_db + return frappe.database.mariadb.setup_db.setup_help_database(help_db_name) diff --git a/frappe/database/database.py b/frappe/database/database.py index 7551c5f628..424bcbbc63 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -28,27 +28,28 @@ from .query import Query class Database(object): """ - Open a database connection with the given parmeters, if use_default is True, use the - login details from `conf.py`. This is called by the request handler and is accessible using - the `db` global variable. the `sql` method is also global to run queries + Open a database connection with the given parmeters, if use_default is True, use the + login details from `conf.py`. This is called by the request handler and is accessible using + the `db` global variable. the `sql` method is also global to run queries """ + VARCHAR_LEN = 140 MAX_COLUMN_LENGTH = 64 OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"] - DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"] - STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by') - DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'idx'] - CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield') + DEFAULT_SHORTCUTS = ["_Login", "__user", "_Full Name", "Today", "__today", "now", "Now"] + STANDARD_VARCHAR_COLUMNS = ("name", "owner", "modified_by") + DEFAULT_COLUMNS = ["name", "creation", "modified", "modified_by", "owner", "docstatus", "idx"] + CHILD_TABLE_COLUMNS = ("parent", "parenttype", "parentfield") MAX_WRITES_PER_TRANSACTION = 200_000 - class InvalidColumnName(frappe.ValidationError): pass - + class InvalidColumnName(frappe.ValidationError): + pass def __init__(self, host=None, user=None, password=None, ac_name=None, use_default=0, port=None): self.setup_type_map() - self.host = host or frappe.conf.db_host or '127.0.0.1' - self.port = port or frappe.conf.db_port or '' + self.host = host or frappe.conf.db_host or "127.0.0.1" + self.port = port or frappe.conf.db_port or "" self.user = user or frappe.conf.db_name self.db_name = frappe.conf.db_name self._conn = None @@ -86,9 +87,22 @@ class Database(object): def get_database_size(self): pass - def sql(self, query, values=(), as_dict = 0, as_list = 0, formatted = 0, - debug=0, ignore_ddl=0, as_utf8=0, auto_commit=0, update=None, - explain=False, run=True, pluck=False): + def sql( + self, + query, + values=(), + as_dict=0, + as_list=0, + formatted=0, + debug=0, + ignore_ddl=0, + as_utf8=0, + auto_commit=0, + update=None, + explain=False, + run=True, + pluck=False, + ): """Execute a SQL query and fetch all rows. :param query: SQL query. @@ -104,15 +118,15 @@ class Database(object): :param run: Returns query without executing it if False. Examples: - # return customer names as dicts - frappe.db.sql("select name from tabCustomer", as_dict=True) + # return customer names as dicts + frappe.db.sql("select name from tabCustomer", as_dict=True) - # return names beginning with a - frappe.db.sql("select name from tabCustomer where name like %s", "a%") + # return names beginning with a + frappe.db.sql("select name from tabCustomer where name like %s", "a%") - # values as dict - frappe.db.sql("select name from tabCustomer where name like %(name)s and owner=%(owner)s", - {"name": "a%", "owner":"test@example.com"}) + # values as dict + frappe.db.sql("select name from tabCustomer where name like %(name)s and owner=%(owner)s", + {"name": "a%", "owner":"test@example.com"}) """ debug = debug or getattr(self, "debug", False) @@ -123,9 +137,9 @@ class Database(object): # remove whitespace / indentation from start and end of query query = query.strip() - if re.search(r'ifnull\(', query, flags=re.IGNORECASE): + if re.search(r"ifnull\(", query, flags=re.IGNORECASE): # replaces ifnull in query with coalesce - query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE) + query = re.sub(r"ifnull\(", "coalesce(", query, flags=re.IGNORECASE) if not self._conn: self.connect() @@ -136,7 +150,8 @@ class Database(object): self.clear_db_table_cache(query) # autocommit - if auto_commit: self.commit() + if auto_commit: + self.commit() # execute try: @@ -145,7 +160,7 @@ class Database(object): self.log_query(query, values, debug, explain) - if values!=(): + if values != (): # MySQL-python==1.2.5 hack! if not isinstance(values, (dict, tuple, list)): @@ -169,7 +184,7 @@ class Database(object): except Exception as e: if self.is_syntax_error(e): # only for mariadb - frappe.errprint('Syntax error in query:') + frappe.errprint("Syntax error in query:") frappe.errprint(query) elif self.is_deadlocked(e): @@ -178,17 +193,20 @@ class Database(object): elif self.is_timedout(e): raise frappe.QueryTimeoutError(e) - elif frappe.conf.db_type == 'postgres': + elif frappe.conf.db_type == "postgres": # TODO: added temporarily print(e) raise - if ignore_ddl and (self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e)): + if ignore_ddl and ( + self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e) + ): pass else: raise - if auto_commit: self.commit() + if auto_commit: + self.commit() if not self._cursor.description: return () @@ -212,29 +230,29 @@ class Database(object): def log_query(self, query, values, debug, explain): # for debugging in tests - if frappe.conf.get('allow_tests') and frappe.cache().get_value('flag_print_sql'): + if frappe.conf.get("allow_tests") and frappe.cache().get_value("flag_print_sql"): print(self.mogrify(query, values)) # debug if debug: - if explain and query.strip().lower().startswith('select'): + if explain and query.strip().lower().startswith("select"): self.explain_query(query, values) frappe.errprint(self.mogrify(query, values)) # info - if (frappe.conf.get("logging") or False)==2: + if (frappe.conf.get("logging") or False) == 2: frappe.log("<<<< query") frappe.log(self.mogrify(query, values)) frappe.log(">>>>") def mogrify(self, query, values): - '''build the query string with values''' + """build the query string with values""" if not values: return query else: try: return self._cursor.mogrify(query, values) - except: # noqa: E722 + except: # noqa: E722 return (query, values) def explain_query(self, query, values=None): @@ -246,6 +264,7 @@ class Database(object): else: self._cursor.execute("explain " + query, values) import json + frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) frappe.errprint("--- query explain end ---") except Exception: @@ -256,8 +275,8 @@ class Database(object): Example: - # doctypes = ["DocType", "DocField", "User", ...] - doctypes = frappe.db.sql_list("select name from DocType") + # doctypes = ["DocType", "DocField", "User", ...] + doctypes = frappe.db.sql_list("select name from DocType") """ return [r[0] for r in self.sql(query, values, **kwargs, debug=debug)] @@ -267,17 +286,16 @@ class Database(object): self.commit() self.sql(query, debug=debug) - def check_transaction_status(self, query): """Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are executed in one transaction. This is to ensure that writes are always flushed otherwise this could cause the system to hang.""" self.check_implicit_commit(query) - if query and query.strip().lower() in ('commit', 'rollback'): + if query and query.strip().lower() in ("commit", "rollback"): self.transaction_writes = 0 - if query[:6].lower() in ('update', 'insert', 'delete'): + if query[:6].lower() in ("update", "insert", "delete"): self.transaction_writes += 1 if self.transaction_writes > self.MAX_WRITES_PER_TRANSACTION: if self.auto_commit_on_many_writes: @@ -288,9 +306,13 @@ class Database(object): raise frappe.TooManyWritesError(msg) def check_implicit_commit(self, query): - if self.transaction_writes and \ - query and query.strip().split()[0].lower() in ['start', 'alter', 'drop', 'create', "begin", "truncate"]: - raise Exception('This statement can cause implicit commit') + if ( + self.transaction_writes + and query + and query.strip().split()[0].lower() + in ["start", "alter", "drop", "create", "begin", "truncate"] + ): + raise Exception("This statement can cause implicit commit") def fetch_as_dict(self, formatted=0, as_utf8=0): """Internal. Converts results to dict.""" @@ -303,7 +325,7 @@ class Database(object): values = [] for value in r: if as_utf8 and isinstance(value, str): - value = value.encode('utf-8') + value = value.encode("utf-8") values.append(value) ret.append(frappe._dict(zip(keys, values))) @@ -311,8 +333,8 @@ class Database(object): @staticmethod def clear_db_table_cache(query): - if query and query.strip().split()[0].lower() in {'drop', 'create'}: - frappe.cache().delete_key('db_tables') + if query and query.strip().split()[0].lower() in {"drop", "create"}: + frappe.cache().delete_key("db_tables") @staticmethod def needs_formatting(result, formatted): @@ -338,7 +360,7 @@ class Database(object): nr = [] for val in r: if as_utf8 and isinstance(val, str): - val = val.encode('utf-8') + val = val.encode("utf-8") nr.append(val) nres.append(nr) return nres @@ -375,21 +397,34 @@ class Database(object): Example: - # return first customer starting with a - frappe.db.get_value("Customer", {"name": ("like a%")}) + # return first customer starting with a + frappe.db.get_value("Customer", {"name": ("like a%")}) - # return last login of **User** `test@example.com` - frappe.db.get_value("User", "test@example.com", "last_login") + # return last login of **User** `test@example.com` + frappe.db.get_value("User", "test@example.com", "last_login") - last_login, last_ip = frappe.db.get_value("User", "test@example.com", - ["last_login", "last_ip"]) + last_login, last_ip = frappe.db.get_value("User", "test@example.com", + ["last_login", "last_ip"]) - # returns default date_format - frappe.db.get_value("System Settings", None, "date_format") + # returns default date_format + frappe.db.get_value("System Settings", None, "date_format") """ - result = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug, - order_by, cache=cache, for_update=for_update, run=run, pluck=pluck, distinct=distinct, limit=1) + result = self.get_values( + doctype, + filters, + fieldname, + ignore, + as_dict, + debug, + order_by, + cache=cache, + for_update=for_update, + run=run, + pluck=pluck, + distinct=distinct, + limit=1, + ) if not run: return result @@ -405,10 +440,24 @@ class Database(object): # single field is requested, send it without wrapping in containers return row[0] - - def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False, - *, run=True, pluck=False, distinct=False, limit=None): + def get_values( + self, + doctype, + filters=None, + fieldname="name", + ignore=None, + as_dict=False, + debug=False, + order_by="KEEP_DEFAULT_ORDERING", + update=None, + cache=False, + for_update=False, + *, + run=True, + pluck=False, + distinct=False, + limit=None, + ): """Returns multiple document properties. :param doctype: DocType name. @@ -422,15 +471,14 @@ class Database(object): Example: - # return first customer starting with a - customers = frappe.db.get_values("Customer", {"name": ("like a%")}) + # return first customer starting with a + customers = frappe.db.get_values("Customer", {"name": ("like a%")}) - # return last login of **User** `test@example.com` - user = frappe.db.get_values("User", "test@example.com", "*")[0] + # return last login of **User** `test@example.com` + user = frappe.db.get_values("User", "test@example.com", "*")[0] """ out = None - if cache and isinstance(filters, str) and \ - (doctype, filters, fieldname) in self.value_cache: + if cache and isinstance(filters, str) and (doctype, filters, fieldname) in self.value_cache: return self.value_cache[(doctype, filters, fieldname)] if distinct: @@ -456,7 +504,7 @@ class Database(object): if isinstance(fieldname, str): fields = [fieldname] - if (filters is not None) and (filters!=doctype or doctype=="DocType"): + if (filters is not None) and (filters != doctype or doctype == "DocType"): try: if order_by: order_by = "modified" if order_by == "KEEP_DEFAULT_ORDERING" else order_by @@ -480,12 +528,16 @@ class Database(object): out = None elif (not ignore) and frappe.db.is_table_missing(e): # table not found, look in singles - out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run, distinct=distinct) + out = self.get_values_from_single( + fields, filters, doctype, as_dict, debug, update, run=run, distinct=distinct + ) else: raise else: - out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run, pluck=pluck, distinct=distinct) + out = self.get_values_from_single( + fields, filters, doctype, as_dict, debug, update, run=run, pluck=pluck, distinct=distinct + ) if cache and isinstance(filters, str): self.value_cache[(doctype, filters, fieldname)] = out @@ -515,7 +567,7 @@ class Database(object): # if not frappe.model.meta.is_single(doctype): # raise frappe.DoesNotExistError("DocType", doctype) - if fields=="*" or isinstance(filters, dict): + if fields == "*" or isinstance(filters, dict): # check if single doc matches with filters values = self.get_singles_dict(doctype) if isinstance(filters, dict): @@ -550,7 +602,6 @@ class Database(object): else: return r and [[i[1] for i in r]] or [] - def get_singles_dict(self, doctype, debug=False, *, for_update=False): """Get Single DocType as dict. @@ -558,8 +609,8 @@ class Database(object): Example: - # Get coulmn and value of the single doctype Accounts Settings - account_settings = frappe.db.get_singles_dict("Accounts Settings") + # Get coulmn and value of the single doctype Accounts Settings + account_settings = frappe.db.get_singles_dict("Accounts Settings") """ result = self.query.get_sql( "Singles", @@ -578,7 +629,14 @@ class Database(object): def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) - def set_single_value(self, doctype: str, fieldname: Union[str, Dict], value: Optional[Union[str, int]] = None, *args, **kwargs): + def set_single_value( + self, + doctype: str, + fieldname: Union[str, Dict], + value: Optional[Union[str, int]] = None, + *args, + **kwargs, + ): """Set field value of Single DocType. :param doctype: DocType of the single object @@ -587,8 +645,8 @@ class Database(object): Example: - # Update the `deny_multiple_sessions` field in System Settings DocType. - company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) + # Update the `deny_multiple_sessions` field in System Settings DocType. + company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) """ return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs) @@ -600,8 +658,8 @@ class Database(object): Example: - # Get the default value of the company from the Global Defaults doctype. - company = frappe.db.get_single_value('Global Defaults', 'default_company') + # Get the default value of the company from the Global Defaults doctype. + company = frappe.db.get_single_value('Global Defaults', 'default_company') """ if doctype not in self.value_cache: @@ -620,7 +678,9 @@ class Database(object): df = frappe.get_meta(doctype).get_field(fieldname) if not df: - frappe.throw(_('Invalid field name: {0}').format(frappe.bold(fieldname)), self.InvalidColumnName) + frappe.throw( + _("Invalid field name: {0}").format(frappe.bold(fieldname)), self.InvalidColumnName + ) val = cast(df.fieldtype, val) @@ -667,16 +727,10 @@ class Database(object): distinct=distinct, limit=limit, ) - if ( - fields == "*" - and not isinstance(fields, (list, tuple)) - and not isinstance(fields, Criterion) - ): + if fields == "*" and not isinstance(fields, (list, tuple)) and not isinstance(fields, Criterion): as_dict = True - r = self.sql( - query, as_dict=as_dict, debug=debug, update=update, run=run, pluck=pluck - ) + r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, pluck=pluck) return r def _get_value_for_many_names( @@ -691,7 +745,7 @@ class Database(object): pluck=False, distinct=False, limit=None, - as_dict=False + as_dict=False, ): names = list(filter(None, names)) if names: @@ -705,7 +759,7 @@ class Database(object): as_list=not as_dict, run=run, distinct=distinct, - limit_page_length=limit + limit_page_length=limit, ) else: return {} @@ -714,8 +768,18 @@ class Database(object): """Update multiple values. Alias for `set_value`.""" return self.set_value(*args, **kwargs) - def set_value(self, dt, dn, field, val=None, modified=None, modified_by=None, - update_modified=True, debug=False, for_update=True): + def set_value( + self, + dt, + dn, + field, + val=None, + modified=None, + modified_by=None, + update_modified=True, + debug=False, + for_update=True, + ): """Set a single value in the database, do not call the ORM triggers but update the modified timestamp (unless specified not to). @@ -741,15 +805,12 @@ class Database(object): if is_single_doctype: frappe.db.delete( - "Singles", - filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug + "Singles", filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug ) singles_data = ((dt, key, sbool(value)) for key, value in to_update.items()) query = ( - frappe.qb.into("Singles") - .columns("doctype", "field", "value") - .insert(*singles_data) + frappe.qb.into("Singles").columns("doctype", "field", "value").insert(*singles_data) ).run(debug=debug) frappe.clear_document_cache(dt, dt) @@ -770,7 +831,7 @@ class Database(object): # TODO: Fix this; doesn't work rn - gavin@frappe.io # frappe.cache().hdel_keys(dt, "document_cache") # Workaround: clear all document caches - frappe.cache().delete_value('document_cache') + frappe.cache().delete_value("document_cache") for column, value in to_update.items(): query = query.set(column, value) @@ -788,8 +849,13 @@ class Database(object): def touch(self, doctype, docname): """Update the modified timestamp of this document.""" modified = now() - self.sql("""update `tab{doctype}` set `modified`=%s - where name=%s""".format(doctype=doctype), (modified, docname)) + self.sql( + """update `tab{doctype}` set `modified`=%s + where name=%s""".format( + doctype=doctype + ), + (modified, docname), + ) return modified @staticmethod @@ -804,11 +870,11 @@ class Database(object): """Return the temperory value and delete it.""" return frappe.cache().hget("temp", key) - def set_global(self, key, val, user='__global'): + def set_global(self, key, val, user="__global"): """Save a global key value. Global values will be automatically set if they match fieldname.""" self.set_default(key, val, user) - def get_global(self, key, user='__global'): + def get_global(self, key, user="__global"): """Returns a global key value.""" return self.get_default(key, user) @@ -827,14 +893,13 @@ class Database(object): """Append a default value for a key, there can be multiple default values for a particular key.""" frappe.defaults.add_default(key, val, parent, parenttype) - @staticmethod def get_defaults(key=None, parent="__default"): """Get all defaults""" if key: defaults = frappe.defaults.get_defaults(parent) d = defaults.get(key, None) - if(not d and key != frappe.scrub(key)): + if not d and key != frappe.scrub(key): d = defaults.get(frappe.scrub(key), None) return d else: @@ -871,8 +936,8 @@ class Database(object): Changes can be undone to a save point by doing frappe.db.rollback(save_point) Note: rollback watchers can not work with save points. - so only changes to database are undone when rolling back to a savepoint. - Avoid using savepoints when writing to filesystem.""" + so only changes to database are undone when rolling back to a savepoint. + Avoid using savepoints when writing to filesystem.""" self.sql(f"savepoint {save_point}") def release_savepoint(self, save_point): @@ -892,10 +957,7 @@ class Database(object): def field_exists(self, dt, fn): """Return true of field exists.""" - return self.exists('DocField', { - 'fieldname': fn, - 'parent': dt - }) + return self.exists("DocField", {"fieldname": fn, "parent": dt}) def table_exists(self, doctype, cached=True): """Returns True if table for given doctype exists.""" @@ -905,15 +967,17 @@ class Database(object): return self.table_exists(doctype) def get_tables(self, cached=True): - tables = frappe.cache().get_value('db_tables') + tables = frappe.cache().get_value("db_tables") if not tables or not cached: - table_rows = self.sql(""" + table_rows = self.sql( + """ SELECT table_name FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog', 'information_schema') - """) + """ + ) tables = {d[0] for d in table_rows} - frappe.cache().set_value('db_tables', tables) + frappe.cache().set_value("db_tables", tables) return tables def a_row_exists(self, doctype): @@ -950,7 +1014,7 @@ class Database(object): return dn if isinstance(dt, dict): - dt = dt.copy() # don't modify the original dict + dt = dt.copy() # don't modify the original dict dt, dn = dt.pop("doctype"), dt return self.get_value(dt, dn, ignore=True, cache=cache) @@ -958,7 +1022,7 @@ class Database(object): def count(self, dt, filters=None, debug=False, cache=False): """Returns `COUNT(*)` for given DocType and filters.""" if cache and not filters: - cache_count = frappe.cache().get_value('doctype:count:{}'.format(dt)) + cache_count = frappe.cache().get_value("doctype:count:{}".format(dt)) if cache_count is not None: return cache_count query = self.query.get_sql(table=dt, filters=filters, fields=Count("*")) @@ -968,7 +1032,7 @@ class Database(object): else: count = self.sql(query, debug=debug)[0][0] if cache: - frappe.cache().set_value('doctype:count:{}'.format(dt), count, expires_in_sec = 86400) + frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400) return count @staticmethod @@ -978,11 +1042,11 @@ class Database(object): @staticmethod def format_datetime(datetime): if not datetime: - return '0001-01-01 00:00:00.000000' + return "0001-01-01 00:00:00.000000" if isinstance(datetime, str): - if ':' not in datetime: - datetime = datetime + ' 00:00:00.000000' + if ":" not in datetime: + datetime = datetime + " 00:00:00.000000" else: datetime = datetime.strftime("%Y-%m-%d %H:%M:%S.%f") @@ -990,32 +1054,43 @@ class Database(object): def get_creation_count(self, doctype, minutes): """Get count of records created in the last x minutes""" - from frappe.utils import now_datetime from dateutil.relativedelta import relativedelta - return self.sql("""select count(name) from `tab{doctype}` - where creation >= %s""".format(doctype=doctype), - now_datetime() - relativedelta(minutes=minutes))[0][0] + from frappe.utils import now_datetime + + return self.sql( + """select count(name) from `tab{doctype}` + where creation >= %s""".format( + doctype=doctype + ), + now_datetime() - relativedelta(minutes=minutes), + )[0][0] def get_db_table_columns(self, table): """Returns list of column names from given table.""" - columns = frappe.cache().hget('table_columns', table) + columns = frappe.cache().hget("table_columns", table) if columns is None: - columns = [r[0] for r in self.sql(''' + columns = [ + r[0] + for r in self.sql( + """ select column_name from information_schema.columns - where table_name = %s ''', table)] + where table_name = %s """, + table, + ) + ] if columns: - frappe.cache().hset('table_columns', table, columns) + frappe.cache().hset("table_columns", table, columns) return columns def get_table_columns(self, doctype): """Returns list of column names from given doctype.""" - columns = self.get_db_table_columns('tab' + doctype) + columns = self.get_db_table_columns("tab" + doctype) if not columns: - raise self.TableMissingError('DocType', doctype) + raise self.TableMissingError("DocType", doctype) return columns def has_column(self, doctype, column): @@ -1023,8 +1098,12 @@ class Database(object): return column in self.get_table_columns(doctype) def get_column_type(self, doctype, column): - return self.sql('''SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = 'tab{0}' AND column_name = '{1}' '''.format(doctype, column))[0][0] + return self.sql( + """SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = 'tab{0}' AND column_name = '{1}' """.format( + doctype, column + ) + )[0][0] def has_index(self, table_name, index_name): raise NotImplementedError @@ -1045,6 +1124,7 @@ class Database(object): def get_system_setting(self, key): def _load_system_settings(): return self.get_singles_dict("System Settings") + return frappe.cache().get_value("system_settings", _load_system_settings).get(key) def close(self): @@ -1066,12 +1146,16 @@ class Database(object): return frappe.db.is_missing_column(e) def get_descendants(self, doctype, name): - '''Return descendants of the current record''' - node_location_indexes = self.get_value(doctype, name, ('lft', 'rgt')) + """Return descendants of the current record""" + node_location_indexes = self.get_value(doctype, name, ("lft", "rgt")) if node_location_indexes: lft, rgt = node_location_indexes - return self.sql_list('''select name from `tab{doctype}` - where lft > {lft} and rgt < {rgt}'''.format(doctype=doctype, lft=lft, rgt=rgt)) + return self.sql_list( + """select name from `tab{doctype}` + where lft > {lft} and rgt < {rgt}""".format( + doctype=doctype, lft=lft, rgt=rgt + ) + ) else: # when document does not exist return [] @@ -1080,7 +1164,7 @@ class Database(object): return self.is_missing_column(e) or self.is_table_missing(e) def multisql(self, sql_dict, values=(), **kwargs): - current_dialect = frappe.db.db_type or 'mariadb' + current_dialect = frappe.db.db_type or "mariadb" query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) @@ -1110,7 +1194,7 @@ class Database(object): return self.truncate(doctype) def get_last_created(self, doctype): - last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') + last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc") if last_record: return get_datetime(last_record[0].creation) else: @@ -1119,7 +1203,7 @@ class Database(object): def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) - if query.strip().lower().split()[0] in ('insert', 'delete', 'update', 'alter', 'drop', 'rename'): + if query.strip().lower().split()[0] in ("insert", "delete", "update", "alter", "drop", "rename"): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" @@ -1146,24 +1230,27 @@ class Database(object): def bulk_insert(self, doctype, fields, values, ignore_duplicates=False): """ - Insert multiple records at a time + Insert multiple records at a time - :param doctype: Doctype name - :param fields: list of fields - :params values: list of list of values + :param doctype: Doctype name + :param fields: list of fields + :params values: list of list of values """ insert_list = [] - fields = ", ".join("`"+field+"`" for field in fields) + fields = ", ".join("`" + field + "`" for field in fields) for idx, value in enumerate(values): insert_list.append(tuple(value)) - if idx and (idx%10000 == 0 or idx < len(values)-1): - self.sql("""INSERT {ignore_duplicates} INTO `tab{doctype}` ({fields}) VALUES {values}""".format( + if idx and (idx % 10000 == 0 or idx < len(values) - 1): + self.sql( + """INSERT {ignore_duplicates} INTO `tab{doctype}` ({fields}) VALUES {values}""".format( ignore_duplicates="IGNORE" if ignore_duplicates else "", doctype=doctype, fields=fields, - values=", ".join(['%s'] * len(insert_list)) - ), tuple(insert_list)) + values=", ".join(["%s"] * len(insert_list)), + ), + tuple(insert_list), + ) insert_list = [] @@ -1173,30 +1260,30 @@ def enqueue_jobs_after_commit(): if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: for job in frappe.flags.enqueue_after_commit: q = get_queue(job.get("queue"), is_async=job.get("is_async")) - q.enqueue_call(execute_job, timeout=job.get("timeout"), - kwargs=job.get("queue_args")) + q.enqueue_call(execute_job, timeout=job.get("timeout"), kwargs=job.get("queue_args")) frappe.flags.enqueue_after_commit = [] + @contextmanager def savepoint(catch: Union[type, Tuple[type, ...]] = Exception): - """ Wrapper for wrapping blocks of DB operations in a savepoint. + """Wrapper for wrapping blocks of DB operations in a savepoint. - as contextmanager: + as contextmanager: - for doc in docs: - with savepoint(catch=DuplicateError): - doc.insert() + for doc in docs: + with savepoint(catch=DuplicateError): + doc.insert() - as decorator (wraps FULL function call): + as decorator (wraps FULL function call): - @savepoint(catch=DuplicateError) - def process_doc(doc): - doc.insert() + @savepoint(catch=DuplicateError) + def process_doc(doc): + doc.insert() """ try: - savepoint = ''.join(random.sample(string.ascii_lowercase, 10)) + savepoint = "".join(random.sample(string.ascii_lowercase, 10)) frappe.db.savepoint(savepoint) - yield # control back to calling function + yield # control back to calling function except catch: frappe.db.rollback(save_point=savepoint) else: diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index b8ffae519b..8f810fe54b 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -1,4 +1,5 @@ import os + import frappe @@ -11,7 +12,7 @@ class DbManager: self.db = db def get_current_host(self): - return self.db.sql("select user()")[0][0].split('@')[1] + return self.db.sql("select user()")[0][0].split("@")[1] def create_user(self, user, password, host=None): # Create user if it doesn't exist. @@ -47,8 +48,11 @@ class DbManager: if not host: host = self.get_current_host() - if frappe.conf.get('rds_db', 0) == 1: - self.db.sql("GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES ON `%s`.* TO '%s'@'%s';" % (target, user, host)) + if frappe.conf.get("rds_db", 0) == 1: + self.db.sql( + "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES ON `%s`.* TO '%s'@'%s';" + % (target, user, host) + ) else: self.db.sql("GRANT ALL PRIVILEGES ON `%s`.* TO '%s'@'%s';" % (target, user, host)) @@ -62,24 +66,27 @@ class DbManager: @staticmethod def restore_database(target, source, user, password): from frappe.utils import make_esc - esc = make_esc('$ ') + + esc = make_esc("$ ") from distutils.spawn import find_executable - pv = find_executable('pv') + + pv = find_executable("pv") if pv: - pipe = '{pv} {source} |'.format( - pv=pv, - source=source - ) - source = '' + pipe = "{pv} {source} |".format(pv=pv, source=source) + source = "" else: - pipe = '' - source = '< {source}'.format(source=source) + pipe = "" + source = "< {source}".format(source=source) if pipe: - print('Restoring Database file...') + print("Restoring Database file...") - command = '{pipe} mysql -u {user} -p{password} -h{host} ' + ('-P{port}' if frappe.db.port else '') + ' {target} {source}' + command = ( + "{pipe} mysql -u {user} -p{password} -h{host} " + + ("-P{port}" if frappe.db.port else "") + + " {target} {source}" + ) command = command.format( pipe=pipe, user=esc(user), @@ -87,6 +94,6 @@ class DbManager: host=esc(frappe.db.host), target=esc(target), source=source, - port=frappe.db.port + port=frappe.db.port, ) os.system(command) diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index a6d5e7b3f2..dcb88d648b 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -17,43 +17,43 @@ class MariaDBDatabase(Database): InternalError = pymysql.err.InternalError SQLError = pymysql.err.ProgrammingError DataError = pymysql.err.DataError - REGEX_CHARACTER = 'regexp' + REGEX_CHARACTER = "regexp" def setup_type_map(self): - self.db_type = 'mariadb' + self.db_type = "mariadb" self.type_map = { - 'Currency': ('decimal', '21,9'), - 'Int': ('int', '11'), - 'Long Int': ('bigint', '20'), - 'Float': ('decimal', '21,9'), - 'Percent': ('decimal', '21,9'), - 'Check': ('int', '1'), - 'Small Text': ('text', ''), - 'Long Text': ('longtext', ''), - 'Code': ('longtext', ''), - 'Text Editor': ('longtext', ''), - 'Markdown Editor': ('longtext', ''), - 'HTML Editor': ('longtext', ''), - 'Date': ('date', ''), - 'Datetime': ('datetime', '6'), - 'Time': ('time', '6'), - 'Text': ('text', ''), - 'Data': ('varchar', self.VARCHAR_LEN), - 'Link': ('varchar', self.VARCHAR_LEN), - 'Dynamic Link': ('varchar', self.VARCHAR_LEN), - 'Password': ('text', ''), - 'Select': ('varchar', self.VARCHAR_LEN), - 'Rating': ('decimal', '3,2'), - 'Read Only': ('varchar', self.VARCHAR_LEN), - 'Attach': ('text', ''), - 'Attach Image': ('text', ''), - 'Signature': ('longtext', ''), - 'Color': ('varchar', self.VARCHAR_LEN), - 'Barcode': ('longtext', ''), - 'Geolocation': ('longtext', ''), - 'Duration': ('decimal', '21,9'), - 'Icon': ('varchar', self.VARCHAR_LEN), - 'Autocomplete': ('varchar', self.VARCHAR_LEN), + "Currency": ("decimal", "21,9"), + "Int": ("int", "11"), + "Long Int": ("bigint", "20"), + "Float": ("decimal", "21,9"), + "Percent": ("decimal", "21,9"), + "Check": ("int", "1"), + "Small Text": ("text", ""), + "Long Text": ("longtext", ""), + "Code": ("longtext", ""), + "Text Editor": ("longtext", ""), + "Markdown Editor": ("longtext", ""), + "HTML Editor": ("longtext", ""), + "Date": ("date", ""), + "Datetime": ("datetime", "6"), + "Time": ("time", "6"), + "Text": ("text", ""), + "Data": ("varchar", self.VARCHAR_LEN), + "Link": ("varchar", self.VARCHAR_LEN), + "Dynamic Link": ("varchar", self.VARCHAR_LEN), + "Password": ("text", ""), + "Select": ("varchar", self.VARCHAR_LEN), + "Rating": ("decimal", "3,2"), + "Read Only": ("varchar", self.VARCHAR_LEN), + "Attach": ("text", ""), + "Attach Image": ("text", ""), + "Signature": ("longtext", ""), + "Color": ("varchar", self.VARCHAR_LEN), + "Barcode": ("longtext", ""), + "Geolocation": ("longtext", ""), + "Duration": ("decimal", "21,9"), + "Icon": ("varchar", self.VARCHAR_LEN), + "Autocomplete": ("varchar", self.VARCHAR_LEN), } def get_connection(self): @@ -61,46 +61,52 @@ class MariaDBDatabase(Database): if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: usessl = 1 ssl_params = { - 'ca':frappe.conf.db_ssl_ca, - 'cert':frappe.conf.db_ssl_cert, - 'key':frappe.conf.db_ssl_key + "ca": frappe.conf.db_ssl_ca, + "cert": frappe.conf.db_ssl_cert, + "key": frappe.conf.db_ssl_key, } - conversions.update({ - FIELD_TYPE.NEWDECIMAL: float, - FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: conversions[str] - }) + conversions.update( + { + FIELD_TYPE.NEWDECIMAL: float, + FIELD_TYPE.DATETIME: get_datetime, + UnicodeWithAttrs: conversions[str], + } + ) conn = pymysql.connect( - user=self.user or '', - password=self.password or '', + user=self.user or "", + password=self.password or "", host=self.host, port=self.port, - charset='utf8mb4', + charset="utf8mb4", use_unicode=True, ssl=ssl_params if usessl else None, conv=conversions, - local_infile=frappe.conf.local_infile + local_infile=frappe.conf.local_infile, ) # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) - if self.user != 'root': + if self.user != "root": conn.select_db(self.user) return conn def get_database_size(self): - ''''Returns database size in MB''' - db_size = self.sql(''' + """'Returns database size in MB""" + db_size = self.sql( + """ SELECT `table_schema` as `database_name`, SUM(`data_length` + `index_length`) / 1024 / 1024 AS `database_size` FROM information_schema.tables WHERE `table_schema` = %s GROUP BY `table_schema` - ''', self.db_name, as_dict=True) + """, + self.db_name, + as_dict=True, + ) - return db_size[0].get('database_size') + return db_size[0].get("database_size") @staticmethod def escape(s, percent=True): @@ -136,7 +142,9 @@ class MariaDBDatabase(Database): table_name = get_table_name(doctype) return self.sql(f"DESC `{table_name}`") - def change_column_type(self, doctype: str, column: str, type: str, nullable: bool = False) -> Union[List, Tuple]: + def change_column_type( + self, doctype: str, column: str, type: str, nullable: bool = False + ) -> Union[List, Tuple]: table_name = get_table_name(doctype) null_constraint = "NOT NULL" if not nullable else "" return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} {null_constraint}") @@ -171,7 +179,7 @@ class MariaDBDatabase(Database): return e.args[0] == ER.DUP_ENTRY @staticmethod - def is_access_denied( e): + def is_access_denied(e): return e.args[0] == ER.ACCESS_DENIED_ERROR @staticmethod @@ -187,25 +195,27 @@ class MariaDBDatabase(Database): return e.args[0] == ER.DATA_TOO_LONG def is_primary_key_violation(self, e): - return self.is_duplicate_entry(e) and 'PRIMARY' in cstr(e.args[1]) + return self.is_duplicate_entry(e) and "PRIMARY" in cstr(e.args[1]) def is_unique_key_violation(self, e): - return self.is_duplicate_entry(e) and 'Duplicate' in cstr(e.args[1]) - + return self.is_duplicate_entry(e) and "Duplicate" in cstr(e.args[1]) def create_auth_table(self): - self.sql_ddl("""create table if not exists `__Auth` ( + self.sql_ddl( + """create table if not exists `__Auth` ( `doctype` VARCHAR(140) NOT NULL, `name` VARCHAR(255) NOT NULL, `fieldname` VARCHAR(140) NOT NULL, `password` TEXT NOT NULL, `encrypted` INT(1) NOT NULL DEFAULT 0, PRIMARY KEY (`doctype`, `name`, `fieldname`) - ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""") + ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""" + ) def create_global_search_table(self): - if not '__global_search' in self.get_tables(): - self.sql('''create table __global_search( + if not "__global_search" in self.get_tables(): + self.sql( + """create table __global_search( doctype varchar(100), name varchar({0}), title varchar({0}), @@ -216,18 +226,24 @@ class MariaDBDatabase(Database): unique `doctype_name` (doctype, name)) COLLATE=utf8mb4_unicode_ci ENGINE=MyISAM - CHARACTER SET=utf8mb4'''.format(self.VARCHAR_LEN)) + CHARACTER SET=utf8mb4""".format( + self.VARCHAR_LEN + ) + ) def create_user_settings_table(self): - self.sql_ddl("""create table if not exists __UserSettings ( + self.sql_ddl( + """create table if not exists __UserSettings ( `user` VARCHAR(180) NOT NULL, `doctype` VARCHAR(180) NOT NULL, `data` TEXT, UNIQUE(user, doctype) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8""") + ) ENGINE=InnoDB DEFAULT CHARSET=utf8""" + ) def create_help_table(self): - self.sql('''create table help( + self.sql( + """create table help( path varchar(255), content text, title text, @@ -238,15 +254,17 @@ class MariaDBDatabase(Database): index (path)) COLLATE=utf8mb4_unicode_ci ENGINE=MyISAM - CHARACTER SET=utf8mb4''') + CHARACTER SET=utf8mb4""" + ) @staticmethod def get_on_duplicate_update(key=None): - return 'ON DUPLICATE key UPDATE ' + return "ON DUPLICATE key UPDATE " def get_table_columns_description(self, table_name): """Returns list of column and its description""" - return self.sql('''select + return self.sql( + """select column_name as 'name', column_type as 'type', column_default as 'default', @@ -260,14 +278,19 @@ class MariaDBDatabase(Database): ), 0) as 'index', column_key = 'UNI' as 'unique' from information_schema.columns as columns - where table_name = '{table_name}' '''.format(table_name=table_name), as_dict=1) + where table_name = '{table_name}' """.format( + table_name=table_name + ), + as_dict=1, + ) def has_index(self, table_name, index_name): - return self.sql("""SHOW INDEX FROM `{table_name}` + return self.sql( + """SHOW INDEX FROM `{table_name}` WHERE Key_name='{index_name}'""".format( - table_name=table_name, - index_name=index_name - )) + table_name=table_name, index_name=index_name + ) + ) def add_index(self, doctype: str, fields: List, index_name: str = None): """Creates an index with given fields if not already created. @@ -276,8 +299,11 @@ class MariaDBDatabase(Database): table_name = get_table_name(doctype) if not self.has_index(table_name, index_name): self.commit() - self.sql("""ALTER TABLE `%s` - ADD INDEX `%s`(%s)""" % (table_name, index_name, ", ".join(fields))) + self.sql( + """ALTER TABLE `%s` + ADD INDEX `%s`(%s)""" + % (table_name, index_name, ", ".join(fields)) + ) def add_unique(self, doctype, fields, constraint_name=None): if isinstance(fields, str): @@ -285,12 +311,17 @@ class MariaDBDatabase(Database): if not constraint_name: constraint_name = "unique_" + "_".join(fields) - if not self.sql("""select CONSTRAINT_NAME from information_schema.TABLE_CONSTRAINTS + if not self.sql( + """select CONSTRAINT_NAME from information_schema.TABLE_CONSTRAINTS where table_name=%s and constraint_type='UNIQUE' and CONSTRAINT_NAME=%s""", - ('tab' + doctype, constraint_name)): - self.commit() - self.sql("""alter table `tab%s` - add unique `%s`(%s)""" % (doctype, constraint_name, ", ".join(fields))) + ("tab" + doctype, constraint_name), + ): + self.commit() + self.sql( + """alter table `tab%s` + add unique `%s`(%s)""" + % (doctype, constraint_name, ", ".join(fields)) + ) def updatedb(self, doctype, meta=None): """ @@ -301,7 +332,7 @@ class MariaDBDatabase(Database): """ res = self.sql("select issingle from `tabDocType` where name=%s", (doctype,)) if not res: - raise Exception('Wrong doctype {0} in updatedb'.format(doctype)) + raise Exception("Wrong doctype {0} in updatedb".format(doctype)) if not res[0][0]: db_table = MariaDBTable(doctype, meta) diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 3b7aa443f2..7c95e9ffcb 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -15,27 +15,31 @@ class MariaDBTable(DBTable): # columns column_defs = self.get_column_definitions() if column_defs: - additional_definitions += ',\n'.join(column_defs) + ',\n' + additional_definitions += ",\n".join(column_defs) + ",\n" # index index_defs = self.get_index_definitions() if index_defs: - additional_definitions += ',\n'.join(index_defs) + ',\n' + additional_definitions += ",\n".join(index_defs) + ",\n" # child table columns if self.meta.get("istable") or 0: - additional_definitions += ',\n'.join( - ( - f"parent varchar({varchar_len})", - f"parentfield varchar({varchar_len})", - f"parenttype varchar({varchar_len})", - "index parent(parent)" + additional_definitions += ( + ",\n".join( + ( + f"parent varchar({varchar_len})", + f"parentfield varchar({varchar_len})", + f"parenttype varchar({varchar_len})", + "index parent(parent)", + ) ) - ) + ',\n' + + ",\n" + ) # creating sequence(s) - if (not self.meta.issingle and self.meta.autoname == "autoincrement")\ - or self.doctype in log_types: + if ( + not self.meta.issingle and self.meta.autoname == "autoincrement" + ) or self.doctype in log_types: # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, # it drops the cache and uses the next non cached value in setval func and @@ -78,7 +82,7 @@ class MariaDBTable(DBTable): add_index_query = [] drop_index_query = [] - columns_to_modify = set(self.change_type + self.add_unique + self.set_default) + columns_to_modify = set(self.change_type + self.add_unique + self.set_default) for col in self.add_column: add_column_query.append("ADD COLUMN `{}` {}".format(col.fieldname, col.get_definition())) @@ -88,31 +92,43 @@ class MariaDBTable(DBTable): for col in self.add_index: # if index key does not exists - if not frappe.db.has_index(self.table_name, col.fieldname + '_index'): + if not frappe.db.has_index(self.table_name, col.fieldname + "_index"): add_index_query.append("ADD INDEX `{}_index`(`{}`)".format(col.fieldname, col.fieldname)) for col in self.drop_index + self.drop_unique: - if col.fieldname != 'name': # primary key + if col.fieldname != "name": # primary key current_column = self.current_columns.get(col.fieldname.lower()) unique_constraint_changed = current_column.unique != col.unique if unique_constraint_changed and not col.unique: # nosemgrep - unique_index_record = frappe.db.sql(""" + unique_index_record = frappe.db.sql( + """ SHOW INDEX FROM `{0}` WHERE Key_name=%s AND Non_unique=0 - """.format(self.table_name), (col.fieldname), as_dict=1) + """.format( + self.table_name + ), + (col.fieldname), + as_dict=1, + ) if unique_index_record: drop_index_query.append("DROP INDEX `{}`".format(unique_index_record[0].Key_name)) index_constraint_changed = current_column.index != col.set_index # if index key exists if index_constraint_changed and not col.set_index: # nosemgrep - index_record = frappe.db.sql(""" + index_record = frappe.db.sql( + """ SHOW INDEX FROM `{0}` WHERE Key_name=%s AND Non_unique=1 - """.format(self.table_name), (col.fieldname + '_index'), as_dict=1) + """.format( + self.table_name + ), + (col.fieldname + "_index"), + as_dict=1, + ) if index_record: drop_index_query.append("DROP INDEX `{}`".format(index_record[0].Key_name)) @@ -125,13 +141,16 @@ class MariaDBTable(DBTable): except Exception as e: # sanitize - if e.args[0]==1060: + if e.args[0] == 1060: frappe.throw(str(e)) - elif e.args[0]==1062: + elif e.args[0] == 1062: fieldname = str(e).split("'")[-2] - frappe.throw(_("{0} field cannot be set as unique in {1}, as there are non-unique existing values").format( - fieldname, self.table_name)) - elif e.args[0]==1067: + frappe.throw( + _("{0} field cannot be set as unique in {1}, as there are non-unique existing values").format( + fieldname, self.table_name + ) + ) + elif e.args[0] == 1067: frappe.throw(str(e.args[1])) else: raise e diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py index 1585e4537b..4399ccfa6a 100644 --- a/frappe/database/mariadb/setup_db.py +++ b/frappe/database/mariadb/setup_db.py @@ -1,5 +1,6 @@ -import frappe import os + +import frappe from frappe.database.db_manager import DbManager expected_settings_10_2_earlier = { @@ -7,12 +8,12 @@ expected_settings_10_2_earlier = { "innodb_file_per_table": "ON", "innodb_large_prefix": "ON", "character_set_server": "utf8mb4", - "collation_server": "utf8mb4_unicode_ci" + "collation_server": "utf8mb4_unicode_ci", } expected_settings_10_3_later = { "character_set_server": "utf8mb4", - "collation_server": "utf8mb4_unicode_ci" + "collation_server": "utf8mb4_unicode_ci", } @@ -20,16 +21,15 @@ def get_mariadb_versions(): # MariaDB classifies their versions as Major (1st and 2nd number), and Minor (3rd number) # Example: Version 10.3.13 is Major Version = 10.3, Minor Version = 13 mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) - version_string = mariadb_variables.get('version').split('-')[0] + version_string = mariadb_variables.get("version").split("-")[0] versions = {} - versions['major'] = version_string.split( - '.')[0] + '.' + version_string.split('.')[1] - versions['minor'] = version_string.split('.')[2] + versions["major"] = version_string.split(".")[0] + "." + version_string.split(".")[1] + versions["minor"] = version_string.split(".")[2] return versions def setup_database(force, source_sql, verbose, no_mariadb_socket=False): - frappe.local.session = frappe._dict({'user':'Administrator'}) + frappe.local.session = frappe._dict({"user": "Administrator"}) db_name = frappe.local.conf.db_name root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) @@ -45,20 +45,24 @@ def setup_database(force, source_sql, verbose, no_mariadb_socket=False): raise Exception("Database %s already exists" % (db_name,)) dbman.create_user(db_name, frappe.conf.db_password, **dbman_kwargs) - if verbose: print("Created user %s" % db_name) + if verbose: + print("Created user %s" % db_name) dbman.create_database(db_name) - if verbose: print("Created database %s" % db_name) + if verbose: + print("Created database %s" % db_name) dbman.grant_all_privileges(db_name, db_name, **dbman_kwargs) dbman.flush_privileges() - if verbose: print("Granted privileges to user %s and database %s" % (db_name, db_name)) + if verbose: + print("Granted privileges to user %s and database %s" % (db_name, db_name)) # close root connection root_conn.close() bootstrap_database(db_name, verbose, source_sql) + def setup_help_database(help_db_name): dbman = DbManager(get_root_connection(frappe.flags.root_login, frappe.flags.root_password)) dbman.drop_database(help_db_name) @@ -69,11 +73,13 @@ def setup_help_database(help_db_name): dbman.create_user(help_db_name, help_db_name) except Exception as e: # user already exists - if e.args[0] != 1396: raise + if e.args[0] != 1396: + raise dbman.create_database(help_db_name) dbman.grant_all_privileges(help_db_name, help_db_name) dbman.flush_privileges() + def drop_user_and_database(db_name, root_login, root_password): frappe.local.db = get_root_connection(root_login, root_password) dbman = DbManager(frappe.local.db) @@ -81,18 +87,19 @@ def drop_user_and_database(db_name, root_login, root_password): dbman.delete_user(db_name) dbman.drop_database(db_name) + def bootstrap_database(db_name, verbose, source_sql=None): import sys frappe.connect(db_name=db_name) if not check_database_settings(): - print('Database settings do not match expected values; stopping database setup.') + print("Database settings do not match expected values; stopping database setup.") sys.exit(1) import_db_from_sql(source_sql, verbose) frappe.connect(db_name=db_name) - if 'tabDefaultValue' not in frappe.db.get_tables(cached=False): + if "tabDefaultValue" not in frappe.db.get_tables(cached=False): from click import secho secho( @@ -101,22 +108,25 @@ def bootstrap_database(db_name, verbose, source_sql=None): "permission, or that the database name exists. Check your mysql" " root password, validity of the backup file or use --force to" " reinstall", - fg="red" + fg="red", ) sys.exit(1) + def import_db_from_sql(source_sql=None, verbose=False): - if verbose: print("Starting database import...") + if verbose: + print("Starting database import...") db_name = frappe.conf.db_name if not source_sql: - source_sql = os.path.join(os.path.dirname(__file__), 'framework_mariadb.sql') + source_sql = os.path.join(os.path.dirname(__file__), "framework_mariadb.sql") DbManager(frappe.local.db).restore_database(db_name, source_sql, db_name, frappe.conf.db_password) - if verbose: print("Imported from database %s" % source_sql) + if verbose: + print("Imported from database %s" % source_sql) def check_database_settings(): versions = get_mariadb_versions() - if versions['major'] <= '10.2': + if versions["major"] <= "10.2": expected_variables = expected_settings_10_2_earlier else: expected_variables = expected_settings_10_3_later @@ -126,26 +136,31 @@ def check_database_settings(): result = True for key, expected_value in expected_variables.items(): if mariadb_variables.get(key) != expected_value: - print("For key %s. Expected value %s, found value %s" % - (key, expected_value, mariadb_variables.get(key))) + print( + "For key %s. Expected value %s, found value %s" + % (key, expected_value, mariadb_variables.get(key)) + ) result = False if not result: site = frappe.local.site - msg = ("Creation of your site - {x} failed because MariaDB is not properly {sep}" - "configured. If using version 10.2.x or earlier, make sure you use the {sep}" - "the Barracuda storage engine. {sep}{sep}" - "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" - "then run `bench new-site {x}` again.{sep2}" - "").format(x=site, sep2="\n"*2, sep="\n") + msg = ( + "Creation of your site - {x} failed because MariaDB is not properly {sep}" + "configured. If using version 10.2.x or earlier, make sure you use the {sep}" + "the Barracuda storage engine. {sep}{sep}" + "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" + "then run `bench new-site {x}` again.{sep2}" + "" + ).format(x=site, sep2="\n" * 2, sep="\n") print_db_config(msg) return result def get_root_connection(root_login, root_password): import getpass + if not frappe.local.flags.root_connection: if not root_login: - root_login = 'root' + root_login = "root" if not root_password: root_password = frappe.conf.get("root_password") or None @@ -153,12 +168,14 @@ def get_root_connection(root_login, root_password): if not root_password: root_password = getpass.getpass("MySQL root password: ") - frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password) + frappe.local.flags.root_connection = frappe.database.get_db( + user=root_login, password=root_password + ) return frappe.local.flags.root_connection def print_db_config(explanation): - print("="*80) + print("=" * 80) print(explanation) - print("="*80) + print("=" * 80) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index eb3e33d39c..c34c20ee7b 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -3,8 +3,8 @@ from typing import List, Tuple, Union import psycopg2 import psycopg2.extensions -from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION +from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe from frappe.database.database import Database @@ -14,11 +14,13 @@ from frappe.utils import cstr, get_table_name # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( psycopg2.extensions.DECIMAL.values, - 'DEC2FLOAT', - lambda value, curs: float(value) if value is not None else None) + "DEC2FLOAT", + lambda value, curs: float(value) if value is not None else None, +) psycopg2.extensions.register_type(DEC2FLOAT) + class PostgresDatabase(Database): ProgrammingError = psycopg2.ProgrammingError TableMissingError = psycopg2.ProgrammingError @@ -27,49 +29,51 @@ class PostgresDatabase(Database): SQLError = psycopg2.ProgrammingError DataError = psycopg2.DataError InterfaceError = psycopg2.InterfaceError - REGEX_CHARACTER = '~' + REGEX_CHARACTER = "~" def setup_type_map(self): - self.db_type = 'postgres' + self.db_type = "postgres" self.type_map = { - 'Currency': ('decimal', '21,9'), - 'Int': ('bigint', None), - 'Long Int': ('bigint', None), - 'Float': ('decimal', '21,9'), - 'Percent': ('decimal', '21,9'), - 'Check': ('smallint', None), - 'Small Text': ('text', ''), - 'Long Text': ('text', ''), - 'Code': ('text', ''), - 'Text Editor': ('text', ''), - 'Markdown Editor': ('text', ''), - 'HTML Editor': ('text', ''), - 'Date': ('date', ''), - 'Datetime': ('timestamp', None), - 'Time': ('time', '6'), - 'Text': ('text', ''), - 'Data': ('varchar', self.VARCHAR_LEN), - 'Link': ('varchar', self.VARCHAR_LEN), - 'Dynamic Link': ('varchar', self.VARCHAR_LEN), - 'Password': ('text', ''), - 'Select': ('varchar', self.VARCHAR_LEN), - 'Rating': ('decimal', '3,2'), - 'Read Only': ('varchar', self.VARCHAR_LEN), - 'Attach': ('text', ''), - 'Attach Image': ('text', ''), - 'Signature': ('text', ''), - 'Color': ('varchar', self.VARCHAR_LEN), - 'Barcode': ('text', ''), - 'Geolocation': ('text', ''), - 'Duration': ('decimal', '21,9'), - 'Icon': ('varchar', self.VARCHAR_LEN), - 'Autocomplete': ('varchar', self.VARCHAR_LEN), + "Currency": ("decimal", "21,9"), + "Int": ("bigint", None), + "Long Int": ("bigint", None), + "Float": ("decimal", "21,9"), + "Percent": ("decimal", "21,9"), + "Check": ("smallint", None), + "Small Text": ("text", ""), + "Long Text": ("text", ""), + "Code": ("text", ""), + "Text Editor": ("text", ""), + "Markdown Editor": ("text", ""), + "HTML Editor": ("text", ""), + "Date": ("date", ""), + "Datetime": ("timestamp", None), + "Time": ("time", "6"), + "Text": ("text", ""), + "Data": ("varchar", self.VARCHAR_LEN), + "Link": ("varchar", self.VARCHAR_LEN), + "Dynamic Link": ("varchar", self.VARCHAR_LEN), + "Password": ("text", ""), + "Select": ("varchar", self.VARCHAR_LEN), + "Rating": ("decimal", "3,2"), + "Read Only": ("varchar", self.VARCHAR_LEN), + "Attach": ("text", ""), + "Attach Image": ("text", ""), + "Signature": ("text", ""), + "Color": ("varchar", self.VARCHAR_LEN), + "Barcode": ("text", ""), + "Geolocation": ("text", ""), + "Duration": ("decimal", "21,9"), + "Icon": ("varchar", self.VARCHAR_LEN), + "Autocomplete": ("varchar", self.VARCHAR_LEN), } def get_connection(self): - conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format( - self.host, self.user, self.user, self.password, self.port - )) + conn = psycopg2.connect( + "host='{}' dbname='{}' user='{}' password='{}' port={}".format( + self.host, self.user, self.user, self.password, self.port + ) + ) conn.set_isolation_level(ISOLATION_LEVEL_REPEATABLE_READ) return conn @@ -77,49 +81,54 @@ class PostgresDatabase(Database): def escape(self, s, percent=True): """Escape quotes and percent in given string.""" if isinstance(s, bytes): - s = s.decode('utf-8') + s = s.decode("utf-8") # MariaDB's driver treats None as an empty string # So Postgres should do the same if s is None: - s = '' + s = "" if percent: s = s.replace("%", "%%") - s = s.encode('utf-8') + s = s.encode("utf-8") return str(psycopg2.extensions.QuotedString(s)) def get_database_size(self): - ''''Returns database size in MB''' - db_size = self.sql("SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", - self.db_name, as_dict=True) - return db_size[0].get('database_size') + """'Returns database size in MB""" + db_size = self.sql( + "SELECT (pg_database_size(%s) / 1024 / 1024) as database_size", self.db_name, as_dict=True + ) + return db_size[0].get("database_size") # pylint: disable=W0221 def sql(self, query, values=(), *args, **kwargs): return super(PostgresDatabase, self).sql( - modify_query(query), - modify_values(values), - *args, - **kwargs + modify_query(query), modify_values(values), *args, **kwargs ) def get_tables(self, cached=True): - return [d[0] for d in self.sql("""select table_name + return [ + d[0] + for d in self.sql( + """select table_name from information_schema.tables where table_catalog='{0}' and table_type = 'BASE TABLE' - and table_schema='{1}'""".format(frappe.conf.db_name, frappe.conf.get("db_schema", "public")))] + and table_schema='{1}'""".format( + frappe.conf.db_name, frappe.conf.get("db_schema", "public") + ) + ) + ] def format_date(self, date): if not date: - return '0001-01-01' + return "0001-01-01" if not isinstance(date, str): - date = date.strftime('%Y-%m-%d') + date = date.strftime("%Y-%m-%d") return date @@ -135,7 +144,7 @@ class PostgresDatabase(Database): # exception type @staticmethod def is_deadlocked(e): - return e.pgcode == '40P01' + return e.pgcode == "40P01" @staticmethod def is_timedout(e): @@ -148,7 +157,7 @@ class PostgresDatabase(Database): @staticmethod def is_table_missing(e): - return getattr(e, 'pgcode', None) == '42P01' + return getattr(e, "pgcode", None) == "42P01" @staticmethod def is_missing_table(e): @@ -156,31 +165,31 @@ class PostgresDatabase(Database): @staticmethod def is_missing_column(e): - return getattr(e, 'pgcode', None) == '42703' + return getattr(e, "pgcode", None) == "42703" @staticmethod def is_access_denied(e): - return e.pgcode == '42501' + return e.pgcode == "42501" @staticmethod def cant_drop_field_or_key(e): - return e.pgcode.startswith('23') + return e.pgcode.startswith("23") @staticmethod def is_duplicate_entry(e): - return e.pgcode == '23505' + return e.pgcode == "23505" @staticmethod def is_primary_key_violation(e): - return getattr(e, "pgcode", None) == '23505' and '_pkey' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == "23505" and "_pkey" in cstr(e.args[0]) @staticmethod def is_unique_key_violation(e): - return getattr(e, "pgcode", None) == '23505' and '_key' in cstr(e.args[0]) + return getattr(e, "pgcode", None) == "23505" and "_key" in cstr(e.args[0]) @staticmethod def is_duplicate_fieldname(e): - return e.pgcode == '42701' + return e.pgcode == "42701" @staticmethod def is_data_too_long(e): @@ -191,54 +200,70 @@ class PostgresDatabase(Database): new_name = get_table_name(new_name) return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`") - def describe(self, doctype: str)-> Union[List, Tuple]: + def describe(self, doctype: str) -> Union[List, Tuple]: table_name = get_table_name(doctype) - return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'") + return self.sql( + f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'" + ) - def change_column_type(self, doctype: str, column: str, type: str, nullable: bool = False) -> Union[List, Tuple]: + def change_column_type( + self, doctype: str, column: str, type: str, nullable: bool = False + ) -> Union[List, Tuple]: table_name = get_table_name(doctype) null_constraint = "SET NOT NULL" if not nullable else "DROP NOT NULL" - return self.sql(f"""ALTER TABLE "{table_name}" + return self.sql( + f"""ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}, - ALTER COLUMN "{column}" {null_constraint}""") + ALTER COLUMN "{column}" {null_constraint}""" + ) def create_auth_table(self): - self.sql_ddl("""create table if not exists "__Auth" ( + self.sql_ddl( + """create table if not exists "__Auth" ( "doctype" VARCHAR(140) NOT NULL, "name" VARCHAR(255) NOT NULL, "fieldname" VARCHAR(140) NOT NULL, "password" TEXT NOT NULL, "encrypted" INT NOT NULL DEFAULT 0, PRIMARY KEY ("doctype", "name", "fieldname") - )""") + )""" + ) def create_global_search_table(self): - if not '__global_search' in self.get_tables(): - self.sql('''create table "__global_search"( + if not "__global_search" in self.get_tables(): + self.sql( + """create table "__global_search"( doctype varchar(100), name varchar({0}), title varchar({0}), content text, route varchar({0}), published int not null default 0, - unique (doctype, name))'''.format(self.VARCHAR_LEN)) + unique (doctype, name))""".format( + self.VARCHAR_LEN + ) + ) def create_user_settings_table(self): - self.sql_ddl("""create table if not exists "__UserSettings" ( + self.sql_ddl( + """create table if not exists "__UserSettings" ( "user" VARCHAR(180) NOT NULL, "doctype" VARCHAR(180) NOT NULL, "data" TEXT, UNIQUE ("user", "doctype") - )""") + )""" + ) def create_help_table(self): - self.sql('''CREATE TABLE "help"( + self.sql( + """CREATE TABLE "help"( "path" varchar(255), "content" text, "title" text, "intro" text, - "full_path" text)''') - self.sql('''CREATE INDEX IF NOT EXISTS "help_index" ON "help" ("path")''') + "full_path" text)""" + ) + self.sql("""CREATE INDEX IF NOT EXISTS "help_index" ON "help" ("path")""") def updatedb(self, doctype, meta=None): """ @@ -249,7 +274,7 @@ class PostgresDatabase(Database): """ res = self.sql("select issingle from `tabDocType` where name='{}'".format(doctype)) if not res: - raise Exception('Wrong doctype {0} in updatedb'.format(doctype)) + raise Exception("Wrong doctype {0} in updatedb".format(doctype)) if not res[0][0]: db_table = PostgresTable(doctype, meta) @@ -260,19 +285,21 @@ class PostgresDatabase(Database): self.begin() @staticmethod - def get_on_duplicate_update(key='name'): + def get_on_duplicate_update(key="name"): if isinstance(key, list): key = '", "'.join(key) - return 'ON CONFLICT ("{key}") DO UPDATE SET '.format( - key=key - ) + return 'ON CONFLICT ("{key}") DO UPDATE SET '.format(key=key) def check_implicit_commit(self, query): - pass # postgres can run DDL in transactions without implicit commits + pass # postgres can run DDL in transactions without implicit commits def has_index(self, table_name, index_name): - return self.sql("""SELECT 1 FROM pg_indexes WHERE tablename='{table_name}' - and indexname='{index_name}' limit 1""".format(table_name=table_name, index_name=index_name)) + return self.sql( + """SELECT 1 FROM pg_indexes WHERE tablename='{table_name}' + and indexname='{index_name}' limit 1""".format( + table_name=table_name, index_name=index_name + ) + ) def add_index(self, doctype: str, fields: List, index_name: str = None): """Creates an index with given fields if not already created. @@ -289,21 +316,27 @@ class PostgresDatabase(Database): if not constraint_name: constraint_name = "unique_" + "_".join(fields) - if not self.sql(""" + if not self.sql( + """ SELECT CONSTRAINT_NAME FROM information_schema.TABLE_CONSTRAINTS WHERE table_name=%s AND constraint_type='UNIQUE' AND CONSTRAINT_NAME=%s""", - ('tab' + doctype, constraint_name)): - self.commit() - self.sql("""ALTER TABLE `tab%s` - ADD CONSTRAINT %s UNIQUE (%s)""" % (doctype, constraint_name, ", ".join(fields))) + ("tab" + doctype, constraint_name), + ): + self.commit() + self.sql( + """ALTER TABLE `tab%s` + ADD CONSTRAINT %s UNIQUE (%s)""" + % (doctype, constraint_name, ", ".join(fields)) + ) def get_table_columns_description(self, table_name): """Returns list of column and its description""" # pylint: disable=W1401 - return self.sql(''' + return self.sql( + """ SELECT a.column_name AS name, CASE LOWER(a.data_type) WHEN 'character varying' THEN CONCAT('varchar(', a.character_maximum_length ,')') @@ -323,20 +356,25 @@ class PostgresDatabase(Database): ON SUBSTRING(b.indexdef, '(.*)') LIKE CONCAT('%', a.column_name, '%') WHERE a.table_name = '{table_name}' GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length; - '''.format(table_name=table_name), as_dict=1) + """.format( + table_name=table_name + ), + as_dict=1, + ) def get_database_list(self, target): return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] + def modify_query(query): - """"Modifies query according to the requirements of postgres""" + """ "Modifies query according to the requirements of postgres""" # replace ` with " for definitions query = str(query) - query = query.replace('`', '"') + query = query.replace("`", '"') query = replace_locate_with_strpos(query) # select from requires "" - if re.search('from tab', query, flags=re.IGNORECASE): - query = re.sub(r'from tab([\w-]*)', r'from "tab\1"', query, flags=re.IGNORECASE) + if re.search("from tab", query, flags=re.IGNORECASE): + query = re.sub(r"from tab([\w-]*)", r'from "tab\1"', query, flags=re.IGNORECASE) # only find int (with/without signs), ignore decimals (with/without signs), ignore hashes (which start with numbers), # drop .0 from decimals and add quotes around them @@ -345,9 +383,12 @@ def modify_query(query): # >>> re.sub(r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query) # "c='abcd' , a >= '45', b = '-45', c = '40', d= '4500', e=3500.53, f=40psdfsd, g= '9092094312', h=12.00023 - query = re.sub(r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query) + query = re.sub( + r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query + ) return query + def modify_values(values): def stringify_value(value): if isinstance(value, int): @@ -375,8 +416,11 @@ def modify_values(values): return values + def replace_locate_with_strpos(query): # strpos is the locate equivalent in postgres - if re.search(r'locate\(', query, flags=re.IGNORECASE): - query = re.sub(r'locate\(([^,]+),([^)]+)(\)?)\)', r'strpos(\2\3, \1)', query, flags=re.IGNORECASE) + if re.search(r"locate\(", query, flags=re.IGNORECASE): + query = re.sub( + r"locate\(([^,]+),([^)]+)(\)?)\)", r"strpos(\2\3, \1)", query, flags=re.IGNORECASE + ) return query diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index b09f73300e..3432c8b548 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -1,9 +1,9 @@ import frappe from frappe import _ -from frappe.utils import cint, flt from frappe.database.schema import DBTable, get_definition from frappe.database.sequence import create_sequence from frappe.model import log_types +from frappe.utils import cint, flt class PostgresTable(DBTable): @@ -31,8 +31,9 @@ class PostgresTable(DBTable): ) # creating sequence(s) - if (not self.meta.issingle and self.meta.autoname == "autoincrement")\ - or self.doctype in log_types: + if ( + not self.meta.issingle and self.meta.autoname == "autoincrement" + ) or self.doctype in log_types: # The sequence cache is per connection. # Since we're opening and closing connections for every transaction this results in skipping the cache @@ -43,7 +44,8 @@ class PostgresTable(DBTable): # TODO: set docstatus length # create table - frappe.db.sql(f"""create table `{self.table_name}` ( + frappe.db.sql( + f"""create table `{self.table_name}` ( {name_column}, creation timestamp(6), modified timestamp(6), @@ -61,14 +63,15 @@ class PostgresTable(DBTable): def create_indexes(self): create_index_query = "" for key, col in self.columns.items(): - if (col.set_index + if ( + col.set_index and col.fieldtype in frappe.db.type_map - and frappe.db.type_map.get(col.fieldtype)[0] - not in ('text', 'longtext')): - create_index_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( - index_name=col.fieldname, - table_name=self.table_name, - field=col.fieldname + and frappe.db.type_map.get(col.fieldtype)[0] not in ("text", "longtext") + ): + create_index_query += ( + 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, table_name=self.table_name, field=col.fieldname + ) ) if create_index_query: # nosemgrep @@ -93,14 +96,16 @@ class PostgresTable(DBTable): elif col.fieldtype in ("Check"): using_clause = "USING {}::smallint".format(col.fieldname) - query.append("ALTER COLUMN `{0}` TYPE {1} {2}".format( - col.fieldname, - get_definition(col.fieldtype, precision=col.precision, length=col.length), - using_clause - )) + query.append( + "ALTER COLUMN `{0}` TYPE {1} {2}".format( + col.fieldname, + get_definition(col.fieldtype, precision=col.precision, length=col.length), + using_clause, + ) + ) for col in self.set_default: - if col.fieldname=="name": + if col.fieldname == "name": continue if col.fieldtype in ("Check", "Int"): @@ -120,29 +125,30 @@ class PostgresTable(DBTable): create_contraint_query = "" for col in self.add_index: # if index key not exists - create_contraint_query += 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( - index_name=col.fieldname, - table_name=self.table_name, - field=col.fieldname) + create_contraint_query += ( + 'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, table_name=self.table_name, field=col.fieldname + ) + ) for col in self.add_unique: # if index key not exists - create_contraint_query += 'CREATE UNIQUE INDEX IF NOT EXISTS "unique_{index_name}" ON `{table_name}`(`{field}`);'.format( - index_name=col.fieldname, - table_name=self.table_name, - field=col.fieldname + create_contraint_query += ( + 'CREATE UNIQUE INDEX IF NOT EXISTS "unique_{index_name}" ON `{table_name}`(`{field}`);'.format( + index_name=col.fieldname, table_name=self.table_name, field=col.fieldname + ) ) drop_contraint_query = "" for col in self.drop_index: # primary key - if col.fieldname != 'name': + if col.fieldname != "name": # if index key exists drop_contraint_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) for col in self.drop_unique: # primary key - if col.fieldname != 'name': + if col.fieldname != "name": # if index key exists drop_contraint_query += 'DROP INDEX IF EXISTS "unique_{}" ;'.format(col.fieldname) try: @@ -163,8 +169,9 @@ class PostgresTable(DBTable): elif frappe.db.is_duplicate_entry(e): fieldname = str(e).split("'")[-2] frappe.throw( - _("{0} field cannot be set as unique in {1}, as there are non-unique existing values") - .format(fieldname, self.table_name) + _("{0} field cannot be set as unique in {1}, as there are non-unique existing values").format( + fieldname, self.table_name + ) ) else: raise e diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 4b265e7660..90d5f72c16 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -9,45 +9,49 @@ def setup_database(force, source_sql=None, verbose=False): root_conn.sql("DROP DATABASE IF EXISTS `{0}`".format(frappe.conf.db_name)) root_conn.sql("DROP USER IF EXISTS {0}".format(frappe.conf.db_name)) root_conn.sql("CREATE DATABASE `{0}`".format(frappe.conf.db_name)) - root_conn.sql("CREATE user {0} password '{1}'".format(frappe.conf.db_name, - frappe.conf.db_password)) + root_conn.sql( + "CREATE user {0} password '{1}'".format(frappe.conf.db_name, frappe.conf.db_password) + ) root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name)) root_conn.close() bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql) frappe.connect() + def bootstrap_database(db_name, verbose, source_sql=None): frappe.connect(db_name=db_name) import_db_from_sql(source_sql, verbose) frappe.connect(db_name=db_name) - if 'tabDefaultValue' not in frappe.db.get_tables(): + if "tabDefaultValue" not in frappe.db.get_tables(): import sys + from click import secho secho( "Table 'tabDefaultValue' missing in the restored site. " "This may be due to incorrect permissions or the result of a restore from a bad backup file. " "Database not installed correctly.", - fg="red" + fg="red", ) sys.exit(1) + def import_db_from_sql(source_sql=None, verbose=False): from shutil import which - from subprocess import run, PIPE + from subprocess import PIPE, run # we can't pass psql password in arguments in postgresql as mysql. So # set password connection parameter in environment variable subprocess_env = os.environ.copy() - subprocess_env['PGPASSWORD'] = str(frappe.conf.db_password) + subprocess_env["PGPASSWORD"] = str(frappe.conf.db_password) # bootstrap db if not source_sql: - source_sql = os.path.join(os.path.dirname(__file__), 'framework_postgres.sql') + source_sql = os.path.join(os.path.dirname(__file__), "framework_postgres.sql") - pv = which('pv') + pv = which("pv") _command = ( f"psql {frappe.conf.db_name} " @@ -67,7 +71,10 @@ def import_db_from_sql(source_sql=None, verbose=False): restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE) if verbose: - print(f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}") + print( + f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}" + ) + def setup_help_database(help_db_name): root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) @@ -77,6 +84,7 @@ def setup_help_database(help_db_name): root_conn.sql("CREATE user {0} password '{1}'".format(help_db_name, help_db_name)) root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name)) + def get_root_connection(root_login=None, root_password=None): if not frappe.local.flags.root_connection: if not root_login: @@ -90,16 +98,24 @@ def get_root_connection(root_login=None, root_password=None): if not root_password: from getpass import getpass + root_password = getpass("Postgres super user password: ") - frappe.local.flags.root_connection = frappe.database.get_db(user=root_login, password=root_password) + frappe.local.flags.root_connection = frappe.database.get_db( + user=root_login, password=root_password + ) return frappe.local.flags.root_connection def drop_user_and_database(db_name, root_login, root_password): - root_conn = get_root_connection(frappe.flags.root_login or root_login, frappe.flags.root_password or root_password) + root_conn = get_root_connection( + frappe.flags.root_login or root_login, frappe.flags.root_password or root_password + ) root_conn.commit() - root_conn.sql(f"SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = %s", (db_name, )) + root_conn.sql( + f"SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = %s", + (db_name,), + ) root_conn.sql(f"DROP DATABASE IF EXISTS {db_name}") root_conn.sql(f"DROP USER IF EXISTS {db_name}") diff --git a/frappe/database/query.py b/frappe/database/query.py index 641b584932..8d8a767370 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -11,11 +11,11 @@ def like(key: str, value: str) -> frappe.qb: """Wrapper method for `LIKE` Args: - key (str): field - value (str): criterion + key (str): field + value (str): criterion Returns: - frappe.qb: `frappe.qb object with `LIKE` + frappe.qb: `frappe.qb object with `LIKE` """ return Field(key).like(value) @@ -24,11 +24,11 @@ def func_in(key: str, value: Union[List, Tuple]) -> frappe.qb: """Wrapper method for `IN` Args: - key (str): field - value (Union[int, str]): criterion + key (str): field + value (Union[int, str]): criterion Returns: - frappe.qb: `frappe.qb object with `IN` + frappe.qb: `frappe.qb object with `IN` """ return Field(key).isin(value) @@ -37,11 +37,11 @@ def not_like(key: str, value: str) -> frappe.qb: """Wrapper method for `NOT LIKE` Args: - key (str): field - value (str): criterion + key (str): field + value (str): criterion Returns: - frappe.qb: `frappe.qb object with `NOT LIKE` + frappe.qb: `frappe.qb object with `NOT LIKE` """ return Field(key).not_like(value) @@ -50,11 +50,11 @@ def func_not_in(key: str, value: Union[List, Tuple]): """Wrapper method for `NOT IN` Args: - key (str): field - value (Union[int, str]): criterion + key (str): field + value (Union[int, str]): criterion Returns: - frappe.qb: `frappe.qb object with `NOT IN` + frappe.qb: `frappe.qb object with `NOT IN` """ return Field(key).notin(value) @@ -63,11 +63,11 @@ def func_regex(key: str, value: str) -> frappe.qb: """Wrapper method for `REGEX` Args: - key (str): field - value (str): criterion + key (str): field + value (str): criterion Returns: - frappe.qb: `frappe.qb object with `REGEX` + frappe.qb: `frappe.qb object with `REGEX` """ return Field(key).regex(value) @@ -76,23 +76,24 @@ def func_between(key: str, value: Union[List, Tuple]) -> frappe.qb: """Wrapper method for `BETWEEN` Args: - key (str): field - value (Union[int, str]): criterion + key (str): field + value (Union[int, str]): criterion Returns: - frappe.qb: `frappe.qb object with `BETWEEN` + frappe.qb: `frappe.qb object with `BETWEEN` """ return Field(key)[slice(*value)] + def make_function(key: Any, value: Union[int, str]): """returns fucntion query Args: - key (Any): field - value (Union[int, str]): criterion + key (Any): field + value (Union[int, str]): criterion Returns: - frappe.qb: frappe.qb object + frappe.qb: frappe.qb object """ return OPERATOR_MAP[value[0]](key, value[1]) @@ -101,10 +102,10 @@ def change_orderby(order: str): """Convert orderby to standart Order object Args: - order (str): Field, order + order (str): Field, order Returns: - tuple: field, order + tuple: field, order """ order = order.split() if order[1].lower() == "asc": @@ -139,10 +140,10 @@ class Query: """Get initial table object Args: - table (str): DocType + table (str): DocType Returns: - frappe.qb: DocType with initial condition + frappe.qb: DocType with initial condition """ if kwargs.get("update"): return frappe.qb.update(table) @@ -154,11 +155,11 @@ class Query: """Generate filters from Criterion objects Args: - table (str): DocType - criterion (Criterion): Filters + table (str): DocType + criterion (Criterion): Filters Returns: - frappe.qb: condition object + frappe.qb: condition object """ condition = self.add_conditions(self.get_condition(table, **kwargs), **kwargs) return condition.where(criterion) @@ -167,10 +168,10 @@ class Query: """Adding additional conditions Args: - conditions (frappe.qb): built conditions + conditions (frappe.qb): built conditions Returns: - conditions (frappe.qb): frappe.qb object + conditions (frappe.qb): frappe.qb object """ if kwargs.get("orderby"): orderby = kwargs.get("orderby") @@ -194,8 +195,8 @@ class Query: """Build conditions using the given Lists or Tuple filters Args: - table (str): DocType - filters (Union[List, Tuple], optional): Filters. Defaults to None. + table (str): DocType + filters (Union[List, Tuple], optional): Filters. Defaults to None. """ conditions = self.get_condition(table, **kwargs) if not filters: @@ -215,15 +216,17 @@ class Query: return self.add_conditions(conditions, **kwargs) - def dict_query(self, table: str, filters: Dict[str, Union[str, int]] = None, **kwargs) -> frappe.qb: + def dict_query( + self, table: str, filters: Dict[str, Union[str, int]] = None, **kwargs + ) -> frappe.qb: """Build conditions using the given dictionary filters Args: - table (str): DocType - filters (Dict[str, Union[str, int]], optional): Filters. Defaults to None. + table (str): DocType + filters (Dict[str, Union[str, int]], optional): Filters. Defaults to None. Returns: - frappe.qb: conditions object + frappe.qb: conditions object """ conditions = self.get_condition(table, **kwargs) if not filters: @@ -255,19 +258,16 @@ class Query: return self.add_conditions(conditions, **kwargs) def build_conditions( - self, - table: str, - filters: Union[Dict[str, Union[str, int]], str, int] = None, - **kwargs + self, table: str, filters: Union[Dict[str, Union[str, int]], str, int] = None, **kwargs ) -> frappe.qb: """Build conditions for sql query Args: - filters (Union[Dict[str, Union[str, int]], str, int]): conditions in Dict - table (str): DocType + filters (Union[Dict[str, Union[str, int]], str, int]): conditions in Dict + table (str): DocType Returns: - frappe.qb: frappe.qb conditions object + frappe.qb: frappe.qb conditions object """ if isinstance(filters, int) or isinstance(filters, str): filters = {"name": str(filters)} @@ -326,9 +326,7 @@ class Permission: user=kwargs.get("user"), parent_doctype=kwargs.get("parent_doctype"), ): - frappe.throw( - _("Insufficient Permission for {0}").format(frappe.bold(dt)) - ) + frappe.throw(_("Insufficient Permission for {0}").format(frappe.bold(dt))) @staticmethod def get_tables_from_query(query: str): diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 7cab8d42b2..19af447aae 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -1,16 +1,18 @@ import re -import frappe +import frappe from frappe import _ -from frappe.utils import cstr, cint, flt +from frappe.utils import cint, cstr, flt + +class InvalidColumnName(frappe.ValidationError): + pass -class InvalidColumnName(frappe.ValidationError): pass class DBTable: def __init__(self, doctype, meta=None): self.doctype = doctype - self.table_name = 'tab{}'.format(doctype) + self.table_name = "tab{}".format(doctype) self.meta = meta or frappe.get_meta(doctype, False) self.columns = {} self.current_columns = {} @@ -29,13 +31,13 @@ class DBTable: self.get_columns_from_docfields() def sync(self): - if self.meta.get('is_virtual'): + if self.meta.get("is_virtual"): # no schema to sync for virtual doctypes return if self.is_new(): self.create() else: - frappe.cache().hdel('table_columns', self.table_name) + frappe.cache().hdel("table_columns", self.table_name) self.alter() def create(self): @@ -48,56 +50,51 @@ class DBTable: if k not in column_list: d = self.columns[k].get_definition() if d: - ret.append('`'+ k + '` ' + d) + ret.append("`" + k + "` " + d) column_list.append(k) return ret def get_index_definitions(self): ret = [] for key, col in self.columns.items(): - if (col.set_index + if ( + col.set_index and not col.unique and col.fieldtype in frappe.db.type_map - and frappe.db.type_map.get(col.fieldtype)[0] - not in ('text', 'longtext')): - ret.append('index `' + key + '`(`' + key + '`)') + and frappe.db.type_map.get(col.fieldtype)[0] not in ("text", "longtext") + ): + ret.append("index `" + key + "`(`" + key + "`)") return ret def get_columns_from_docfields(self): """ - get columns from docfields and custom fields + get columns from docfields and custom fields """ fields = self.meta.get_fieldnames_with_value(with_field_meta=True) # optional fields like _comments - if not self.meta.get('istable'): + if not self.meta.get("istable"): for fieldname in frappe.db.OPTIONAL_COLUMNS: - fields.append({ - "fieldname": fieldname, - "fieldtype": "Text" - }) + fields.append({"fieldname": fieldname, "fieldtype": "Text"}) # add _seen column if track_seen - if self.meta.get('track_seen'): - fields.append({ - 'fieldname': '_seen', - 'fieldtype': 'Text' - }) + if self.meta.get("track_seen"): + fields.append({"fieldname": "_seen", "fieldtype": "Text"}) for field in fields: if field.get("is_virtual"): continue - self.columns[field.get('fieldname')] = DbColumn( + self.columns[field.get("fieldname")] = DbColumn( self, - field.get('fieldname'), - field.get('fieldtype'), - field.get('length'), - field.get('default'), - field.get('search_index'), - field.get('options'), - field.get('unique'), - field.get('precision') + field.get("fieldname"), + field.get("fieldtype"), + field.get("length"), + field.get("default"), + field.get("search_index"), + field.get("options"), + field.get("unique"), + field.get("precision"), ) def validate(self): @@ -107,19 +104,22 @@ class DBTable: self.setup_table_columns() - columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in - frappe.db.STANDARD_VARCHAR_COLUMNS] + columns = [ + frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in frappe.db.STANDARD_VARCHAR_COLUMNS + ] if self.meta.get("istable"): - columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in - frappe.db.CHILD_TABLE_COLUMNS] + columns += [ + frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in frappe.db.CHILD_TABLE_COLUMNS + ] columns += self.columns.values() for col in columns: if len(col.fieldname) >= 64: - frappe.throw(_("Fieldname is limited to 64 characters ({0})") - .format(frappe.bold(col.fieldname))) + frappe.throw( + _("Fieldname is limited to 64 characters ({0})").format(frappe.bold(col.fieldname)) + ) - if 'varchar' in frappe.db.type_map.get(col.fieldtype, ()): + if "varchar" in frappe.db.type_map.get(col.fieldtype, ()): # validate length range new_length = cint(col.length) or cint(frappe.db.VARCHAR_LEN) @@ -130,7 +130,7 @@ class DBTable: if not current_col: continue current_type = self.current_columns[col.fieldname]["type"] - current_length = re.findall(r'varchar\(([\d]+)\)', current_type) + current_length = re.findall(r"varchar\(([\d]+)\)", current_type) if not current_length: # case when the field is no longer a varchar continue @@ -138,8 +138,11 @@ class DBTable: if cint(current_length) != cint(new_length): try: # check for truncation - max_length = frappe.db.sql("""SELECT MAX(CHAR_LENGTH(`{fieldname}`)) FROM `tab{doctype}`""" - .format(fieldname=col.fieldname, doctype=self.doctype)) + max_length = frappe.db.sql( + """SELECT MAX(CHAR_LENGTH(`{fieldname}`)) FROM `tab{doctype}`""".format( + fieldname=col.fieldname, doctype=self.doctype + ) + ) except frappe.db.InternalError as e: if frappe.db.is_missing_column(e): @@ -150,8 +153,9 @@ class DBTable: if max_length and max_length[0][0] and max_length[0][0] > new_length: if col.fieldname in self.columns: self.columns[col.fieldname].length = current_length - info_message = _("Reverting length to {0} for '{1}' in '{2}'. Setting the length as {3} will cause truncation of data.") \ - .format(current_length, col.fieldname, self.doctype, new_length) + info_message = _( + "Reverting length to {0} for '{1}' in '{2}'. Setting the length as {3} will cause truncation of data." + ).format(current_length, col.fieldname, self.doctype, new_length) frappe.msgprint(info_message) def is_new(self): @@ -167,8 +171,9 @@ class DBTable: class DbColumn: - def __init__(self, table, fieldname, fieldtype, length, default, - set_index, options, unique, precision): + def __init__( + self, table, fieldname, fieldtype, length, default, set_index, options, unique, precision + ): self.table = table self.fieldname = fieldname self.fieldtype = fieldtype @@ -187,18 +192,22 @@ class DbColumn: if self.fieldtype in ("Check", "Int"): default_value = cint(self.default) or 0 - column_def += ' not null default {0}'.format(default_value) + column_def += " not null default {0}".format(default_value) elif self.fieldtype in ("Currency", "Float", "Percent"): default_value = flt(self.default) or 0 - column_def += ' not null default {0}'.format(default_value) + column_def += " not null default {0}".format(default_value) - elif self.default and (self.default not in frappe.db.DEFAULT_SHORTCUTS) \ - and not cstr(self.default).startswith(":") and column_def not in ('text', 'longtext'): + elif ( + self.default + and (self.default not in frappe.db.DEFAULT_SHORTCUTS) + and not cstr(self.default).startswith(":") + and column_def not in ("text", "longtext") + ): column_def += " default {}".format(frappe.db.escape(self.default)) - if self.unique and (column_def not in ('text', 'longtext')): - column_def += ' unique' + if self.unique and (column_def not in ("text", "longtext")): + column_def += " unique" return column_def @@ -214,7 +223,7 @@ class DbColumn: self.fieldname = validate_column_name(self.fieldname) self.table.add_column.append(self) - if column_type not in ('text', 'longtext'): + if column_type not in ("text", "longtext"): if self.unique: self.table.add_unique.append(self) if self.set_index: @@ -222,34 +231,36 @@ class DbColumn: return # type - if (current_def['type'] != column_type): + if current_def["type"] != column_type: self.table.change_type.append(self) # unique - if ((self.unique and not current_def['unique']) and column_type not in ('text', 'longtext')): + if (self.unique and not current_def["unique"]) and column_type not in ("text", "longtext"): self.table.add_unique.append(self) - elif (current_def['unique'] and not self.unique) and column_type not in ('text', 'longtext'): + elif (current_def["unique"] and not self.unique) and column_type not in ("text", "longtext"): self.table.drop_unique.append(self) # default - if (self.default_changed(current_def) + if ( + self.default_changed(current_def) and (self.default not in frappe.db.DEFAULT_SHORTCUTS) and not cstr(self.default).startswith(":") - and not (column_type in ['text','longtext'])): + and not (column_type in ["text", "longtext"]) + ): self.table.set_default.append(self) # index should be applied or dropped irrespective of type change - if (current_def['index'] and not self.set_index) and column_type not in ('text', 'longtext'): + if (current_def["index"] and not self.set_index) and column_type not in ("text", "longtext"): self.table.drop_index.append(self) - elif (not current_def['index'] and self.set_index) and not (column_type in ('text', 'longtext')): + elif (not current_def["index"] and self.set_index) and not (column_type in ("text", "longtext")): self.table.add_index.append(self) def default_changed(self, current_def): - if "decimal" in current_def['type']: + if "decimal" in current_def["type"]: return self.default_changed_for_decimal(current_def) else: - cur_default = current_def['default'] + cur_default = current_def["default"] new_default = self.default if cur_default == "NULL" or cur_default is None: cur_default = None @@ -259,21 +270,21 @@ class DbColumn: cur_default = cur_default.lstrip("'").rstrip("'") fieldtype = self.fieldtype - if fieldtype in ['Int', 'Check']: + if fieldtype in ["Int", "Check"]: cur_default = cint(cur_default) new_default = cint(new_default) - elif fieldtype in ['Currency', 'Float', 'Percent']: + elif fieldtype in ["Currency", "Float", "Percent"]: cur_default = flt(cur_default) new_default = flt(new_default) return cur_default != new_default def default_changed_for_decimal(self, current_def): try: - if current_def['default'] in ("", None) and self.default in ("", None): + if current_def["default"] in ("", None) and self.default in ("", None): # both none, empty return False - elif current_def['default'] in ("", None): + elif current_def["default"] in ("", None): try: # check if new default value is valid float(self.default) @@ -287,22 +298,29 @@ class DbColumn: else: # NOTE float() raise ValueError when "" or None is passed - return float(current_def['default'])!=float(self.default) + return float(current_def["default"]) != float(self.default) except TypeError: return True + def validate_column_name(n): special_characters = re.findall(r"[\W]", n, re.UNICODE) if special_characters: special_characters = ", ".join('"{0}"'.format(c) for c in special_characters) - frappe.throw(_("Fieldname {0} cannot have special characters like {1}").format( - frappe.bold(cstr(n)), special_characters), frappe.db.InvalidColumnName) + frappe.throw( + _("Fieldname {0} cannot have special characters like {1}").format( + frappe.bold(cstr(n)), special_characters + ), + frappe.db.InvalidColumnName, + ) return n + def validate_column_length(fieldname): if len(fieldname) > frappe.db.MAX_COLUMN_LENGTH: frappe.throw(_("Fieldname is limited to 64 characters ({0})").format(fieldname)) + def get_definition(fieldtype, precision=None, length=None): d = frappe.db.type_map.get(fieldtype) @@ -320,7 +338,7 @@ def get_definition(fieldtype, precision=None, length=None): # This check needs to exist for backward compatibility. # Till V13, default size used for float, currency and percent are (18, 6). if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6: - size = '21,9' + size = "21,9" if length: if coltype == "varchar": @@ -336,14 +354,9 @@ def get_definition(fieldtype, precision=None, length=None): return coltype + def add_column( - doctype, - column_name, - fieldtype, - precision=None, - length=None, - default=None, - not_null=False + doctype, column_name, fieldtype, precision=None, length=None, default=None, not_null=False ): if column_name in frappe.db.get_table_columns(doctype): # already exists @@ -354,7 +367,7 @@ def add_column( query = "alter table `tab%s` add column %s %s" % ( doctype, column_name, - get_definition(fieldtype, precision, length) + get_definition(fieldtype, precision, length), ) if not_null: diff --git a/frappe/database/sequence.py b/frappe/database/sequence.py index 334fd3d71e..c4789dbdaf 100644 --- a/frappe/database/sequence.py +++ b/frappe/database/sequence.py @@ -11,7 +11,7 @@ def create_sequence( start_value: int = 0, increment_by: int = 0, min_value: int = 0, - max_value: int = 0 + max_value: int = 0, ) -> str: query = "create sequence" @@ -57,16 +57,12 @@ def create_sequence( def get_next_val(doctype_name: str, slug: str = "_id_seq") -> int: if db.db_type == "postgres": - return db.sql(f"select nextval(\'\"{scrub(doctype_name + slug)}\"\')")[0][0] + return db.sql(f"select nextval('\"{scrub(doctype_name + slug)}\"')")[0][0] return db.sql(f"select nextval(`{scrub(doctype_name + slug)}`)")[0][0] def set_next_val( - doctype_name: str, - next_val: int, - *, - slug: str = "_id_seq", - is_val_used :bool = False + doctype_name: str, next_val: int, *, slug: str = "_id_seq", is_val_used: bool = False ) -> None: if not is_val_used: diff --git a/frappe/defaults.py b/frappe/defaults.py index e249ef2099..4bbdcf25c6 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -2,25 +2,28 @@ # License: MIT. See LICENSE import frappe -from frappe.desk.notifications import clear_notifications from frappe.cache_manager import clear_defaults_cache, common_default_keys +from frappe.desk.notifications import clear_notifications from frappe.query_builder import DocType # Note: DefaultValue records are identified by parenttype # __default, __global or 'User Permission' + def set_user_default(key, value, user=None, parenttype=None): set_default(key, value, user or frappe.session.user, parenttype) + def add_user_default(key, value, user=None, parenttype=None): add_default(key, value, user or frappe.session.user, parenttype) + def get_user_default(key, user=None): user_defaults = get_defaults(user or frappe.session.user) d = user_defaults.get(key, None) if is_a_user_permission_key(key): - if d and isinstance(d, (list, tuple)) and len(d)==1: + if d and isinstance(d, (list, tuple)) and len(d) == 1: # Use User Permission value when only when it has a single value d = d[0] @@ -33,12 +36,13 @@ def get_user_default(key, user=None): return value + def get_user_default_as_list(key, user=None): user_defaults = get_defaults(user or frappe.session.user) d = user_defaults.get(key, None) if is_a_user_permission_key(key): - if d and isinstance(d, (list, tuple)) and len(d)==1: + if d and isinstance(d, (list, tuple)) and len(d) == 1: # Use User Permission value when only when it has a single value d = [d[0]] @@ -52,9 +56,11 @@ def get_user_default_as_list(key, user=None): return values + def is_a_user_permission_key(key): return ":" not in key and key != frappe.scrub(key) + def not_in_user_permission(key, value, user=None): # returns true or false based on if value exist in user permission user = user or frappe.session.user @@ -62,17 +68,22 @@ def not_in_user_permission(key, value, user=None): for perm in user_permission: # doc found in user permission - if perm.get('doc') == value: return False + if perm.get("doc") == value: + return False # return true only if user_permission exists return True if user_permission else False + def get_user_permissions(user=None): - from frappe.core.doctype.user_permission.user_permission \ - import get_user_permissions as _get_user_permissions - '''Return frappe.core.doctype.user_permissions.user_permissions._get_user_permissions (kept for backward compatibility)''' + from frappe.core.doctype.user_permission.user_permission import ( + get_user_permissions as _get_user_permissions, + ) + + """Return frappe.core.doctype.user_permissions.user_permissions._get_user_permissions (kept for backward compatibility)""" return _get_user_permissions(user) + def get_defaults(user=None): globald = get_defaults_for() @@ -87,17 +98,22 @@ def get_defaults(user=None): return globald + def clear_user_default(key, user=None): clear_default(key, parent=user or frappe.session.user) + # Global + def set_global_default(key, value): set_default(key, value, "__default") + def add_global_default(key, value): add_default(key, value, "__default") + def get_global_default(key): d = get_defaults().get(key, None) @@ -107,8 +123,10 @@ def get_global_default(key): return value + # Common + def set_default(key, value, parent, parenttype="__default"): """Override or add a default value. Adds default value in table `tabDefaultValue`. @@ -118,31 +136,36 @@ def set_default(key, value, parent, parenttype="__default"): :param parent: Usually, **User** to whom the default belongs. :param parenttype: [optional] default is `__default`.""" table = DocType("DefaultValue") - key_exists = frappe.qb.from_(table).where( - (table.defkey == key) & (table.parent == parent) - ).select(table.defkey).for_update().run() + key_exists = ( + frappe.qb.from_(table) + .where((table.defkey == key) & (table.parent == parent)) + .select(table.defkey) + .for_update() + .run() + ) if key_exists: - frappe.db.delete("DefaultValue", { - "defkey": key, - "parent": parent - }) + frappe.db.delete("DefaultValue", {"defkey": key, "parent": parent}) if value is not None: add_default(key, value, parent) else: _clear_cache(parent) + def add_default(key, value, parent, parenttype=None): - d = frappe.get_doc({ - "doctype": "DefaultValue", - "parent": parent, - "parenttype": parenttype or "__default", - "parentfield": "system_defaults", - "defkey": key, - "defvalue": value - }) + d = frappe.get_doc( + { + "doctype": "DefaultValue", + "parent": parent, + "parenttype": parenttype or "__default", + "parentfield": "system_defaults", + "defkey": key, + "defvalue": value, + } + ) d.insert(ignore_permissions=True) _clear_cache(parent) + def clear_default(key=None, value=None, parent=None, name=None, parenttype=None): """Clear a default value by any of the given parameters and delete caches. @@ -183,6 +206,7 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) _clear_cache(parent) + def get_defaults_for(parent="__default"): """get all defaults""" defaults = frappe.cache().hget("defaults", parent) @@ -190,11 +214,13 @@ def get_defaults_for(parent="__default"): if defaults is None: # sort descending because first default must get precedence table = DocType("DefaultValue") - res = frappe.qb.from_(table).where( - table.parent == parent - ).select( - table.defkey, table.defvalue - ).orderby("creation").run(as_dict=True) + res = ( + frappe.qb.from_(table) + .where(table.parent == parent) + .select(table.defkey, table.defvalue) + .orderby("creation") + .run(as_dict=True) + ) defaults = frappe._dict({}) for d in res: @@ -213,6 +239,7 @@ def get_defaults_for(parent="__default"): return defaults + def _clear_cache(parent): if parent in common_default_keys: frappe.clear_cache() diff --git a/frappe/deferred_insert.py b/frappe/deferred_insert.py index b1338a73b0..48f8ddae26 100644 --- a/frappe/deferred_insert.py +++ b/frappe/deferred_insert.py @@ -1,13 +1,15 @@ import json -import frappe +import frappe from frappe.utils import cstr -queue_prefix = 'insert_queue_for_' +queue_prefix = "insert_queue_for_" + def deferred_insert(doctype, records): frappe.cache().rpush(queue_prefix + doctype, records) + def save_to_db(): queue_keys = frappe.cache().get_keys(queue_prefix) for key in queue_keys: @@ -16,7 +18,7 @@ def save_to_db(): doctype = get_doctype_name(key) while frappe.cache().llen(queue_key) > 0 and record_count <= 500: records = frappe.cache().lpop(queue_key) - records = json.loads(records.decode('utf-8')) + records = json.loads(records.decode("utf-8")) if isinstance(records, dict): record_count += 1 insert_record(records, doctype) @@ -27,17 +29,20 @@ def save_to_db(): frappe.db.commit() + def insert_record(record, doctype): - if not record.get('doctype'): - record['doctype'] = doctype + if not record.get("doctype"): + record["doctype"] = doctype try: doc = frappe.get_doc(record) doc.insert() except Exception as e: print(e, doctype) + def get_key_name(key): - return cstr(key).split('|')[1] + return cstr(key).split("|")[1] + def get_doctype_name(key): return cstr(key).split(queue_prefix)[1] diff --git a/frappe/desk/__init__.py b/frappe/desk/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/desk/__init__.py +++ b/frappe/desk/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/desk/calendar.py b/frappe/desk/calendar.py index 66e6dd8434..d8c058536d 100644 --- a/frappe/desk/calendar.py +++ b/frappe/desk/calendar.py @@ -1,9 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -import json + @frappe.whitelist() def update_event(args, field_map): @@ -15,13 +17,16 @@ def update_event(args, field_map): w.set(field_map.end, args.get(field_map.end)) w.save() + def get_event_conditions(doctype, filters=None): """Returns SQL conditions with user permissions and filters for event queries""" from frappe.desk.reportview import get_filters_cond + if not frappe.has_permission(doctype): frappe.throw(_("Not Permitted"), frappe.PermissionError) - return get_filters_cond(doctype, filters, [], with_match_conditions = True) + return get_filters_cond(doctype, filters, [], with_match_conditions=True) + @frappe.whitelist() def get_events(doctype, start, end, field_map, filters=None, fields=None): @@ -31,14 +36,12 @@ def get_events(doctype, start, end, field_map, filters=None, fields=None): doc_meta = frappe.get_meta(doctype) for d in doc_meta.fields: if d.fieldtype == "Color": - field_map.update({ - "color": d.fieldname - }) + field_map.update({"color": d.fieldname}) filters = json.loads(filters) if filters else [] if not fields: - fields = [field_map.start, field_map.end, field_map.title, 'name'] + fields = [field_map.start, field_map.end, field_map.title, "name"] if field_map.color: fields.append(field_map.color) @@ -47,8 +50,8 @@ def get_events(doctype, start, end, field_map, filters=None, fields=None): end_date = "ifnull(%s, '2199-12-31 00:00:00')" % field_map.end filters += [ - [doctype, start_date, '<=', end], - [doctype, end_date, '>=', start], + [doctype, start_date, "<=", end], + [doctype, end_date, ">=", start], ] fields = list({field for field in fields if field}) return frappe.get_list(doctype, fields=fields, filters=filters) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index a01008280c..ad0bd549d8 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -4,30 +4,31 @@ import frappe from frappe.translate import send_translations + @frappe.whitelist() def get(name): """ - Return the :term:`doclist` of the `Page` specified by `name` + Return the :term:`doclist` of the `Page` specified by `name` """ - page = frappe.get_doc('Page', name) + page = frappe.get_doc("Page", name) if page.is_permitted(): page.load_assets() docs = frappe._dict(page.as_dict()) - if getattr(page, '_dynamic_page', None): - docs['_dynamic_page'] = 1 + if getattr(page, "_dynamic_page", None): + docs["_dynamic_page"] = 1 return docs else: - frappe.response['403'] = 1 - raise frappe.PermissionError('No read permission for Page %s' %(page.title or name)) + frappe.response["403"] = 1 + raise frappe.PermissionError("No read permission for Page %s" % (page.title or name)) @frappe.whitelist(allow_guest=True) def getpage(): """ - Load the page from `frappe.form` and send it via `frappe.response` + Load the page from `frappe.form` and send it via `frappe.response` """ - page = frappe.form_dict.get('name') + page = frappe.form_dict.get("name") doc = get(page) # load translations @@ -36,6 +37,7 @@ def getpage(): frappe.response.docs.append(doc) + def has_permission(page): if frappe.session.user == "Administrator" or "System Manager" in frappe.get_roles(): return True diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 4164db679d..385151f754 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -2,17 +2,19 @@ # License: MIT. See LICENSE # Author - Shivam Mishra +from functools import wraps +from json import dumps, loads + import frappe -from json import loads, dumps -from frappe import _, DoesNotExistError, ValidationError, _dict +from frappe import DoesNotExistError, ValidationError, _, _dict from frappe.boot import get_allowed_pages, get_allowed_reports -from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles -from functools import wraps from frappe.cache_manager import ( build_domain_restriced_doctype_cache, build_domain_restriced_page_cache, - build_table_count_cache + build_table_count_cache, ) +from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles + def handle_not_exist(fn): @wraps(fn) @@ -29,40 +31,53 @@ def handle_not_exist(fn): class Workspace: def __init__(self, page, minimal=False): - self.page_name = page.get('name') - self.page_title = page.get('title') - self.public_page = page.get('public') + self.page_name = page.get("name") + self.page_title = page.get("title") + self.public_page = page.get("public") self.workspace_manager = "Workspace Manager" in frappe.get_roles() self.user = frappe.get_user() - self.allowed_modules = self.get_cached('user_allowed_modules', self.get_allowed_modules) + self.allowed_modules = self.get_cached("user_allowed_modules", self.get_allowed_modules) self.doc = frappe.get_cached_doc("Workspace", self.page_name) - if self.doc and self.doc.module and self.doc.module not in self.allowed_modules and not self.workspace_manager: + if ( + self.doc + and self.doc.module + and self.doc.module not in self.allowed_modules + and not self.workspace_manager + ): raise frappe.PermissionError - self.can_read = self.get_cached('user_perm_can_read', self.get_can_read_items) + self.can_read = self.get_cached("user_perm_can_read", self.get_can_read_items) self.allowed_pages = get_allowed_pages(cache=True) self.allowed_reports = get_allowed_reports(cache=True) if not minimal: if self.doc.content: - self.onboarding_list = [x['data']['onboarding_name'] for x in loads(self.doc.content) if x['type'] == 'onboarding'] + self.onboarding_list = [ + x["data"]["onboarding_name"] for x in loads(self.doc.content) if x["type"] == "onboarding" + ] self.onboardings = [] self.table_counts = get_table_with_counts() - self.restricted_doctypes = frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache() - self.restricted_pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache() + self.restricted_doctypes = ( + frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache() + ) + self.restricted_pages = ( + frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache() + ) def is_permitted(self): """Returns true if Has Role is not set or the user is allowed.""" from frappe.utils import has_common - allowed = [d.role for d in frappe.get_all("Has Role", fields=["role"], filters={"parent": self.doc.name})] + allowed = [ + d.role for d in frappe.get_all("Has Role", fields=["role"], filters={"parent": self.doc.name}) + ] - custom_roles = get_custom_allowed_roles('page', self.doc.name) + custom_roles = get_custom_allowed_roles("page", self.doc.name) allowed.extend(custom_roles) if not allowed: @@ -130,9 +145,9 @@ class Workspace: item_type = item_type.lower() if item_type == "doctype": - return (name in self.can_read or [] and name in self.restricted_doctypes or []) + return name in self.can_read or [] and name in self.restricted_doctypes or [] if item_type == "page": - return (name in self.allowed_pages and name in self.restricted_pages) + return name in self.allowed_pages and name in self.restricted_pages if item_type == "report": return name in self.allowed_reports if item_type == "help": @@ -143,27 +158,19 @@ class Workspace: return False def build_workspace(self): - self.cards = { - 'items': self.get_links() - } + self.cards = {"items": self.get_links()} - self.charts = { - 'items': self.get_charts() - } + self.charts = {"items": self.get_charts()} - self.shortcuts = { - 'items': self.get_shortcuts() - } + self.shortcuts = {"items": self.get_shortcuts()} - self.onboardings = { - 'items': self.get_onboardings() - } + self.onboardings = {"items": self.get_onboardings()} def _doctype_contains_a_record(self, name): exists = self.table_counts.get(name, False) if not exists and frappe.db.exists(name): - if not frappe.db.get_value('DocType', name, 'issingle'): + if not frappe.db.get_value("DocType", name, "issingle"): exists = bool(frappe.db.get_all(name, limit=1)) else: exists = True @@ -210,7 +217,7 @@ class Workspace: new_items = [] card = _dict(card) - links = card.get('links', []) + links = card.get("links", []) for item in links: item = _dict(item) @@ -242,7 +249,7 @@ class Workspace: charts = self.doc.charts for chart in charts: - if frappe.has_permission('Dashboard Chart', doc=chart.chart_name): + if frappe.has_permission("Dashboard Chart", doc=chart.chart_name): # Translate label chart.label = _(chart.label) if chart.label else _(chart.chart_name) all_charts.append(chart) @@ -251,7 +258,6 @@ class Workspace: @handle_not_exist def get_shortcuts(self): - def _in_active_domains(item): if not item.restrict_to_domain: return True @@ -267,9 +273,9 @@ class Workspace: if item.type == "Report": report = self.allowed_reports.get(item.link_to, {}) if report.get("report_type") in ["Query Report", "Script Report", "Custom Report"]: - new_item['is_query_report'] = 1 + new_item["is_query_report"] = 1 else: - new_item['ref_doctype'] = report.get('ref_doctype') + new_item["ref_doctype"] = report.get("ref_doctype") # Translate label new_item["label"] = _(item.label) if item.label else _(item.link_to) @@ -285,12 +291,12 @@ class Workspace: onboarding_doc = self.get_onboarding_doc(onboarding) if onboarding_doc: item = { - 'label': _(onboarding), - 'title': _(onboarding_doc.title), - 'subtitle': _(onboarding_doc.subtitle), - 'success': _(onboarding_doc.success_message), - 'docs_url': onboarding_doc.documentation_url, - 'items': self.get_onboarding_steps(onboarding_doc) + "label": _(onboarding), + "title": _(onboarding_doc.title), + "subtitle": _(onboarding_doc.subtitle), + "success": _(onboarding_doc.success_message), + "docs_url": onboarding_doc.documentation_url, + "items": self.get_onboarding_steps(onboarding_doc), } self.onboardings.append(item) return self.onboardings @@ -302,7 +308,9 @@ class Workspace: step = doc.as_dict().copy() step.label = _(doc.title) if step.action == "Create Entry": - step.is_submittable = frappe.db.get_value("DocType", step.reference_document, 'is_submittable', cache=True) + step.is_submittable = frappe.db.get_value( + "DocType", step.reference_document, "is_submittable", cache=True + ) steps.append(step) return steps @@ -315,36 +323,37 @@ def get_desktop_page(page): on desk. Args: - page (json): page data + page (json): page data Returns: - dict: dictionary of cards, charts and shortcuts to be displayed on website + dict: dictionary of cards, charts and shortcuts to be displayed on website """ try: workspace = Workspace(loads(page)) workspace.build_workspace() return { - 'charts': workspace.charts, - 'shortcuts': workspace.shortcuts, - 'cards': workspace.cards, - 'onboardings': workspace.onboardings + "charts": workspace.charts, + "shortcuts": workspace.shortcuts, + "cards": workspace.cards, + "onboardings": workspace.onboardings, } except DoesNotExistError: frappe.log_error(frappe.get_traceback()) return {} + @frappe.whitelist() def get_workspace_sidebar_items(): """Get list of sidebar items for desk""" has_access = "Workspace Manager" in frappe.get_roles() # don't get domain restricted pages - blocked_modules = frappe.get_doc('User', frappe.session.user).get_blocked_modules() - blocked_modules.append('Dummy Module') + blocked_modules = frappe.get_doc("User", frappe.session.user).get_blocked_modules() + blocked_modules.append("Dummy Module") filters = { - 'restrict_to_domain': ['in', frappe.get_active_domains()], - 'module': ['not in', blocked_modules] + "restrict_to_domain": ["in", frappe.get_active_domains()], + "module": ["not in", blocked_modules], } if has_access: @@ -352,8 +361,10 @@ def get_workspace_sidebar_items(): # pages sorted based on sequence id order_by = "sequence_id asc" - fields = ["name", "title", "for_user", "parent_page", "content", "public", "module", "icon"] - all_pages = frappe.get_all("Workspace", fields=fields, filters=filters, order_by=order_by, ignore_permissions=True) + fields = ["name", "title", "for_user", "parent_page", "content", "public", "module", "icon"] + all_pages = frappe.get_all( + "Workspace", fields=fields, filters=filters, order_by=order_by, ignore_permissions=True + ) pages = [] private_pages = [] @@ -366,13 +377,14 @@ def get_workspace_sidebar_items(): pages.append(page) elif page.for_user == frappe.session.user: private_pages.append(page) - page['label'] = _(page.get('name')) + page["label"] = _(page.get("name")) except frappe.PermissionError: pass if private_pages: pages.extend(private_pages) - return {'pages': pages, 'has_access': has_access} + return {"pages": pages, "has_access": has_access} + def get_table_with_counts(): counts = frappe.cache().get_value("information_schema:counts") @@ -381,52 +393,57 @@ def get_table_with_counts(): return counts + def get_custom_reports_and_doctypes(module): return [ - _dict({ - "label": _("Custom Documents"), - "links": get_custom_doctype_list(module) - }), - _dict({ - "label": _("Custom Reports"), - "links": get_custom_report_list(module) - }), + _dict({"label": _("Custom Documents"), "links": get_custom_doctype_list(module)}), + _dict({"label": _("Custom Reports"), "links": get_custom_report_list(module)}), ] + def get_custom_doctype_list(module): - doctypes = frappe.get_all("DocType", fields=["name"], filters={"custom": 1, "istable": 0, "module": module}, order_by="name") + doctypes = frappe.get_all( + "DocType", + fields=["name"], + filters={"custom": 1, "istable": 0, "module": module}, + order_by="name", + ) out = [] for d in doctypes: - out.append({ - "type": "Link", - "link_type": "doctype", - "link_to": d.name, - "label": _(d.name) - }) + out.append({"type": "Link", "link_type": "doctype", "link_to": d.name, "label": _(d.name)}) return out + def get_custom_report_list(module): """Returns list on new style reports for modules.""" - reports = frappe.get_all("Report", fields=["name", "ref_doctype", "report_type"], filters= - {"is_standard": "No", "disabled": 0, "module": module}, - order_by="name") + reports = frappe.get_all( + "Report", + fields=["name", "ref_doctype", "report_type"], + filters={"is_standard": "No", "disabled": 0, "module": module}, + order_by="name", + ) out = [] for r in reports: - out.append({ - "type": "Link", - "link_type": "report", - "doctype": r.ref_doctype, - "dependencies": r.ref_doctype, - "is_query_report": 1 if r.report_type in ("Query Report", "Script Report", "Custom Report") else 0, - "label": _(r.name), - "link_to": r.name, - }) + out.append( + { + "type": "Link", + "link_type": "report", + "doctype": r.ref_doctype, + "dependencies": r.ref_doctype, + "is_query_report": 1 + if r.report_type in ("Query Report", "Script Report", "Custom Report") + else 0, + "label": _(r.name), + "link_to": r.name, + } + ) return out + def save_new_widget(doc, page, blocks, new_widgets): if loads(new_widgets): widgets = _dict(loads(new_widgets)) @@ -448,38 +465,41 @@ def save_new_widget(doc, page, blocks, new_widgets): json_config = widgets and dumps(widgets, sort_keys=True, indent=4) # Error log body - log = \ - """ + log = """ page: {0} config: {1} exception: {2} - """.format(page, json_config, e) + """.format( + page, json_config, e + ) frappe.log_error(log, _("Could not save customization")) return False return True + def clean_up(original_page, blocks): page_widgets = {} - for wid in ['shortcut', 'card', 'chart']: + for wid in ["shortcut", "card", "chart"]: # get list of widget's name from blocks - page_widgets[wid] = [x['data'][wid + '_name'] for x in loads(blocks) if x['type'] == wid] + page_widgets[wid] = [x["data"][wid + "_name"] for x in loads(blocks) if x["type"] == wid] # shortcut & chart cleanup - for wid in ['shortcut', 'chart']: + for wid in ["shortcut", "chart"]: updated_widgets = [] - original_page.get(wid+'s').reverse() + original_page.get(wid + "s").reverse() - for w in original_page.get(wid+'s'): + for w in original_page.get(wid + "s"): if w.label in page_widgets[wid] and w.label not in [x.label for x in updated_widgets]: updated_widgets.append(w) - original_page.set(wid+'s', updated_widgets) + original_page.set(wid + "s", updated_widgets) # card cleanup for i, v in enumerate(original_page.links): - if v.type == 'Card Break' and v.label not in page_widgets['card']: - del original_page.links[i : i+v.link_count+1] + if v.type == "Card Break" and v.label not in page_widgets["card"]: + del original_page.links[i : i + v.link_count + 1] + def new_widget(config, doctype, parentfield): if not config: @@ -502,21 +522,22 @@ def new_widget(config, doctype, parentfield): prepare_widget_list.append(doc) return prepare_widget_list + def prepare_widget(config, doctype, parentfield): """Create widget child table entries with parent details Args: - config (dict): Dictionary containing widget config - doctype (string): Doctype name of the child table - parentfield (string): Parent field for the child table + config (dict): Dictionary containing widget config + doctype (string): Doctype name of the child table + parentfield (string): Parent field for the child table Returns: - TYPE: List of Document objects + TYPE: List of Document objects """ if not config: return [] - order = config.get('order') - widgets = config.get('widgets') + order = config.get("order") + widgets = config.get("widgets") prepare_widget_list = [] for idx, name in enumerate(order): wid_config = widgets[name].copy() @@ -536,14 +557,15 @@ def prepare_widget(config, doctype, parentfield): prepare_widget_list.append(doc) return prepare_widget_list + @frappe.whitelist() def update_onboarding_step(name, field, value): """Update status of onboaridng step Args: - name (string): Name of the doc - field (string): field to be updated - value: Value to be updated + name (string): Name of the doc + field (string): field to be updated + value: Value to be updated """ frappe.db.set_value("Onboarding Step", name, field, value) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 20887f8886..4d4e83a242 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -3,34 +3,36 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document from frappe.utils import cint class BulkUpdate(Document): pass + @frappe.whitelist() -def update(doctype, field, value, condition='', limit=500): +def update(doctype, field, value, condition="", limit=500): if not limit or cint(limit) > 500: limit = 500 if condition: - condition = ' where ' + condition + condition = " where " + condition - if ';' in condition: - frappe.throw(_('; not allowed in condition')) + if ";" in condition: + frappe.throw(_("; not allowed in condition")) docnames = frappe.db.sql_list( - '''select name from `tab{0}`{1} limit {2} offset 0'''.format(doctype, condition, limit) + """select name from `tab{0}`{1} limit {2} offset 0""".format(doctype, condition, limit) ) data = {} data[field] = value - return submit_cancel_or_update_docs(doctype, docnames, 'update', data) + return submit_cancel_or_update_docs(doctype, docnames, "update", data) + @frappe.whitelist() -def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None): +def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): docnames = frappe.parse_json(docnames) if data: @@ -41,17 +43,17 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None): for i, d in enumerate(docnames, 1): doc = frappe.get_doc(doctype, d) try: - message = '' - if action == 'submit' and doc.docstatus.is_draft(): + message = "" + if action == "submit" and doc.docstatus.is_draft(): doc.submit() - message = _('Submiting {0}').format(doctype) - elif action == 'cancel' and doc.docstatus.is_submitted(): + message = _("Submiting {0}").format(doctype) + elif action == "cancel" and doc.docstatus.is_submitted(): doc.cancel() - message = _('Cancelling {0}').format(doctype) - elif action == 'update' and not doc.docstatus.is_cancelled(): + message = _("Cancelling {0}").format(doctype) + elif action == "update" and not doc.docstatus.is_cancelled(): doc.update(data) doc.save() - message = _('Updating {0}').format(doctype) + message = _("Updating {0}").format(doctype) else: failed.append(d) frappe.db.commit() @@ -63,11 +65,8 @@ def submit_cancel_or_update_docs(doctype, docnames, action='submit', data=None): return failed + def show_progress(docnames, message, i, description): n = len(docnames) if n >= 10: - frappe.publish_progress( - float(i) * 100 / n, - title = message, - description = description - ) + frappe.publish_progress(float(i) * 100 / n, title=message, description=description) diff --git a/frappe/desk/doctype/calendar_view/calendar_view.py b/frappe/desk/doctype/calendar_view/calendar_view.py index 11612f5587..01968e835d 100644 --- a/frappe/desk/doctype/calendar_view/calendar_view.py +++ b/frappe/desk/doctype/calendar_view/calendar_view.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class CalendarView(Document): pass diff --git a/frappe/desk/doctype/console_log/console_log.py b/frappe/desk/doctype/console_log/console_log.py index e0b552ebfd..ebe93f535d 100644 --- a/frappe/desk/doctype/console_log/console_log.py +++ b/frappe/desk/doctype/console_log/console_log.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ConsoleLog(Document): pass diff --git a/frappe/desk/doctype/console_log/test_console_log.py b/frappe/desk/doctype/console_log/test_console_log.py index c41b9d68c8..409ac88299 100644 --- a/frappe/desk/doctype/console_log/test_console_log.py +++ b/frappe/desk/doctype/console_log/test_console_log.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestConsoleLog(unittest.TestCase): pass diff --git a/frappe/desk/doctype/dashboard/dashboard.py b/frappe/desk/doctype/dashboard/dashboard.py index ac62796dc2..61e997836c 100644 --- a/frappe/desk/doctype/dashboard/dashboard.py +++ b/frappe/desk/doctype/dashboard/dashboard.py @@ -17,16 +17,13 @@ class Dashboard(Document): # make all other dashboards non-default DashBoard = DocType("Dashboard") - frappe.qb.update(DashBoard).set( - DashBoard.is_default, 0 - ).where( + frappe.qb.update(DashBoard).set(DashBoard.is_default, 0).where( DashBoard.name != self.name ).run() if frappe.conf.developer_mode and self.is_standard: export_to_files( - record_list=[["Dashboard", self.name, f"{self.module} Dashboard"]], - record_module=self.module + record_list=[["Dashboard", self.name, f"{self.module} Dashboard"]], record_module=self.module ) def validate(self): @@ -35,11 +32,11 @@ class Dashboard(Document): if self.is_standard: non_standard_docs_map = { - 'Dashboard Chart': get_non_standard_charts_in_dashboard(self), - 'Number Card': get_non_standard_cards_in_dashboard(self) + "Dashboard Chart": get_non_standard_charts_in_dashboard(self), + "Number Card": get_non_standard_cards_in_dashboard(self), } - if non_standard_docs_map['Dashboard Chart'] or non_standard_docs_map['Number Card']: + if non_standard_docs_map["Dashboard Chart"] or non_standard_docs_map["Number Card"]: message = get_non_standard_warning_message(non_standard_docs_map) frappe.throw(message, title=_("Standard Not Set"), is_minimizable=True) @@ -57,62 +54,76 @@ def get_permission_query_conditions(user): if not user: user = frappe.session.user - if user == 'Administrator': + if user == "Administrator": return roles = frappe.get_roles(user) if "System Manager" in roles: return None - allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()] - module_condition = '`tabDashboard`.`module` in ({allowed_modules}) or `tabDashboard`.`module` is NULL'.format( - allowed_modules=','.join(allowed_modules)) + allowed_modules = [ + frappe.db.escape(module.get("module_name")) for module in get_modules_from_all_apps_for_user() + ] + module_condition = ( + "`tabDashboard`.`module` in ({allowed_modules}) or `tabDashboard`.`module` is NULL".format( + allowed_modules=",".join(allowed_modules) + ) + ) return module_condition + @frappe.whitelist() def get_permitted_charts(dashboard_name): permitted_charts = [] - dashboard = frappe.get_doc('Dashboard', dashboard_name) + dashboard = frappe.get_doc("Dashboard", dashboard_name) for chart in dashboard.charts: - if frappe.has_permission('Dashboard Chart', doc=chart.chart): + if frappe.has_permission("Dashboard Chart", doc=chart.chart): chart_dict = frappe._dict() chart_dict.update(chart.as_dict()) - if dashboard.get('chart_options'): - chart_dict.custom_options = dashboard.get('chart_options') + if dashboard.get("chart_options"): + chart_dict.custom_options = dashboard.get("chart_options") permitted_charts.append(chart_dict) return permitted_charts + @frappe.whitelist() def get_permitted_cards(dashboard_name): permitted_cards = [] - dashboard = frappe.get_doc('Dashboard', dashboard_name) + dashboard = frappe.get_doc("Dashboard", dashboard_name) for card in dashboard.cards: - if frappe.has_permission('Number Card', doc=card.card): + if frappe.has_permission("Number Card", doc=card.card): permitted_cards.append(card) return permitted_cards + def get_non_standard_charts_in_dashboard(dashboard): - non_standard_charts = [doc.name for doc in frappe.get_list('Dashboard Chart', {'is_standard': 0})] - return [chart_link.chart for chart_link in dashboard.charts if chart_link.chart in non_standard_charts] + non_standard_charts = [doc.name for doc in frappe.get_list("Dashboard Chart", {"is_standard": 0})] + return [ + chart_link.chart for chart_link in dashboard.charts if chart_link.chart in non_standard_charts + ] + def get_non_standard_cards_in_dashboard(dashboard): - non_standard_cards = [doc.name for doc in frappe.get_list('Number Card', {'is_standard': 0})] + non_standard_cards = [doc.name for doc in frappe.get_list("Number Card", {"is_standard": 0})] return [card_link.card for card_link in dashboard.cards if card_link.card in non_standard_cards] + def get_non_standard_warning_message(non_standard_docs_map): - message = _('''Please set the following documents in this Dashboard as standard first.''') + message = _("""Please set the following documents in this Dashboard as standard first.""") def get_html(docs, doctype): - html = '

{}

'.format(frappe.bold(doctype)) + html = "

{}

".format(frappe.bold(doctype)) for doc in docs: - html += ''.format(doctype=doctype, doc=doc) - html += '
' + html += ''.format( + doctype=doctype, doc=doc + ) + html += "
" return html - html = message + '
' + html = message + "
" for doctype in non_standard_docs_map: if non_standard_docs_map[doctype]: diff --git a/frappe/desk/doctype/dashboard/test_dashboard.py b/frappe/desk/doctype/dashboard/test_dashboard.py index 15c132c027..ee3d1848e2 100644 --- a/frappe/desk/doctype/dashboard/test_dashboard.py +++ b/frappe/desk/doctype/dashboard/test_dashboard.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestDashboard(unittest.TestCase): pass diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index cb77ef7a1a..246c9ad4cd 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -2,25 +2,31 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe -from frappe import _ import datetime import json -from frappe.utils.dashboard import cache_source -from frappe.utils import nowdate, getdate, get_datetime, cint, now_datetime -from frappe.utils.dateutils import get_period, get_period_beginning, get_from_date_from_timespan, get_dates_from_timegrain -from frappe.model.naming import append_number_if_name_exists + +import frappe +from frappe import _ from frappe.boot import get_allowed_reports from frappe.config import get_modules_from_all_apps_for_user from frappe.model.document import Document +from frappe.model.naming import append_number_if_name_exists from frappe.modules.export_file import export_to_files +from frappe.utils import cint, get_datetime, getdate, now_datetime, nowdate +from frappe.utils.dashboard import cache_source +from frappe.utils.dateutils import ( + get_dates_from_timegrain, + get_from_date_from_timespan, + get_period, + get_period_beginning, +) def get_permission_query_conditions(user): if not user: user = frappe.session.user - if user == 'Administrator': + if user == "Administrator": return roles = frappe.get_roles(user) @@ -31,22 +37,32 @@ def get_permission_query_conditions(user): report_condition = False module_condition = False - allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] - allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] - allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()] + allowed_doctypes = [ + frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read() + ] + allowed_reports = [ + frappe.db.escape(key) if type(key) == str else key.encode("UTF8") + for key in get_allowed_reports() + ] + allowed_modules = [ + frappe.db.escape(module.get("module_name")) for module in get_modules_from_all_apps_for_user() + ] if allowed_doctypes: - doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format( - allowed_doctypes=','.join(allowed_doctypes)) + doctype_condition = "`tabDashboard Chart`.`document_type` in ({allowed_doctypes})".format( + allowed_doctypes=",".join(allowed_doctypes) + ) if allowed_reports: - report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format( - allowed_reports=','.join(allowed_reports)) + report_condition = "`tabDashboard Chart`.`report_name` in ({allowed_reports})".format( + allowed_reports=",".join(allowed_reports) + ) if allowed_modules: - module_condition = '''`tabDashboard Chart`.`module` in ({allowed_modules}) - or `tabDashboard Chart`.`module` is NULL'''.format( - allowed_modules=','.join(allowed_modules)) + module_condition = """`tabDashboard Chart`.`module` in ({allowed_modules}) + or `tabDashboard Chart`.`module` is NULL""".format( + allowed_modules=",".join(allowed_modules) + ) - return ''' + return """ ((`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average') and {doctype_condition}) or @@ -54,20 +70,22 @@ def get_permission_query_conditions(user): and {report_condition})) and ({module_condition}) - '''.format( + """.format( doctype_condition=doctype_condition, report_condition=report_condition, - module_condition=module_condition + module_condition=module_condition, ) + def has_permission(doc, ptype, user): roles = frappe.get_roles(user) if "System Manager" in roles: return True - - if doc.chart_type == 'Report': - allowed_reports = [key if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + if doc.chart_type == "Report": + allowed_reports = [ + key if type(key) == str else key.encode("UTF8") for key in get_allowed_reports() + ] if doc.report_name in allowed_reports: return True else: @@ -77,19 +95,30 @@ def has_permission(doc, ptype, user): return False + @frappe.whitelist() @cache_source -def get(chart_name = None, chart = None, no_cache = None, filters = None, from_date = None, - to_date = None, timespan = None, time_interval = None, heatmap_year=None, refresh = None): +def get( + chart_name=None, + chart=None, + no_cache=None, + filters=None, + from_date=None, + to_date=None, + timespan=None, + time_interval=None, + heatmap_year=None, + refresh=None, +): if chart_name: - chart = frappe.get_doc('Dashboard Chart', chart_name) + chart = frappe.get_doc("Dashboard Chart", chart_name) else: chart = frappe._dict(frappe.parse_json(chart)) heatmap_year = heatmap_year or chart.heatmap_year timespan = timespan or chart.timespan - if timespan == 'Select Date Range': + if timespan == "Select Date Range": if from_date and len(from_date): from_date = get_datetime(from_date) else: @@ -106,34 +135,36 @@ def get(chart_name = None, chart = None, no_cache = None, filters = None, from_d filters = [] # don't include cancelled documents - filters.append([chart.document_type, 'docstatus', '<', 2, False]) + filters.append([chart.document_type, "docstatus", "<", 2, False]) - if chart.chart_type == 'Group By': + if chart.chart_type == "Group By": chart_config = get_group_by_chart_config(chart, filters) else: - if chart.type == 'Heatmap': + if chart.type == "Heatmap": chart_config = get_heatmap_chart_config(chart, filters, heatmap_year) else: - chart_config = get_chart_config(chart, filters, timespan, timegrain, from_date, to_date) + chart_config = get_chart_config(chart, filters, timespan, timegrain, from_date, to_date) return chart_config + @frappe.whitelist() def create_dashboard_chart(args): args = frappe.parse_json(args) - doc = frappe.new_doc('Dashboard Chart') + doc = frappe.new_doc("Dashboard Chart") doc.update(args) - if args.get('custom_options'): - doc.custom_options = json.dumps(args.get('custom_options')) + if args.get("custom_options"): + doc.custom_options = json.dumps(args.get("custom_options")) - if frappe.db.exists('Dashboard Chart', args.chart_name): - args.chart_name = append_number_if_name_exists('Dashboard Chart', args.chart_name) + if frappe.db.exists("Dashboard Chart", args.chart_name): + args.chart_name = append_number_if_name_exists("Dashboard Chart", args.chart_name) doc.chart_name = args.chart_name doc.insert(ignore_permissions=True) return doc + @frappe.whitelist() def create_report_chart(args): doc = create_dashboard_chart(args) @@ -142,24 +173,26 @@ def create_report_chart(args): if args.dashboard: add_chart_to_dashboard(json.dumps(args)) + @frappe.whitelist() def add_chart_to_dashboard(args): args = frappe.parse_json(args) - dashboard = frappe.get_doc('Dashboard', args.dashboard) - dashboard_link = frappe.new_doc('Dashboard Chart Link') + dashboard = frappe.get_doc("Dashboard", args.dashboard) + dashboard_link = frappe.new_doc("Dashboard Chart Link") dashboard_link.chart = args.chart_name or args.name if args.set_standard and dashboard.is_standard: - chart = frappe.get_doc('Dashboard Chart', dashboard_link.chart) + chart = frappe.get_doc("Dashboard Chart", dashboard_link.chart) chart.is_standard = 1 chart.module = dashboard.module chart.save() - dashboard.append('charts', dashboard_link) + dashboard.append("charts", dashboard_link) dashboard.save() frappe.db.commit() + def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): if not from_date: from_date = get_from_date_from_timespan(to_date, timespan) @@ -169,108 +202,106 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): doctype = chart.document_type datefield = chart.based_on - value_field = chart.value_based_on or '1' - from_date = from_date.strftime('%Y-%m-%d') + value_field = chart.value_based_on or "1" + from_date = from_date.strftime("%Y-%m-%d") to_date = to_date - filters.append([doctype, datefield, '>=', from_date, False]) - filters.append([doctype, datefield, '<=', to_date, False]) + filters.append([doctype, datefield, ">=", from_date, False]) + filters.append([doctype, datefield, "<=", to_date, False]) data = frappe.db.get_list( doctype, - fields = [ - '{} as _unit'.format(datefield), - 'SUM({})'.format(value_field), - 'COUNT(*)' - ], - filters = filters, - group_by = '_unit', - order_by = '_unit asc', - as_list = True, - ignore_ifnull = True + fields=["{} as _unit".format(datefield), "SUM({})".format(value_field), "COUNT(*)"], + filters=filters, + group_by="_unit", + order_by="_unit asc", + as_list=True, + ignore_ifnull=True, ) result = get_result(data, timegrain, from_date, to_date, chart.chart_type) chart_config = { "labels": [get_period(r[0], timegrain) for r in result], - "datasets": [{ - "name": chart.name, - "values": [r[1] for r in result] - }] + "datasets": [{"name": chart.name, "values": [r[1] for r in result]}], } return chart_config + def get_heatmap_chart_config(chart, filters, heatmap_year): aggregate_function = get_aggregate_function(chart.chart_type) - value_field = chart.value_based_on or '1' + value_field = chart.value_based_on or "1" doctype = chart.document_type datefield = chart.based_on year = cint(heatmap_year) if heatmap_year else getdate(nowdate()).year - year_start_date = datetime.date(year, 1, 1).strftime('%Y-%m-%d') - next_year_start_date = datetime.date(year + 1, 1, 1).strftime('%Y-%m-%d') + year_start_date = datetime.date(year, 1, 1).strftime("%Y-%m-%d") + next_year_start_date = datetime.date(year + 1, 1, 1).strftime("%Y-%m-%d") - filters.append([doctype, datefield, '>', "{date}".format(date=year_start_date), False]) - filters.append([doctype, datefield, '<', "{date}".format(date=next_year_start_date), False]) + filters.append([doctype, datefield, ">", "{date}".format(date=year_start_date), False]) + filters.append([doctype, datefield, "<", "{date}".format(date=next_year_start_date), False]) - if frappe.db.db_type == 'mariadb': - timestamp_field = 'unix_timestamp({datefield})'.format(datefield=datefield) + if frappe.db.db_type == "mariadb": + timestamp_field = "unix_timestamp({datefield})".format(datefield=datefield) else: - timestamp_field = 'extract(epoch from timestamp {datefield})'.format(datefield=datefield) - - data = dict(frappe.db.get_all( - doctype, - fields = [ - timestamp_field, - '{aggregate_function}({value_field})'.format(aggregate_function=aggregate_function, value_field=value_field), - ], - filters = filters, - group_by = 'date({datefield})'.format(datefield=datefield), - as_list = 1, - order_by = '{datefield} asc'.format(datefield=datefield), - ignore_ifnull = True - )) + timestamp_field = "extract(epoch from timestamp {datefield})".format(datefield=datefield) + + data = dict( + frappe.db.get_all( + doctype, + fields=[ + timestamp_field, + "{aggregate_function}({value_field})".format( + aggregate_function=aggregate_function, value_field=value_field + ), + ], + filters=filters, + group_by="date({datefield})".format(datefield=datefield), + as_list=1, + order_by="{datefield} asc".format(datefield=datefield), + ignore_ifnull=True, + ) + ) chart_config = { - 'labels': [], - 'dataPoints': data, + "labels": [], + "dataPoints": data, } return chart_config + def get_group_by_chart_config(chart, filters): aggregate_function = get_aggregate_function(chart.group_by_type) - value_field = chart.aggregate_function_based_on or '1' + value_field = chart.aggregate_function_based_on or "1" group_by_field = chart.group_by_based_on doctype = chart.document_type data = frappe.db.get_list( doctype, - fields = [ - '{} as name'.format(group_by_field), - '{aggregate_function}({value_field}) as count'.format(aggregate_function=aggregate_function, value_field=value_field), + fields=[ + "{} as name".format(group_by_field), + "{aggregate_function}({value_field}) as count".format( + aggregate_function=aggregate_function, value_field=value_field + ), ], - filters = filters, - group_by = group_by_field, - order_by = 'count desc', - ignore_ifnull = True + filters=filters, + group_by=group_by_field, + order_by="count desc", + ignore_ifnull=True, ) if data: if chart.number_of_groups and chart.number_of_groups < len(data): other_count = 0 for i in range(chart.number_of_groups - 1, len(data)): - other_count += data[i]['count'] - data = data[0: chart.number_of_groups - 1] - data.append({'name': 'Other', 'count': other_count}) + other_count += data[i]["count"] + data = data[0 : chart.number_of_groups - 1] + data.append({"name": "Other", "count": other_count}) chart_config = { - "labels": [item['name'] if item['name'] else 'Not Specified' for item in data], - "datasets": [{ - "name": chart.name, - "values": [item['count'] for item in data] - }] + "labels": [item["name"] if item["name"] else "Not Specified" for item in data], + "datasets": [{"name": chart.name, "values": [item["count"] for item in data]}], } return chart_config @@ -297,35 +328,33 @@ def get_result(data, timegrain, from_date, to_date, chart_type): d[1] += data[data_index][1] count += data[data_index][2] data_index += 1 - if chart_type == 'Average' and not count == 0: - d[1] = d[1]/count - if chart_type == 'Count': + if chart_type == "Average" and not count == 0: + d[1] = d[1] / count + if chart_type == "Count": d[1] = count return result + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): - or_filters = {'owner': frappe.session.user, 'is_public': 1} - return frappe.db.get_list('Dashboard Chart', - fields=['name'], - filters=filters, - or_filters=or_filters, - as_list = 1) + or_filters = {"owner": frappe.session.user, "is_public": 1} + return frappe.db.get_list( + "Dashboard Chart", fields=["name"], filters=filters, or_filters=or_filters, as_list=1 + ) -class DashboardChart(Document): +class DashboardChart(Document): def on_update(self): - frappe.cache().delete_key('chart-data:{}'.format(self.name)) + frappe.cache().delete_key("chart-data:{}".format(self.name)) if frappe.conf.developer_mode and self.is_standard: - export_to_files(record_list=[['Dashboard Chart', self.name]], record_module=self.module) - + export_to_files(record_list=[["Dashboard Chart", self.name]], record_module=self.module) def validate(self): if not frappe.conf.developer_mode and self.is_standard: frappe.throw(_("Cannot edit Standard charts")) - if self.chart_type != 'Custom' and self.chart_type != 'Report': + if self.chart_type != "Custom" and self.chart_type != "Report": self.check_required_field() self.check_document_type() @@ -335,13 +364,17 @@ class DashboardChart(Document): if not self.document_type: frappe.throw(_("Document type is required to create a dashboard chart")) - if self.document_type and frappe.get_meta(self.document_type).istable and not self.parent_document_type: + if ( + self.document_type + and frappe.get_meta(self.document_type).istable + and not self.parent_document_type + ): frappe.throw(_("Parent document type is required to create a dashboard chart")) - if self.chart_type == 'Group By': + if self.chart_type == "Group By": if not self.group_by_based_on: frappe.throw(_("Group By field is required to create a dashboard chart")) - if self.group_by_type in ['Sum', 'Average'] and not self.aggregate_function_based_on: + if self.group_by_type in ["Sum", "Average"] and not self.aggregate_function_based_on: frappe.throw(_("Aggregate Function field is required to create a dashboard chart")) else: if not self.based_on: diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 5c986b5b7c..94ea1af35c 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -1,182 +1,186 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest, frappe -from frappe.utils import getdate, formatdate, get_last_day -from frappe.utils.dateutils import get_period_ending, get_period -from frappe.desk.doctype.dashboard_chart.dashboard_chart import get - +import unittest from datetime import datetime -from dateutil.relativedelta import relativedelta from unittest.mock import patch +from dateutil.relativedelta import relativedelta + +import frappe +from frappe.desk.doctype.dashboard_chart.dashboard_chart import get +from frappe.utils import formatdate, get_last_day, getdate +from frappe.utils.dateutils import get_period, get_period_ending + + class TestDashboardChart(unittest.TestCase): def test_period_ending(self): - self.assertEqual(get_period_ending('2019-04-10', 'Daily'), - getdate('2019-04-10')) + self.assertEqual(get_period_ending("2019-04-10", "Daily"), getdate("2019-04-10")) # week starts on monday with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): - self.assertEqual(get_period_ending('2019-04-10', 'Weekly'), - getdate('2019-04-14')) - - self.assertEqual(get_period_ending('2019-04-10', 'Monthly'), - getdate('2019-04-30')) - self.assertEqual(get_period_ending('2019-04-30', 'Monthly'), - getdate('2019-04-30')) - self.assertEqual(get_period_ending('2019-03-31', 'Monthly'), - getdate('2019-03-31')) - - self.assertEqual(get_period_ending('2019-04-10', 'Quarterly'), - getdate('2019-06-30')) - self.assertEqual(get_period_ending('2019-06-30', 'Quarterly'), - getdate('2019-06-30')) - self.assertEqual(get_period_ending('2019-10-01', 'Quarterly'), - getdate('2019-12-31')) + self.assertEqual(get_period_ending("2019-04-10", "Weekly"), getdate("2019-04-14")) + + self.assertEqual(get_period_ending("2019-04-10", "Monthly"), getdate("2019-04-30")) + self.assertEqual(get_period_ending("2019-04-30", "Monthly"), getdate("2019-04-30")) + self.assertEqual(get_period_ending("2019-03-31", "Monthly"), getdate("2019-03-31")) + + self.assertEqual(get_period_ending("2019-04-10", "Quarterly"), getdate("2019-06-30")) + self.assertEqual(get_period_ending("2019-06-30", "Quarterly"), getdate("2019-06-30")) + self.assertEqual(get_period_ending("2019-10-01", "Quarterly"), getdate("2019-12-31")) def test_dashboard_chart(self): - if frappe.db.exists('Dashboard Chart', 'Test Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Dashboard Chart') - - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Dashboard Chart', - chart_type = 'Count', - document_type = 'DocType', - based_on = 'creation', - timespan = 'Last Year', - time_interval = 'Monthly', - filters_json = '{}', - timeseries = 1 - )).insert() + if frappe.db.exists("Dashboard Chart", "Test Dashboard Chart"): + frappe.delete_doc("Dashboard Chart", "Test Dashboard Chart") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Dashboard Chart", + chart_type="Count", + document_type="DocType", + based_on="creation", + timespan="Last Year", + time_interval="Monthly", + filters_json="{}", + timeseries=1, + ) + ).insert() cur_date = datetime.now() - relativedelta(years=1) - result = get(chart_name='Test Dashboard Chart', refresh=1) + result = get(chart_name="Test Dashboard Chart", refresh=1) for idx in range(13): month = get_last_day(cur_date) - month = formatdate(month.strftime('%Y-%m-%d')) - self.assertEqual(result.get('labels')[idx], get_period(month)) + month = formatdate(month.strftime("%Y-%m-%d")) + self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) frappe.db.rollback() def test_empty_dashboard_chart(self): - if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart') + if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart"): + frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart") frappe.db.delete("Error Log") - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Empty Dashboard Chart', - chart_type = 'Count', - document_type = 'Error Log', - based_on = 'creation', - timespan = 'Last Year', - time_interval = 'Monthly', - filters_json = '[]', - timeseries = 1 - )).insert() + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Empty Dashboard Chart", + chart_type="Count", + document_type="Error Log", + based_on="creation", + timespan="Last Year", + time_interval="Monthly", + filters_json="[]", + timeseries=1, + ) + ).insert() cur_date = datetime.now() - relativedelta(years=1) - result = get(chart_name ='Test Empty Dashboard Chart', refresh=1) + result = get(chart_name="Test Empty Dashboard Chart", refresh=1) for idx in range(13): month = get_last_day(cur_date) - month = formatdate(month.strftime('%Y-%m-%d')) - self.assertEqual(result.get('labels')[idx], get_period(month)) + month = formatdate(month.strftime("%Y-%m-%d")) + self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) frappe.db.rollback() def test_chart_wih_one_value(self): - if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart 2'): - frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2') + if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart 2"): + frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart 2") frappe.db.delete("Error Log") # create one data point - frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert() - - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Empty Dashboard Chart 2', - chart_type = 'Count', - document_type = 'Error Log', - based_on = 'creation', - timespan = 'Last Year', - time_interval = 'Monthly', - filters_json = '[]', - timeseries = 1 - )).insert() + frappe.get_doc(dict(doctype="Error Log", creation="2018-06-01 00:00:00")).insert() + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Empty Dashboard Chart 2", + chart_type="Count", + document_type="Error Log", + based_on="creation", + timespan="Last Year", + time_interval="Monthly", + filters_json="[]", + timeseries=1, + ) + ).insert() cur_date = datetime.now() - relativedelta(years=1) - result = get(chart_name ='Test Empty Dashboard Chart 2', refresh = 1) + result = get(chart_name="Test Empty Dashboard Chart 2", refresh=1) for idx in range(13): month = get_last_day(cur_date) - month = formatdate(month.strftime('%Y-%m-%d')) - self.assertEqual(result.get('labels')[idx], get_period(month)) + month = formatdate(month.strftime("%Y-%m-%d")) + self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) # only 1 data point with value - self.assertEqual(result.get('datasets')[0].get('values')[2], 0) + self.assertEqual(result.get("datasets")[0].get("values")[2], 0) frappe.db.rollback() def test_group_by_chart_type(self): - if frappe.db.exists('Dashboard Chart', 'Test Group By Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Group By Dashboard Chart') - - frappe.get_doc({"doctype":"ToDo", "description": "test"}).insert() - - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Group By Dashboard Chart', - chart_type = 'Group By', - document_type = 'ToDo', - group_by_based_on = 'status', - filters_json = '[]', - )).insert() + if frappe.db.exists("Dashboard Chart", "Test Group By Dashboard Chart"): + frappe.delete_doc("Dashboard Chart", "Test Group By Dashboard Chart") + + frappe.get_doc({"doctype": "ToDo", "description": "test"}).insert() + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Group By Dashboard Chart", + chart_type="Group By", + document_type="ToDo", + group_by_based_on="status", + filters_json="[]", + ) + ).insert() - result = get(chart_name ='Test Group By Dashboard Chart', refresh = 1) - todo_status_count = frappe.db.count('ToDo', {'status': result.get('labels')[0]}) + result = get(chart_name="Test Group By Dashboard Chart", refresh=1) + todo_status_count = frappe.db.count("ToDo", {"status": result.get("labels")[0]}) - self.assertEqual(result.get('datasets')[0].get('values')[0], todo_status_count) + self.assertEqual(result.get("datasets")[0].get("values")[0], todo_status_count) frappe.db.rollback() def test_daily_dashboard_chart(self): insert_test_records() - if frappe.db.exists('Dashboard Chart', 'Test Daily Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Daily Dashboard Chart') - - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Daily Dashboard Chart', - chart_type = 'Sum', - document_type = 'Communication', - based_on = 'communication_date', - value_based_on = 'rating', - timespan = 'Select Date Range', - time_interval = 'Daily', - from_date = datetime(2019, 1, 6), - to_date = datetime(2019, 1, 11), - filters_json = '[]', - timeseries = 1 - )).insert() - - result = get(chart_name = 'Test Daily Dashboard Chart', refresh = 1) - - self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0]) + if frappe.db.exists("Dashboard Chart", "Test Daily Dashboard Chart"): + frappe.delete_doc("Dashboard Chart", "Test Daily Dashboard Chart") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Daily Dashboard Chart", + chart_type="Sum", + document_type="Communication", + based_on="communication_date", + value_based_on="rating", + timespan="Select Date Range", + time_interval="Daily", + from_date=datetime(2019, 1, 6), + to_date=datetime(2019, 1, 11), + filters_json="[]", + timeseries=1, + ) + ).insert() + + result = get(chart_name="Test Daily Dashboard Chart", refresh=1) + + self.assertEqual(result.get("datasets")[0].get("values"), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0]) self.assertEqual( - result.get('labels'), - ['06-01-19', '07-01-19', '08-01-19', '09-01-19', '10-01-19', '11-01-19'] + result.get("labels"), ["06-01-19", "07-01-19", "08-01-19", "09-01-19", "10-01-19", "11-01-19"] ) frappe.db.rollback() @@ -184,81 +188,81 @@ class TestDashboardChart(unittest.TestCase): def test_weekly_dashboard_chart(self): insert_test_records() - if frappe.db.exists('Dashboard Chart', 'Test Weekly Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Weekly Dashboard Chart') - - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Weekly Dashboard Chart', - chart_type = 'Sum', - document_type = 'Communication', - based_on = 'communication_date', - value_based_on = 'rating', - timespan = 'Select Date Range', - time_interval = 'Weekly', - from_date = datetime(2018, 12, 30), - to_date = datetime(2019, 1, 15), - filters_json = '[]', - timeseries = 1 - )).insert() + if frappe.db.exists("Dashboard Chart", "Test Weekly Dashboard Chart"): + frappe.delete_doc("Dashboard Chart", "Test Weekly Dashboard Chart") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Weekly Dashboard Chart", + chart_type="Sum", + document_type="Communication", + based_on="communication_date", + value_based_on="rating", + timespan="Select Date Range", + time_interval="Weekly", + from_date=datetime(2018, 12, 30), + to_date=datetime(2019, 1, 15), + filters_json="[]", + timeseries=1, + ) + ).insert() with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): - result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1) + result = get(chart_name="Test Weekly Dashboard Chart", refresh=1) - self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0]) - self.assertEqual( - result.get('labels'), - ['30-12-18', '06-01-19', '13-01-19', '20-01-19'] - ) + self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 300.0, 800.0, 0.0]) + self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) frappe.db.rollback() def test_avg_dashboard_chart(self): insert_test_records() - if frappe.db.exists('Dashboard Chart', 'Test Average Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Average Dashboard Chart') - - frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Average Dashboard Chart', - chart_type = 'Average', - document_type = 'Communication', - based_on = 'communication_date', - value_based_on = 'rating', - timespan = 'Select Date Range', - time_interval = 'Weekly', - from_date = datetime(2018, 12, 30), - to_date = datetime(2019, 1, 15), - filters_json = '[]', - timeseries = 1 - )).insert() + if frappe.db.exists("Dashboard Chart", "Test Average Dashboard Chart"): + frappe.delete_doc("Dashboard Chart", "Test Average Dashboard Chart") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Average Dashboard Chart", + chart_type="Average", + document_type="Communication", + based_on="communication_date", + value_based_on="rating", + timespan="Select Date Range", + time_interval="Weekly", + from_date=datetime(2018, 12, 30), + to_date=datetime(2019, 1, 15), + filters_json="[]", + timeseries=1, + ) + ).insert() with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): - result = get(chart_name='Test Average Dashboard Chart', refresh = 1) - self.assertEqual( - result.get('labels'), - ['30-12-18', '06-01-19', '13-01-19', '20-01-19'] - ) - self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 150.0, 266.6666666666667, 0.0]) + result = get(chart_name="Test Average Dashboard Chart", refresh=1) + self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) + self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 150.0, 266.6666666666667, 0.0]) frappe.db.rollback() + def insert_test_records(): - create_new_communication('Communication 1', datetime(2018, 12, 30), 50) - create_new_communication('Communication 2', datetime(2019, 1, 4), 100) - create_new_communication('Communication 3', datetime(2019, 1, 6), 200) - create_new_communication('Communication 4', datetime(2019, 1, 7), 400) - create_new_communication('Communication 5', datetime(2019, 1, 8), 300) - create_new_communication('Communication 6', datetime(2019, 1, 10), 100) + create_new_communication("Communication 1", datetime(2018, 12, 30), 50) + create_new_communication("Communication 2", datetime(2019, 1, 4), 100) + create_new_communication("Communication 3", datetime(2019, 1, 6), 200) + create_new_communication("Communication 4", datetime(2019, 1, 7), 400) + create_new_communication("Communication 5", datetime(2019, 1, 8), 300) + create_new_communication("Communication 6", datetime(2019, 1, 10), 100) + def create_new_communication(subject, date, rating): communication = { - 'doctype': 'Communication', - 'subject': subject, - 'rating': rating, - 'communication_date': date + "doctype": "Communication", + "subject": subject, + "rating": rating, + "communication_date": date, } comm = frappe.get_doc(communication) - if not frappe.db.exists("Communication", {'subject' : comm.subject}): + if not frappe.db.exists("Communication", {"subject": comm.subject}): comm.insert() diff --git a/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py b/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py index 8b2fba2e58..41f35d2ee4 100644 --- a/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py +++ b/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class DashboardChartField(Document): pass diff --git a/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py b/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py index 87d095d5d1..b2a7caefeb 100644 --- a/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py +++ b/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class DashboardChartLink(Document): pass diff --git a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py index 71ded32837..29c1b6ee7d 100644 --- a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py @@ -2,21 +2,29 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe, os +import os + +import frappe from frappe import _ from frappe.model.document import Document -from frappe.modules.export_file import export_to_files from frappe.modules import get_module_path, scrub +from frappe.modules.export_file import export_to_files @frappe.whitelist() def get_config(name): - doc = frappe.get_doc('Dashboard Chart Source', name) - with open(os.path.join(get_module_path(doc.module), - 'dashboard_chart_source', scrub(doc.name), scrub(doc.name) + '.js'), 'r') as f: + doc = frappe.get_doc("Dashboard Chart Source", name) + with open( + os.path.join( + get_module_path(doc.module), "dashboard_chart_source", scrub(doc.name), scrub(doc.name) + ".js" + ), + "r", + ) as f: return f.read() + class DashboardChartSource(Document): def on_update(self): - export_to_files(record_list=[[self.doctype, self.name]], - record_module=self.module, create_init=True) + export_to_files( + record_list=[[self.doctype, self.name]], record_module=self.module, create_init=True + ) diff --git a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py index 6d6773d52e..0c219c08cc 100644 --- a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestDashboardChartSource(unittest.TestCase): pass diff --git a/frappe/desk/doctype/dashboard_settings/dashboard_settings.py b/frappe/desk/doctype/dashboard_settings/dashboard_settings.py index 2f29b3e989..d43476ad7d 100644 --- a/frappe/desk/doctype/dashboard_settings/dashboard_settings.py +++ b/frappe/desk/doctype/dashboard_settings/dashboard_settings.py @@ -2,10 +2,13 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + +import frappe + # import frappe from frappe.model.document import Document -import frappe -import json + class DashboardSettings(Document): pass @@ -14,21 +17,24 @@ class DashboardSettings(Document): @frappe.whitelist() def create_dashboard_settings(user): if not frappe.db.exists("Dashboard Settings", user): - doc = frappe.new_doc('Dashboard Settings') + doc = frappe.new_doc("Dashboard Settings") doc.name = user doc.insert(ignore_permissions=True) frappe.db.commit() return doc + def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user + + return """(`tabDashboard Settings`.name = '{user}')""".format(user=user) - return '''(`tabDashboard Settings`.name = '{user}')'''.format(user=user) @frappe.whitelist() def save_chart_config(reset, config, chart_name): reset = frappe.parse_json(reset) - doc = frappe.get_doc('Dashboard Settings', frappe.session.user) + doc = frappe.get_doc("Dashboard Settings", frappe.session.user) chart_config = frappe.parse_json(doc.chart_config) or {} if reset: @@ -39,4 +45,6 @@ def save_chart_config(reset, config, chart_name): chart_config[chart_name] = {} chart_config[chart_name].update(config) - frappe.db.set_value('Dashboard Settings', frappe.session.user, 'chart_config', json.dumps(chart_config)) \ No newline at end of file + frappe.db.set_value( + "Dashboard Settings", frappe.session.user, "chart_config", json.dumps(chart_config) + ) diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 194b0d0ca4..29de1f56d9 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -2,13 +2,15 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import frappe -from frappe import _ import json import random + +import frappe +from frappe import _ from frappe.model.document import Document from frappe.utils.user import UserPermissions + class DesktopIcon(Document): def validate(self): if not self.label: @@ -17,30 +19,50 @@ class DesktopIcon(Document): def on_trash(self): clear_desktop_icons_cache() + def after_doctype_insert(): - frappe.db.add_unique('Desktop Icon', ('module_name', 'owner', 'standard')) + frappe.db.add_unique("Desktop Icon", ("module_name", "owner", "standard")) + def get_desktop_icons(user=None): - '''Return desktop icons for user''' + """Return desktop icons for user""" if not user: user = frappe.session.user - user_icons = frappe.cache().hget('desktop_icons', user) + user_icons = frappe.cache().hget("desktop_icons", user) if not user_icons: - fields = ['module_name', 'hidden', 'label', 'link', 'type', 'icon', 'color', 'description', 'category', - '_doctype', '_report', 'idx', 'force_show', 'reverse', 'custom', 'standard', 'blocked'] + fields = [ + "module_name", + "hidden", + "label", + "link", + "type", + "icon", + "color", + "description", + "category", + "_doctype", + "_report", + "idx", + "force_show", + "reverse", + "custom", + "standard", + "blocked", + ] active_domains = frappe.get_active_domains() - blocked_doctypes = frappe.get_all("DocType", filters={ - "ifnull(restrict_to_domain, '')": ("not in", ",".join(active_domains)) - }, fields=["name"]) + blocked_doctypes = frappe.get_all( + "DocType", + filters={"ifnull(restrict_to_domain, '')": ("not in", ",".join(active_domains))}, + fields=["name"], + ) - blocked_doctypes = [ d.get("name") for d in blocked_doctypes ] + blocked_doctypes = [d.get("name") for d in blocked_doctypes] - standard_icons = frappe.db.get_all('Desktop Icon', - fields=fields, filters={'standard': 1}) + standard_icons = frappe.db.get_all("Desktop Icon", fields=fields, filters={"standard": 1}) standard_map = {} for icon in standard_icons: @@ -48,8 +70,9 @@ def get_desktop_icons(user=None): icon.blocked = 1 standard_map[icon.module_name] = icon - user_icons = frappe.db.get_all('Desktop Icon', fields=fields, - filters={'standard': 0, 'owner': user}) + user_icons = frappe.db.get_all( + "Desktop Icon", fields=fields, filters={"standard": 0, "owner": user} + ) # update hidden property for icon in user_icons: @@ -60,11 +83,10 @@ def get_desktop_icons(user=None): # override properties from standard icon if standard_icon: - for key in ('route', 'label', 'color', 'icon', 'link'): + for key in ("route", "label", "color", "icon", "link"): if standard_icon.get(key): icon[key] = standard_icon.get(key) - if standard_icon.blocked: icon.hidden = 1 @@ -86,52 +108,68 @@ def get_desktop_icons(user=None): user_icons.append(standard_icon) - user_blocked_modules = frappe.get_doc('User', user).get_blocked_modules() + user_blocked_modules = frappe.get_doc("User", user).get_blocked_modules() for icon in user_icons: if icon.module_name in user_blocked_modules: icon.hidden = 1 # sort by idx - user_icons.sort(key = lambda a: a.idx) + user_icons.sort(key=lambda a: a.idx) # translate for d in user_icons: - if d.label: d.label = _(d.label) + if d.label: + d.label = _(d.label) - frappe.cache().hset('desktop_icons', user, user_icons) + frappe.cache().hset("desktop_icons", user, user_icons) return user_icons + @frappe.whitelist() -def add_user_icon(_doctype, _report=None, label=None, link=None, type='link', standard=0): - '''Add a new user desktop icon to the desktop''' +def add_user_icon(_doctype, _report=None, label=None, link=None, type="link", standard=0): + """Add a new user desktop icon to the desktop""" - if not label: label = _doctype or _report - if not link: link = 'List/{0}'.format(_doctype) + if not label: + label = _doctype or _report + if not link: + link = "List/{0}".format(_doctype) # find if a standard icon exists - icon_name = frappe.db.exists('Desktop Icon', {'standard': standard, 'link': link, - 'owner': frappe.session.user}) + icon_name = frappe.db.exists( + "Desktop Icon", {"standard": standard, "link": link, "owner": frappe.session.user} + ) if icon_name: - if frappe.db.get_value('Desktop Icon', icon_name, 'hidden'): + if frappe.db.get_value("Desktop Icon", icon_name, "hidden"): # if it is hidden, unhide it - frappe.db.set_value('Desktop Icon', icon_name, 'hidden', 0) + frappe.db.set_value("Desktop Icon", icon_name, "hidden", 0) clear_desktop_icons_cache() else: - idx = frappe.db.sql('select max(idx) from `tabDesktop Icon` where owner=%s', - frappe.session.user)[0][0] or \ - frappe.db.sql('select count(*) from `tabDesktop Icon` where standard=1')[0][0] + idx = ( + frappe.db.sql("select max(idx) from `tabDesktop Icon` where owner=%s", frappe.session.user)[0][ + 0 + ] + or frappe.db.sql("select count(*) from `tabDesktop Icon` where standard=1")[0][0] + ) if not frappe.db.get_value("Report", _report): _report = None - userdefined_icon = frappe.db.get_value('DocType', _doctype, ['icon','color','module'], as_dict=True) + userdefined_icon = frappe.db.get_value( + "DocType", _doctype, ["icon", "color", "module"], as_dict=True + ) else: - userdefined_icon = frappe.db.get_value('Report', _report, ['icon','color','module'], as_dict=True) + userdefined_icon = frappe.db.get_value( + "Report", _report, ["icon", "color", "module"], as_dict=True + ) - module_icon = frappe.get_value('Desktop Icon', {'standard':1, 'module_name':userdefined_icon.module}, - ['name', 'icon', 'color', 'reverse'], as_dict=True) + module_icon = frappe.get_value( + "Desktop Icon", + {"standard": 1, "module_name": userdefined_icon.module}, + ["name", "icon", "color", "reverse"], + as_dict=True, + ) if not module_icon: module_icon = frappe._dict() @@ -140,27 +178,29 @@ def add_user_icon(_doctype, _report=None, label=None, link=None, type='link', st module_icon.reverse = 0 if (len(opts) > 1) else 1 try: - new_icon = frappe.get_doc({ - 'doctype': 'Desktop Icon', - 'label': label, - 'module_name': label, - 'link': link, - 'type': type, - '_doctype': _doctype, - '_report': _report, - 'icon': userdefined_icon.icon or module_icon.icon, - 'color': userdefined_icon.color or module_icon.color, - 'reverse': module_icon.reverse, - 'idx': idx + 1, - 'custom': 1, - 'standard': standard - }).insert(ignore_permissions=True) + new_icon = frappe.get_doc( + { + "doctype": "Desktop Icon", + "label": label, + "module_name": label, + "link": link, + "type": type, + "_doctype": _doctype, + "_report": _report, + "icon": userdefined_icon.icon or module_icon.icon, + "color": userdefined_icon.color or module_icon.color, + "reverse": module_icon.reverse, + "idx": idx + 1, + "custom": 1, + "standard": standard, + } + ).insert(ignore_permissions=True) clear_desktop_icons_cache() icon_name = new_icon.name except frappe.UniqueValidationError as e: - frappe.throw(_('Desktop Icon already exists')) + frappe.throw(_("Desktop Icon already exists")) except Exception as e: raise e @@ -169,31 +209,31 @@ def add_user_icon(_doctype, _report=None, label=None, link=None, type='link', st @frappe.whitelist() def set_order(new_order, user=None): - '''set new order by duplicating user icons (if user is set) or set global order''' + """set new order by duplicating user icons (if user is set) or set global order""" if isinstance(new_order, str): new_order = json.loads(new_order) for i, module_name in enumerate(new_order): - if module_name not in ('Explore',): + if module_name not in ("Explore",): if user: icon = get_user_copy(module_name, user) else: - name = frappe.db.get_value('Desktop Icon', - {'standard': 1, 'module_name': module_name}) + name = frappe.db.get_value("Desktop Icon", {"standard": 1, "module_name": module_name}) if name: - icon = frappe.get_doc('Desktop Icon', name) + icon = frappe.get_doc("Desktop Icon", name) else: # standard icon missing, create one for DocType name = add_user_icon(module_name, standard=1) - icon = frappe.get_doc('Desktop Icon', name) + icon = frappe.get_doc("Desktop Icon", name) - icon.db_set('idx', i) + icon.db_set("idx", i) clear_desktop_icons_cache() + def set_desktop_icons(visible_list, ignore_duplicate=True): - '''Resets all lists and makes only the given one standard, + """Resets all lists and makes only the given one standard, if the desktop icon does not exist and the name is a DocType, then will create - an icon for the doctype''' + an icon for the doctype""" # clear all custom only if setup is not complete if not int(frappe.defaults.get_defaults().setup_complete or 0): @@ -201,15 +241,15 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: - frappe.db.sql('update `tabDesktop Icon` set blocked=0, hidden=1 where standard=1') + frappe.db.sql("update `tabDesktop Icon` set blocked=0, hidden=1 where standard=1") # set as visible if present, or add icon for module_name in visible_list: - name = frappe.db.get_value('Desktop Icon', {'module_name': module_name}) + name = frappe.db.get_value("Desktop Icon", {"module_name": module_name}) if name: - frappe.db.set_value('Desktop Icon', name, 'hidden', 0) + frappe.db.set_value("Desktop Icon", name, "hidden", 0) else: - if frappe.db.exists('DocType', module_name): + if frappe.db.exists("DocType", module_name): try: add_user_icon(module_name, standard=1) except frappe.UniqueValidationError as e: @@ -225,10 +265,11 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): clear_desktop_icons_cache() + def set_hidden_list(hidden_list, user=None): - '''Sets property `hidden`=1 in **Desktop Icon** for given user. + """Sets property `hidden`=1 in **Desktop Icon** for given user. If user is None then it will set global values. - It will also set the rest of the icons as shown (`hidden` = 0)''' + It will also set the rest of the icons as shown (`hidden` = 0)""" if isinstance(hidden_list, str): hidden_list = json.loads(hidden_list) @@ -245,9 +286,10 @@ def set_hidden_list(hidden_list, user=None): else: frappe.clear_cache() + def set_hidden(module_name, user=None, hidden=1): - '''Set module hidden property for given user. If user is not specified, - hide/unhide it globally''' + """Set module hidden property for given user. If user is not specified, + hide/unhide it globally""" if user: icon = get_user_copy(module_name, user) @@ -256,55 +298,71 @@ def set_hidden(module_name, user=None, hidden=1): return # hidden by user - icon.db_set('hidden', hidden) + icon.db_set("hidden", hidden) else: - icon = frappe.get_doc('Desktop Icon', {'standard': 1, 'module_name': module_name}) + icon = frappe.get_doc("Desktop Icon", {"standard": 1, "module_name": module_name}) # blocked is globally hidden - icon.db_set('blocked', hidden) + icon.db_set("blocked", hidden) + def get_all_icons(): - return [d.module_name for d in frappe.get_all('Desktop Icon', - filters={'standard': 1}, fields=['module_name'])] + return [ + d.module_name + for d in frappe.get_all("Desktop Icon", filters={"standard": 1}, fields=["module_name"]) + ] + def clear_desktop_icons_cache(user=None): - frappe.cache().hdel('desktop_icons', user or frappe.session.user) - frappe.cache().hdel('bootinfo', user or frappe.session.user) + frappe.cache().hdel("desktop_icons", user or frappe.session.user) + frappe.cache().hdel("bootinfo", user or frappe.session.user) + def get_user_copy(module_name, user=None): - '''Return user copy (Desktop Icon) of the given module_name. If user copy does not exist, create one. + """Return user copy (Desktop Icon) of the given module_name. If user copy does not exist, create one. :param module_name: Name of the module :param user: User for which the copy is required (optional) - ''' + """ if not user: user = frappe.session.user - desktop_icon_name = frappe.db.get_value('Desktop Icon', - {'module_name': module_name, 'owner': user, 'standard': 0}) + desktop_icon_name = frappe.db.get_value( + "Desktop Icon", {"module_name": module_name, "owner": user, "standard": 0} + ) if desktop_icon_name: - return frappe.get_doc('Desktop Icon', desktop_icon_name) + return frappe.get_doc("Desktop Icon", desktop_icon_name) else: return make_user_copy(module_name, user) + def make_user_copy(module_name, user): - '''Insert and return the user copy of a standard Desktop Icon''' - standard_name = frappe.db.get_value('Desktop Icon', {'module_name': module_name, 'standard': 1}) + """Insert and return the user copy of a standard Desktop Icon""" + standard_name = frappe.db.get_value("Desktop Icon", {"module_name": module_name, "standard": 1}) if not standard_name: - frappe.throw(_('{0} not found').format(module_name), frappe.DoesNotExistError) - - original = frappe.get_doc('Desktop Icon', standard_name) - - desktop_icon = frappe.get_doc({ - 'doctype': 'Desktop Icon', - 'standard': 0, - 'owner': user, - 'module_name': module_name - }) - - for key in ('app', 'label', 'route', 'type', '_doctype', 'idx', 'reverse', 'force_show', 'link', 'icon', 'color'): + frappe.throw(_("{0} not found").format(module_name), frappe.DoesNotExistError) + + original = frappe.get_doc("Desktop Icon", standard_name) + + desktop_icon = frappe.get_doc( + {"doctype": "Desktop Icon", "standard": 0, "owner": user, "module_name": module_name} + ) + + for key in ( + "app", + "label", + "route", + "type", + "_doctype", + "idx", + "reverse", + "force_show", + "link", + "icon", + "color", + ): if original.get(key): desktop_icon.set(key, original.get(key)) @@ -312,43 +370,42 @@ def make_user_copy(module_name, user): return desktop_icon + def sync_desktop_icons(): - '''Sync desktop icons from all apps''' + """Sync desktop icons from all apps""" for app in frappe.get_installed_apps(): sync_from_app(app) + def sync_from_app(app): - '''Sync desktop icons from app. To be called during install''' + """Sync desktop icons from app. To be called during install""" try: - modules = frappe.get_attr(app + '.config.desktop.get_data')() or {} + modules = frappe.get_attr(app + ".config.desktop.get_data")() or {} except ImportError: return [] if isinstance(modules, dict): modules_list = [] for m, desktop_icon in modules.items(): - desktop_icon['module_name'] = m + desktop_icon["module_name"] = m modules_list.append(desktop_icon) else: modules_list = modules for i, m in enumerate(modules_list): - desktop_icon_name = frappe.db.get_value('Desktop Icon', - {'module_name': m['module_name'], 'app': app, 'standard': 1}) + desktop_icon_name = frappe.db.get_value( + "Desktop Icon", {"module_name": m["module_name"], "app": app, "standard": 1} + ) if desktop_icon_name: - desktop_icon = frappe.get_doc('Desktop Icon', desktop_icon_name) + desktop_icon = frappe.get_doc("Desktop Icon", desktop_icon_name) else: # new icon - desktop_icon = frappe.get_doc({ - 'doctype': 'Desktop Icon', - 'idx': i, - 'standard': 1, - 'app': app, - 'owner': 'Administrator' - }) + desktop_icon = frappe.get_doc( + {"doctype": "Desktop Icon", "idx": i, "standard": 1, "app": app, "owner": "Administrator"} + ) - if 'doctype' in m: - m['_doctype'] = m.pop('doctype') + if "doctype" in m: + m["_doctype"] = m.pop("doctype") desktop_icon.update(m) try: @@ -358,46 +415,53 @@ def sync_from_app(app): return modules_list + @frappe.whitelist() def update_icons(hidden_list, user=None): """update modules""" if not user: - frappe.only_for('System Manager') + frappe.only_for("System Manager") set_hidden_list(hidden_list, user) - frappe.msgprint(frappe._('Updated'), indicator='green', title=_('Success'), alert=True) + frappe.msgprint(frappe._("Updated"), indicator="green", title=_("Success"), alert=True) + def get_context(context): context.icons = get_user_icons(frappe.session.user) context.user = frappe.session.user - if 'System Manager' in frappe.get_roles(): - context.users = frappe.db.get_all('User', filters={'user_type': 'System User', 'enabled': 1}, - fields = ['name', 'first_name', 'last_name']) + if "System Manager" in frappe.get_roles(): + context.users = frappe.db.get_all( + "User", + filters={"user_type": "System User", "enabled": 1}, + fields=["name", "first_name", "last_name"], + ) + @frappe.whitelist() def get_module_icons(user=None): if user != frappe.session.user: - frappe.only_for('System Manager') + frappe.only_for("System Manager") if not user: - icons = frappe.db.get_all('Desktop Icon', - fields='*', filters={'standard': 1}, order_by='idx') + icons = frappe.db.get_all("Desktop Icon", fields="*", filters={"standard": 1}, order_by="idx") else: - frappe.cache().hdel('desktop_icons', user) + frappe.cache().hdel("desktop_icons", user) icons = get_user_icons(user) for icon in icons: icon.value = frappe.db.escape(_(icon.label or icon.module_name)) - return {'icons': icons, 'user': user} + return {"icons": icons, "user": user} + def get_user_icons(user): - '''Get user icons for module setup page''' + """Get user icons for module setup page""" user_perms = UserPermissions(user) user_perms.build_permissions() from frappe.boot import get_allowed_pages + allowed_pages = get_allowed_pages() icons = [] @@ -407,13 +471,13 @@ def get_user_icons(user): add = False if not icon.custom: - if icon.module_name==['Help', 'Settings']: + if icon.module_name == ["Help", "Settings"]: pass - elif icon.type=="page" and icon.link not in allowed_pages: + elif icon.type == "page" and icon.link not in allowed_pages: add = False - elif icon.type=="module" and icon.module_name not in user_perms.allow_modules: + elif icon.type == "module" and icon.module_name not in user_perms.allow_modules: add = False if add: @@ -421,64 +485,66 @@ def get_user_icons(user): return icons + palette = ( - ('#FFC4C4',), - ('#FFE8CD',), - ('#FFD2C2',), - ('#FF8989',), - ('#FFD19C',), - ('#FFA685',), - ('#FF4D4D', 1), - ('#FFB868',), - ('#FF7846', 1), - ('#A83333', 1), - ('#A87945', 1), - ('#A84F2E', 1), - ('#D2D2FF',), - ('#F8D4F8',), - ('#DAC7FF',), - ('#A3A3FF',), - ('#F3AAF0',), - ('#B592FF',), - ('#7575FF', 1), - ('#EC7DEA', 1), - ('#8E58FF', 1), - ('#4D4DA8', 1), - ('#934F92', 1), - ('#5E3AA8', 1), - ('#EBF8CC',), - ('#FFD7D7',), - ('#D2F8ED',), - ('#D9F399',), - ('#FFB1B1',), - ('#A4F3DD',), - ('#C5EC63',), - ('#FF8989', 1), - ('#77ECCA',), - ('#7B933D', 1), - ('#A85B5B', 1), - ('#49937E', 1), - ('#FFFACD',), - ('#D2F1FF',), - ('#CEF6D1',), - ('#FFF69C',), - ('#A6E4FF',), - ('#9DECA2',), - ('#FFF168',), - ('#78D6FF',), - ('#6BE273',), - ('#A89F45', 1), - ('#4F8EA8', 1), - ('#428B46', 1) + ("#FFC4C4",), + ("#FFE8CD",), + ("#FFD2C2",), + ("#FF8989",), + ("#FFD19C",), + ("#FFA685",), + ("#FF4D4D", 1), + ("#FFB868",), + ("#FF7846", 1), + ("#A83333", 1), + ("#A87945", 1), + ("#A84F2E", 1), + ("#D2D2FF",), + ("#F8D4F8",), + ("#DAC7FF",), + ("#A3A3FF",), + ("#F3AAF0",), + ("#B592FF",), + ("#7575FF", 1), + ("#EC7DEA", 1), + ("#8E58FF", 1), + ("#4D4DA8", 1), + ("#934F92", 1), + ("#5E3AA8", 1), + ("#EBF8CC",), + ("#FFD7D7",), + ("#D2F8ED",), + ("#D9F399",), + ("#FFB1B1",), + ("#A4F3DD",), + ("#C5EC63",), + ("#FF8989", 1), + ("#77ECCA",), + ("#7B933D", 1), + ("#A85B5B", 1), + ("#49937E", 1), + ("#FFFACD",), + ("#D2F1FF",), + ("#CEF6D1",), + ("#FFF69C",), + ("#A6E4FF",), + ("#9DECA2",), + ("#FFF168",), + ("#78D6FF",), + ("#6BE273",), + ("#A89F45", 1), + ("#4F8EA8", 1), + ("#428B46", 1), ) + @frappe.whitelist() -def hide(name, user = None): +def hide(name, user=None): if not user: user = frappe.session.user try: - set_hidden(name, user, hidden = 1) + set_hidden(name, user, hidden=1) clear_desktop_icons_cache() except Exception: return False diff --git a/frappe/desk/doctype/event/__init__.py b/frappe/desk/doctype/event/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/desk/doctype/event/__init__.py +++ b/frappe/desk/doctype/event/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 86f0656bc6..531cc69c57 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -2,19 +2,40 @@ # License: MIT. See LICENSE -import frappe import json -from frappe.utils import (getdate, cint, add_months, date_diff, add_days, - nowdate, get_datetime_str, cstr, get_datetime, now_datetime, format_datetime) +import frappe from frappe import _ +from frappe.desk.doctype.notification_settings.notification_settings import ( + is_email_notifications_enabled_for_type, +) +from frappe.desk.reportview import get_filters_cond from frappe.model.document import Document +from frappe.utils import ( + add_days, + add_months, + cint, + cstr, + date_diff, + format_datetime, + get_datetime, + get_datetime_str, + getdate, + now_datetime, + nowdate, +) from frappe.utils.user import get_enabled_system_users -from frappe.desk.reportview import get_filters_cond -from frappe.desk.doctype.notification_settings.notification_settings import is_email_notifications_enabled_for_type weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] -communication_mapping = {"": "Event", "Event": "Event", "Meeting": "Meeting", "Call": "Phone", "Sent/Received Email": "Email", "Other": "Other"} +communication_mapping = { + "": "Event", + "Event": "Event", + "Meeting": "Meeting", + "Call": "Phone", + "Sent/Received Email": "Email", + "Other": "Other", +} + class Event(Document): def validate(self): @@ -27,7 +48,9 @@ class Event(Document): if self.starts_on and self.ends_on: self.validate_from_to_dates("starts_on", "ends_on") - if self.repeat_on == "Daily" and self.ends_on and getdate(self.starts_on) != getdate(self.ends_on): + if ( + self.repeat_on == "Daily" and self.ends_on and getdate(self.starts_on) != getdate(self.ends_on) + ): frappe.throw(_("Daily Events should finish on the Same Day.")) if self.sync_with_google_calendar and not self.google_calendar: @@ -37,7 +60,9 @@ class Event(Document): self.sync_communication() def on_trash(self): - communications = frappe.get_all("Communication", dict(reference_doctype=self.doctype, reference_name=self.name)) + communications = frappe.get_all( + "Communication", dict(reference_doctype=self.doctype, reference_name=self.name) + ) if communications: for communication in communications: frappe.delete_doc_if_exists("Communication", communication.name) @@ -49,7 +74,7 @@ class Event(Document): ["Communication", "reference_doctype", "=", self.doctype], ["Communication", "reference_name", "=", self.name], ["Communication Link", "link_doctype", "=", participant.reference_doctype], - ["Communication Link", "link_name", "=", participant.reference_docname] + ["Communication Link", "link_name", "=", participant.reference_docname], ] comms = frappe.get_all("Communication", filters=filters, fields=["name"]) @@ -59,7 +84,7 @@ class Event(Document): self.update_communication(participant, communication) else: meta = frappe.get_meta(participant.reference_doctype) - if hasattr(meta, "allow_events_in_timeline") and meta.allow_events_in_timeline==1: + if hasattr(meta, "allow_events_in_timeline") and meta.allow_events_in_timeline == 1: self.create_communication(participant) def create_communication(self, participant): @@ -76,7 +101,9 @@ class Event(Document): communication.sender_full_name = frappe.utils.get_fullname(self.owner) communication.reference_doctype = self.doctype communication.reference_name = self.name - communication.communication_medium = communication_mapping.get(self.event_category) if self.event_category else "" + communication.communication_medium = ( + communication_mapping.get(self.event_category) if self.event_category else "" + ) communication.status = "Linked" communication.add_link(participant.reference_doctype, participant.reference_docname) communication.save(ignore_permissions=True) @@ -85,23 +112,27 @@ class Event(Document): """Add a single participant to event participants Args: - doctype (string): Reference Doctype - docname (string): Reference Docname + doctype (string): Reference Doctype + docname (string): Reference Docname """ - self.append("event_participants", { - "reference_doctype": doctype, - "reference_docname": docname, - }) + self.append( + "event_participants", + { + "reference_doctype": doctype, + "reference_docname": docname, + }, + ) def add_participants(self, participants): """Add participant entry Args: - participants ([Array]): Array of a dict with doctype and docname + participants ([Array]): Array of a dict with doctype and docname """ - for participant in participants: + for participant in participants: self.add_participant(participant["doctype"], participant["docname"]) + @frappe.whitelist() def delete_communication(event, reference_doctype, reference_docname): deleted_participant = frappe.get_doc(reference_doctype, reference_docname) @@ -112,7 +143,7 @@ def delete_communication(event, reference_doctype, reference_docname): ["Communication", "reference_doctype", "=", event.get("doctype")], ["Communication", "reference_name", "=", event.get("name")], ["Communication Link", "link_doctype", "=", deleted_participant.reference_doctype], - ["Communication Link", "link_name", "=", deleted_participant.reference_docname] + ["Communication Link", "link_name", "=", deleted_participant.reference_docname], ] comms = frappe.get_list("Communication", filters=filters, fields=["name"]) @@ -129,23 +160,29 @@ def delete_communication(event, reference_doctype, reference_docname): def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user return """(`tabEvent`.`event_type`='Public' or `tabEvent`.`owner`=%(user)s)""" % { - "user": frappe.db.escape(user), - } + "user": frappe.db.escape(user), + } + def has_permission(doc, user): - if doc.event_type=="Public" or doc.owner==user: + if doc.event_type == "Public" or doc.owner == user: return True return False + def send_event_digest(): today = nowdate() # select only those users that have event reminder email notifications enabled - users = [user for user in get_enabled_system_users() if - is_email_notifications_enabled_for_type(user.name, 'Event Reminders')] + users = [ + user + for user in get_enabled_system_users() + if is_email_notifications_enabled_for_type(user.name, "Event Reminders") + ] for user in users: events = get_events(today, today, user.name, for_reminder=True) @@ -153,7 +190,7 @@ def send_event_digest(): frappe.set_user_lang(user.name, user.language) for e in events: - e.starts_on = format_datetime(e.starts_on, 'hh:mm a') + e.starts_on = format_datetime(e.starts_on, "hh:mm a") if e.all_day: e.starts_on = "All Day" @@ -162,11 +199,12 @@ def send_event_digest(): subject=frappe._("Upcoming Events for Today"), template="upcoming_events", args={ - 'events': events, + "events": events, }, - header=[frappe._("Events in Today's Calendar"), 'blue'] + header=[frappe._("Events in Today's Calendar"), "blue"], ) + @frappe.whitelist() def get_events(start, end, user=None, for_reminder=False, filters=None): if not user: @@ -175,13 +213,14 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): if isinstance(filters, str): filters = json.loads(filters) - filter_condition = get_filters_cond('Event', filters, []) + filter_condition = get_filters_cond("Event", filters, []) tables = ["`tabEvent`"] if "`tabEvent Participants`" in filter_condition: tables.append("`tabEvent Participants`") - events = frappe.db.sql(""" + events = frappe.db.sql( + """ SELECT `tabEvent`.name, `tabEvent`.subject, `tabEvent`.description, @@ -234,12 +273,15 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): ORDER BY `tabEvent`.starts_on""".format( tables=", ".join(tables), filter_condition=filter_condition, - reminder_condition="AND coalesce(`tabEvent`.send_reminder, 0)=1" if for_reminder else "" - ), { + reminder_condition="AND coalesce(`tabEvent`.send_reminder, 0)=1" if for_reminder else "", + ), + { "start": start, "end": end, "user": user, - }, as_dict=1) + }, + as_dict=1, + ) # process recurring events start = start.split(" ")[0] @@ -250,11 +292,16 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): def add_event(e, date): new_event = e.copy() - enddate = add_days(date,int(date_diff(e.ends_on.split(" ")[0], e.starts_on.split(" ")[0]))) \ - if (e.starts_on and e.ends_on) else date + enddate = ( + add_days(date, int(date_diff(e.ends_on.split(" ")[0], e.starts_on.split(" ")[0]))) + if (e.starts_on and e.ends_on) + else date + ) new_event.starts_on = date + " " + e.starts_on.split(" ")[1] - new_event.ends_on = new_event.ends_on = enddate + " " + e.ends_on.split(" ")[1] if e.ends_on else None + new_event.ends_on = new_event.ends_on = ( + enddate + " " + e.ends_on.split(" ")[1] if e.ends_on else None + ) add_events.append(new_event) @@ -275,9 +322,13 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): event_start = "-".join(event_start.split("-")[1:]) # repeat for all years in period - for year in range(start_year, end_year+1): + for year in range(start_year, end_year + 1): date = str(year) + "-" + event_start - if getdate(date) >= getdate(start) and getdate(date) <= getdate(end) and getdate(date) <= getdate(repeat): + if ( + getdate(date) >= getdate(start) + and getdate(date) <= getdate(end) + and getdate(date) <= getdate(repeat) + ): add_event(e, date) remove_events.append(e) @@ -295,19 +346,27 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): start_from = date for i in range(int(date_diff(end, start) / 30) + 3): - if getdate(date) >= getdate(start) and getdate(date) <= getdate(end) \ - and getdate(date) <= getdate(repeat) and getdate(date) >= getdate(event_start): + if ( + getdate(date) >= getdate(start) + and getdate(date) <= getdate(end) + and getdate(date) <= getdate(repeat) + and getdate(date) >= getdate(event_start) + ): add_event(e, date) - date = add_months(start_from, i+1) + date = add_months(start_from, i + 1) remove_events.append(e) if e.repeat_on == "Weekly": for cnt in range(date_diff(end, start) + 1): date = add_days(start, cnt) - if getdate(date) >= getdate(start) and getdate(date) <= getdate(end) \ - and getdate(date) <= getdate(repeat) and getdate(date) >= getdate(event_start) \ - and e[weekdays[getdate(date).weekday()]]: + if ( + getdate(date) >= getdate(start) + and getdate(date) <= getdate(end) + and getdate(date) <= getdate(repeat) + and getdate(date) >= getdate(event_start) + and e[weekdays[getdate(date).weekday()]] + ): add_event(e, date) remove_events.append(e) @@ -315,7 +374,11 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): if e.repeat_on == "Daily": for cnt in range(date_diff(end, start) + 1): date = add_days(start, cnt) - if getdate(date) >= getdate(event_start) and getdate(date) <= getdate(end) and getdate(date) <= getdate(repeat): + if ( + getdate(date) >= getdate(event_start) + and getdate(date) <= getdate(end) + and getdate(date) <= getdate(repeat) + ): add_event(e, date) remove_events.append(e) @@ -332,26 +395,36 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): return events + def delete_events(ref_type, ref_name, delete_event=False): - participations = frappe.get_all("Event Participants", filters={"reference_doctype": ref_type, "reference_docname": ref_name, - "parenttype": "Event"}, fields=["parent", "name"]) + participations = frappe.get_all( + "Event Participants", + filters={"reference_doctype": ref_type, "reference_docname": ref_name, "parenttype": "Event"}, + fields=["parent", "name"], + ) if participations: for participation in participations: if delete_event: frappe.delete_doc("Event", participation.parent, for_reload=True) else: - total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent}) + total_participants = frappe.get_all( + "Event Participants", filters={"parenttype": "Event", "parent": participation.parent} + ) if len(total_participants) <= 1: frappe.db.delete("Event", {"name": participation.parent}) frappe.db.delete("Event Participants", {"name": participation.name}) + # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): - events = frappe.get_list("Event", filters={"status": "Open"}, fields=["name", "ends_on", "repeat_till"]) + events = frappe.get_list( + "Event", filters={"status": "Open"}, fields=["name", "ends_on", "repeat_till"] + ) for event in events: - if (event.ends_on and getdate(event.ends_on) < getdate(nowdate())) \ - or (event.repeat_till and getdate(event.repeat_till) < getdate(nowdate())): + if (event.ends_on and getdate(event.ends_on) < getdate(nowdate())) or ( + event.repeat_till and getdate(event.repeat_till) < getdate(nowdate()) + ): frappe.db.set_value("Event", event.name, "status", "Closed") diff --git a/frappe/desk/doctype/event/test_event.py b/frappe/desk/doctype/event/test_event.py index b0269a80cc..041bda643e 100644 --- a/frappe/desk/doctype/event/test_event.py +++ b/frappe/desk/doctype/event/test_event.py @@ -2,22 +2,23 @@ # License: MIT. See LICENSE """Use blog post test to test user permissions logic""" -import frappe -import frappe.defaults -import unittest import json +import unittest +import frappe +import frappe.defaults from frappe.desk.doctype.event.event import get_events from frappe.test_runner import make_test_objects -test_records = frappe.get_test_records('Event') +test_records = frappe.get_test_records("Event") + class TestEvent(unittest.TestCase): def setUp(self): frappe.db.delete("Event") - make_test_objects('Event', reset=True) + make_test_objects("Event", reset=True) - self.test_records = frappe.get_test_records('Event') + self.test_records = frappe.get_test_records("Event") self.test_user = "test1@example.com" def tearDown(self): @@ -25,16 +26,16 @@ class TestEvent(unittest.TestCase): def test_allowed_public(self): frappe.set_user(self.test_user) - doc = frappe.get_doc("Event", frappe.db.get_value("Event", {"subject":"_Test Event 1"})) + doc = frappe.get_doc("Event", frappe.db.get_value("Event", {"subject": "_Test Event 1"})) self.assertTrue(frappe.has_permission("Event", doc=doc)) def test_not_allowed_private(self): frappe.set_user(self.test_user) - doc = frappe.get_doc("Event", frappe.db.get_value("Event", {"subject":"_Test Event 2"})) + doc = frappe.get_doc("Event", frappe.db.get_value("Event", {"subject": "_Test Event 2"})) self.assertFalse(frappe.has_permission("Event", doc=doc)) def test_allowed_private_if_in_event_user(self): - name = frappe.db.get_value("Event", {"subject":"_Test Event 3"}) + name = frappe.db.get_value("Event", {"subject": "_Test Event 3"}) frappe.share.add("Event", name, self.test_user, "read") frappe.set_user(self.test_user) doc = frappe.get_doc("Event", name) @@ -44,7 +45,9 @@ class TestEvent(unittest.TestCase): def test_event_list(self): frappe.set_user(self.test_user) - res = frappe.get_list("Event", filters=[["Event", "subject", "like", "_Test Event%"]], fields=["name", "subject"]) + res = frappe.get_list( + "Event", filters=[["Event", "subject", "like", "_Test Event%"]], fields=["name", "subject"] + ) self.assertEqual(len(res), 1) subjects = [r.subject for r in res] self.assertTrue("_Test Event 1" in subjects) @@ -68,32 +71,38 @@ class TestEvent(unittest.TestCase): ev = frappe.get_doc(self.test_records[0]).insert() - add({ - "assign_to": ["test@example.com"], - "doctype": "Event", - "name": ev.name, - "description": "Test Assignment" - }) + add( + { + "assign_to": ["test@example.com"], + "doctype": "Event", + "name": ev.name, + "description": "Test Assignment", + } + ) ev = frappe.get_doc("Event", ev.name) self.assertEqual(ev._assign, json.dumps(["test@example.com"])) # add another one - add({ - "assign_to": [self.test_user], - "doctype": "Event", - "name": ev.name, - "description": "Test Assignment" - }) + add( + { + "assign_to": [self.test_user], + "doctype": "Event", + "name": ev.name, + "description": "Test Assignment", + } + ) ev = frappe.get_doc("Event", ev.name) self.assertEqual(set(json.loads(ev._assign)), set(["test@example.com", self.test_user])) # Remove an assignment - todo = frappe.get_doc("ToDo", {"reference_type": ev.doctype, "reference_name": ev.name, - "allocated_to": self.test_user}) + todo = frappe.get_doc( + "ToDo", + {"reference_type": ev.doctype, "reference_name": ev.name, "allocated_to": self.test_user}, + ) todo.status = "Cancelled" todo.save() @@ -104,24 +113,26 @@ class TestEvent(unittest.TestCase): ev.delete() def test_recurring(self): - ev = frappe.get_doc({ - "doctype":"Event", - "subject": "_Test Event", - "starts_on": "2014-02-01", - "event_type": "Public", - "repeat_this_event": 1, - "repeat_on": "Yearly" - }) + ev = frappe.get_doc( + { + "doctype": "Event", + "subject": "_Test Event", + "starts_on": "2014-02-01", + "event_type": "Public", + "repeat_this_event": 1, + "repeat_on": "Yearly", + } + ) ev.insert() ev_list = get_events("2014-02-01", "2014-02-01", "Administrator", for_reminder=True) - self.assertTrue(bool(list(filter(lambda e: e.name==ev.name, ev_list)))) + self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list)))) ev_list1 = get_events("2015-01-20", "2015-01-20", "Administrator", for_reminder=True) - self.assertFalse(bool(list(filter(lambda e: e.name==ev.name, ev_list1)))) + self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list1)))) ev_list2 = get_events("2014-02-20", "2014-02-20", "Administrator", for_reminder=True) - self.assertFalse(bool(list(filter(lambda e: e.name==ev.name, ev_list2)))) + self.assertFalse(bool(list(filter(lambda e: e.name == ev.name, ev_list2)))) ev_list3 = get_events("2015-02-01", "2015-02-01", "Administrator", for_reminder=True) - self.assertTrue(bool(list(filter(lambda e: e.name==ev.name, ev_list3)))) + self.assertTrue(bool(list(filter(lambda e: e.name == ev.name, ev_list3)))) diff --git a/frappe/desk/doctype/event_participants/event_participants.py b/frappe/desk/doctype/event_participants/event_participants.py index b834ba3a82..fdb834b285 100644 --- a/frappe/desk/doctype/event_participants/event_participants.py +++ b/frappe/desk/doctype/event_participants/event_participants.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE from frappe.model.document import Document + class EventParticipants(Document): - pass \ No newline at end of file + pass diff --git a/frappe/desk/doctype/form_tour/test_form_tour.py b/frappe/desk/doctype/form_tour/test_form_tour.py index 3670cbc218..cb0c4ef33a 100644 --- a/frappe/desk/doctype/form_tour/test_form_tour.py +++ b/frappe/desk/doctype/form_tour/test_form_tour.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestFormTour(unittest.TestCase): pass diff --git a/frappe/desk/doctype/form_tour_step/form_tour_step.py b/frappe/desk/doctype/form_tour_step/form_tour_step.py index bbc8edea08..9d82ae8527 100644 --- a/frappe/desk/doctype/form_tour_step/form_tour_step.py +++ b/frappe/desk/doctype/form_tour_step/form_tour_step.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class FormTourStep(Document): pass diff --git a/frappe/desk/doctype/global_search_doctype/global_search_doctype.py b/frappe/desk/doctype/global_search_doctype/global_search_doctype.py index 30a31f959f..8bdc05cd71 100644 --- a/frappe/desk/doctype/global_search_doctype/global_search_doctype.py +++ b/frappe/desk/doctype/global_search_doctype/global_search_doctype.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class GlobalSearchDocType(Document): pass diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.py b/frappe/desk/doctype/global_search_settings/global_search_settings.py index e9a47cecd1..b7ffd7faf7 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.py +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.py @@ -3,11 +3,11 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document -class GlobalSearchSettings(Document): +class GlobalSearchSettings(Document): def validate(self): dts, core_dts, repeated_dts = [], [], [] @@ -25,11 +25,12 @@ class GlobalSearchSettings(Document): frappe.throw(_("Core Modules {0} cannot be searched in Global Search.").format(core_dts)) if repeated_dts: - repeated_dts = (", ".join([frappe.bold(dt) for dt in repeated_dts])) + repeated_dts = ", ".join([frappe.bold(dt) for dt in repeated_dts]) frappe.throw(_("Document Type {0} has been repeated.").format(repeated_dts)) # reset cache - frappe.cache().hdel('global_search', 'search_priorities') + frappe.cache().hdel("global_search", "search_priorities") + def get_doctypes_for_global_search(): def get_from_db(): @@ -43,6 +44,7 @@ def get_doctypes_for_global_search(): def reset_global_search_settings_doctypes(): update_global_search_doctypes() + def update_global_search_doctypes(): global_search_doctypes = [] show_message(1, _("Fetching default Global Search documents.")) @@ -77,11 +79,14 @@ def update_global_search_doctypes(): if dt not in doctype_list: continue - global_search_settings.append("allowed_in_global_search", { - "document_type": dt - }) + global_search_settings.append("allowed_in_global_search", {"document_type": dt}) global_search_settings.save(ignore_permissions=True) show_message(3, "Global Search Documents have been reset.") + def show_message(progress, msg): - frappe.publish_realtime('global_search_settings', {"progress":progress, "total":3, "msg": msg}, user=frappe.session.user) + frappe.publish_realtime( + "global_search_settings", + {"progress": progress, "total": 3, "msg": msg}, + user=frappe.session.user, + ) diff --git a/frappe/desk/doctype/kanban_board/kanban_board.py b/frappe/desk/doctype/kanban_board/kanban_board.py index 97f529a061..ed936bb79e 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.py +++ b/frappe/desk/doctype/kanban_board/kanban_board.py @@ -2,8 +2,9 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import json + +import frappe from frappe import _ from frappe.model.document import Document @@ -24,14 +25,17 @@ class KanbanBoard(Document): if not column.column_name: frappe.msgprint(frappe._("Column Name cannot be empty"), raise_exception=True) + def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user if user == "Administrator": return "" return """(`tabKanban Board`.private=0 or `tabKanban Board`.owner='{user}')""".format(user=user) + def has_permission(doc, ptype, user): if doc.private == 0 or user == "Administrator": return True @@ -41,32 +45,33 @@ def has_permission(doc, ptype, user): return False + @frappe.whitelist() def get_kanban_boards(doctype): - '''Get Kanban Boards for doctype to show in List View''' - return frappe.get_list('Kanban Board', - fields=['name', 'filters', 'reference_doctype', 'private'], - filters={ 'reference_doctype': doctype } + """Get Kanban Boards for doctype to show in List View""" + return frappe.get_list( + "Kanban Board", + fields=["name", "filters", "reference_doctype", "private"], + filters={"reference_doctype": doctype}, ) + @frappe.whitelist() def add_column(board_name, column_title): - '''Adds new column to Kanban Board''' + """Adds new column to Kanban Board""" doc = frappe.get_doc("Kanban Board", board_name) for col in doc.columns: if column_title == col.column_name: frappe.throw(_("Column {0} already exist.").format(column_title)) - doc.append("columns", dict( - column_name=column_title - )) + doc.append("columns", dict(column_name=column_title)) doc.save() return doc.columns @frappe.whitelist() def archive_restore_column(board_name, column_title, status): - '''Set column's status to status''' + """Set column's status to status""" doc = frappe.get_doc("Kanban Board", board_name) for col in doc.columns: if column_title == col.column_name: @@ -78,8 +83,8 @@ def archive_restore_column(board_name, column_title, status): @frappe.whitelist() def update_order(board_name, order): - '''Save the order of cards in columns''' - board = frappe.get_doc('Kanban Board', board_name) + """Save the order of cards in columns""" + board = frappe.get_doc("Kanban Board", board_name) doctype = board.reference_doctype fieldname = board.field_name order_dict = json.loads(order) @@ -88,17 +93,10 @@ def update_order(board_name, order): for col_name, cards in order_dict.items(): order_list = [] for card in cards: - column = frappe.get_value( - doctype, - {'name': card}, - fieldname - ) + column = frappe.get_value(doctype, {"name": card}, fieldname) if column != col_name: frappe.set_value(doctype, card, fieldname, col_name) - updated_cards.append(dict( - name=card, - column=col_name - )) + updated_cards.append(dict(name=card, column=col_name)) for column in board.columns: if column.column_name == col_name: @@ -107,10 +105,13 @@ def update_order(board_name, order): board.save() return board, updated_cards + @frappe.whitelist() -def update_order_for_single_card(board_name, docname, from_colname, to_colname, old_index, new_index): - '''Save the order of cards in columns''' - board = frappe.get_doc('Kanban Board', board_name) +def update_order_for_single_card( + board_name, docname, from_colname, to_colname, old_index, new_index +): + """Save the order of cards in columns""" + board = frappe.get_doc("Kanban Board", board_name) doctype = board.reference_doctype fieldname = board.field_name old_index = frappe.parse_json(old_index) @@ -135,6 +136,7 @@ def update_order_for_single_card(board_name, docname, from_colname, to_colname, return board + def get_kanban_column_order_and_index(board, colname): for i, col in enumerate(board.columns): if col.column_name == colname: @@ -143,9 +145,10 @@ def get_kanban_column_order_and_index(board, colname): return col_order, col_idx + @frappe.whitelist() def add_card(board_name, docname, colname): - board = frappe.get_doc('Kanban Board', board_name) + board = frappe.get_doc("Kanban Board", board_name) col_order, col_idx = get_kanban_column_order_and_index(board, colname) col_order.insert(0, docname) @@ -155,11 +158,12 @@ def add_card(board_name, docname, colname): board.save() return board + @frappe.whitelist() def quick_kanban_board(doctype, board_name, field_name, project=None): - '''Create new KanbanBoard quickly with default options''' + """Create new KanbanBoard quickly with default options""" - doc = frappe.new_doc('Kanban Board') + doc = frappe.new_doc("Kanban Board") meta = frappe.get_meta(doctype) doc.kanban_board_name = board_name @@ -169,40 +173,39 @@ def quick_kanban_board(doctype, board_name, field_name, project=None): if project: doc.filters = '[["Task","project","=","{0}"]]'.format(project) - options = '' + options = "" for field in meta.fields: if field.fieldname == field_name: options = field.options columns = [] if options: - columns = options.split('\n') + columns = options.split("\n") for column in columns: if not column: continue - doc.append("columns", dict( - column_name=column - )) - + doc.append("columns", dict(column_name=column)) - if doctype in ['Note', 'ToDo']: + if doctype in ["Note", "ToDo"]: doc.private = 1 doc.save() return doc + def get_order_for_column(board, colname): - filters = [[board.reference_doctype, board.field_name, '=', colname]] + filters = [[board.reference_doctype, board.field_name, "=", colname]] if board.filters: filters.append(frappe.parse_json(board.filters)[0]) - return frappe.as_json(frappe.get_list(board.reference_doctype, filters=filters, pluck='name')) + return frappe.as_json(frappe.get_list(board.reference_doctype, filters=filters, pluck="name")) + @frappe.whitelist() def update_column_order(board_name, order): - '''Set the order of columns in Kanban Board''' - board = frappe.get_doc('Kanban Board', board_name) + """Set the order of columns in Kanban Board""" + board = frappe.get_doc("Kanban Board", board_name) order = json.loads(order) old_columns = board.columns new_columns = [] @@ -217,20 +220,24 @@ def update_column_order(board_name, order): board.columns = [] for col in new_columns: - board.append("columns", dict( - column_name=col.column_name, - status=col.status, - order=col.order, - indicator=col.indicator, - )) + board.append( + "columns", + dict( + column_name=col.column_name, + status=col.status, + order=col.order, + indicator=col.indicator, + ), + ) board.save() return board + @frappe.whitelist() def set_indicator(board_name, column_name, indicator): - '''Set the indicator color of column''' - board = frappe.get_doc('Kanban Board', board_name) + """Set the indicator color of column""" + board = frappe.get_doc("Kanban Board", board_name) for column in board.columns: if column.column_name == column_name: @@ -242,6 +249,5 @@ def set_indicator(board_name, column_name, indicator): @frappe.whitelist() def save_filters(board_name, filters): - '''Save filters silently''' - frappe.db.set_value('Kanban Board', board_name, 'filters', - filters, update_modified=False) + """Save filters silently""" + frappe.db.set_value("Kanban Board", board_name, "filters", filters, update_modified=False) diff --git a/frappe/desk/doctype/kanban_board/test_kanban_board.py b/frappe/desk/doctype/kanban_board/test_kanban_board.py index f00446141a..179e6c71e5 100644 --- a/frappe/desk/doctype/kanban_board/test_kanban_board.py +++ b/frappe/desk/doctype/kanban_board/test_kanban_board.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Kanban Board') + class TestKanbanBoard(unittest.TestCase): pass diff --git a/frappe/desk/doctype/kanban_board_column/kanban_board_column.py b/frappe/desk/doctype/kanban_board_column/kanban_board_column.py index d919fd6aed..8a1f839c98 100644 --- a/frappe/desk/doctype/kanban_board_column/kanban_board_column.py +++ b/frappe/desk/doctype/kanban_board_column/kanban_board_column.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class KanbanBoardColumn(Document): pass diff --git a/frappe/desk/doctype/list_filter/list_filter.py b/frappe/desk/doctype/list_filter/list_filter.py index d2b01d301e..ac434a760a 100644 --- a/frappe/desk/doctype/list_filter/list_filter.py +++ b/frappe/desk/doctype/list_filter/list_filter.py @@ -2,8 +2,11 @@ # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe from frappe.model.document import Document + class ListFilter(Document): pass diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.py b/frappe/desk/doctype/list_view_settings/list_view_settings.py index 78b56fe7d5..7d25f57acf 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.py @@ -5,11 +5,12 @@ import frappe from frappe.model.document import Document -class ListViewSettings(Document): +class ListViewSettings(Document): def on_update(self): frappe.clear_document_cache(self.doctype, self.name) + @frappe.whitelist() def save_listview_settings(doctype, listview_settings, removed_listview_fields): @@ -28,15 +29,15 @@ def save_listview_settings(doctype, listview_settings, removed_listview_fields): set_listview_fields(doctype, listview_settings.get("fields"), removed_listview_fields) - return { - "meta": frappe.get_meta(doctype, False), - "listview_settings": doc - } + return {"meta": frappe.get_meta(doctype, False), "listview_settings": doc} + def set_listview_fields(doctype, listview_fields, removed_listview_fields): meta = frappe.get_meta(doctype) - listview_fields = [f.get("fieldname") for f in frappe.parse_json(listview_fields) if f.get("fieldname")] + listview_fields = [ + f.get("fieldname") for f in frappe.parse_json(listview_fields) if f.get("fieldname") + ] for field in removed_listview_fields: set_in_list_view_property(doctype, meta.get_field(field), "0") @@ -44,29 +45,39 @@ def set_listview_fields(doctype, listview_fields, removed_listview_fields): for field in listview_fields: set_in_list_view_property(doctype, meta.get_field(field), "1") + def set_in_list_view_property(doctype, field, value): if not field or field.fieldname == "status_field": return - property_setter = frappe.db.get_value("Property Setter", {"doc_type": doctype, "field_name": field.fieldname, "property": "in_list_view"}) + property_setter = frappe.db.get_value( + "Property Setter", + {"doc_type": doctype, "field_name": field.fieldname, "property": "in_list_view"}, + ) if property_setter: doc = frappe.get_doc("Property Setter", property_setter) doc.value = value doc.save() else: - frappe.make_property_setter({ - "doctype": doctype, - "doctype_or_field": "DocField", - "fieldname": field.fieldname, - "property": "in_list_view", - "value": value, - "property_type": "Check" - }, ignore_validate=True) + frappe.make_property_setter( + { + "doctype": doctype, + "doctype_or_field": "DocField", + "fieldname": field.fieldname, + "property": "in_list_view", + "value": value, + "property_type": "Check", + }, + ignore_validate=True, + ) + @frappe.whitelist() def get_default_listview_fields(doctype): meta = frappe.get_meta(doctype) - path = frappe.get_module_path(frappe.scrub(meta.module), "doctype", frappe.scrub(meta.name), frappe.scrub(meta.name) + ".json") + path = frappe.get_module_path( + frappe.scrub(meta.module), "doctype", frappe.scrub(meta.name), frappe.scrub(meta.name) + ".json" + ) doctype_json = frappe.get_file_json(path) fields = [f.get("fieldname") for f in doctype_json.get("fields") if f.get("in_list_view")] diff --git a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py index 85872dd36e..0b6a0773e3 100644 --- a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestListViewSettings(unittest.TestCase): pass diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.py b/frappe/desk/doctype/module_onboarding/module_onboarding.py index aa268c792c..7a12328ee0 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.py @@ -10,10 +10,10 @@ from frappe.modules.export_file import export_to_files class ModuleOnboarding(Document): def on_update(self): if frappe.conf.developer_mode: - export_to_files(record_list=[['Module Onboarding', self.name]], record_module=self.module) + export_to_files(record_list=[["Module Onboarding", self.name]], record_module=self.module) for step in self.steps: - export_to_files(record_list=[['Onboarding Step', step.step]], record_module=self.module) + export_to_files(record_list=[["Onboarding Step", step.step]], record_module=self.module) def get_steps(self): return [frappe.get_doc("Onboarding Step", step.step) for step in self.steps] diff --git a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py index 42f472abc1..8def3ac40e 100644 --- a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestModuleOnboarding(unittest.TestCase): pass diff --git a/frappe/desk/doctype/note/note.py b/frappe/desk/doctype/note/note.py index ae7af07cd9..de019d9898 100644 --- a/frappe/desk/doctype/note/note.py +++ b/frappe/desk/doctype/note/note.py @@ -4,10 +4,12 @@ import frappe from frappe.model.document import Document + class Note(Document): def autoname(self): # replace forbidden characters import re + self.name = re.sub("[%'\"#*?`]", "", self.title.strip()) def validate(self): @@ -20,21 +22,25 @@ class Note(Document): self.print_heading = self.name self.sub_heading = "" + @frappe.whitelist() def mark_as_seen(note): - note = frappe.get_doc('Note', note) + note = frappe.get_doc("Note", note) if frappe.session.user not in [d.user for d in note.seen_by]: - note.append('seen_by', {'user': frappe.session.user}) + note.append("seen_by", {"user": frappe.session.user}) note.save(ignore_version=True) + def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user if user == "Administrator": return "" return """(`tabNote`.public=1 or `tabNote`.owner="{user}")""".format(user=user) + def has_permission(doc, ptype, user): if doc.public == 1 or user == "Administrator": return True diff --git a/frappe/desk/doctype/note/test_note.py b/frappe/desk/doctype/note/test_note.py index ac2116c38a..d8bdb9efc4 100644 --- a/frappe/desk/doctype/note/test_note.py +++ b/frappe/desk/doctype/note/test_note.py @@ -1,10 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and Contributors # License: MIT. See LICENSE -import frappe import unittest -test_records = frappe.get_test_records('Note') +import frappe + +test_records = frappe.get_test_records("Note") + class TestNote(unittest.TestCase): def insert_note(self): @@ -12,64 +14,65 @@ class TestNote(unittest.TestCase): frappe.db.delete("Note") frappe.db.delete("Note Seen By") - return frappe.get_doc(dict(doctype='Note', title='test note', - content='test note content')).insert() + return frappe.get_doc( + dict(doctype="Note", title="test note", content="test note content") + ).insert() def test_version(self): note = self.insert_note() - note.title = 'test note 1' - note.content = '1' + note.title = "test note 1" + note.content = "1" note.save(ignore_version=False) - version = frappe.get_doc('Version', dict(docname=note.name)) + version = frappe.get_doc("Version", dict(docname=note.name)) data = version.get_data() - self.assertTrue(('title', 'test note', 'test note 1'), data['changed']) - self.assertTrue(('content', 'test note content', '1'), data['changed']) + self.assertTrue(("title", "test note", "test note 1"), data["changed"]) + self.assertTrue(("content", "test note content", "1"), data["changed"]) def test_rows(self): note = self.insert_note() # test add - note.append('seen_by', {'user': 'Administrator'}) + note.append("seen_by", {"user": "Administrator"}) note.save(ignore_version=False) - version = frappe.get_doc('Version', dict(docname=note.name)) + version = frappe.get_doc("Version", dict(docname=note.name)) data = version.get_data() - self.assertEqual(len(data.get('added')), 1) - self.assertEqual(len(data.get('removed')), 0) - self.assertEqual(len(data.get('changed')), 0) + self.assertEqual(len(data.get("added")), 1) + self.assertEqual(len(data.get("removed")), 0) + self.assertEqual(len(data.get("changed")), 0) - for row in data.get('added'): - self.assertEqual(row[0], 'seen_by') - self.assertEqual(row[1]['user'], 'Administrator') + for row in data.get("added"): + self.assertEqual(row[0], "seen_by") + self.assertEqual(row[1]["user"], "Administrator") # test row change - note.seen_by[0].user = 'Guest' + note.seen_by[0].user = "Guest" note.save(ignore_version=False) - version = frappe.get_doc('Version', dict(docname=note.name)) + version = frappe.get_doc("Version", dict(docname=note.name)) data = version.get_data() - self.assertEqual(len(data.get('row_changed')), 1) - for row in data.get('row_changed'): - self.assertEqual(row[0], 'seen_by') + self.assertEqual(len(data.get("row_changed")), 1) + for row in data.get("row_changed"): + self.assertEqual(row[0], "seen_by") self.assertEqual(row[1], 0) self.assertEqual(row[2], note.seen_by[0].name) - self.assertEqual(row[3], [['user', 'Administrator', 'Guest']]) + self.assertEqual(row[3], [["user", "Administrator", "Guest"]]) # test remove note.seen_by = [] note.save(ignore_version=False) - version = frappe.get_doc('Version', dict(docname=note.name)) + version = frappe.get_doc("Version", dict(docname=note.name)) data = version.get_data() - self.assertEqual(len(data.get('removed')), 1) - for row in data.get('removed'): - self.assertEqual(row[0], 'seen_by') - self.assertEqual(row[1]['user'], 'Guest') + self.assertEqual(len(data.get("removed")), 1) + for row in data.get("removed"): + self.assertEqual(row[0], "seen_by") + self.assertEqual(row[1]["user"], "Guest") # self.assertTrue(('title', 'test note', 'test note 1'), data['changed']) # self.assertTrue(('content', 'test note content', '1'), data['changed']) diff --git a/frappe/desk/doctype/note_seen_by/note_seen_by.py b/frappe/desk/doctype/note_seen_by/note_seen_by.py index 01bee05a9f..7b87cf13b2 100644 --- a/frappe/desk/doctype/note_seen_by/note_seen_by.py +++ b/frappe/desk/doctype/note_seen_by/note_seen_by.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class NoteSeenBy(Document): pass diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 12e628ada2..1a466ea78b 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -4,12 +4,17 @@ import frappe from frappe import _ +from frappe.desk.doctype.notification_settings.notification_settings import ( + is_email_notifications_enabled_for_type, + is_notifications_enabled, + set_seen_value, +) from frappe.model.document import Document -from frappe.desk.doctype.notification_settings.notification_settings import (is_notifications_enabled, is_email_notifications_enabled_for_type, set_seen_value) + class NotificationLog(Document): def after_insert(self): - frappe.publish_realtime('notification', after_commit=True, user=self.for_user) + frappe.publish_realtime("notification", after_commit=True, user=self.for_user) set_notifications_as_unseen(self.for_user) if is_email_notifications_enabled_for_type(self.for_user, self.type): try: @@ -22,61 +27,67 @@ def get_permission_query_conditions(for_user): if not for_user: for_user = frappe.session.user - if for_user == 'Administrator': + if for_user == "Administrator": return - return '''(`tabNotification Log`.for_user = '{user}')'''.format(user=for_user) + return """(`tabNotification Log`.for_user = '{user}')""".format(user=for_user) + def get_title(doctype, docname, title_field=None): if not title_field: title_field = frappe.get_meta(doctype).get_title_field() - title = docname if title_field == "name" else \ - frappe.db.get_value(doctype, docname, title_field) + title = docname if title_field == "name" else frappe.db.get_value(doctype, docname, title_field) return title + def get_title_html(title): return '{0}'.format(title) + def enqueue_create_notification(users, doc): - ''' + """ During installation of new site, enqueue_create_notification tries to connect to Redis. This breaks new site creation if Redis server is not running. We do not need any notifications in fresh installation - ''' + """ if frappe.flags.in_install: return doc = frappe._dict(doc) if isinstance(users, str): - users = [user.strip() for user in users.split(',') if user.strip()] + users = [user.strip() for user in users.split(",") if user.strip()] users = list(set(users)) frappe.enqueue( - 'frappe.desk.doctype.notification_log.notification_log.make_notification_logs', + "frappe.desk.doctype.notification_log.notification_log.make_notification_logs", doc=doc, users=users, - now=frappe.flags.in_test + now=frappe.flags.in_test, ) + def make_notification_logs(doc, users): - from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled + from frappe.social.doctype.energy_point_settings.energy_point_settings import ( + is_energy_point_enabled, + ) for user in users: - if frappe.db.exists('User', {"email": user, "enabled": 1}): + if frappe.db.exists("User", {"email": user, "enabled": 1}): if is_notifications_enabled(user): - if doc.type == 'Energy Point' and not is_energy_point_enabled(): + if doc.type == "Energy Point" and not is_energy_point_enabled(): return - _doc = frappe.new_doc('Notification Log') + _doc = frappe.new_doc("Notification Log") _doc.update(doc) _doc.for_user = user - if _doc.for_user != _doc.from_user or doc.type == 'Energy Point' or doc.type == 'Alert': + if _doc.for_user != _doc.from_user or doc.type == "Energy Point" or doc.type == "Alert": _doc.insert(ignore_permissions=True) + def send_notification_email(doc): - if doc.type == 'Energy Point' and doc.email_content is None: + if doc.type == "Energy Point" and doc.email_content is None: return from frappe.utils import get_url_to_form, strip_html @@ -86,52 +97,58 @@ def send_notification_email(doc): email_subject = strip_html(doc.subject) frappe.sendmail( - recipients = doc.for_user, - subject = email_subject, - template = "new_notification", - args = { - 'body_content': doc.subject, - 'description': doc.email_content, - 'document_type': doc.document_type, - 'document_name': doc.document_name, - 'doc_link': doc_link + recipients=doc.for_user, + subject=email_subject, + template="new_notification", + args={ + "body_content": doc.subject, + "description": doc.email_content, + "document_type": doc.document_type, + "document_name": doc.document_name, + "doc_link": doc_link, }, - header = [header, 'orange'], - now=frappe.flags.in_test + header=[header, "orange"], + now=frappe.flags.in_test, ) + def get_email_header(doc): docname = doc.document_name header_map = { - 'Default': _('New Notification'), - 'Mention': _('New Mention on {0}').format(docname), - 'Assignment': _('Assignment Update on {0}').format(docname), - 'Share': _('New Document Shared {0}').format(docname), - 'Energy Point': _('Energy Point Update on {0}').format(docname), + "Default": _("New Notification"), + "Mention": _("New Mention on {0}").format(docname), + "Assignment": _("Assignment Update on {0}").format(docname), + "Share": _("New Document Shared {0}").format(docname), + "Energy Point": _("Energy Point Update on {0}").format(docname), } - return header_map[doc.type or 'Default'] + return header_map[doc.type or "Default"] + @frappe.whitelist() def mark_all_as_read(): - unread_docs_list = frappe.db.get_all('Notification Log', filters = {'read': 0, 'for_user': frappe.session.user}) + unread_docs_list = frappe.db.get_all( + "Notification Log", filters={"read": 0, "for_user": frappe.session.user} + ) unread_docnames = [doc.name for doc in unread_docs_list] if unread_docnames: - filters = {'name': ['in', unread_docnames]} - frappe.db.set_value('Notification Log', filters, 'read', 1, update_modified=False) + filters = {"name": ["in", unread_docnames]} + frappe.db.set_value("Notification Log", filters, "read", 1, update_modified=False) @frappe.whitelist() def mark_as_read(docname): if docname: - frappe.db.set_value('Notification Log', docname, 'read', 1, update_modified=False) + frappe.db.set_value("Notification Log", docname, "read", 1, update_modified=False) + @frappe.whitelist() def trigger_indicator_hide(): - frappe.publish_realtime('indicator_hide', user=frappe.session.user) + frappe.publish_realtime("indicator_hide", user=frappe.session.user) + def set_notifications_as_unseen(user): try: - frappe.db.set_value('Notification Settings', user, 'seen', 0) + frappe.db.set_value("Notification Settings", user, "seen", 0) except frappe.DoesNotExistError: return diff --git a/frappe/desk/doctype/notification_log/test_notification_log.py b/frappe/desk/doctype/notification_log/test_notification_log.py index 4c415a860c..44b1b53ead 100644 --- a/frappe/desk/doctype/notification_log/test_notification_log.py +++ b/frappe/desk/doctype/notification_log/test_notification_log.py @@ -1,58 +1,55 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE +import unittest + import frappe from frappe.core.doctype.user.user import get_system_users from frappe.desk.form.assign_to import add as assign_task -import unittest + class TestNotificationLog(unittest.TestCase): def test_assignment(self): todo = get_todo() user = get_user() - assign_task({ - "assign_to": [user], - "doctype": 'ToDo', - "name": todo.name, - "description": todo.description - }) - log_type = frappe.db.get_value('Notification Log', { - 'document_type': 'ToDo', - 'document_name': todo.name - }, 'type') - self.assertEqual(log_type, 'Assignment') + assign_task( + {"assign_to": [user], "doctype": "ToDo", "name": todo.name, "description": todo.description} + ) + log_type = frappe.db.get_value( + "Notification Log", {"document_type": "ToDo", "document_name": todo.name}, "type" + ) + self.assertEqual(log_type, "Assignment") def test_share(self): todo = get_todo() user = get_user() - frappe.share.add('ToDo', todo.name, user, notify=1) - log_type = frappe.db.get_value('Notification Log', { - 'document_type': 'ToDo', - 'document_name': todo.name - }, 'type') - self.assertEqual(log_type, 'Share') + frappe.share.add("ToDo", todo.name, user, notify=1) + log_type = frappe.db.get_value( + "Notification Log", {"document_type": "ToDo", "document_name": todo.name}, "type" + ) + self.assertEqual(log_type, "Share") email = get_last_email_queue() - content = 'Subject: {} shared a document ToDo'.format(frappe.utils.get_fullname(frappe.session.user)) + content = "Subject: {} shared a document ToDo".format( + frappe.utils.get_fullname(frappe.session.user) + ) self.assertTrue(content in email.message) def get_last_email_queue(): - res = frappe.db.get_all('Email Queue', - fields=['message'], - order_by='creation desc', - limit=1 - ) + res = frappe.db.get_all("Email Queue", fields=["message"], order_by="creation desc", limit=1) return res[0] + def get_todo(): - if not frappe.get_all('ToDo'): - return frappe.get_doc({ 'doctype': 'ToDo', 'description': 'Test for Notification' }).insert() + if not frappe.get_all("ToDo"): + return frappe.get_doc({"doctype": "ToDo", "description": "Test for Notification"}).insert() + + res = frappe.get_all("ToDo", limit=1) + return frappe.get_cached_doc("ToDo", res[0].name) - res = frappe.get_all('ToDo', limit=1) - return frappe.get_cached_doc('ToDo', res[0].name) def get_user(): return get_system_users(limit=1)[0] diff --git a/frappe/desk/doctype/notification_settings/notification_settings.py b/frappe/desk/doctype/notification_settings/notification_settings.py index cf6bb2d78d..bbb4a62154 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.py +++ b/frappe/desk/doctype/notification_settings/notification_settings.py @@ -5,47 +5,52 @@ import frappe from frappe.model.document import Document + class NotificationSettings(Document): def on_update(self): from frappe.desk.notifications import clear_notification_config + clear_notification_config(frappe.session.user) def is_notifications_enabled(user): - enabled = frappe.db.get_value('Notification Settings', user, 'enabled') + enabled = frappe.db.get_value("Notification Settings", user, "enabled") if enabled is None: return True return enabled + def is_email_notifications_enabled(user): - enabled = frappe.db.get_value('Notification Settings', user, 'enable_email_notifications') + enabled = frappe.db.get_value("Notification Settings", user, "enable_email_notifications") if enabled is None: return True return enabled + def is_email_notifications_enabled_for_type(user, notification_type): if not is_email_notifications_enabled(user): return False - if notification_type == 'Alert': + if notification_type == "Alert": return False - fieldname = 'enable_email_' + frappe.scrub(notification_type) - enabled = frappe.db.get_value('Notification Settings', user, fieldname) + fieldname = "enable_email_" + frappe.scrub(notification_type) + enabled = frappe.db.get_value("Notification Settings", user, fieldname) if enabled is None: return True return enabled + def create_notification_settings(user): if not frappe.db.exists("Notification Settings", user): - _doc = frappe.new_doc('Notification Settings') + _doc = frappe.new_doc("Notification Settings") _doc.name = user _doc.insert(ignore_permissions=True) def toggle_notifications(user, enable=False): if frappe.db.exists("Notification Settings", user): - frappe.db.set_value("Notification Settings", user, 'enabled', enable) + frappe.db.set_value("Notification Settings", user, "enabled", enable) @frappe.whitelist() @@ -54,8 +59,8 @@ def get_subscribed_documents(): return [] try: - if frappe.db.exists('Notification Settings', frappe.session.user): - doc = frappe.get_doc('Notification Settings', frappe.session.user) + if frappe.db.exists("Notification Settings", frappe.session.user): + doc = frappe.get_doc("Notification Settings", frappe.session.user) return [item.document for item in doc.subscribed_documents] # Notification Settings is fetched even before sync doctype is called # but it will throw an ImportError, we can ignore it in migrate @@ -66,17 +71,19 @@ def get_subscribed_documents(): def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - if user == 'Administrator': + if user == "Administrator": return roles = frappe.get_roles(user) if "System Manager" in roles: - return '''(`tabNotification Settings`.name != 'Administrator')''' + return """(`tabNotification Settings`.name != 'Administrator')""" + + return """(`tabNotification Settings`.name = '{user}')""".format(user=user) - return '''(`tabNotification Settings`.name = '{user}')'''.format(user=user) @frappe.whitelist() def set_seen_value(value, user): - frappe.db.set_value('Notification Settings', user, 'seen', value, update_modified=False) + frappe.db.set_value("Notification Settings", user, "seen", value, update_modified=False) diff --git a/frappe/desk/doctype/notification_settings/test_notification_settings.py b/frappe/desk/doctype/notification_settings/test_notification_settings.py index e3dac0af5f..966b923567 100644 --- a/frappe/desk/doctype/notification_settings/test_notification_settings.py +++ b/frappe/desk/doctype/notification_settings/test_notification_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestNotificationSettings(unittest.TestCase): pass diff --git a/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py b/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py index 1fdba22779..b72f827cd7 100644 --- a/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py +++ b/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class NotificationSubscribedDocument(Document): pass diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 784f46bb19..370b187ffe 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -4,11 +4,12 @@ import frappe from frappe import _ +from frappe.config import get_modules_from_all_apps_for_user from frappe.model.document import Document -from frappe.utils import cint from frappe.model.naming import append_number_if_name_exists from frappe.modules.export_file import export_to_files -from frappe.config import get_modules_from_all_apps_for_user +from frappe.utils import cint + class NumberCard(Document): def autoname(self): @@ -16,24 +17,29 @@ class NumberCard(Document): self.name = self.label if frappe.db.exists("Number Card", self.name): - self.name = append_number_if_name_exists('Number Card', self.name) + self.name = append_number_if_name_exists("Number Card", self.name) def validate(self): if not self.document_type: frappe.throw(_("Document type is required to create a number card")) - if self.document_type and frappe.get_meta(self.document_type).istable and not self.parent_document_type: + if ( + self.document_type + and frappe.get_meta(self.document_type).istable + and not self.parent_document_type + ): frappe.throw(_("Parent document type is required to create a number card")) def on_update(self): if frappe.conf.developer_mode and self.is_standard: - export_to_files(record_list=[['Number Card', self.name]], record_module=self.module) + export_to_files(record_list=[["Number Card", self.name]], record_module=self.module) + def get_permission_query_conditions(user=None): if not user: user = frappe.session.user - if user == 'Administrator': + if user == "Administrator": return roles = frappe.get_roles(user) @@ -43,22 +49,31 @@ def get_permission_query_conditions(user=None): doctype_condition = False module_condition = False - allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] - allowed_modules = [frappe.db.escape(module.get('module_name')) for module in get_modules_from_all_apps_for_user()] + allowed_doctypes = [ + frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read() + ] + allowed_modules = [ + frappe.db.escape(module.get("module_name")) for module in get_modules_from_all_apps_for_user() + ] if allowed_doctypes: - doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format( - allowed_doctypes=','.join(allowed_doctypes)) + doctype_condition = "`tabNumber Card`.`document_type` in ({allowed_doctypes})".format( + allowed_doctypes=",".join(allowed_doctypes) + ) if allowed_modules: - module_condition = '''`tabNumber Card`.`module` in ({allowed_modules}) - or `tabNumber Card`.`module` is NULL'''.format( - allowed_modules=','.join(allowed_modules)) + module_condition = """`tabNumber Card`.`module` in ({allowed_modules}) + or `tabNumber Card`.`module` is NULL""".format( + allowed_modules=",".join(allowed_modules) + ) - return ''' + return """ {doctype_condition} and {module_condition} - '''.format(doctype_condition=doctype_condition, module_condition=module_condition) + """.format( + doctype_condition=doctype_condition, module_condition=module_condition + ) + def has_permission(doc, ptype, user): roles = frappe.get_roles(user) @@ -71,24 +86,29 @@ def has_permission(doc, ptype, user): return False + @frappe.whitelist() def get_result(doc, filters, to_date=None): doc = frappe.parse_json(doc) fields = [] sql_function_map = { - 'Count': 'count', - 'Sum': 'sum', - 'Average': 'avg', - 'Minimum': 'min', - 'Maximum': 'max' + "Count": "count", + "Sum": "sum", + "Average": "avg", + "Minimum": "min", + "Maximum": "max", } function = sql_function_map[doc.function] - if function == 'count': - fields = ['{function}(*) as result'.format(function=function)] + if function == "count": + fields = ["{function}(*) as result".format(function=function)] else: - fields = ['{function}({based_on}) as result'.format(function=function, based_on=doc.aggregate_function_based_on)] + fields = [ + "{function}({based_on}) as result".format( + function=function, based_on=doc.aggregate_function_based_on + ) + ] filters = frappe.parse_json(filters) @@ -96,21 +116,22 @@ def get_result(doc, filters, to_date=None): filters = [] if to_date: - filters.append([doc.document_type, 'creation', '<', to_date]) + filters.append([doc.document_type, "creation", "<", to_date]) res = frappe.db.get_list(doc.document_type, fields=fields, filters=filters) - number = res[0]['result'] if res else 0 + number = res[0]["result"] if res else 0 return cint(number) + @frappe.whitelist() def get_percentage_difference(doc, filters, result): doc = frappe.parse_json(doc) result = frappe.parse_json(result) - doc = frappe.get_doc('Number Card', doc.name) + doc = frappe.get_doc("Number Card", doc.name) - if not doc.get('show_percentage_stats'): + if not doc.get("show_percentage_stats"): return previous_result = calculate_previous_result(doc, filters) @@ -120,18 +141,18 @@ def get_percentage_difference(doc, filters, result): if result == previous_result: return 0 else: - return ((result/previous_result)-1)*100.0 + return ((result / previous_result) - 1) * 100.0 def calculate_previous_result(doc, filters): from frappe.utils import add_to_date current_date = frappe.utils.now() - if doc.stats_time_interval == 'Daily': + if doc.stats_time_interval == "Daily": previous_date = add_to_date(current_date, days=-1) - elif doc.stats_time_interval == 'Weekly': + elif doc.stats_time_interval == "Weekly": previous_date = add_to_date(current_date, weeks=-1) - elif doc.stats_time_interval == 'Monthly': + elif doc.stats_time_interval == "Monthly": previous_date = add_to_date(current_date, months=-1) else: previous_date = add_to_date(current_date, years=-1) @@ -139,15 +160,17 @@ def calculate_previous_result(doc, filters): number = get_result(doc, filters, previous_date) return number + @frappe.whitelist() def create_number_card(args): args = frappe.parse_json(args) - doc = frappe.new_doc('Number Card') + doc = frappe.new_doc("Number Card") doc.update(args) doc.insert(ignore_permissions=True) return doc + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): @@ -155,21 +178,23 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): searchfields = meta.get_search_fields() search_conditions = [] - if not frappe.db.exists('DocType', doctype): + if not frappe.db.exists("DocType", doctype): return if txt: for field in searchfields: - search_conditions.append('`tab{doctype}`.`{field}` like %(txt)s'.format(field=field, doctype=doctype, txt=txt)) + search_conditions.append( + "`tab{doctype}`.`{field}` like %(txt)s".format(field=field, doctype=doctype, txt=txt) + ) - search_conditions = ' or '.join(search_conditions) + search_conditions = " or ".join(search_conditions) - search_conditions = 'and (' + search_conditions +')' if search_conditions else '' + search_conditions = "and (" + search_conditions + ")" if search_conditions else "" conditions, values = frappe.db.build_conditions(filters) - values['txt'] = '%' + txt + '%' + values["txt"] = "%" + txt + "%" return frappe.db.sql( - '''select + """select `tabNumber Card`.name, `tabNumber Card`.label, `tabNumber Card`.document_type from `tabNumber Card` @@ -178,12 +203,15 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): (`tabNumber Card`.owner = '{user}' or `tabNumber Card`.is_public = 1) {search_conditions} - '''.format( - filters=filters, - user=frappe.session.user, - search_conditions=search_conditions, - conditions=conditions - ), values) + """.format( + filters=filters, + user=frappe.session.user, + search_conditions=search_conditions, + conditions=conditions, + ), + values, + ) + @frappe.whitelist() def create_report_number_card(args): @@ -193,19 +221,20 @@ def create_report_number_card(args): if args.dashboard: add_card_to_dashboard(frappe.as_json(args)) + @frappe.whitelist() def add_card_to_dashboard(args): args = frappe.parse_json(args) - dashboard = frappe.get_doc('Dashboard', args.dashboard) - dashboard_link = frappe.new_doc('Number Card Link') + dashboard = frappe.get_doc("Dashboard", args.dashboard) + dashboard_link = frappe.new_doc("Number Card Link") dashboard_link.card = args.name if args.set_standard and dashboard.is_standard: - card = frappe.get_doc('Number Card', dashboard_link.card) + card = frappe.get_doc("Number Card", dashboard_link.card) card.is_standard = 1 card.module = dashboard.module card.save() - dashboard.append('cards', dashboard_link) + dashboard.append("cards", dashboard_link) dashboard.save() diff --git a/frappe/desk/doctype/number_card/test_number_card.py b/frappe/desk/doctype/number_card/test_number_card.py index cc92e63341..817ea2fad4 100644 --- a/frappe/desk/doctype/number_card/test_number_card.py +++ b/frappe/desk/doctype/number_card/test_number_card.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestNumberCard(unittest.TestCase): pass diff --git a/frappe/desk/doctype/number_card_link/number_card_link.py b/frappe/desk/doctype/number_card_link/number_card_link.py index 0b55ae6dcd..b630d7caa7 100644 --- a/frappe/desk/doctype/number_card_link/number_card_link.py +++ b/frappe/desk/doctype/number_card_link/number_card_link.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class NumberCardLink(Document): pass diff --git a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py index c13fb29678..9a12b0aab9 100644 --- a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py +++ b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestOnboardingPermission(unittest.TestCase): pass diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.py b/frappe/desk/doctype/onboarding_step/onboarding_step.py index 45e0ca34fd..4a4d487cc8 100644 --- a/frappe/desk/doctype/onboarding_step/onboarding_step.py +++ b/frappe/desk/doctype/onboarding_step/onboarding_step.py @@ -2,11 +2,13 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -import json from frappe.model.document import Document + class OnboardingStep(Document): def before_export(self, doc): doc.is_complete = 0 @@ -17,11 +19,13 @@ class OnboardingStep(Document): def get_onboarding_steps(ob_steps): steps = [] for s in json.loads(ob_steps): - doc = frappe.get_doc('Onboarding Step', s.get('step')) + doc = frappe.get_doc("Onboarding Step", s.get("step")) step = doc.as_dict().copy() step.label = _(doc.title) if step.action == "Create Entry": - step.is_submittable = frappe.db.get_value("DocType", step.reference_document, 'is_submittable', cache=True) + step.is_submittable = frappe.db.get_value( + "DocType", step.reference_document, "is_submittable", cache=True + ) steps.append(step) return steps diff --git a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py index b0651da4da..2342656a72 100644 --- a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py +++ b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestOnboardingStep(unittest.TestCase): pass diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index f0aa867c8a..cea1ed79e3 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -16,29 +16,31 @@ def flush_old_route_records(): """Deletes all route records except last 500 records per user""" records_to_keep_limit = 500 - users = frappe.db.sql(''' + users = frappe.db.sql( + """ SELECT `user` FROM `tabRoute History` GROUP BY `user` HAVING count(`name`) > %(limit)s - ''', { - "limit": records_to_keep_limit - }) + """, + {"limit": records_to_keep_limit}, + ) for user in users: user = user[0] - last_record_to_keep = frappe.db.get_all('Route History', - filters={'user': user}, + last_record_to_keep = frappe.db.get_all( + "Route History", + filters={"user": user}, limit=1, limit_start=500, - fields=['modified'], - order_by='modified desc' + fields=["modified"], + order_by="modified desc", + ) + + frappe.db.delete( + "Route History", {"modified": ("<=", last_record_to_keep[0].modified), "user": user} ) - frappe.db.delete("Route History", { - "modified": ("<=", last_record_to_keep[0].modified), - "user": user - }) @frappe.whitelist() def deferred_insert(routes): @@ -53,8 +55,14 @@ def deferred_insert(routes): _deferred_insert("Route History", json.dumps(routes)) + @frappe.whitelist() def frequently_visited_links(): - return frappe.get_all('Route History', fields=['route', 'count(name) as count'], filters={ - 'user': frappe.session.user - }, group_by="route", order_by="count desc", limit=5) + return frappe.get_all( + "Route History", + fields=["route", "count(name) as count"], + filters={"user": frappe.session.user}, + group_by="route", + order_by="count desc", + limit=5, + ) diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py index bf0925e2d7..ffebfe2dd0 100644 --- a/frappe/desk/doctype/system_console/system_console.py +++ b/frappe/desk/doctype/system_console/system_console.py @@ -5,20 +5,21 @@ import json import frappe -from frappe.utils.safe_exec import safe_exec, read_sql from frappe.model.document import Document +from frappe.utils.safe_exec import read_sql, safe_exec + class SystemConsole(Document): def run(self): - frappe.only_for('System Manager') + frappe.only_for("System Manager") try: frappe.debug_log = [] - if self.type == 'Python': + if self.type == "Python": safe_exec(self.console) - self.output = '\n'.join(frappe.debug_log) - elif self.type == 'SQL': + self.output = "\n".join(frappe.debug_log) + elif self.type == "SQL": self.output = frappe.as_json(read_sql(self.console, as_dict=1)) - except: # noqa: E722 + except: # noqa: E722 self.output = frappe.get_traceback() if self.commit: @@ -26,29 +27,31 @@ class SystemConsole(Document): else: frappe.db.rollback() - frappe.get_doc(dict( - doctype='Console Log', - script=self.console, - output=self.output)).insert() + frappe.get_doc(dict(doctype="Console Log", script=self.console, output=self.output)).insert() frappe.db.commit() + @frappe.whitelist() def execute_code(doc): console = frappe.get_doc(json.loads(doc)) console.run() return console.as_dict() + @frappe.whitelist() def show_processlist(): - frappe.only_for('System Manager') + frappe.only_for("System Manager") - return frappe.db.multisql({ - "postgres": """ + return frappe.db.multisql( + { + "postgres": """ SELECT pid AS "Id", query_start AS "Time", state AS "State", query AS "Info", wait_event AS "Progress" FROM pg_stat_activity""", - "mariadb": "show full processlist" - }, as_dict=True) + "mariadb": "show full processlist", + }, + as_dict=True, + ) diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py index fa7c577faa..372cbbc1f4 100644 --- a/frappe/desk/doctype/system_console/test_system_console.py +++ b/frappe/desk/doctype/system_console/test_system_console.py @@ -1,18 +1,20 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestSystemConsole(unittest.TestCase): def test_system_console(self): - system_console = frappe.get_doc('System Console') + system_console = frappe.get_doc("System Console") system_console.console = 'log("hello")' system_console.run() - self.assertEqual(system_console.output, 'hello') + self.assertEqual(system_console.output, "hello") system_console.console = 'log(frappe.db.get_value("DocType", "DocType", "module"))' system_console.run() - self.assertEqual(system_console.output, 'Core') + self.assertEqual(system_console.output, "Core") diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index d44c481210..aabf0351a5 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -3,12 +3,14 @@ import frappe from frappe.model.document import Document -from frappe.utils import unique from frappe.query_builder import DocType +from frappe.utils import unique + class Tag(Document): pass + def check_user_tags(dt): "if the user does not have a tags column, then it creates one" try: @@ -18,6 +20,7 @@ def check_user_tags(dt): if frappe.db.is_column_missing(e): DocTags(dt).setup() + @frappe.whitelist() def add_tag(tag, dt, dn, color=None): "adds a new tag to a record, and creates the Tag master" @@ -25,6 +28,7 @@ def add_tag(tag, dt, dn, color=None): return tag + @frappe.whitelist() def add_tags(tags, dt, docs, color=None): "adds a new tag to a record, and creates the Tag master" @@ -36,20 +40,19 @@ def add_tags(tags, dt, docs, color=None): # return tag + @frappe.whitelist() def remove_tag(tag, dt, dn): "removes tag from the record" DocTags(dt).remove(dn, tag) + @frappe.whitelist() def get_tagged_docs(doctype, tag): frappe.has_permission(doctype, throw=True) doctype = DocType(doctype) - return ( - frappe.qb.from_(doctype) - .where(doctype._user_tags.like(tag)) - .select(doctype.name) - ).run() + return (frappe.qb.from_(doctype).where(doctype._user_tags.like(tag)).select(doctype.name)).run() + @frappe.whitelist() def get_tags(doctype, txt): @@ -58,22 +61,24 @@ def get_tags(doctype, txt): return sorted(filter(lambda t: t and txt.lower() in t.lower(), list(set(tags)))) + class DocTags: """Tags for a particular doctype""" + def __init__(self, dt): self.dt = dt def get_tag_fields(self): """returns tag_fields property""" - return frappe.db.get_value('DocType', self.dt, 'tag_fields') + return frappe.db.get_value("DocType", self.dt, "tag_fields") def get_tags(self, dn): """returns tag for a particular item""" - return (frappe.db.get_value(self.dt, dn, '_user_tags', ignore=1) or '').strip() + return (frappe.db.get_value(self.dt, dn, "_user_tags", ignore=1) or "").strip() def add(self, dn, tag): """add a new user tag""" - tl = self.get_tags(dn).split(',') + tl = self.get_tags(dn).split(",") if not tag in tl: tl.append(tag) if not frappe.db.exists("Tag", tag): @@ -82,8 +87,8 @@ class DocTags: def remove(self, dn, tag): """remove a user tag""" - tl = self.get_tags(dn).split(',') - self.update(dn, filter(lambda x:x.lower()!=tag.lower(), tl)) + tl = self.get_tags(dn).split(",") + self.update(dn, filter(lambda x: x.lower() != tag.lower(), tl)) def remove_all(self, dn): """remove all user tags (call before delete)""" @@ -93,14 +98,15 @@ class DocTags: """updates the _user_tag column in the table""" if not tl: - tags = '' + tags = "" else: tl = unique(filter(lambda x: x, tl)) - tags = ',' + ','.join(tl) + tags = "," + ",".join(tl) try: - frappe.db.sql("update `tab%s` set _user_tags=%s where name=%s" % \ - (self.dt,'%s','%s'), (tags , dn)) - doc= frappe.get_doc(self.dt, dn) + frappe.db.sql( + "update `tab%s` set _user_tags=%s where name=%s" % (self.dt, "%s", "%s"), (tags, dn) + ) + doc = frappe.get_doc(self.dt, dn) update_tags(doc, tags) except Exception as e: if frappe.db.is_column_missing(e): @@ -110,26 +116,27 @@ class DocTags: self.setup() self.update(dn, tl) - else: raise + else: + raise def setup(self): """adds the _user_tags column if not exists""" from frappe.database.schema import add_column + add_column(self.dt, "_user_tags", "Data") + def delete_tags_for_document(doc): """ - Delete the Tag Link entry of a document that has - been deleted - :param doc: Deleted document + Delete the Tag Link entry of a document that has + been deleted + :param doc: Deleted document """ if not frappe.db.table_exists("Tag Link"): return - frappe.db.delete("Tag Link", { - "document_type": doc.doctype, - "document_name": doc.name - }) + frappe.db.delete("Tag Link", {"document_type": doc.doctype, "document_name": doc.name}) + def update_tags(doc, tags): """Adds tags for documents @@ -137,50 +144,52 @@ def update_tags(doc, tags): :param doc: Document to be added to global tags """ new_tags = {tag.strip() for tag in tags.split(",") if tag} - existing_tags = [tag.tag for tag in frappe.get_list("Tag Link", filters={ - "document_type": doc.doctype, - "document_name": doc.name - }, fields=["tag"])] + existing_tags = [ + tag.tag + for tag in frappe.get_list( + "Tag Link", filters={"document_type": doc.doctype, "document_name": doc.name}, fields=["tag"] + ) + ] added_tags = set(new_tags) - set(existing_tags) for tag in added_tags: - frappe.get_doc({ - "doctype": "Tag Link", - "document_type": doc.doctype, - "document_name": doc.name, - "title": doc.get_title() or '', - "tag": tag - }).insert(ignore_permissions=True) + frappe.get_doc( + { + "doctype": "Tag Link", + "document_type": doc.doctype, + "document_name": doc.name, + "title": doc.get_title() or "", + "tag": tag, + } + ).insert(ignore_permissions=True) deleted_tags = list(set(existing_tags) - set(new_tags)) for tag in deleted_tags: - frappe.db.delete("Tag Link", { - "document_type": doc.doctype, - "document_name": doc.name, - "tag": tag - }) + frappe.db.delete( + "Tag Link", {"document_type": doc.doctype, "document_name": doc.name, "tag": tag} + ) + @frappe.whitelist() def get_documents_for_tag(tag): """ - Search for given text in Tag Link - :param tag: tag to be searched + Search for given text in Tag Link + :param tag: tag to be searched """ # remove hastag `#` from tag tag = tag[1:] results = [] - result = frappe.get_list("Tag Link", filters={"tag": tag}, fields=["document_type", "document_name", "title", "tag"]) + result = frappe.get_list( + "Tag Link", filters={"tag": tag}, fields=["document_type", "document_name", "title", "tag"] + ) for res in result: - results.append({ - "doctype": res.document_type, - "name": res.document_name, - "content": res.title - }) + results.append({"doctype": res.document_type, "name": res.document_name, "content": res.title}) return results + @frappe.whitelist() def get_tags_list_for_awesomebar(): return [t.name for t in frappe.get_list("Tag")] diff --git a/frappe/desk/doctype/tag/test_tag.py b/frappe/desk/doctype/tag/test_tag.py index b9c6e0b744..8719da8c21 100644 --- a/frappe/desk/doctype/tag/test_tag.py +++ b/frappe/desk/doctype/tag/test_tag.py @@ -1,8 +1,9 @@ import unittest -import frappe -from frappe.desk.reportview import get_stats +import frappe from frappe.desk.doctype.tag.tag import add_tag +from frappe.desk.reportview import get_stats + class TestTag(unittest.TestCase): def setUp(self) -> None: @@ -10,17 +11,25 @@ class TestTag(unittest.TestCase): frappe.db.sql("UPDATE `tabDocType` set _user_tags=''") def test_tag_count_query(self): - self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'), - {'_user_tags': [['No Tags', frappe.db.count('DocType')]]}) - add_tag('Standard', 'DocType', 'User') - add_tag('Standard', 'DocType', 'ToDo') + self.assertDictEqual( + get_stats('["_user_tags"]', "DocType"), + {"_user_tags": [["No Tags", frappe.db.count("DocType")]]}, + ) + add_tag("Standard", "DocType", "User") + add_tag("Standard", "DocType", "ToDo") # count with no filter - self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'), - {'_user_tags': [['Standard', 2], ['No Tags', frappe.db.count('DocType') - 2]]}) + self.assertDictEqual( + get_stats('["_user_tags"]', "DocType"), + {"_user_tags": [["Standard", 2], ["No Tags", frappe.db.count("DocType") - 2]]}, + ) # count with child table field filter - self.assertDictEqual(get_stats('["_user_tags"]', - 'DocType', - filters='[["DocField", "fieldname", "like", "%last_name%"], ["DocType", "name", "like", "%use%"]]'), - {'_user_tags': [['Standard', 1], ['No Tags', 0]]}) \ No newline at end of file + self.assertDictEqual( + get_stats( + '["_user_tags"]', + "DocType", + filters='[["DocField", "fieldname", "like", "%last_name%"], ["DocType", "name", "like", "%use%"]]', + ), + {"_user_tags": [["Standard", 1], ["No Tags", 0]]}, + ) diff --git a/frappe/desk/doctype/tag_link/tag_link.py b/frappe/desk/doctype/tag_link/tag_link.py index d07894989d..ec816352ca 100644 --- a/frappe/desk/doctype/tag_link/tag_link.py +++ b/frappe/desk/doctype/tag_link/tag_link.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class TagLink(Document): pass diff --git a/frappe/desk/doctype/tag_link/test_tag_link.py b/frappe/desk/doctype/tag_link/test_tag_link.py index fa6a22903f..d4d1dd61fa 100644 --- a/frappe/desk/doctype/tag_link/test_tag_link.py +++ b/frappe/desk/doctype/tag_link/test_tag_link.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestTagLink(unittest.TestCase): pass diff --git a/frappe/desk/doctype/todo/__init__.py b/frappe/desk/doctype/todo/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/desk/doctype/todo/__init__.py +++ b/frappe/desk/doctype/todo/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/desk/doctype/todo/test_todo.py b/frappe/desk/doctype/todo/test_todo.py index 34d3cee191..5c54889e00 100644 --- a/frappe/desk/doctype/todo/test_todo.py +++ b/frappe/desk/doctype/todo/test_todo.py @@ -1,79 +1,90 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest + +import frappe +from frappe.core.doctype.doctype.doctype import clear_permissions_cache from frappe.model.db_query import DatabaseQuery from frappe.permissions import add_permission, reset_perms -from frappe.core.doctype.doctype.doctype import clear_permissions_cache -test_dependencies = ['User'] +test_dependencies = ["User"] + class TestToDo(unittest.TestCase): def test_delete(self): - todo = frappe.get_doc(dict(doctype='ToDo', description='test todo', - assigned_by='Administrator')).insert() + todo = frappe.get_doc( + dict(doctype="ToDo", description="test todo", assigned_by="Administrator") + ).insert() frappe.db.delete("Deleted Document") todo.delete() - deleted = frappe.get_doc('Deleted Document', dict(deleted_doctype=todo.doctype, deleted_name=todo.name)) + deleted = frappe.get_doc( + "Deleted Document", dict(deleted_doctype=todo.doctype, deleted_name=todo.name) + ) self.assertEqual(todo.as_json(), deleted.data) def test_fetch(self): - todo = frappe.get_doc(dict(doctype='ToDo', description='test todo', - assigned_by='Administrator')).insert() - self.assertEqual(todo.assigned_by_full_name, - frappe.db.get_value('User', todo.assigned_by, 'full_name')) + todo = frappe.get_doc( + dict(doctype="ToDo", description="test todo", assigned_by="Administrator") + ).insert() + self.assertEqual( + todo.assigned_by_full_name, frappe.db.get_value("User", todo.assigned_by, "full_name") + ) def test_fetch_setup(self): frappe.db.delete("ToDo") - todo_meta = frappe.get_doc('DocType', 'ToDo') - todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_from = '' + todo_meta = frappe.get_doc("DocType", "ToDo") + todo_meta.get("fields", dict(fieldname="assigned_by_full_name"))[0].fetch_from = "" todo_meta.save() - frappe.clear_cache(doctype='ToDo') + frappe.clear_cache(doctype="ToDo") - todo = frappe.get_doc(dict(doctype='ToDo', description='test todo', - assigned_by='Administrator')).insert() + todo = frappe.get_doc( + dict(doctype="ToDo", description="test todo", assigned_by="Administrator") + ).insert() self.assertFalse(todo.assigned_by_full_name) - todo_meta = frappe.get_doc('DocType', 'ToDo') - todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_from = 'assigned_by.full_name' + todo_meta = frappe.get_doc("DocType", "ToDo") + todo_meta.get("fields", dict(fieldname="assigned_by_full_name"))[ + 0 + ].fetch_from = "assigned_by.full_name" todo_meta.save() todo.reload() - self.assertEqual(todo.assigned_by_full_name, - frappe.db.get_value('User', todo.assigned_by, 'full_name')) + self.assertEqual( + todo.assigned_by_full_name, frappe.db.get_value("User", todo.assigned_by, "full_name") + ) def test_todo_list_access(self): - create_new_todo('Test1', 'testperm@example.com') + create_new_todo("Test1", "testperm@example.com") - frappe.set_user('test4@example.com') - create_new_todo('Test2', 'test4@example.com') - test_user_data = DatabaseQuery('ToDo').execute() + frappe.set_user("test4@example.com") + create_new_todo("Test2", "test4@example.com") + test_user_data = DatabaseQuery("ToDo").execute() - frappe.set_user('testperm@example.com') - system_manager_data = DatabaseQuery('ToDo').execute() + frappe.set_user("testperm@example.com") + system_manager_data = DatabaseQuery("ToDo").execute() self.assertNotEqual(test_user_data, system_manager_data) - frappe.set_user('Administrator') + frappe.set_user("Administrator") frappe.db.rollback() def test_doc_read_access(self): - #owner and assigned_by is testperm - todo1 = create_new_todo('Test1', 'testperm@example.com') - test_user = frappe.get_doc('User', 'test4@example.com') + # owner and assigned_by is testperm + todo1 = create_new_todo("Test1", "testperm@example.com") + test_user = frappe.get_doc("User", "test4@example.com") - #owner is testperm, but assigned_by is test4 - todo2 = create_new_todo('Test2', 'test4@example.com') + # owner is testperm, but assigned_by is test4 + todo2 = create_new_todo("Test2", "test4@example.com") - frappe.set_user('test4@example.com') - #owner and assigned_by is test4 - todo3 = create_new_todo('Test3', 'test4@example.com') + frappe.set_user("test4@example.com") + # owner and assigned_by is test4 + todo3 = create_new_todo("Test3", "test4@example.com") # user without any role to read or write todo document self.assertFalse(todo1.has_permission("read")) @@ -87,54 +98,58 @@ class TestToDo(unittest.TestCase): self.assertTrue(todo3.has_permission("read")) self.assertTrue(todo3.has_permission("write")) - frappe.set_user('Administrator') + frappe.set_user("Administrator") - test_user.add_roles('Blogger') - add_permission('ToDo', 'Blogger') + test_user.add_roles("Blogger") + add_permission("ToDo", "Blogger") - frappe.set_user('test4@example.com') + frappe.set_user("test4@example.com") # user with only read access to todo document, not an owner or assigned_by self.assertTrue(todo1.has_permission("read")) self.assertFalse(todo1.has_permission("write")) - frappe.set_user('Administrator') - test_user.remove_roles('Blogger') - reset_perms('ToDo') - clear_permissions_cache('ToDo') + frappe.set_user("Administrator") + test_user.remove_roles("Blogger") + reset_perms("ToDo") + clear_permissions_cache("ToDo") frappe.db.rollback() def test_fetch_if_empty(self): frappe.db.delete("ToDo") # Allow user changes - todo_meta = frappe.get_doc('DocType', 'ToDo') - field = todo_meta.get('fields', dict(fieldname='assigned_by_full_name'))[0] - field.fetch_from = 'assigned_by.full_name' + todo_meta = frappe.get_doc("DocType", "ToDo") + field = todo_meta.get("fields", dict(fieldname="assigned_by_full_name"))[0] + field.fetch_from = "assigned_by.full_name" field.fetch_if_empty = 1 todo_meta.save() - frappe.clear_cache(doctype='ToDo') + frappe.clear_cache(doctype="ToDo") - todo = frappe.get_doc(dict(doctype='ToDo', description='test todo', - assigned_by='Administrator', assigned_by_full_name='Admin')).insert() + todo = frappe.get_doc( + dict( + doctype="ToDo", + description="test todo", + assigned_by="Administrator", + assigned_by_full_name="Admin", + ) + ).insert() - self.assertEqual(todo.assigned_by_full_name, 'Admin') + self.assertEqual(todo.assigned_by_full_name, "Admin") # Overwrite user changes - todo.meta.get('fields', dict(fieldname='assigned_by_full_name'))[0].fetch_if_empty = 0 + todo.meta.get("fields", dict(fieldname="assigned_by_full_name"))[0].fetch_if_empty = 0 todo.meta.save() todo.reload() todo.save() - self.assertEqual(todo.assigned_by_full_name, - frappe.db.get_value('User', todo.assigned_by, 'full_name')) + self.assertEqual( + todo.assigned_by_full_name, frappe.db.get_value("User", todo.assigned_by, "full_name") + ) + def create_new_todo(description, assigned_by): - todo = { - 'doctype': 'ToDo', - 'description': description, - 'assigned_by': assigned_by - } + todo = {"doctype": "ToDo", "description": description, "assigned_by": assigned_by} return frappe.get_doc(todo).insert() diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 3b84b94754..e076f3384a 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -1,45 +1,46 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import json +import frappe from frappe.model.document import Document from frappe.utils import get_fullname, parse_addr exclude_from_linked_with = True + class ToDo(Document): - DocType = 'ToDo' + DocType = "ToDo" def validate(self): self._assignment = None if self.is_new(): if self.assigned_by == self.allocated_to: - assignment_message = frappe._("{0} self assigned this task: {1}").format(get_fullname(self.assigned_by), self.description) + assignment_message = frappe._("{0} self assigned this task: {1}").format( + get_fullname(self.assigned_by), self.description + ) else: - assignment_message = frappe._("{0} assigned {1}: {2}").format(get_fullname(self.assigned_by), get_fullname(self.allocated_to), self.description) + assignment_message = frappe._("{0} assigned {1}: {2}").format( + get_fullname(self.assigned_by), get_fullname(self.allocated_to), self.description + ) - self._assignment = { - "text": assignment_message, - "comment_type": "Assigned" - } + self._assignment = {"text": assignment_message, "comment_type": "Assigned"} else: # NOTE the previous value is only available in validate method if self.get_db_value("status") != self.status: if self.allocated_to == frappe.session.user: removal_message = frappe._("{0} removed their assignment.").format( - get_fullname(frappe.session.user)) + get_fullname(frappe.session.user) + ) else: removal_message = frappe._("Assignment of {0} removed by {1}").format( - get_fullname(self.allocated_to), get_fullname(frappe.session.user)) + get_fullname(self.allocated_to), get_fullname(frappe.session.user) + ) - self._assignment = { - "text": removal_message, - "comment_type": "Assignment Completed" - } + self._assignment = {"text": removal_message, "comment_type": "Assignment Completed"} def on_update(self): if self._assignment: @@ -59,26 +60,34 @@ class ToDo(Document): def delete_communication_links(self): # unlink todo from linked comments - return frappe.db.delete("Communication Link", { - "link_doctype": self.doctype, - "link_name": self.name - }) + return frappe.db.delete( + "Communication Link", {"link_doctype": self.doctype, "link_name": self.name} + ) def update_in_reference(self): if not (self.reference_type and self.reference_name): return try: - assignments = frappe.get_all("ToDo", filters={ - "reference_type": self.reference_type, - "reference_name": self.reference_name, - "status": ("!=", "Cancelled"), - "allocated_to": ("is", "set") - }, pluck="allocated_to") + assignments = frappe.get_all( + "ToDo", + filters={ + "reference_type": self.reference_type, + "reference_name": self.reference_name, + "status": ("!=", "Cancelled"), + "allocated_to": ("is", "set"), + }, + pluck="allocated_to", + ) assignments.reverse() - frappe.db.set_value(self.reference_type, self.reference_name, - "_assign", json.dumps(assignments), update_modified=False) + frappe.db.set_value( + self.reference_type, + self.reference_name, + "_assign", + json.dumps(assignments), + update_modified=False, + ) except Exception as e: if frappe.db.is_table_missing(e) and frappe.flags.in_install: @@ -87,6 +96,7 @@ class ToDo(Document): elif frappe.db.is_column_missing(e): from frappe.database.schema import add_column + add_column(self.reference_type, "_assign", "Text") self.update_in_reference() @@ -95,42 +105,44 @@ class ToDo(Document): @classmethod def get_owners(cls, filters=None): - """Returns list of owners after applying filters on todo's. - """ - rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['allocated_to']) + """Returns list of owners after applying filters on todo's.""" + rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=["allocated_to"]) return [parse_addr(row.allocated_to)[1] for row in rows if row.allocated_to] + # NOTE: todo is viewable if a user is an owner, or set as assigned_to value, or has any role that is allowed to access ToDo doctype. def on_doctype_update(): frappe.db.add_index("ToDo", ["reference_type", "reference_name"]) + def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - todo_roles = frappe.permissions.get_doctype_roles('ToDo') - if 'All' in todo_roles: - todo_roles.remove('All') + todo_roles = frappe.permissions.get_doctype_roles("ToDo") + if "All" in todo_roles: + todo_roles.remove("All") if any(check in todo_roles for check in frappe.get_roles(user)): return None else: - return """(`tabToDo`.allocated_to = {user} or `tabToDo`.assigned_by = {user})"""\ - .format(user=frappe.db.escape(user)) + return """(`tabToDo`.allocated_to = {user} or `tabToDo`.assigned_by = {user})""".format( + user=frappe.db.escape(user) + ) + def has_permission(doc, ptype="read", user=None): user = user or frappe.session.user - todo_roles = frappe.permissions.get_doctype_roles('ToDo', ptype) - if 'All' in todo_roles: - todo_roles.remove('All') + todo_roles = frappe.permissions.get_doctype_roles("ToDo", ptype) + if "All" in todo_roles: + todo_roles.remove("All") if any(check in todo_roles for check in frappe.get_roles(user)): return True else: - return doc.allocated_to==user or doc.assigned_by==user + return doc.allocated_to == user or doc.assigned_by == user + @frappe.whitelist() def new_todo(description): - frappe.get_doc({ - 'doctype': 'ToDo', - 'description': description - }).insert() + frappe.get_doc({"doctype": "ToDo", "description": description}).insert() diff --git a/frappe/desk/doctype/workspace/test_workspace.py b/frappe/desk/doctype/workspace/test_workspace.py index 6c16e69afe..9281240e08 100644 --- a/frappe/desk/doctype/workspace/test_workspace.py +++ b/frappe/desk/doctype/workspace/test_workspace.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest + +import frappe + + class TestWorkspace(unittest.TestCase): def setUp(self): create_module("Test Module") @@ -27,16 +30,16 @@ class TestWorkspace(unittest.TestCase): # else: # self.assertEqual(len(cards), 1) + def create_module(module_name): - module = frappe.get_doc({ - "doctype": "Module Def", - "module_name": module_name, - "app_name": "frappe" - }) - module.insert(ignore_if_duplicate = True) + module = frappe.get_doc( + {"doctype": "Module Def", "module_name": module_name, "app_name": "frappe"} + ) + module.insert(ignore_if_duplicate=True) return module + def create_workspace(**args): workspace = frappe.new_doc("Workspace") args = frappe._dict(args) @@ -49,47 +52,51 @@ def create_workspace(**args): return workspace + def insert_card(workspace, card_label, doctype1, doctype2, country=None): - workspace.append("links", { - "type": "Card Break", - "label": card_label, - "only_for": country - }) + workspace.append("links", {"type": "Card Break", "label": card_label, "only_for": country}) create_doctype(doctype1, "Test Module") - workspace.append("links", { - "type": "Link", - "label": doctype1, - "only_for": country, - "link_type": "DocType", - "link_to": doctype1 - }) + workspace.append( + "links", + { + "type": "Link", + "label": doctype1, + "only_for": country, + "link_type": "DocType", + "link_to": doctype1, + }, + ) create_doctype(doctype2, "Test Module") - workspace.append("links", { - "type": "Link", - "label": doctype2, - "only_for": country, - "link_type": "DocType", - "link_to": doctype2 - }) + workspace.append( + "links", + { + "type": "Link", + "label": doctype2, + "only_for": country, + "link_type": "DocType", + "link_to": doctype2, + }, + ) + def create_doctype(doctype_name, module): - frappe.get_doc({ - 'doctype': 'DocType', - 'name': doctype_name, - 'module': module, - 'custom': 1, - 'autoname': 'field:title', - 'fields': [ - {'label': 'Title', 'fieldname': 'title', 'reqd': 1, 'fieldtype': 'Data'}, - {'label': 'Description', 'fieldname': 'description', 'fieldtype': 'Small Text'}, - {'label': 'Date', 'fieldname': 'date', 'fieldtype': 'Date'}, - {'label': 'Duration', 'fieldname': 'duration', 'fieldtype': 'Duration'}, - {'label': 'Number', 'fieldname': 'number', 'fieldtype': 'Int'}, - {'label': 'Number', 'fieldname': 'another_number', 'fieldtype': 'Int'} - ], - 'permissions': [ - {'role': 'System Manager'} - ] - }).insert(ignore_if_duplicate = True) + frappe.get_doc( + { + "doctype": "DocType", + "name": doctype_name, + "module": module, + "custom": 1, + "autoname": "field:title", + "fields": [ + {"label": "Title", "fieldname": "title", "reqd": 1, "fieldtype": "Data"}, + {"label": "Description", "fieldname": "description", "fieldtype": "Small Text"}, + {"label": "Date", "fieldname": "date", "fieldtype": "Date"}, + {"label": "Duration", "fieldname": "duration", "fieldtype": "Duration"}, + {"label": "Number", "fieldname": "number", "fieldtype": "Int"}, + {"label": "Number", "fieldname": "another_number", "fieldtype": "Int"}, + ], + "permissions": [{"role": "System Manager"}], + } + ).insert(ignore_if_duplicate=True) diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py index ba3319b591..a2dbcbfbe2 100644 --- a/frappe/desk/doctype/workspace/workspace.py +++ b/frappe/desk/doctype/workspace/workspace.py @@ -2,19 +2,20 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +from json import loads + import frappe from frappe import _ -from frappe.modules.export_file import export_to_files -from frappe.model.document import Document -from frappe.model.rename_doc import rename_doc from frappe.desk.desktop import save_new_widget from frappe.desk.utils import validate_route_conflict +from frappe.model.document import Document +from frappe.model.rename_doc import rename_doc +from frappe.modules.export_file import export_to_files -from json import loads class Workspace(Document): def validate(self): - if (self.public and not is_workspace_manager() and not disable_saving_as_public()): + if self.public and not is_workspace_manager() and not disable_saving_as_public(): frappe.throw(_("You need to be Workspace Manager to edit this document")) validate_route_conflict(self.doctype, self.name) @@ -29,30 +30,37 @@ class Workspace(Document): return if frappe.conf.developer_mode and self.module and self.public: - export_to_files(record_list=[['Workspace', self.name]], record_module=self.module) + export_to_files(record_list=[["Workspace", self.name]], record_module=self.module) @staticmethod def get_module_page_map(): - pages = frappe.get_all("Workspace", fields=["name", "module"], filters={'for_user': ''}, as_list=1) + pages = frappe.get_all( + "Workspace", fields=["name", "module"], filters={"for_user": ""}, as_list=1 + ) - return { page[1]: page[0] for page in pages if page[1] } + return {page[1]: page[0] for page in pages if page[1]} def get_link_groups(self): cards = [] - current_card = frappe._dict({ - "label": "Link", - "type": "Card Break", - "icon": None, - "hidden": False, - }) + current_card = frappe._dict( + { + "label": "Link", + "type": "Card Break", + "icon": None, + "hidden": False, + } + ) card_links = [] for link in self.links: link = link.as_dict() if link.type == "Card Break": - if card_links and (not current_card.get('only_for') or current_card.get('only_for') == frappe.get_system_settings('country')): - current_card['links'] = card_links + if card_links and ( + not current_card.get("only_for") + or current_card.get("only_for") == frappe.get_system_settings("country") + ): + current_card["links"] = card_links cards.append(current_card) current_card = link @@ -60,7 +68,7 @@ class Workspace(Document): else: card_links.append(link) - current_card['links'] = card_links + current_card["links"] = card_links cards.append(current_card) return cards @@ -68,60 +76,68 @@ class Workspace(Document): def build_links_table_from_card(self, config): for idx, card in enumerate(config): - links = loads(card.get('links')) + links = loads(card.get("links")) # remove duplicate before adding for idx, link in enumerate(self.links): - if link.label == card.get('label') and link.type == 'Card Break': + if link.label == card.get("label") and link.type == "Card Break": del self.links[idx : idx + link.link_count + 1] - self.append('links', { - "label": card.get('label'), - "type": "Card Break", - "icon": card.get('icon'), - "hidden": card.get('hidden') or False, - "link_count": card.get('link_count'), - "idx": 1 if not self.links else self.links[-1].idx + 1 - }) + self.append( + "links", + { + "label": card.get("label"), + "type": "Card Break", + "icon": card.get("icon"), + "hidden": card.get("hidden") or False, + "link_count": card.get("link_count"), + "idx": 1 if not self.links else self.links[-1].idx + 1, + }, + ) for link in links: - self.append('links', { - "label": link.get('label'), - "type": "Link", - "link_type": link.get('link_type'), - "link_to": link.get('link_to'), - "onboard": link.get('onboard'), - "only_for": link.get('only_for'), - "dependencies": link.get('dependencies'), - "is_query_report": link.get('is_query_report'), - "idx": self.links[-1].idx + 1 - }) + self.append( + "links", + { + "label": link.get("label"), + "type": "Link", + "link_type": link.get("link_type"), + "link_to": link.get("link_to"), + "onboard": link.get("onboard"), + "only_for": link.get("only_for"), + "dependencies": link.get("dependencies"), + "is_query_report": link.get("is_query_report"), + "idx": self.links[-1].idx + 1, + }, + ) + def disable_saving_as_public(): - return frappe.flags.in_install or \ - frappe.flags.in_patch or \ - frappe.flags.in_test or \ - frappe.flags.in_fixtures or \ - frappe.flags.in_migrate + return ( + frappe.flags.in_install + or frappe.flags.in_patch + or frappe.flags.in_test + or frappe.flags.in_fixtures + or frappe.flags.in_migrate + ) + def get_link_type(key): key = key.lower() - link_type_map = { - "doctype": "DocType", - "page": "Page", - "report": "Report" - } + link_type_map = {"doctype": "DocType", "page": "Page", "report": "Report"} if key in link_type_map: return link_type_map[key] return "DocType" + def get_report_type(report): report_type = frappe.get_value("Report", report, "report_type") return report_type in ["Query Report", "Script Report", "Custom Report"] + @frappe.whitelist() def new_page(new_page): if not loads(new_page): @@ -132,33 +148,28 @@ def new_page(new_page): if page.get("public") and not is_workspace_manager(): return - doc = frappe.new_doc('Workspace') - doc.title = page.get('title') - doc.icon = page.get('icon') - doc.content = page.get('content') - doc.parent_page = page.get('parent_page') - doc.label = page.get('label') - doc.for_user = page.get('for_user') - doc.public = page.get('public') + doc = frappe.new_doc("Workspace") + doc.title = page.get("title") + doc.icon = page.get("icon") + doc.content = page.get("content") + doc.parent_page = page.get("parent_page") + doc.label = page.get("label") + doc.for_user = page.get("for_user") + doc.public = page.get("public") doc.sequence_id = last_sequence_id(doc) + 1 doc.save(ignore_permissions=True) return doc + @frappe.whitelist() def save_page(title, public, new_widgets, blocks): public = frappe.parse_json(public) - filters = { - 'public': public, - 'label': title - } + filters = {"public": public, "label": title} if not public: - filters = { - 'for_user': frappe.session.user, - 'label': title + "-" + frappe.session.user - } + filters = {"for_user": frappe.session.user, "label": title + "-" + frappe.session.user} pages = frappe.get_list("Workspace", filters=filters) if pages: doc = frappe.get_doc("Workspace", pages[0]) @@ -170,16 +181,14 @@ def save_page(title, public, new_widgets, blocks): return {"name": title, "public": public, "label": doc.label} + @frappe.whitelist() def update_page(name, title, icon, parent, public): public = frappe.parse_json(public) doc = frappe.get_doc("Workspace", name) - filters = { - 'parent_page': doc.title, - 'public': doc.public - } + filters = {"parent_page": doc.title, "public": doc.public} child_docs = frappe.get_list("Workspace", filters=filters) if doc: @@ -187,10 +196,10 @@ def update_page(name, title, icon, parent, public): doc.icon = icon doc.parent_page = parent if doc.public != public: - doc.sequence_id = frappe.db.count('Workspace', {'public':public}, cache=True) + doc.sequence_id = frappe.db.count("Workspace", {"public": public}, cache=True) doc.public = public - doc.for_user = '' if public else doc.for_user or frappe.session.user - doc.label = '{0}-{1}'.format(title, doc.for_user) if doc.for_user else title + doc.for_user = "" if public else doc.for_user or frappe.session.user + doc.label = "{0}-{1}".format(title, doc.for_user) if doc.for_user else title doc.save(ignore_permissions=True) if name != doc.label: @@ -206,6 +215,7 @@ def update_page(name, title, icon, parent, public): return {"name": doc.title, "public": doc.public, "label": doc.label} + @frappe.whitelist() def duplicate_page(page_name, new_page): if not loads(new_page): @@ -218,15 +228,15 @@ def duplicate_page(page_name, new_page): old_doc = frappe.get_doc("Workspace", page_name) doc = frappe.copy_doc(old_doc) - doc.title = new_page.get('title') - doc.icon = new_page.get('icon') - doc.parent_page = new_page.get('parent') or '' - doc.public = new_page.get('is_public') - doc.for_user = '' + doc.title = new_page.get("title") + doc.icon = new_page.get("icon") + doc.parent_page = new_page.get("parent") or "" + doc.public = new_page.get("is_public") + doc.for_user = "" doc.label = doc.title if not doc.public: doc.for_user = doc.for_user or frappe.session.user - doc.label = '{0}-{1}'.format(doc.title, doc.for_user) + doc.label = "{0}-{1}".format(doc.title, doc.for_user) doc.name = doc.label if old_doc.public == doc.public: doc.sequence_id += 0.1 @@ -236,6 +246,7 @@ def duplicate_page(page_name, new_page): return doc + @frappe.whitelist() def delete_page(page): if not loads(page): @@ -251,6 +262,7 @@ def delete_page(page): return {"name": page.get("name"), "public": page.get("public"), "title": page.get("title")} + @frappe.whitelist() def sort_pages(sb_public_items, sb_private_items): if not loads(sb_public_items) and not loads(sb_private_items): @@ -259,8 +271,8 @@ def sort_pages(sb_public_items, sb_private_items): sb_public_items = loads(sb_public_items) sb_private_items = loads(sb_private_items) - workspace_public_pages = get_page_list(['name', 'title'], {'public': 1}) - workspace_private_pages = get_page_list(['name', 'title'], {'for_user': frappe.session.user}) + workspace_public_pages = get_page_list(["name", "title"], {"public": 1}) + workspace_private_pages = get_page_list(["name", "title"], {"for_user": frappe.session.user}) if sb_private_items: return sort_page(workspace_private_pages, sb_private_items) @@ -270,40 +282,40 @@ def sort_pages(sb_public_items, sb_private_items): return False + def sort_page(workspace_pages, pages): for seq, d in enumerate(pages): for page in workspace_pages: - if page.title == d.get('title'): - doc = frappe.get_doc('Workspace', page.name) + if page.title == d.get("title"): + doc = frappe.get_doc("Workspace", page.name) doc.sequence_id = seq + 1 - doc.parent_page = d.get('parent_page') or "" + doc.parent_page = d.get("parent_page") or "" doc.flags.ignore_links = True doc.save(ignore_permissions=True) break return True + def last_sequence_id(doc): - doc_exists = frappe.db.exists({ - 'doctype': 'Workspace', - 'public': doc.public, - 'for_user': doc.for_user - }) + doc_exists = frappe.db.exists( + {"doctype": "Workspace", "public": doc.public, "for_user": doc.for_user} + ) if not doc_exists: return 0 - return frappe.db.get_list('Workspace', - fields=['sequence_id'], - filters={ - 'public': doc.public, - 'for_user': doc.for_user - }, - order_by="sequence_id desc" + return frappe.db.get_list( + "Workspace", + fields=["sequence_id"], + filters={"public": doc.public, "for_user": doc.for_user}, + order_by="sequence_id desc", )[0].sequence_id + def get_page_list(fields, filters): - return frappe.get_list("Workspace", fields=fields, filters=filters, order_by='sequence_id asc') + return frappe.get_list("Workspace", fields=fields, filters=filters, order_by="sequence_id asc") + def is_workspace_manager(): return "Workspace Manager" in frappe.get_roles() diff --git a/frappe/desk/doctype/workspace_chart/workspace_chart.py b/frappe/desk/doctype/workspace_chart/workspace_chart.py index a3b66d99ab..e02cf06ee0 100644 --- a/frappe/desk/doctype/workspace_chart/workspace_chart.py +++ b/frappe/desk/doctype/workspace_chart/workspace_chart.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WorkspaceChart(Document): pass diff --git a/frappe/desk/doctype/workspace_link/workspace_link.py b/frappe/desk/doctype/workspace_link/workspace_link.py index 72256ba490..5e55a7c2bd 100644 --- a/frappe/desk/doctype/workspace_link/workspace_link.py +++ b/frappe/desk/doctype/workspace_link/workspace_link.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WorkspaceLink(Document): pass diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py index 1dad4cca05..4ca86c8146 100644 --- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py +++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WorkspaceShortcut(Document): pass diff --git a/frappe/desk/form/__init__.py b/frappe/desk/form/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/desk/form/__init__.py +++ b/frappe/desk/form/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py index d79927a506..4107f95827 100644 --- a/frappe/desk/form/assign_to.py +++ b/frappe/desk/form/assign_to.py @@ -3,38 +3,51 @@ """assign/unassign to ToDo""" +import json + import frappe +import frappe.share +import frappe.utils from frappe import _ +from frappe.desk.doctype.notification_log.notification_log import ( + enqueue_create_notification, + get_title, + get_title_html, +) from frappe.desk.form.document_follow import follow_document -from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\ - get_title, get_title_html -import frappe.utils -import frappe.share -import json -class DuplicateToDoError(frappe.ValidationError): pass + +class DuplicateToDoError(frappe.ValidationError): + pass + def get(args=None): """get assigned to""" if not args: args = frappe.local.form_dict - return frappe.get_all("ToDo", fields=["allocated_to as owner", "name"], filters={ - "reference_type": args.get("doctype"), - "reference_name": args.get("name"), - "status": ("!=", "Cancelled") - }, limit=5) + return frappe.get_all( + "ToDo", + fields=["allocated_to as owner", "name"], + filters={ + "reference_type": args.get("doctype"), + "reference_name": args.get("name"), + "status": ("!=", "Cancelled"), + }, + limit=5, + ) + @frappe.whitelist() def add(args=None): """add in someone's to do list - args = { - "assign_to": [], - "doctype": , - "name": , - "description": , - "assignment_rule": - } + args = { + "assign_to": [], + "doctype": , + "name": , + "description": , + "assignment_rule": + } """ if not args: @@ -45,10 +58,10 @@ def add(args=None): for assign_to in frappe.parse_json(args.get("assign_to")): filters = { - "reference_type": args['doctype'], - "reference_name": args['name'], + "reference_type": args["doctype"], + "reference_name": args["name"], "status": "Open", - "allocated_to": assign_to + "allocated_to": assign_to, } if frappe.get_all("ToDo", filters=filters): @@ -56,27 +69,29 @@ def add(args=None): else: from frappe.utils import nowdate - if not args.get('description'): - args['description'] = _('Assignment for {0} {1}').format(args['doctype'], args['name']) - - d = frappe.get_doc({ - "doctype": "ToDo", - "allocated_to": assign_to, - "reference_type": args['doctype'], - "reference_name": args['name'], - "description": args.get('description'), - "priority": args.get("priority", "Medium"), - "status": "Open", - "date": args.get('date', nowdate()), - "assigned_by": args.get('assigned_by', frappe.session.user), - 'assignment_rule': args.get('assignment_rule') - }).insert(ignore_permissions=True) + if not args.get("description"): + args["description"] = _("Assignment for {0} {1}").format(args["doctype"], args["name"]) + + d = frappe.get_doc( + { + "doctype": "ToDo", + "allocated_to": assign_to, + "reference_type": args["doctype"], + "reference_name": args["name"], + "description": args.get("description"), + "priority": args.get("priority", "Medium"), + "status": "Open", + "date": args.get("date", nowdate()), + "assigned_by": args.get("assigned_by", frappe.session.user), + "assignment_rule": args.get("assignment_rule"), + } + ).insert(ignore_permissions=True) # set assigned_to if field exists - if frappe.get_meta(args['doctype']).get_field("assigned_to"): - frappe.db.set_value(args['doctype'], args['name'], "assigned_to", assign_to) + if frappe.get_meta(args["doctype"]).get_field("assigned_to"): + frappe.db.set_value(args["doctype"], args["name"], "assigned_to", assign_to) - doc = frappe.get_doc(args['doctype'], args['name']) + doc = frappe.get_doc(args["doctype"], args["name"]) # if assignee does not have permissions, share if not frappe.has_permission(doc=doc, user=assign_to): @@ -85,15 +100,23 @@ def add(args=None): # make this document followed by assigned user if frappe.get_cached_value("User", assign_to, "follow_assigned_documents"): - follow_document(args['doctype'], args['name'], assign_to) + follow_document(args["doctype"], args["name"], assign_to) # notify - notify_assignment(d.assigned_by, d.allocated_to, d.reference_type, d.reference_name, action='ASSIGN', - description=args.get("description")) + notify_assignment( + d.assigned_by, + d.allocated_to, + d.reference_type, + d.reference_name, + action="ASSIGN", + description=args.get("description"), + ) if shared_with_users: user_list = format_message_for_assign_to(shared_with_users) - frappe.msgprint(_("Shared with the following Users with Read access:{0}").format(user_list, alert=True)) + frappe.msgprint( + _("Shared with the following Users with Read access:{0}").format(user_list, alert=True) + ) if users_with_duplicate_todo: user_list = format_message_for_assign_to(users_with_duplicate_todo) @@ -101,20 +124,25 @@ def add(args=None): return get(args) + @frappe.whitelist() def add_multiple(args=None): if not args: args = frappe.local.form_dict - docname_list = json.loads(args['name']) + docname_list = json.loads(args["name"]) for docname in docname_list: args.update({"name": docname}) add(args) + def close_all_assignments(doctype, name): - assignments = frappe.db.get_all('ToDo', fields=['allocated_to'], filters = - dict(reference_type = doctype, reference_name = name, status=('!=', 'Cancelled'))) + assignments = frappe.db.get_all( + "ToDo", + fields=["allocated_to"], + filters=dict(reference_type=doctype, reference_name=name, status=("!=", "Cancelled")), + ) if not assignments: return False @@ -123,15 +151,24 @@ def close_all_assignments(doctype, name): return True + @frappe.whitelist() def remove(doctype, name, assign_to): return set_status(doctype, name, assign_to, status="Cancelled") + def set_status(doctype, name, assign_to, status="Cancelled"): """remove from todo""" try: - todo = frappe.db.get_value("ToDo", {"reference_type":doctype, - "reference_name":name, "allocated_to":assign_to, "status": ('!=', status)}) + todo = frappe.db.get_value( + "ToDo", + { + "reference_type": doctype, + "reference_name": name, + "allocated_to": assign_to, + "status": ("!=", status), + }, + ) if todo: todo = frappe.get_doc("ToDo", todo) todo.status = status @@ -142,17 +179,19 @@ def set_status(doctype, name, assign_to, status="Cancelled"): pass # clear assigned_to if field exists - if frappe.get_meta(doctype).get_field("assigned_to") and status=="Cancelled": + if frappe.get_meta(doctype).get_field("assigned_to") and status == "Cancelled": frappe.db.set_value(doctype, name, "assigned_to", None) return get({"doctype": doctype, "name": name}) + def clear(doctype, name): - ''' + """ Clears assignments, return False if not assigned. - ''' - assignments = frappe.db.get_all('ToDo', fields=['allocated_to'], filters = - dict(reference_type = doctype, reference_name = name)) + """ + assignments = frappe.db.get_all( + "ToDo", fields=["allocated_to"], filters=dict(reference_type=doctype, reference_name=name) + ) if not assignments: return False @@ -161,42 +200,46 @@ def clear(doctype, name): return True -def notify_assignment(assigned_by, allocated_to, doc_type, doc_name, action='CLOSE', - description=None): + +def notify_assignment( + assigned_by, allocated_to, doc_type, doc_name, action="CLOSE", description=None +): """ - Notify assignee that there is a change in assignment + Notify assignee that there is a change in assignment """ if not (assigned_by and allocated_to and doc_type and doc_name): return # return if self assigned or user disabled - if assigned_by == allocated_to or not frappe.db.get_value('User', allocated_to, 'enabled'): + if assigned_by == allocated_to or not frappe.db.get_value("User", allocated_to, "enabled"): return # Search for email address in description -- i.e. assignee - user_name = frappe.get_cached_value('User', frappe.session.user, 'full_name') + user_name = frappe.get_cached_value("User", frappe.session.user, "full_name") title = get_title(doc_type, doc_name) - description_html = "
{0}
".format(description) if description else None + description_html = "
{0}
".format(description) if description else None - if action == 'CLOSE': - subject = _('Your assignment on {0} {1} has been removed by {2}')\ - .format(frappe.bold(doc_type), get_title_html(title), frappe.bold(user_name)) + if action == "CLOSE": + subject = _("Your assignment on {0} {1} has been removed by {2}").format( + frappe.bold(doc_type), get_title_html(title), frappe.bold(user_name) + ) else: user_name = frappe.bold(user_name) document_type = frappe.bold(doc_type) title = get_title_html(title) - subject = _('{0} assigned a new task {1} {2} to you').format(user_name, document_type, title) + subject = _("{0} assigned a new task {1} {2} to you").format(user_name, document_type, title) notification_doc = { - 'type': 'Assignment', - 'document_type': doc_type, - 'subject': subject, - 'document_name': doc_name, - 'from_user': frappe.session.user, - 'email_content': description_html + "type": "Assignment", + "document_type": doc_type, + "subject": subject, + "document_name": doc_name, + "from_user": frappe.session.user, + "email_content": description_html, } enqueue_create_notification(allocated_to, notification_doc) + def format_message_for_assign_to(users): - return "

" + "
".join(users) \ No newline at end of file + return "

" + "
".join(users) diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py index 7dd2b64c21..527b9da036 100644 --- a/frappe/desk/form/document_follow.py +++ b/frappe/desk/form/document_follow.py @@ -3,10 +3,11 @@ import frappe import frappe.utils -from frappe.utils import get_url_to_form -from frappe.model import log_types from frappe import _ +from frappe.model import log_types from frappe.query_builder import DocType +from frappe.utils import get_url_to_form + @frappe.whitelist() def update_follow(doctype, doc_name, following): @@ -18,22 +19,32 @@ def update_follow(doctype, doc_name, following): @frappe.whitelist() def follow_document(doctype, doc_name, user): - ''' - param: - Doctype name - doc name - user email - - condition: - avoided for some doctype - follow only if track changes are set to 1 - ''' - if (doctype in ("Communication", "ToDo", "Email Unsubscribe", "File", "Comment", "Email Account", "Email Domain") - or doctype in log_types): + """ + param: + Doctype name + doc name + user email + + condition: + avoided for some doctype + follow only if track changes are set to 1 + """ + if ( + doctype + in ( + "Communication", + "ToDo", + "Email Unsubscribe", + "File", + "Comment", + "Email Account", + "Email Domain", + ) + or doctype in log_types + ): return - if ((not frappe.get_meta(doctype).track_changes) - or user == "Administrator"): + if (not frappe.get_meta(doctype).track_changes) or user == "Administrator": return if not frappe.db.get_value("User", user, "document_follow_notify", ignore=True, cache=True): @@ -41,35 +52,32 @@ def follow_document(doctype, doc_name, user): if not is_document_followed(doctype, doc_name, user): doc = frappe.new_doc("Document Follow") - doc.update({ - "ref_doctype": doctype, - "ref_docname": doc_name, - "user": user - }) + doc.update({"ref_doctype": doctype, "ref_docname": doc_name, "user": user}) doc.save() return doc + @frappe.whitelist() def unfollow_document(doctype, doc_name, user): doc = frappe.get_all( "Document Follow", - filters={ - "ref_doctype": doctype, - "ref_docname": doc_name, - "user": user - }, + filters={"ref_doctype": doctype, "ref_docname": doc_name, "user": user}, fields=["name"], - limit=1 + limit=1, ) if doc: frappe.delete_doc("Document Follow", doc[0].name) return 1 return 0 + def get_message(doc_name, doctype, frequency, user): - activity_list = get_version(doctype, doc_name, frequency, user) + get_comments(doctype, doc_name, frequency, user) + activity_list = get_version(doctype, doc_name, frequency, user) + get_comments( + doctype, doc_name, frequency, user + ) return sorted(activity_list, key=lambda k: k["time"], reverse=True) + def send_email_alert(receiver, docinfo, timeline): if receiver: frappe.sendmail( @@ -79,20 +87,21 @@ def send_email_alert(receiver, docinfo, timeline): args={ "docinfo": docinfo, "timeline": timeline, - } + }, ) + def send_document_follow_mails(frequency): - ''' - param: - frequency for sanding mails + """ + param: + frequency for sanding mails - task: - set receiver according to frequency - group document list according to user - get changes, activity, comments on doctype - call method to send mail - ''' + task: + set receiver according to frequency + group document list according to user + get changes, activity, comments on doctype + call method to send mail + """ user_list = get_user_list(frequency) @@ -104,15 +113,20 @@ def send_document_follow_mails(frequency): # nosemgrep frappe.db.commit() + def get_user_list(frequency): - DocumentFollow = DocType('Document Follow') - User = DocType('User') - return (frappe.qb.from_(DocumentFollow).join(User) + DocumentFollow = DocType("Document Follow") + User = DocType("User") + return ( + frappe.qb.from_(DocumentFollow) + .join(User) .on(DocumentFollow.user == User.name) .where(User.document_follow_notify == 1) .where(User.document_follow_frequency == frequency) .select(DocumentFollow.user) - .groupby(DocumentFollow.user)).run(pluck="user") + .groupby(DocumentFollow.user) + ).run(pluck="user") + def get_message_for_user(frequency, user): message = [] @@ -123,28 +137,33 @@ def get_message_for_user(frequency, user): content = get_message(document_follow.ref_docname, document_follow.ref_doctype, frequency, user) if content: message = message + content - valid_document_follows.append({ - "reference_docname": document_follow.ref_docname, - "reference_doctype": document_follow.ref_doctype, - "reference_url": get_url_to_form(document_follow.ref_doctype, document_follow.ref_docname) - }) + valid_document_follows.append( + { + "reference_docname": document_follow.ref_docname, + "reference_doctype": document_follow.ref_doctype, + "reference_url": get_url_to_form(document_follow.ref_doctype, document_follow.ref_docname), + } + ) return message, valid_document_follows + def get_document_followed_by_user(user): - DocumentFollow = DocType('Document Follow') + DocumentFollow = DocType("Document Follow") # at max 20 documents are sent for each user - return (frappe.qb.from_(DocumentFollow) + return ( + frappe.qb.from_(DocumentFollow) .where(DocumentFollow.user == user) .select(DocumentFollow.ref_doctype, DocumentFollow.ref_docname) .orderby(DocumentFollow.modified) - .limit(20)).run(as_dict=True) + .limit(20) + ).run(as_dict=True) + def get_version(doctype, doc_name, frequency, user): timeline = [] filters = get_filters("docname", doc_name, frequency, user) - version = frappe.get_all("Version", - filters=filters, - fields=["ref_doctype", "data", "modified", "modified", "modified_by"] + version = frappe.get_all( + "Version", filters=filters, fields=["ref_doctype", "data", "modified", "modified", "modified_by"] ) if version: for v in version: @@ -162,154 +181,154 @@ def get_version(doctype, doc_name, frequency, user): return timeline + def get_comments(doctype, doc_name, frequency, user): from html2text import html2text timeline = [] filters = get_filters("reference_name", doc_name, frequency, user) - comments = frappe.get_all("Comment", - filters=filters, - fields=["content", "modified", "modified_by", "comment_type"] + comments = frappe.get_all( + "Comment", filters=filters, fields=["content", "modified", "modified_by", "comment_type"] ) for comment in comments: if comment.comment_type == "Like": - by = ''' By : {0}'''.format(comment.modified_by) + by = """ By : {0}""".format(comment.modified_by) elif comment.comment_type == "Comment": - by = '''Commented by : {0}'''.format(comment.modified_by) + by = """Commented by : {0}""".format(comment.modified_by) else: - by = '' + by = "" time = frappe.utils.format_datetime(comment.modified, "hh:mm a") - timeline.append({ - "time": comment.modified, - "data": { - "time": time, - "comment": html2text(str(comment.content)), - "by": by - }, - "doctype": doctype, - "doc_name": doc_name, - "type": "comment" - }) + timeline.append( + { + "time": comment.modified, + "data": {"time": time, "comment": html2text(str(comment.content)), "by": by}, + "doctype": doctype, + "doc_name": doc_name, + "type": "comment", + } + ) return timeline + def is_document_followed(doctype, doc_name, user): return frappe.db.exists( - "Document Follow", - { - "ref_doctype": doctype, - "ref_docname": doc_name, - "user": user - } + "Document Follow", {"ref_doctype": doctype, "ref_docname": doc_name, "user": user} ) + @frappe.whitelist() def get_follow_users(doctype, doc_name): return frappe.get_all( - "Document Follow", - filters={ - "ref_doctype": doctype, - "ref_docname":doc_name - }, - fields=["user"] + "Document Follow", filters={"ref_doctype": doctype, "ref_docname": doc_name}, fields=["user"] ) + def get_row_changed(row_changed, time, doctype, doc_name, v): from html2text import html2text items = [] for d in row_changed: - d[2] = d[2] if d[2] else ' ' - d[0] = d[0] if d[0] else ' ' - d[3][0][1] = d[3][0][1] if d[3][0][1] else ' ' - items.append({ - "time": v.modified, - "data": { + d[2] = d[2] if d[2] else " " + d[0] = d[0] if d[0] else " " + d[3][0][1] = d[3][0][1] if d[3][0][1] else " " + items.append( + { + "time": v.modified, + "data": { "time": time, "table_field": d[0], "row": str(d[1]), "field": d[3][0][0], "from": html2text(str(d[3][0][1])), - "to": html2text(str(d[3][0][2])) + "to": html2text(str(d[3][0][2])), }, - "doctype": doctype, - "doc_name": doc_name, - "type": "row changed", - "by": v.modified_by - }) + "doctype": doctype, + "doc_name": doc_name, + "type": "row changed", + "by": v.modified_by, + } + ) return items + def get_added_row(added, time, doctype, doc_name, v): items = [] for d in added: - items.append({ - "time": v.modified, - "data": { - "to": d[0], - "time": time - }, - "doctype": doctype, - "doc_name": doc_name, - "type": "row added", - "by": v.modified_by - }) + items.append( + { + "time": v.modified, + "data": {"to": d[0], "time": time}, + "doctype": doctype, + "doc_name": doc_name, + "type": "row added", + "by": v.modified_by, + } + ) return items + def get_field_changed(changed, time, doctype, doc_name, v): from html2text import html2text items = [] for d in changed: - d[1] = d[1] if d[1] else ' ' - d[2] = d[2] if d[2] else ' ' - d[0] = d[0] if d[0] else ' ' - items.append({ - "time": v.modified, - "data": { + d[1] = d[1] if d[1] else " " + d[2] = d[2] if d[2] else " " + d[0] = d[0] if d[0] else " " + items.append( + { + "time": v.modified, + "data": { "time": time, "field": d[0], "from": html2text(str(d[1])), - "to": html2text(str(d[2])) + "to": html2text(str(d[2])), }, - "doctype": doctype, - "doc_name": doc_name, - "type": "field changed", - "by": v.modified_by - }) + "doctype": doctype, + "doc_name": doc_name, + "type": "field changed", + "by": v.modified_by, + } + ) return items + def send_hourly_updates(): send_document_follow_mails("Hourly") + def send_daily_updates(): send_document_follow_mails("Daily") + def send_weekly_updates(): send_document_follow_mails("Weekly") + def get_filters(search_by, name, frequency, user): filters = [] if frequency == "Weekly": filters = [ [search_by, "=", name], - ["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(),-7)], + ["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(), -7)], ["modified", "<", frappe.utils.nowdate()], - ["modified_by", "!=", user] + ["modified_by", "!=", user], ] elif frequency == "Daily": filters = [ [search_by, "=", name], - ["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(),-1)], + ["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(), -1)], ["modified", "<", frappe.utils.nowdate()], - ["modified_by", "!=", user] + ["modified_by", "!=", user], ] elif frequency == "Hourly": filters = [ [search_by, "=", name], ["modified", ">", frappe.utils.add_to_date(frappe.utils.now_datetime(), hours=-1)], ["modified", "<", frappe.utils.now_datetime()], - ["modified_by", "!=", user] + ["modified_by", "!=", user], ] return filters diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index 010d65c95b..7a53c8b65a 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -1,9 +1,9 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import itertools import json from collections import defaultdict -import itertools from typing import Dict, List, Optional import frappe @@ -16,7 +16,7 @@ from frappe.modules import load_doctype_module @frappe.whitelist() def get_submitted_linked_docs(doctype: str, name: str) -> List[tuple]: - """ Get all the nested submitted documents those are present in referencing tables (dependent tables). + """Get all the nested submitted documents those are present in referencing tables (dependent tables). :param doctype: Document type :param name: Name of the document @@ -25,31 +25,29 @@ def get_submitted_linked_docs(doctype: str, name: str) -> List[tuple]: * User should be able to cancel the linked documents along with the one user trying to cancel. Case1: If document sd1-n1 (document name n1 from sumittable doctype sd1) is linked to sd2-n2 and sd2-n2 is linked to sd3-n3, - Getting submittable linked docs of `sd1-n1`should give both sd2-n2 and sd3-n3. + Getting submittable linked docs of `sd1-n1`should give both sd2-n2 and sd3-n3. Case2: If document sd1-n1 (document name n1 from sumittable doctype sd1) is linked to d2-n2 and d2-n2 is linked to sd3-n3, - Getting submittable linked docs of `sd1-n1`should give None. (because d2-n2 is not a submittable doctype) + Getting submittable linked docs of `sd1-n1`should give None. (because d2-n2 is not a submittable doctype) Case3: If document sd1-n1 (document name n1 from submittable doctype sd1) is linked to d2-n2 & sd2-n2. d2-n2 is linked to sd3-n3. - Getting submittable linked docs of `sd1-n1`should give sd2-n2. + Getting submittable linked docs of `sd1-n1`should give sd2-n2. Logic: ----- 1. We can find linked documents only if we know how the doctypes are related. 2. As we need only submittable documents, we can limit doctype relations search to submittable doctypes by - finding the relationships(Foreign key references) across submittable doctypes. + finding the relationships(Foreign key references) across submittable doctypes. 3. Searching for links is going to be a tree like structure where at every level, - you will be finding documents using parent document and parent document links. + you will be finding documents using parent document and parent document links. """ tree = SubmittableDocumentTree(doctype, name) visited_documents = tree.get_all_children() docs = [] for dt, names in visited_documents.items(): - docs.extend([{'doctype': dt, 'name': name, 'docstatus': 1} for name in names]) + docs.extend([{"doctype": dt, "name": name, "docstatus": 1} for name in names]) + + return {"docs": docs, "count": len(docs)} - return { - "docs": docs, - "count": len(docs) - } class SubmittableDocumentTree: def __init__(self, doctype: str, name: str): @@ -68,8 +66,8 @@ class SubmittableDocumentTree: self.to_be_visited_documents = {doctype: [name]} self.visited_documents = defaultdict(list) - self._submittable_doctypes = None # All submittable doctypes in the system - self._references_across_doctypes = None # doctype wise links/references + self._submittable_doctypes = None # All submittable doctypes in the system + self._references_across_doctypes = None # doctype wise links/references def get_all_children(self): """Get all nodes of a tree except the root node (all the nested submitted @@ -98,75 +96,86 @@ class SubmittableDocumentTree: return self.visited_documents def get_next_level_children(self, parent_dt, parent_names): - """Get immediate children of a Node(parent_dt, parent_names) - """ + """Get immediate children of a Node(parent_dt, parent_names)""" referencing_fields = self.get_doctype_references(parent_dt) child_docs = defaultdict(list) for field in referencing_fields: - links = get_referencing_documents(parent_dt, parent_names.copy(), field, get_parent_if_child_table_doc=True, - parent_filters=[('docstatus', '=', 1)], allowed_parents=self.get_link_sources()) or {} + links = ( + get_referencing_documents( + parent_dt, + parent_names.copy(), + field, + get_parent_if_child_table_doc=True, + parent_filters=[("docstatus", "=", 1)], + allowed_parents=self.get_link_sources(), + ) + or {} + ) for dt, names in links.items(): child_docs[dt].extend(names) return child_docs def get_doctype_references(self, doctype): - """Get references for a given document. - """ + """Get references for a given document.""" if self._references_across_doctypes is None: get_links_to = self.get_document_sources() limit_link_doctypes = self.get_link_sources() self._references_across_doctypes = get_references_across_doctypes( - get_links_to, limit_link_doctypes) + get_links_to, limit_link_doctypes + ) return self._references_across_doctypes.get(doctype, []) def get_document_sources(self): - """Returns list of doctypes from where we access submittable documents. - """ + """Returns list of doctypes from where we access submittable documents.""" return list(set(self.get_link_sources() + [self.root_doctype])) def get_link_sources(self): - """limit doctype links to these doctypes. - """ + """limit doctype links to these doctypes.""" return list(set(self.get_submittable_doctypes()) - set(get_exempted_doctypes() or [])) def get_submittable_doctypes(self) -> List[str]: - """Returns list of submittable doctypes. - """ + """Returns list of submittable doctypes.""" if not self._submittable_doctypes: - self._submittable_doctypes = frappe.db.get_all('DocType', {'is_submittable': 1}, pluck='name') + self._submittable_doctypes = frappe.db.get_all("DocType", {"is_submittable": 1}, pluck="name") return self._submittable_doctypes -def get_child_tables_of_doctypes(doctypes: List[str]=None): - """Returns child tables by doctype. - """ - filters=[['fieldtype','=', 'Table']] +def get_child_tables_of_doctypes(doctypes: List[str] = None): + """Returns child tables by doctype.""" + filters = [["fieldtype", "=", "Table"]] filters_for_docfield = filters filters_for_customfield = filters if doctypes: - filters_for_docfield = filters + [['parent', 'in', tuple(doctypes)]] - filters_for_customfield = filters + [['dt', 'in', tuple(doctypes)]] + filters_for_docfield = filters + [["parent", "in", tuple(doctypes)]] + filters_for_customfield = filters + [["dt", "in", tuple(doctypes)]] - links = frappe.get_all("DocField", + links = frappe.get_all( + "DocField", fields=["parent", "fieldname", "options as child_table"], filters=filters_for_docfield, - as_list=1) + as_list=1, + ) - links+= frappe.get_all("Custom Field", + links += frappe.get_all( + "Custom Field", fields=["dt as parent", "fieldname", "options as child_table"], filters=filters_for_customfield, - as_list=1) + as_list=1, + ) child_tables_by_doctype = defaultdict(list) for doctype, fieldname, child_table in links: child_tables_by_doctype[doctype].append( - {'doctype': doctype, 'fieldname': fieldname, 'child_table': child_table}) + {"doctype": doctype, "fieldname": fieldname, "child_table": child_table} + ) return child_tables_by_doctype -def get_references_across_doctypes(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None) -> List: +def get_references_across_doctypes( + to_doctypes: List[str] = None, limit_link_doctypes: List[str] = None +) -> List: """Find doctype wise foreign key references. :param to_doctypes: Get links of these doctypes. @@ -176,14 +185,22 @@ def get_references_across_doctypes(to_doctypes: List[str]=None, limit_link_docty """ if limit_link_doctypes: child_tables_by_doctype = get_child_tables_of_doctypes(limit_link_doctypes) - all_child_tables = [each['child_table'] for each in itertools.chain(*child_tables_by_doctype.values())] + all_child_tables = [ + each["child_table"] for each in itertools.chain(*child_tables_by_doctype.values()) + ] limit_link_doctypes = limit_link_doctypes + all_child_tables else: child_tables_by_doctype = get_child_tables_of_doctypes() - all_child_tables = [each['child_table'] for each in itertools.chain(*child_tables_by_doctype.values())] + all_child_tables = [ + each["child_table"] for each in itertools.chain(*child_tables_by_doctype.values()) + ] - references_by_link_fields = get_references_across_doctypes_by_link_field(to_doctypes, limit_link_doctypes) - references_by_dlink_fields = get_references_across_doctypes_by_dynamic_link_field(to_doctypes, limit_link_doctypes) + references_by_link_fields = get_references_across_doctypes_by_link_field( + to_doctypes, limit_link_doctypes + ) + references_by_dlink_fields = get_references_across_doctypes_by_dynamic_link_field( + to_doctypes, limit_link_doctypes + ) references = references_by_link_fields.copy() for k, v in references_by_dlink_fields.items(): @@ -191,118 +208,140 @@ def get_references_across_doctypes(to_doctypes: List[str]=None, limit_link_docty for doctype, links in references.items(): for link in links: - link['is_child'] = (link['doctype'] in all_child_tables) + link["is_child"] = link["doctype"] in all_child_tables return references -def get_references_across_doctypes_by_link_field(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None): +def get_references_across_doctypes_by_link_field( + to_doctypes: List[str] = None, limit_link_doctypes: List[str] = None +): """Find doctype wise foreign key references based on link fields. :param to_doctypes: Get links to these doctypes. :param limit_link_doctypes: limit links to these doctypes. """ - filters=[['fieldtype','=', 'Link']] + filters = [["fieldtype", "=", "Link"]] if to_doctypes: - filters += [['options', 'in', tuple(to_doctypes)]] + filters += [["options", "in", tuple(to_doctypes)]] filters_for_docfield = filters[:] filters_for_customfield = filters[:] if limit_link_doctypes: - filters_for_docfield += [['parent', 'in', tuple(limit_link_doctypes)]] - filters_for_customfield += [['dt', 'in', tuple(limit_link_doctypes)]] + filters_for_docfield += [["parent", "in", tuple(limit_link_doctypes)]] + filters_for_customfield += [["dt", "in", tuple(limit_link_doctypes)]] - links = frappe.get_all("DocField", + links = frappe.get_all( + "DocField", fields=["parent", "fieldname", "options as linked_to"], filters=filters_for_docfield, - as_list=1) + as_list=1, + ) - links+= frappe.get_all("Custom Field", + links += frappe.get_all( + "Custom Field", fields=["dt as parent", "fieldname", "options as linked_to"], filters=filters_for_customfield, - as_list=1) + as_list=1, + ) links_by_doctype = defaultdict(list) for doctype, fieldname, linked_to in links: - links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname}) + links_by_doctype[linked_to].append({"doctype": doctype, "fieldname": fieldname}) return links_by_doctype -def get_references_across_doctypes_by_dynamic_link_field(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None): +def get_references_across_doctypes_by_dynamic_link_field( + to_doctypes: List[str] = None, limit_link_doctypes: List[str] = None +): """Find doctype wise foreign key references based on dynamic link fields. :param to_doctypes: Get links to these doctypes. :param limit_link_doctypes: limit links to these doctypes. """ - filters=[['fieldtype','=', 'Dynamic Link']] + filters = [["fieldtype", "=", "Dynamic Link"]] filters_for_docfield = filters[:] filters_for_customfield = filters[:] if limit_link_doctypes: - filters_for_docfield += [['parent', 'in', tuple(limit_link_doctypes)]] - filters_for_customfield += [['dt', 'in', tuple(limit_link_doctypes)]] + filters_for_docfield += [["parent", "in", tuple(limit_link_doctypes)]] + filters_for_customfield += [["dt", "in", tuple(limit_link_doctypes)]] # find dynamic links of parents - links = frappe.get_all("DocField", + links = frappe.get_all( + "DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters_for_docfield, - as_list=1) + as_list=1, + ) - links += frappe.get_all("Custom Field", + links += frappe.get_all( + "Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters_for_customfield, - as_list=1) + as_list=1, + ) links_by_doctype = defaultdict(list) for doctype, fieldname, doctype_fieldname in links: try: - filters = [[doctype_fieldname, 'in', to_doctypes]] if to_doctypes else [] - for linked_to in frappe.db.get_all(doctype, pluck=doctype_fieldname, filters = filters, distinct=1): + filters = [[doctype_fieldname, "in", to_doctypes]] if to_doctypes else [] + for linked_to in frappe.db.get_all( + doctype, pluck=doctype_fieldname, filters=filters, distinct=1 + ): if linked_to: - links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname, 'doctype_fieldname': doctype_fieldname}) + links_by_doctype[linked_to].append( + {"doctype": doctype, "fieldname": fieldname, "doctype_fieldname": doctype_fieldname} + ) except frappe.db.ProgrammingError: # TODO: FIXME continue return links_by_doctype -def get_referencing_documents(reference_doctype: str, reference_names: List[str], - link_info: dict, get_parent_if_child_table_doc: bool=True, - parent_filters: List[list]=None, child_filters=None, allowed_parents=None): + +def get_referencing_documents( + reference_doctype: str, + reference_names: List[str], + link_info: dict, + get_parent_if_child_table_doc: bool = True, + parent_filters: List[list] = None, + child_filters=None, + allowed_parents=None, +): """Get linked documents based on link_info. :param reference_doctype: reference doctype to find links :param reference_names: reference document names to find links for :param link_info: linking details to get the linked documents - Ex: {'doctype': 'Purchase Invoice Advance', 'fieldname': 'reference_name', - 'doctype_fieldname': 'reference_type', 'is_child': True} + Ex: {'doctype': 'Purchase Invoice Advance', 'fieldname': 'reference_name', + 'doctype_fieldname': 'reference_type', 'is_child': True} :param get_parent_if_child_table_doc: Get parent record incase linked document is a child table record. :param parent_filters: filters to apply on if not a child table. :param child_filters: apply filters if it is a child table. :param allowed_parents: list of parents allowed in case of get_parent_if_child_table_doc - is enabled. + is enabled. """ - from_table = link_info['doctype'] - filters = [[link_info['fieldname'], 'in', tuple(reference_names)]] - if link_info.get('doctype_fieldname'): - filters.append([link_info['doctype_fieldname'], '=', reference_doctype]) + from_table = link_info["doctype"] + filters = [[link_info["fieldname"], "in", tuple(reference_names)]] + if link_info.get("doctype_fieldname"): + filters.append([link_info["doctype_fieldname"], "=", reference_doctype]) - if not link_info.get('is_child'): + if not link_info.get("is_child"): filters.extend(parent_filters or []) - return {from_table: frappe.db.get_all(from_table, filters, pluck='name')} - + return {from_table: frappe.db.get_all(from_table, filters, pluck="name")} filters.extend(child_filters or []) - res = frappe.db.get_all(from_table, filters = filters, fields = ['name', 'parenttype', 'parent']) + res = frappe.db.get_all(from_table, filters=filters, fields=["name", "parenttype", "parent"]) documents = defaultdict(list) - for parent, rows in itertools.groupby(res, key = lambda row: row['parenttype']): + for parent, rows in itertools.groupby(res, key=lambda row: row["parenttype"]): if allowed_parents and parent not in allowed_parents: continue - filters = (parent_filters or []) + [['name', 'in', tuple([row.parent for row in rows])]] - documents[parent].extend(frappe.db.get_all(parent, filters=filters, pluck='name') or []) + filters = (parent_filters or []) + [["name", "in", tuple([row.parent for row in rows])]] + documents[parent].extend(frappe.db.get_all(parent, filters=filters, pluck="name") or []) return documents @@ -312,8 +351,8 @@ def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=None): Cancel all linked doctype, optionally ignore doctypes specified in a list. Arguments: - docs (json str) - It contains list of dictionaries of a linked documents. - ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling. + docs (json str) - It contains list of dictionaries of a linked documents. + ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling. """ if ignore_doctypes_on_cancel_all is None: ignore_doctypes_on_cancel_all = [] @@ -325,7 +364,7 @@ def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=None): if validate_linked_doc(doc, ignore_doctypes_on_cancel_all): linked_doc = frappe.get_doc(doc.get("doctype"), doc.get("name")) linked_doc.cancel() - frappe.publish_progress(percent=i/len(docs) * 100, title=_("Cancelling documents")) + frappe.publish_progress(percent=i / len(docs) * 100, title=_("Cancelling documents")) def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=None): @@ -333,36 +372,36 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=None): Validate a document to be submitted and non-exempted from auto-cancel. Arguments: - docinfo (dict): The document to check for submitted and non-exempt from auto-cancel - ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling. + docinfo (dict): The document to check for submitted and non-exempt from auto-cancel + ignore_doctypes_on_cancel_all (list) - List of doctypes to ignore while cancelling. Returns: - bool: True if linked document passes all validations, else False + bool: True if linked document passes all validations, else False """ - #ignore doctype to cancel + # ignore doctype to cancel if docinfo.get("doctype") in (ignore_doctypes_on_cancel_all or []): return False # skip non-submittable doctypes since they don't need to be cancelled - if not frappe.get_meta(docinfo.get('doctype')).is_submittable: + if not frappe.get_meta(docinfo.get("doctype")).is_submittable: return False # skip draft or cancelled documents - if docinfo.get('docstatus') != 1: + if docinfo.get("docstatus") != 1: return False # skip other doctypes since they don't need to be cancelled auto_cancel_exempt_doctypes = get_exempted_doctypes() - if docinfo.get('doctype') in auto_cancel_exempt_doctypes: + if docinfo.get("doctype") in auto_cancel_exempt_doctypes: return False return True def get_exempted_doctypes(): - """ Get list of doctypes exempted from being auto-cancelled """ + """Get list of doctypes exempted from being auto-cancelled""" auto_cancel_exempt_doctypes = [] - for doctypes in frappe.get_hooks('auto_cancel_exempted_doctypes'): + for doctypes in frappe.get_hooks("auto_cancel_exempted_doctypes"): auto_cancel_exempt_doctypes.append(doctypes) return auto_cancel_exempt_doctypes @@ -394,16 +433,23 @@ def get_linked_docs(doctype: str, name: str, linkinfo: Optional[Dict] = None) -> continue if not linkmeta.get("issingle"): - fields = [d.fieldname for d in linkmeta.get("fields", { - "in_list_view": 1, - "fieldtype": ["not in", ("Image", "HTML", "Button") + frappe.model.table_fields] - })] + ["name", "modified", "docstatus"] + fields = [ + d.fieldname + for d in linkmeta.get( + "fields", + { + "in_list_view": 1, + "fieldtype": ["not in", ("Image", "HTML", "Button") + frappe.model.table_fields], + }, + ) + ] + ["name", "modified", "docstatus"] if link.get("add_fields"): fields += link["add_fields"] - fields = ["`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf - and "`tab" not in sf] + fields = [ + "`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf and "`tab" not in sf + ] try: if link.get("filters"): @@ -418,24 +464,28 @@ def get_linked_docs(doctype: str, name: str, linkinfo: Optional[Dict] = None) -> me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True) if me and me.parenttype == dt: - ret = frappe.get_all(doctype=dt, fields=fields, - filters=[[dt, "name", '=', me.parent]]) + ret = frappe.get_all(doctype=dt, fields=fields, filters=[[dt, "name", "=", me.parent]]) elif link.get("child_doctype"): - or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")] + or_filters = [ + [link.get("child_doctype"), link_fieldnames, "=", name] + for link_fieldnames in link.get("fieldname") + ] # dynamic link if link.get("doctype_fieldname"): - filters.append([link.get('child_doctype'), link.get("doctype_fieldname"), "=", doctype]) + filters.append([link.get("child_doctype"), link.get("doctype_fieldname"), "=", doctype]) - ret = frappe.get_all(doctype=dt, fields=fields, filters=filters, or_filters=or_filters, distinct=True) + ret = frappe.get_all( + doctype=dt, fields=fields, filters=filters, or_filters=or_filters, distinct=True + ) else: link_fieldnames = link.get("fieldname") if link_fieldnames: if isinstance(link_fieldnames, str): link_fieldnames = [link_fieldnames] - or_filters = [[dt, fieldname, '=', name] for fieldname in link_fieldnames] + or_filters = [[dt, fieldname, "=", name] for fieldname in link_fieldnames] # dynamic link if link.get("doctype_fieldname"): filters.append([dt, link.get("doctype_fieldname"), "=", doctype]) @@ -468,11 +518,14 @@ def get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False): Example, for Customer: - {"Address": {"fieldname": "customer"}..} + {"Address": {"fieldname": "customer"}..} """ - if(without_ignore_user_permissions_enabled): - return frappe.cache().hget("linked_doctypes_without_ignore_user_permissions_enabled", - doctype, lambda: _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled)) + if without_ignore_user_permissions_enabled: + return frappe.cache().hget( + "linked_doctypes_without_ignore_user_permissions_enabled", + doctype, + lambda: _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled), + ) else: return frappe.cache().hget("linked_doctypes", doctype, lambda: _get_linked_doctypes(doctype)) @@ -483,14 +536,16 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled)) ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled)) - filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]] - if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) + filters = [["fieldtype", "in", frappe.model.table_fields], ["options", "=", doctype]] + if without_ignore_user_permissions_enabled: + filters.append(["ignore_user_permissions", "!=", 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters) - links+= frappe.get_all("Custom Field", fields=["dt"], filters=filters) + links += frappe.get_all("Custom Field", fields=["dt"], filters=filters) - for dt, in links: - if dt in ret: continue + for (dt,) in links: + if dt in ret: + continue ret[dt] = {"get_parent": True} for dt in list(ret): @@ -509,31 +564,44 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False) def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): - filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]] - if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) + filters = [["fieldtype", "=", "Link"], ["options", "=", doctype]] + if without_ignore_user_permissions_enabled: + filters.append(["ignore_user_permissions", "!=", 1]) # find links of parents links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1) - links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1) + links += frappe.get_all( + "Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1 + ) ret = {} - if not links: return ret + if not links: + return ret links_dict = defaultdict(list) for doctype, fieldname in links: links_dict[doctype].append(fieldname) for doctype_name in links_dict: - ret[doctype_name] = { "fieldname": links_dict.get(doctype_name) } - table_doctypes = frappe.get_all("DocType", filters=[["istable", "=", "1"], ["name", "in", tuple(links_dict)]]) - child_filters = [['fieldtype','in', frappe.model.table_fields], ['options', 'in', tuple(doctype.name for doctype in table_doctypes)]] - if without_ignore_user_permissions_enabled: child_filters.append(['ignore_user_permissions', '!=', 1]) + ret[doctype_name] = {"fieldname": links_dict.get(doctype_name)} + table_doctypes = frappe.get_all( + "DocType", filters=[["istable", "=", "1"], ["name", "in", tuple(links_dict)]] + ) + child_filters = [ + ["fieldtype", "in", frappe.model.table_fields], + ["options", "in", tuple(doctype.name for doctype in table_doctypes)], + ] + if without_ignore_user_permissions_enabled: + child_filters.append(["ignore_user_permissions", "!=", 1]) # find out if linked in a child table - for parent, options in frappe.get_all("DocField", fields=["parent", "options"], filters=child_filters, as_list=1): - ret[parent] = { "child_doctype": options, "fieldname": links_dict[options]} - if options in ret: del ret[options] + for parent, options in frappe.get_all( + "DocField", fields=["parent", "options"], filters=child_filters, as_list=1 + ): + ret[parent] = {"child_doctype": options, "fieldname": links_dict[options]} + if options in ret: + del ret[options] return ret @@ -541,37 +609,45 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False): def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False): ret = {} - filters = [['fieldtype','=', 'Dynamic Link']] - if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1]) + filters = [["fieldtype", "=", "Dynamic Link"]] + if without_ignore_user_permissions_enabled: + filters.append(["ignore_user_permissions", "!=", 1]) # find dynamic links of parents - links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) - links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters) + links = frappe.get_all( + "DocField", + fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], + filters=filters, + ) + links += frappe.get_all( + "Custom Field", + fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], + filters=filters, + ) for df in links: - if is_single(df.doctype): continue + if is_single(df.doctype): + continue is_child = frappe.get_meta(df.doctype).istable possible_link = frappe.get_all( df.doctype, filters={df.doctype_fieldname: doctype}, fields=["parenttype"] if is_child else None, - distinct=True + distinct=True, ) - if not possible_link: continue + if not possible_link: + continue if is_child: for d in possible_link: ret[d.parenttype] = { "child_doctype": df.doctype, "fieldname": [df.fieldname], - "doctype_fieldname": df.doctype_fieldname + "doctype_fieldname": df.doctype_fieldname, } else: - ret[df.doctype] = { - "fieldname": [df.fieldname], - "doctype_fieldname": df.doctype_fieldname - } + ret[df.doctype] = {"fieldname": [df.fieldname], "doctype_fieldname": df.doctype_fieldname} return ret diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 9abf64c2cc..636b662a09 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -1,19 +1,20 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json from typing import Dict, List, Union -import frappe, json -import frappe.utils -import frappe.share +from urllib.parse import quote + +import frappe import frappe.defaults import frappe.desk.form.meta +import frappe.share +import frappe.utils +from frappe import _, _dict +from frappe.desk.form.document_follow import is_document_followed from frappe.model.utils.user_settings import get_user_settings from frappe.permissions import get_doc_permissions -from frappe.desk.form.document_follow import is_document_followed from frappe.utils.data import cstr -from frappe import _ -from frappe import _dict -from urllib.parse import quote @frappe.whitelist() @@ -25,7 +26,7 @@ def getdoc(doctype, name, user=None): """ if not (doctype and name): - raise Exception('doctype and name required!') + raise Exception("doctype and name required!") if not name: name = doctype @@ -38,7 +39,9 @@ def getdoc(doctype, name, user=None): run_onload(doc) if not doc.has_permission("read"): - frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(doctype + ' ' + name)) + frappe.flags.error_message = _("Insufficient Permission for {0}").format( + frappe.bold(doctype + " " + name) + ) raise frappe.PermissionError(("read", doctype, name)) doc.apply_fieldlevel_read_permissions() @@ -70,18 +73,19 @@ def getdoctype(doctype, with_parent=False, cached_timestamp=None): parent_dt = frappe.model.meta.get_parent_dt(doctype) if parent_dt: docs = get_meta_bundle(parent_dt) - frappe.response['parent_dt'] = parent_dt + frappe.response["parent_dt"] = parent_dt if not docs: docs = get_meta_bundle(doctype) - frappe.response['user_settings'] = get_user_settings(parent_dt or doctype) + frappe.response["user_settings"] = get_user_settings(parent_dt or doctype) - if cached_timestamp and docs[0].modified==cached_timestamp: + if cached_timestamp and docs[0].modified == cached_timestamp: return "use_cache" frappe.response.docs.extend(docs) + def get_meta_bundle(doctype): bundle = [frappe.desk.form.meta.get_meta(doctype)] for df in bundle[0].fields: @@ -89,6 +93,7 @@ def get_meta_bundle(doctype): bundle.append(frappe.desk.form.meta.get_meta(df.options, not frappe.conf.developer_mode)) return bundle + @frappe.whitelist() def get_docinfo(doc=None, doctype=None, name=None): if not doc: @@ -97,35 +102,42 @@ def get_docinfo(doc=None, doctype=None, name=None): raise frappe.PermissionError all_communications = _get_communications(doc.doctype, doc.name) - automated_messages = [msg for msg in all_communications if msg['communication_type'] == 'Automated Message'] - communications_except_auto_messages = [msg for msg in all_communications if msg['communication_type'] != 'Automated Message'] + automated_messages = [ + msg for msg in all_communications if msg["communication_type"] == "Automated Message" + ] + communications_except_auto_messages = [ + msg for msg in all_communications if msg["communication_type"] != "Automated Message" + ] - docinfo = frappe._dict(user_info = {}) + docinfo = frappe._dict(user_info={}) add_comments(doc, docinfo) - docinfo.update({ - "attachments": get_attachments(doc.doctype, doc.name), - "communications": communications_except_auto_messages, - "automated_messages": automated_messages, - 'total_comments': len(json.loads(doc.get('_comments') or '[]')), - 'versions': get_versions(doc), - "assignments": get_assignments(doc.doctype, doc.name), - "permissions": get_doc_permissions(doc), - "shared": frappe.share.get_users(doc.doctype, doc.name), - "views": get_view_logs(doc.doctype, doc.name), - "energy_point_logs": get_point_logs(doc.doctype, doc.name), - "additional_timeline_content": get_additional_timeline_content(doc.doctype, doc.name), - "milestones": get_milestones(doc.doctype, doc.name), - "is_document_followed": is_document_followed(doc.doctype, doc.name, frappe.session.user), - "tags": get_tags(doc.doctype, doc.name), - "document_email": get_document_email(doc.doctype, doc.name), - }) + docinfo.update( + { + "attachments": get_attachments(doc.doctype, doc.name), + "communications": communications_except_auto_messages, + "automated_messages": automated_messages, + "total_comments": len(json.loads(doc.get("_comments") or "[]")), + "versions": get_versions(doc), + "assignments": get_assignments(doc.doctype, doc.name), + "permissions": get_doc_permissions(doc), + "shared": frappe.share.get_users(doc.doctype, doc.name), + "views": get_view_logs(doc.doctype, doc.name), + "energy_point_logs": get_point_logs(doc.doctype, doc.name), + "additional_timeline_content": get_additional_timeline_content(doc.doctype, doc.name), + "milestones": get_milestones(doc.doctype, doc.name), + "is_document_followed": is_document_followed(doc.doctype, doc.name, frappe.session.user), + "tags": get_tags(doc.doctype, doc.name), + "document_email": get_document_email(doc.doctype, doc.name), + } + ) update_user_info(docinfo) frappe.response["docinfo"] = docinfo + def add_comments(doc, docinfo): # divide comments into separate lists docinfo.comments = [] @@ -136,12 +148,10 @@ def add_comments(doc, docinfo): docinfo.like_logs = [] docinfo.workflow_logs = [] - comments = frappe.get_all("Comment", + comments = frappe.get_all( + "Comment", fields=["name", "creation", "content", "owner", "comment_type"], - filters={ - "reference_doctype": doc.doctype, - "reference_name": doc.name - } + filters={"reference_doctype": doc.doctype, "reference_name": doc.name}, ) for c in comments: @@ -149,16 +159,16 @@ def add_comments(doc, docinfo): c.content = frappe.utils.markdown(c.content) docinfo.comments.append(c) - elif c.comment_type in ('Shared', 'Unshared'): + elif c.comment_type in ("Shared", "Unshared"): docinfo.shared.append(c) - elif c.comment_type in ('Assignment Completed', 'Assigned'): + elif c.comment_type in ("Assignment Completed", "Assigned"): docinfo.assignment_logs.append(c) - elif c.comment_type in ('Attachment', 'Attachment Removed'): + elif c.comment_type in ("Attachment", "Attachment Removed"): docinfo.attachment_logs.append(c) - elif c.comment_type in ('Info', 'Edit', 'Label'): + elif c.comment_type in ("Info", "Edit", "Label"): docinfo.info_logs.append(c) elif c.comment_type == "Like": @@ -169,21 +179,34 @@ def add_comments(doc, docinfo): frappe.utils.add_user_info(c.owner, docinfo.user_info) - return comments def get_milestones(doctype, name): - return frappe.db.get_all('Milestone', fields = ['creation', 'owner', 'track_field', 'value'], - filters=dict(reference_type=doctype, reference_name=name)) + return frappe.db.get_all( + "Milestone", + fields=["creation", "owner", "track_field", "value"], + filters=dict(reference_type=doctype, reference_name=name), + ) + def get_attachments(dt, dn): - return frappe.get_all("File", fields=["name", "file_name", "file_url", "is_private"], - filters = {"attached_to_name": dn, "attached_to_doctype": dt}) + return frappe.get_all( + "File", + fields=["name", "file_name", "file_url", "is_private"], + filters={"attached_to_name": dn, "attached_to_doctype": dt}, + ) + def get_versions(doc): - return frappe.get_all('Version', filters=dict(ref_doctype=doc.doctype, docname=doc.name), - fields=['name', 'owner', 'creation', 'data'], limit=10, order_by='creation desc') + return frappe.get_all( + "Version", + filters=dict(ref_doctype=doc.doctype, docname=doc.name), + fields=["name", "owner", "creation", "data"], + limit=10, + order_by="creation desc", + ) + @frappe.whitelist() def get_communications(doctype, name, start=0, limit=20): @@ -194,29 +217,32 @@ def get_communications(doctype, name, start=0, limit=20): return _get_communications(doctype, name, start, limit) -def get_comments(doctype: str, name: str, comment_type : Union[str, List[str]] = "Comment") -> List[frappe._dict]: +def get_comments( + doctype: str, name: str, comment_type: Union[str, List[str]] = "Comment" +) -> List[frappe._dict]: if isinstance(comment_type, list): comment_types = comment_type - elif comment_type == 'share': - comment_types = ['Shared', 'Unshared'] + elif comment_type == "share": + comment_types = ["Shared", "Unshared"] - elif comment_type == 'assignment': - comment_types = ['Assignment Completed', 'Assigned'] + elif comment_type == "assignment": + comment_types = ["Assignment Completed", "Assigned"] - elif comment_type == 'attachment': - comment_types = ['Attachment', 'Attachment Removed'] + elif comment_type == "attachment": + comment_types = ["Attachment", "Attachment Removed"] else: comment_types = [comment_type] - comments = frappe.get_all("Comment", + comments = frappe.get_all( + "Comment", fields=["name", "creation", "content", "owner", "comment_type"], filters={ "reference_doctype": doctype, "reference_name": name, - "comment_type": ['in', comment_types], - } + "comment_type": ["in", comment_types], + }, ) # convert to markdown (legacy ?) @@ -226,94 +252,111 @@ def get_comments(doctype: str, name: str, comment_type : Union[str, List[str]] = return comments + def get_point_logs(doctype, docname): - return frappe.db.get_all('Energy Point Log', filters={ - 'reference_doctype': doctype, - 'reference_name': docname, - 'type': ['!=', 'Review'] - }, fields=['*']) + return frappe.db.get_all( + "Energy Point Log", + filters={"reference_doctype": doctype, "reference_name": docname, "type": ["!=", "Review"]}, + fields=["*"], + ) + def _get_communications(doctype, name, start=0, limit=20): communications = get_communication_data(doctype, name, start, limit) for c in communications: - if c.communication_type=="Communication": - c.attachments = json.dumps(frappe.get_all("File", - fields=["file_url", "is_private"], - filters={"attached_to_doctype": "Communication", - "attached_to_name": c.name} - )) + if c.communication_type == "Communication": + c.attachments = json.dumps( + frappe.get_all( + "File", + fields=["file_url", "is_private"], + filters={"attached_to_doctype": "Communication", "attached_to_name": c.name}, + ) + ) return communications -def get_communication_data(doctype, name, start=0, limit=20, after=None, fields=None, - group_by=None, as_dict=True): - '''Returns list of communications for a given document''' + +def get_communication_data( + doctype, name, start=0, limit=20, after=None, fields=None, group_by=None, as_dict=True +): + """Returns list of communications for a given document""" if not fields: - fields = ''' + fields = """ C.name, C.communication_type, C.communication_medium, C.comment_type, C.communication_date, C.content, C.sender, C.sender_full_name, C.cc, C.bcc, C.creation AS creation, C.subject, C.delivery_status, C._liked_by, C.reference_doctype, C.reference_name, C.read_by_recipient, C.rating, C.recipients - ''' + """ - conditions = '' + conditions = "" if after: # find after a particular date - conditions += ''' + conditions += """ AND C.creation > {0} - '''.format(after) + """.format( + after + ) - if doctype=='User': - conditions += ''' + if doctype == "User": + conditions += """ AND NOT (C.reference_doctype='User' AND C.communication_type='Communication') - ''' + """ # communications linked to reference_doctype - part1 = ''' + part1 = """ SELECT {fields} FROM `tabCommunication` as C WHERE C.communication_type IN ('Communication', 'Feedback', 'Automated Message') AND (C.reference_doctype = %(doctype)s AND C.reference_name = %(name)s) {conditions} - '''.format(fields=fields, conditions=conditions) + """.format( + fields=fields, conditions=conditions + ) # communications linked in Timeline Links - part2 = ''' + part2 = """ SELECT {fields} FROM `tabCommunication` as C INNER JOIN `tabCommunication Link` ON C.name=`tabCommunication Link`.parent WHERE C.communication_type IN ('Communication', 'Feedback', 'Automated Message') AND `tabCommunication Link`.link_doctype = %(doctype)s AND `tabCommunication Link`.link_name = %(name)s {conditions} - '''.format(fields=fields, conditions=conditions) + """.format( + fields=fields, conditions=conditions + ) - communications = frappe.db.sql(''' + communications = frappe.db.sql( + """ SELECT * FROM (({part1}) UNION ({part2})) AS combined {group_by} ORDER BY creation DESC LIMIT %(limit)s OFFSET %(start)s - '''.format(part1=part1, part2=part2, group_by=(group_by or '')), dict( - doctype=doctype, - name=name, - start=frappe.utils.cint(start), - limit=limit - ), as_dict=as_dict) + """.format( + part1=part1, part2=part2, group_by=(group_by or "") + ), + dict(doctype=doctype, name=name, start=frappe.utils.cint(start), limit=limit), + as_dict=as_dict, + ) return communications + def get_assignments(dt, dn): - return frappe.get_all("ToDo", - fields=['name', 'allocated_to as owner', 'description', 'status'], + return frappe.get_all( + "ToDo", + fields=["name", "allocated_to as owner", "description", "status"], filters={ - 'reference_type': dt, - 'reference_name': dn, - 'status': ('!=', 'Cancelled'), - 'allocated_to': ("is", "set") - }) + "reference_type": dt, + "reference_name": dn, + "status": ("!=", "Cancelled"), + "allocated_to": ("is", "set"), + }, + ) + @frappe.whitelist() def get_badge_info(doctypes, filters): @@ -326,31 +369,42 @@ def get_badge_info(doctypes, filters): return out + def run_onload(doc): doc.set("__onload", frappe._dict()) doc.run_method("onload") + def get_view_logs(doctype, docname): - """ get and return the latest view logs if available """ + """get and return the latest view logs if available""" logs = [] - if hasattr(frappe.get_meta(doctype), 'track_views') and frappe.get_meta(doctype).track_views: - view_logs = frappe.get_all("View Log", filters={ - "reference_doctype": doctype, - "reference_name": docname, - }, fields=["name", "creation", "owner"], order_by="creation desc") + if hasattr(frappe.get_meta(doctype), "track_views") and frappe.get_meta(doctype).track_views: + view_logs = frappe.get_all( + "View Log", + filters={ + "reference_doctype": doctype, + "reference_name": docname, + }, + fields=["name", "creation", "owner"], + order_by="creation desc", + ) if view_logs: logs = view_logs return logs + def get_tags(doctype, name): - tags = [tag.tag for tag in frappe.get_all("Tag Link", filters={ - "document_type": doctype, - "document_name": name - }, fields=["tag"])] + tags = [ + tag.tag + for tag in frappe.get_all( + "Tag Link", filters={"document_type": doctype, "document_name": name}, fields=["tag"] + ) + ] return ",".join(tags) + def get_document_email(doctype, name): email = get_automatic_email_link() if not email: @@ -359,13 +413,17 @@ def get_document_email(doctype, name): email = email.split("@") return "{0}+{1}+{2}@{3}".format(email[0], quote(doctype), quote(cstr(name)), email[1]) + def get_automatic_email_link(): - return frappe.db.get_value("Email Account", {"enable_incoming": 1, "enable_automatic_linking": 1}, "email_id") + return frappe.db.get_value( + "Email Account", {"enable_incoming": 1, "enable_automatic_linking": 1}, "email_id" + ) + def get_additional_timeline_content(doctype, docname): contents = [] - hooks = frappe.get_hooks().get('additional_timeline_content', {}) - methods_for_all_doctype = hooks.get('*', []) + hooks = frappe.get_hooks().get("additional_timeline_content", {}) + methods_for_all_doctype = hooks.get("*", []) methods_for_current_doctype = hooks.get(doctype, []) for method in methods_for_all_doctype + methods_for_current_doctype: @@ -373,6 +431,7 @@ def get_additional_timeline_content(doctype, docname): return contents + def set_link_titles(doc): link_titles = {} link_titles.update(get_title_values_for_link_and_dynamic_link_fields(doc)) @@ -380,6 +439,7 @@ def set_link_titles(doc): send_link_titles(link_titles) + def get_title_values_for_link_and_dynamic_link_fields(doc, link_fields=None): link_titles = {} @@ -397,13 +457,12 @@ def get_title_values_for_link_and_dynamic_link_fields(doc, link_fields=None): if not meta or not (meta.title_field and meta.show_title_field_in_link): continue - link_title = frappe.db.get_value( - doctype, doc.get(field.fieldname), meta.title_field, cache=True - ) + link_title = frappe.db.get_value(doctype, doc.get(field.fieldname), meta.title_field, cache=True) link_titles.update({doctype + "::" + doc.get(field.fieldname): link_title}) return link_titles + def get_title_values_for_table_and_multiselect_fields(doc, table_fields=None): link_titles = {} @@ -420,6 +479,7 @@ def get_title_values_for_table_and_multiselect_fields(doc, table_fields=None): return link_titles + def send_link_titles(link_titles): """Append link titles dict in `frappe.local.response`.""" if "_link_titles" not in frappe.local.response: @@ -427,6 +487,7 @@ def send_link_titles(link_titles): frappe.local.response["_link_titles"].update(link_titles) + def update_user_info(docinfo): for d in docinfo.communications: frappe.utils.add_user_info(d.sender, docinfo.user_info) @@ -440,6 +501,7 @@ def update_user_info(docinfo): for d in docinfo.views: frappe.utils.add_user_info(d.owner, docinfo.user_info) + @frappe.whitelist() def get_user_info_for_viewers(users): user_info = {} @@ -447,4 +509,3 @@ def get_user_info_for_viewers(users): frappe.utils.add_user_info(user, user_info) return user_info - diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index fa6a1f313b..ba19377c48 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -11,13 +11,24 @@ from frappe.modules import get_module_path, load_doctype_module, scrub from frappe.translate import extract_messages_from_code, make_dict_from_messages from frappe.utils import get_html_format - ASSET_KEYS = ( - "__js", "__css", "__list_js", "__calendar_js", "__map_js", - "__linked_with", "__messages", "__print_formats", "__workflow_docs", - "__form_grid_templates", "__listview_template", "__tree_js", - "__dashboard", "__kanban_column_fields", '__templates', - '__custom_js', '__custom_list_js' + "__js", + "__css", + "__list_js", + "__calendar_js", + "__map_js", + "__linked_with", + "__messages", + "__print_formats", + "__workflow_docs", + "__form_grid_templates", + "__listview_template", + "__tree_js", + "__dashboard", + "__kanban_column_fields", + "__templates", + "__custom_js", + "__custom_list_js", ) @@ -33,11 +44,12 @@ def get_meta(doctype, cached=True): else: meta = FormMeta(doctype) - if frappe.local.lang != 'en': + if frappe.local.lang != "en": meta.set_translations(frappe.local.lang) return meta + class FormMeta(Meta): def __init__(self, doctype): super(FormMeta, self).__init__(doctype) @@ -50,7 +62,7 @@ class FormMeta(Meta): super(FormMeta, self).set(key, value, *args, **kwargs) def load_assets(self): - if self.get('__assets_loaded', False): + if self.get("__assets_loaded", False): return self.add_search_fields() @@ -65,7 +77,7 @@ class FormMeta(Meta): self.load_dashboard() self.load_kanban_meta() - self.set('__assets_loaded', True) + self.set("__assets_loaded", True) def as_dict(self, no_nulls=False): d = super(FormMeta, self).as_dict(no_nulls=no_nulls) @@ -85,25 +97,26 @@ class FormMeta(Meta): if self.custom: return - path = os.path.join(get_module_path(self.module), 'doctype', scrub(self.name)) + path = os.path.join(get_module_path(self.module), "doctype", scrub(self.name)) + def _get_path(fname): return os.path.join(path, scrub(fname)) system_country = frappe.get_system_settings("country") - self._add_code(_get_path(self.name + '.js'), '__js') + self._add_code(_get_path(self.name + ".js"), "__js") if system_country: - self._add_code(_get_path(os.path.join('regional', system_country + '.js')), '__js') + self._add_code(_get_path(os.path.join("regional", system_country + ".js")), "__js") - self._add_code(_get_path(self.name + '.css'), "__css") - self._add_code(_get_path(self.name + '_list.js'), '__list_js') + self._add_code(_get_path(self.name + ".css"), "__css") + self._add_code(_get_path(self.name + "_list.js"), "__list_js") if system_country: - self._add_code(_get_path(os.path.join('regional', system_country + '_list.js')), '__list_js') + self._add_code(_get_path(os.path.join("regional", system_country + "_list.js")), "__list_js") - self._add_code(_get_path(self.name + '_calendar.js'), '__calendar_js') - self._add_code(_get_path(self.name + '_tree.js'), '__tree_js') + self._add_code(_get_path(self.name + "_calendar.js"), "__calendar_js") + self._add_code(_get_path(self.name + "_tree.js"), "__tree_js") - listview_template = _get_path(self.name + '_list.html') + listview_template = _get_path(self.name + "_list.html") if os.path.exists(listview_template): self.set("__listview_template", get_html_format(listview_template)) @@ -126,8 +139,8 @@ class FormMeta(Meta): templates = dict() for fname in os.listdir(path): if fname.endswith(".html"): - with io.open(os.path.join(path, fname), 'r', encoding = 'utf-8') as f: - templates[fname.split('.')[0]] = scrub_html_template(f.read()) + with io.open(os.path.join(path, fname), "r", encoding="utf-8") as f: + templates[fname.split(".")[0]] = scrub_html_template(f.read()) self.set("__templates", templates or None) @@ -138,19 +151,23 @@ class FormMeta(Meta): def add_custom_script(self): """embed all require files""" # custom script - client_scripts = frappe.db.get_all("Client Script", - filters={"dt": self.name, "enabled": 1}, - fields=["script", "view"], - order_by="creation asc" - ) or "" - - list_script = '' - form_script = '' + client_scripts = ( + frappe.db.get_all( + "Client Script", + filters={"dt": self.name, "enabled": 1}, + fields=["script", "view"], + order_by="creation asc", + ) + or "" + ) + + list_script = "" + form_script = "" for script in client_scripts: - if script.view == 'List': + if script.view == "List": list_script += script.script - if script.view == 'Form': + if script.view == "Form": form_script += script.script file = scrub(self.name) @@ -162,7 +179,7 @@ class FormMeta(Meta): def add_search_fields(self): """add search fields found in the doctypes indicated by link fields' options""" - for df in self.get("fields", {"fieldtype": "Link", "options":["!=", "[Select]"]}): + for df in self.get("fields", {"fieldtype": "Link", "options": ["!=", "[Select]"]}): if df.options: search_fields = frappe.get_meta(df.options).search_fields if search_fields: @@ -179,9 +196,13 @@ class FormMeta(Meta): pass def load_print_formats(self): - print_formats = frappe.db.sql("""select * FROM `tabPrint Format` - WHERE doc_type=%s AND docstatus<2 and disabled=0""", (self.name,), as_dict=1, - update={"doctype":"Print Format"}) + print_formats = frappe.db.sql( + """select * FROM `tabPrint Format` + WHERE doc_type=%s AND docstatus<2 and disabled=0""", + (self.name,), + as_dict=1, + update={"doctype": "Print Format"}, + ) self.set("__print_formats", print_formats) @@ -199,7 +220,6 @@ class FormMeta(Meta): self.set("__workflow_docs", workflow_docs) - def load_templates(self): if not self.custom: module = load_doctype_module(self.name) @@ -222,7 +242,7 @@ class FormMeta(Meta): self.get("__messages").update(messages) def load_dashboard(self): - self.set('__dashboard', self.get_dashboard_data()) + self.set("__dashboard", self.get_dashboard_data()) def load_kanban_meta(self): self.load_kanban_column_fields() @@ -230,16 +250,17 @@ class FormMeta(Meta): def load_kanban_column_fields(self): try: values = frappe.get_list( - 'Kanban Board', fields=['field_name'], - filters={'reference_doctype': self.name}) + "Kanban Board", fields=["field_name"], filters={"reference_doctype": self.name} + ) - fields = [x['field_name'] for x in values] + fields = [x["field_name"] for x in values] fields = list(set(fields)) self.set("__kanban_column_fields", fields) except frappe.PermissionError: # no access to kanban board pass + def get_code_files_via_hooks(hook, name): code_files = [] for app_name in frappe.get_installed_apps(): @@ -257,6 +278,7 @@ def get_code_files_via_hooks(hook, name): return code_files + def get_js(path): js = frappe.read_file(path) if js: diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index b580e2c769..cc3865bc60 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -1,9 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe from frappe.desk.form.load import run_onload + @frappe.whitelist() def savedocs(doc, action): """save / submit / update doclist""" @@ -12,9 +15,9 @@ def savedocs(doc, action): set_local_name(doc) # action - doc.docstatus = {"Save":0, "Submit": 1, "Update": 1, "Cancel": 2}[action] + doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action] - if doc.docstatus==1: + if doc.docstatus == 1: doc.submit() else: doc.save() @@ -23,11 +26,12 @@ def savedocs(doc, action): run_onload(doc) send_updated_docs(doc) - frappe.msgprint(frappe._("Saved"), indicator='green', alert=True) + frappe.msgprint(frappe._("Saved"), indicator="green", alert=True) except Exception: frappe.errprint(frappe.utils.get_traceback()) raise + @frappe.whitelist() def cancel(doctype=None, name=None, workflow_state_fieldname=None, workflow_state=None): """cancel a doclist""" @@ -37,25 +41,28 @@ def cancel(doctype=None, name=None, workflow_state_fieldname=None, workflow_stat doc.set(workflow_state_fieldname, workflow_state) doc.cancel() send_updated_docs(doc) - frappe.msgprint(frappe._("Cancelled"), indicator='red', alert=True) + frappe.msgprint(frappe._("Cancelled"), indicator="red", alert=True) except Exception: frappe.errprint(frappe.utils.get_traceback()) raise + def send_updated_docs(doc): from .load import get_docinfo + get_docinfo(doc) d = doc.as_dict() - if hasattr(doc, 'localname'): + if hasattr(doc, "localname"): d["localname"] = doc.localname frappe.response.docs.append(d) + def set_local_name(doc): def _set_local_name(d): - if doc.get('__islocal') or d.get('__islocal'): + if doc.get("__islocal") or d.get("__islocal"): d.localname = d.name d.name = None diff --git a/frappe/desk/form/test_form.py b/frappe/desk/form/test_form.py index 86c3aba29a..c05a932241 100644 --- a/frappe/desk/form/test_form.py +++ b/frappe/desk/form/test_form.py @@ -1,16 +1,19 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest +import frappe from frappe.desk.form.linked_with import get_linked_docs, get_linked_doctypes + class TestForm(unittest.TestCase): def test_linked_with(self): results = get_linked_docs("Role", "System Manager", linkinfo=get_linked_doctypes("Role")) self.assertTrue("User" in results) self.assertTrue("DocType" in results) -if __name__=="__main__": + +if __name__ == "__main__": frappe.connect() unittest.main() diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py index f80b89d2b8..2738d1f74a 100644 --- a/frappe/desk/form/utils.py +++ b/frappe/desk/form/utils.py @@ -1,33 +1,37 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json -import frappe.desk.form.meta +import json + +import frappe import frappe.desk.form.load -from frappe.desk.form.document_follow import follow_document +import frappe.desk.form.meta +from frappe import _ from frappe.core.doctype.file.file import extract_images_from_html +from frappe.desk.form.document_follow import follow_document -from frappe import _ @frappe.whitelist() def remove_attach(): """remove attachment""" - fid = frappe.form_dict.get('fid') - file_name = frappe.form_dict.get('file_name') - frappe.delete_doc('File', fid) + fid = frappe.form_dict.get("fid") + file_name = frappe.form_dict.get("file_name") + frappe.delete_doc("File", fid) @frappe.whitelist() def add_comment(reference_doctype, reference_name, content, comment_email, comment_by): """allow any logged user to post a comment""" - doc = frappe.get_doc(dict( - doctype='Comment', - reference_doctype=reference_doctype, - reference_name=reference_name, - comment_email=comment_email, - comment_type='Comment', - comment_by=comment_by - )) + doc = frappe.get_doc( + dict( + doctype="Comment", + reference_doctype=reference_doctype, + reference_name=reference_name, + comment_email=comment_email, + comment_type="Comment", + comment_by=comment_by, + ) + ) reference_doc = frappe.get_doc(reference_doctype, reference_name) doc.content = extract_images_from_html(reference_doc, content, is_private=True) doc.insert(ignore_permissions=True) @@ -35,22 +39,25 @@ def add_comment(reference_doctype, reference_name, content, comment_email, comme follow_document(doc.reference_doctype, doc.reference_name, frappe.session.user) return doc.as_dict() + @frappe.whitelist() def update_comment(name, content): """allow only owner to update comment""" - doc = frappe.get_doc('Comment', name) + doc = frappe.get_doc("Comment", name) - if frappe.session.user not in ['Administrator', doc.owner]: - frappe.throw(_('Comment can only be edited by the owner'), frappe.PermissionError) + if frappe.session.user not in ["Administrator", doc.owner]: + frappe.throw(_("Comment can only be edited by the owner"), frappe.PermissionError) doc.content = content doc.save(ignore_permissions=True) + @frappe.whitelist() -def get_next(doctype, value, prev, filters=None, sort_order='desc', sort_field='modified'): +def get_next(doctype, value, prev, filters=None, sort_order="desc", sort_field="modified"): prev = int(prev) - if not filters: filters = [] + if not filters: + filters = [] if isinstance(filters, str): filters = json.loads(filters) @@ -65,11 +72,15 @@ def get_next(doctype, value, prev, filters=None, sort_order='desc', sort_field=' # # add condition for next or prev item filters.append([doctype, sort_field, condition, frappe.get_value(doctype, value, sort_field)]) - res = frappe.get_list(doctype, - fields = ["name"], - filters = filters, - order_by = "`tab{0}`.{1}".format(doctype, sort_field) + " " + sort_order, - limit_start=0, limit_page_length=1, as_list=True) + res = frappe.get_list( + doctype, + fields=["name"], + filters=filters, + order_by="`tab{0}`.{1}".format(doctype, sort_field) + " " + sort_order, + limit_start=0, + limit_page_length=1, + as_list=True, + ) if not res: frappe.msgprint(_("No further records")) @@ -77,10 +88,8 @@ def get_next(doctype, value, prev, filters=None, sort_order='desc', sort_field=' else: return res[0][0] -def get_pdf_link(doctype, docname, print_format='Standard', no_letterhead=0): - return '/api/method/frappe.utils.print_format.download_pdf?doctype={doctype}&name={docname}&format={print_format}&no_letterhead={no_letterhead}'.format( - doctype = doctype, - docname = docname, - print_format = print_format, - no_letterhead = no_letterhead + +def get_pdf_link(doctype, docname, print_format="Standard", no_letterhead=0): + return "/api/method/frappe.utils.print_format.download_pdf?doctype={doctype}&name={docname}&format={print_format}&no_letterhead={no_letterhead}".format( + doctype=doctype, docname=docname, print_format=print_format, no_letterhead=no_letterhead ) diff --git a/frappe/desk/gantt.py b/frappe/desk/gantt.py index 58ef3b836e..a09c52dafe 100644 --- a/frappe/desk/gantt.py +++ b/frappe/desk/gantt.py @@ -1,7 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe + @frappe.whitelist() def update_task(args, field_map): @@ -11,4 +14,4 @@ def update_task(args, field_map): d = frappe.get_doc(args.doctype, args.name) d.set(field_map.start, args.start) d.set(field_map.end, args.end) - d.save() \ No newline at end of file + d.save() diff --git a/frappe/desk/leaderboard.py b/frappe/desk/leaderboard.py index a98ae1a1c6..a5f5de3117 100644 --- a/frappe/desk/leaderboard.py +++ b/frappe/desk/leaderboard.py @@ -1,50 +1,54 @@ import frappe from frappe.utils import get_fullname + def get_leaderboards(): leaderboards = { - 'User': { - 'fields': ['points'], - 'method': 'frappe.desk.leaderboard.get_energy_point_leaderboard', - 'company_disabled': 1, - 'icon': 'users' + "User": { + "fields": ["points"], + "method": "frappe.desk.leaderboard.get_energy_point_leaderboard", + "company_disabled": 1, + "icon": "users", } } return leaderboards + @frappe.whitelist() -def get_energy_point_leaderboard(date_range, company = None, field = None, limit = None): - all_users = frappe.db.get_all('User', - filters = { - 'name': ['not in', ['Administrator', 'Guest']], - 'enabled': 1, - 'user_type': ['!=', 'Website User'] +def get_energy_point_leaderboard(date_range, company=None, field=None, limit=None): + all_users = frappe.db.get_all( + "User", + filters={ + "name": ["not in", ["Administrator", "Guest"]], + "enabled": 1, + "user_type": ["!=", "Website User"], }, - order_by = 'name ASC') - all_users_list = list(map(lambda x: x['name'], all_users)) + order_by="name ASC", + ) + all_users_list = list(map(lambda x: x["name"], all_users)) - filters = [ - ['type', '!=', 'Review'], - ['user', 'in', all_users_list] - ] + filters = [["type", "!=", "Review"], ["user", "in", all_users_list]] if date_range: date_range = frappe.parse_json(date_range) - filters.append(['creation', 'between', [date_range[0], date_range[1]]]) - energy_point_users = frappe.db.get_all('Energy Point Log', - fields = ['user as name', 'sum(points) as value'], - filters = filters, - group_by = 'user', - order_by = 'value desc' + filters.append(["creation", "between", [date_range[0], date_range[1]]]) + energy_point_users = frappe.db.get_all( + "Energy Point Log", + fields=["user as name", "sum(points) as value"], + filters=filters, + group_by="user", + order_by="value desc", ) - energy_point_users_list = list(map(lambda x: x['name'], energy_point_users)) + energy_point_users_list = list(map(lambda x: x["name"], energy_point_users)) for user in all_users_list: if user not in energy_point_users_list: - energy_point_users.append({'name': user, 'value': 0}) + energy_point_users.append({"name": user, "value": 0}) for user in energy_point_users: - user_id = user['name'] - user['name'] = get_fullname(user['name']) - user['formatted_name'] = '{}'.format(user_id, get_fullname(user_id)) + user_id = user["name"] + user["name"] = get_fullname(user["name"]) + user["formatted_name"] = '{}'.format( + user_id, get_fullname(user_id) + ) - return energy_point_users \ No newline at end of file + return energy_point_users diff --git a/frappe/desk/like.py b/frappe/desk/like.py index 5e5a789973..9e97cb269c 100644 --- a/frappe/desk/like.py +++ b/frappe/desk/like.py @@ -3,12 +3,15 @@ """Allow adding of likes to documents""" -import frappe, json -from frappe.database.schema import add_column +import json + +import frappe from frappe import _ +from frappe.database.schema import add_column from frappe.desk.form.document_follow import follow_document from frappe.utils import get_link_to_form + @frappe.whitelist() def toggle_like(doctype, name, add=False): """Adds / removes the current user in the `__liked_by` property of the given document. @@ -23,6 +26,7 @@ def toggle_like(doctype, name, add=False): _toggle_like(doctype, name, add) + def _toggle_like(doctype, name, add, user=None): """Same as toggle_like but hides param `user` from API""" @@ -37,7 +41,7 @@ def _toggle_like(doctype, name, add, user=None): else: liked_by = [] - if add=="Yes": + if add == "Yes": if user not in liked_by: liked_by.append(user) add_comment(doctype, name) @@ -57,28 +61,44 @@ def _toggle_like(doctype, name, add, user=None): else: raise + def remove_like(doctype, name): """Remove previous Like""" # remove Comment - frappe.delete_doc("Comment", [c.name for c in frappe.get_all("Comment", - filters={ - "comment_type": "Like", - "reference_doctype": doctype, - "reference_name": name, - "owner": frappe.session.user, - } - )], ignore_permissions=True) + frappe.delete_doc( + "Comment", + [ + c.name + for c in frappe.get_all( + "Comment", + filters={ + "comment_type": "Like", + "reference_doctype": doctype, + "reference_name": name, + "owner": frappe.session.user, + }, + ) + ], + ignore_permissions=True, + ) + def add_comment(doctype, name): doc = frappe.get_doc(doctype, name) - if doctype=="Communication" and doc.reference_doctype and doc.reference_name: - link = get_link_to_form(doc.reference_doctype, doc.reference_name, - "{0} {1}".format(_(doc.reference_doctype), doc.reference_name)) - - doc.add_comment("Like", _("{0}: {1} in {2}").format(_(doc.communication_type), - "" + doc.subject + "", link), - link_doctype=doc.reference_doctype, link_name=doc.reference_name) + if doctype == "Communication" and doc.reference_doctype and doc.reference_name: + link = get_link_to_form( + doc.reference_doctype, + doc.reference_name, + "{0} {1}".format(_(doc.reference_doctype), doc.reference_name), + ) + + doc.add_comment( + "Like", + _("{0}: {1} in {2}").format(_(doc.communication_type), "" + doc.subject + "", link), + link_doctype=doc.reference_doctype, + link_name=doc.reference_name, + ) else: doc.add_comment("Like", _("Liked")) diff --git a/frappe/desk/link_preview.py b/frappe/desk/link_preview.py index 03f8368a3a..374a151505 100644 --- a/frappe/desk/link_preview.py +++ b/frappe/desk/link_preview.py @@ -1,45 +1,52 @@ +import json + import frappe from frappe.model import no_value_fields, table_fields -import json + @frappe.whitelist() def get_preview_data(doctype, docname): preview_fields = [] meta = frappe.get_meta(doctype) - if not meta.show_preview_popup: return + if not meta.show_preview_popup: + return - preview_fields = [field.fieldname for field in meta.fields \ - if field.in_preview and field.fieldtype not in no_value_fields \ - and field.fieldtype not in table_fields] + preview_fields = [ + field.fieldname + for field in meta.fields + if field.in_preview + and field.fieldtype not in no_value_fields + and field.fieldtype not in table_fields + ] # no preview fields defined, build list from mandatory fields if not preview_fields: - preview_fields = [field.fieldname for field in meta.fields if field.reqd \ - and field.fieldtype not in table_fields] + preview_fields = [ + field.fieldname for field in meta.fields if field.reqd and field.fieldtype not in table_fields + ] title_field = meta.get_title_field() image_field = meta.image_field preview_fields.append(title_field) preview_fields.append(image_field) - preview_fields.append('name') + preview_fields.append("name") - preview_data = frappe.get_list(doctype, filters={ - 'name': docname - }, fields=preview_fields, limit=1) + preview_data = frappe.get_list(doctype, filters={"name": docname}, fields=preview_fields, limit=1) - if not preview_data: return + if not preview_data: + return preview_data = preview_data[0] formatted_preview_data = { - 'preview_image': preview_data.get(image_field), - 'preview_title': preview_data.get(title_field), - 'name': preview_data.get('name'), + "preview_image": preview_data.get(image_field), + "preview_title": preview_data.get(title_field), + "name": preview_data.get("name"), } for key, val in preview_data.items(): - if val and meta.has_field(key) and key not in [image_field, title_field, 'name']: + if val and meta.has_field(key) and key not in [image_field, title_field, "name"]: formatted_preview_data[meta.get_field(key).label] = frappe.format( val, meta.get_field(key).fieldtype, diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py index 3d6f1254a2..88216d3998 100644 --- a/frappe/desk/listview.py +++ b/frappe/desk/listview.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import frappe + @frappe.whitelist() def get_list_settings(doctype): try: @@ -9,6 +10,7 @@ def get_list_settings(doctype): except frappe.DoesNotExistError: frappe.clear_messages() + @frappe.whitelist() def set_list_settings(doctype, values): try: @@ -24,12 +26,13 @@ def set_list_settings(doctype, values): @frappe.whitelist() def get_group_by_count(doctype, current_filters, field): current_filters = frappe.parse_json(current_filters) - subquery_condition = '' + subquery_condition = "" subquery = frappe.get_all(doctype, filters=current_filters, run=False) - if field == 'assigned_to': - subquery_condition = ' and `tabToDo`.reference_name in ({subquery})'.format(subquery = subquery) - return frappe.db.sql("""select `tabToDo`.allocated_to as name, count(*) as count + if field == "assigned_to": + subquery_condition = " and `tabToDo`.reference_name in ({subquery})".format(subquery=subquery) + return frappe.db.sql( + """select `tabToDo`.allocated_to as name, count(*) as count from `tabToDo`, `tabUser` where @@ -41,13 +44,17 @@ def get_group_by_count(doctype, current_filters, field): `tabToDo`.allocated_to order by count desc - limit 50""".format(subquery_condition = subquery_condition), as_dict=True) + limit 50""".format( + subquery_condition=subquery_condition + ), + as_dict=True, + ) else: - return frappe.db.get_list(doctype, + return frappe.db.get_list( + doctype, filters=current_filters, - group_by='`tab{0}`.{1}'.format(doctype, field), - fields=['count(*) as count', '`{}` as name'.format(field)], - order_by='count desc', + group_by="`tab{0}`.{1}".format(doctype, field), + fields=["count(*) as count", "`{}` as name".format(field)], + order_by="count desc", limit=50, ) - diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py index 7a9c211c3c..a4fc2ccd1e 100644 --- a/frappe/desk/moduleview.py +++ b/frappe/desk/moduleview.py @@ -1,12 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import json + +import frappe from frappe import _ from frappe.boot import get_allowed_pages, get_allowed_reports -from frappe.desk.doctype.desktop_icon.desktop_icon import set_hidden, clear_desktop_icons_cache -from frappe.cache_manager import build_domain_restriced_doctype_cache, build_domain_restriced_page_cache, build_table_count_cache +from frappe.cache_manager import ( + build_domain_restriced_doctype_cache, + build_domain_restriced_page_cache, + build_table_count_cache, +) +from frappe.desk.doctype.desktop_icon.desktop_icon import clear_desktop_icons_cache, set_hidden + @frappe.whitelist() def get(module): @@ -14,17 +20,17 @@ def get(module): `/desk/#Module/[name]`.""" data = get_data(module) - out = { - "data": data - } + out = {"data": data} return out + @frappe.whitelist() def hide_module(module): set_hidden(module, frappe.session.user, 1) clear_desktop_icons_cache() + def get_table_with_counts(): counts = frappe.cache().get_value("information_schema:counts") if counts: @@ -32,6 +38,7 @@ def get_table_with_counts(): else: return build_table_count_cache() + def get_data(module, build=True): """Get module data for the module view `desk/#Module/[name]`""" doctype_info = get_doctype_info(module) @@ -42,8 +49,7 @@ def get_data(module, build=True): else: add_custom_doctypes(data, doctype_info) - add_section(data, _("Custom Reports"), "fa fa-list-alt", - get_report_list(module)) + add_section(data, _("Custom Reports"), "fa fa-list-alt", get_report_list(module)) data = combine_common_sections(data) data = apply_permissions(data) @@ -52,10 +58,11 @@ def get_data(module, build=True): if build: exists_cache = get_table_with_counts() + def doctype_contains_a_record(name): exists = exists_cache.get(name) if not exists: - if not frappe.db.get_value('DocType', name, 'issingle'): + if not frappe.db.get_value("DocType", name, "issingle"): exists = frappe.db.count(name) else: exists = True @@ -88,6 +95,7 @@ def get_data(module, build=True): return data + def build_config_from_file(module): """Build module info from `app/config/desktop.py` files.""" data = [] @@ -101,9 +109,12 @@ def build_config_from_file(module): return filter_by_restrict_to_domain(data) + def filter_by_restrict_to_domain(data): - """ filter Pages and DocType depending on the Active Module(s) """ - doctypes = frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache() + """filter Pages and DocType depending on the Active Module(s)""" + doctypes = ( + frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache() + ) pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache() for d in data: @@ -113,13 +124,14 @@ def filter_by_restrict_to_domain(data): item_type = item.get("type") item_name = item.get("name") - if (item_name in pages) or (item_name in doctypes) or item_type == 'report': + if (item_name in pages) or (item_name in doctypes) or item_type == "report": _items.append(item) - d.update({ "items": _items }) + d.update({"items": _items}) return data + def build_standard_config(module, doctype_info): """Build standard module data from DocTypes.""" if not frappe.db.get_value("Module Def", module): @@ -127,47 +139,60 @@ def build_standard_config(module, doctype_info): data = [] - add_section(data, _("Documents"), "fa fa-star", - [d for d in doctype_info if d.document_type in ("Document", "Transaction")]) + add_section( + data, + _("Documents"), + "fa fa-star", + [d for d in doctype_info if d.document_type in ("Document", "Transaction")], + ) - add_section(data, _("Setup"), "fa fa-cog", - [d for d in doctype_info if d.document_type in ("Master", "Setup", "")]) + add_section( + data, + _("Setup"), + "fa fa-cog", + [d for d in doctype_info if d.document_type in ("Master", "Setup", "")], + ) - add_section(data, _("Standard Reports"), "fa fa-list", - get_report_list(module, is_standard="Yes")) + add_section(data, _("Standard Reports"), "fa fa-list", get_report_list(module, is_standard="Yes")) return data + def add_section(data, label, icon, items): """Adds a section to the module data.""" - if not items: return - data.append({ - "label": label, - "icon": icon, - "items": items - }) + if not items: + return + data.append({"label": label, "icon": icon, "items": items}) def add_custom_doctypes(data, doctype_info): """Adds Custom DocTypes to modules setup via `config/desktop.py`.""" - add_section(data, _("Documents"), "fa fa-star", - [d for d in doctype_info if (d.custom and d.document_type in ("Document", "Transaction"))]) + add_section( + data, + _("Documents"), + "fa fa-star", + [d for d in doctype_info if (d.custom and d.document_type in ("Document", "Transaction"))], + ) + + add_section( + data, + _("Setup"), + "fa fa-cog", + [d for d in doctype_info if (d.custom and d.document_type in ("Setup", "Master", ""))], + ) - add_section(data, _("Setup"), "fa fa-cog", - [d for d in doctype_info if (d.custom and d.document_type in ("Setup", "Master", ""))]) def get_doctype_info(module): """Returns list of non child DocTypes for given module.""" active_domains = frappe.get_active_domains() - doctype_info = frappe.get_all("DocType", filters={ - "module": module, - "istable": 0 - }, or_filters={ - "ifnull(restrict_to_domain, '')": "", - "restrict_to_domain": ("in", active_domains) - }, fields=["'doctype' as type", "name", "description", "document_type", - "custom", "issingle"], order_by="custom asc, document_type desc, name asc") + doctype_info = frappe.get_all( + "DocType", + filters={"module": module, "istable": 0}, + or_filters={"ifnull(restrict_to_domain, '')": "", "restrict_to_domain": ("in", active_domains)}, + fields=["'doctype' as type", "name", "description", "document_type", "custom", "issingle"], + order_by="custom asc, document_type desc, name asc", + ) for d in doctype_info: d.document_type = d.document_type or "" @@ -175,6 +200,7 @@ def get_doctype_info(module): return doctype_info + def combine_common_sections(data): """Combine sections declared in separate apps.""" sections = [] @@ -188,6 +214,7 @@ def combine_common_sections(data): return sections + def apply_permissions(data): default_country = frappe.db.get_default("country") @@ -201,16 +228,18 @@ def apply_permissions(data): for section in data: new_items = [] - for item in (section.get("items") or []): + for item in section.get("items") or []: item = frappe._dict(item) - if item.country and item.country!=default_country: + if item.country and item.country != default_country: continue - if ((item.type=="doctype" and item.name in user.can_read) - or (item.type=="page" and item.name in allowed_pages) - or (item.type=="report" and item.name in allowed_reports) - or item.type=="help"): + if ( + (item.type == "doctype" and item.name in user.can_read) + or (item.type == "page" and item.name in allowed_pages) + or (item.type == "report" and item.name in allowed_reports) + or item.type == "help" + ): new_items.append(item) @@ -221,11 +250,13 @@ def apply_permissions(data): return new_data + def get_disabled_reports(): if not hasattr(frappe.local, "disabled_reports"): frappe.local.disabled_reports = set(r.name for r in frappe.get_all("Report", {"disabled": 1})) return frappe.local.disabled_reports + def get_config(app, module): """Load module info from `[app].config.[module]`.""" config = frappe.get_module("{app}.config.{module}".format(app=app, module=module)) @@ -237,7 +268,7 @@ def get_config(app, module): for section in sections: items = [] for item in section["items"]: - if item["type"]=="report" and item["name"] in disabled_reports: + if item["type"] == "report" and item["name"] in disabled_reports: continue # some module links might not have name if not item.get("name"): @@ -245,10 +276,11 @@ def get_config(app, module): if not item.get("label"): item["label"] = _(item.get("name")) items.append(item) - section['items'] = items + section["items"] = items return sections + def config_exists(app, module): try: frappe.get_module("{app}.config.{module}".format(app=app, module=module)) @@ -256,6 +288,7 @@ def config_exists(app, module): except ImportError: return False + def add_setup_section(config, app, module, label, icon): """Add common sections to `/desk#Module/Setup`""" try: @@ -265,16 +298,13 @@ def add_setup_section(config, app, module, label, icon): except ImportError: pass + def get_setup_section(app, module, label, icon): """Get the setup section from each module (for global Setup page).""" config = get_config(app, module) for section in config: - if section.get("label")==_("Setup"): - return { - "label": label, - "icon": icon, - "items": section["items"] - } + if section.get("label") == _("Setup"): + return {"label": label, "icon": icon, "items": section["items"]} def get_onboard_items(app, module): @@ -303,9 +333,11 @@ def get_onboard_items(app, module): return onboard_items or fallback_items + @frappe.whitelist() def get_links_for_module(app, module): - return [{'value': l.get('name'), 'label': l.get('label')} for l in get_links(app, module)] + return [{"value": l.get("name"), "label": l.get("label")} for l in get_links(app, module)] + def get_links(app, module): try: @@ -315,21 +347,23 @@ def get_links(app, module): links = [] for section in sections: - for item in section['items']: + for item in section["items"]: links.append(item) return links + @frappe.whitelist() def get_desktop_settings(): from frappe.config import get_modules_from_all_apps_for_user + all_modules = get_modules_from_all_apps_for_user() home_settings = get_home_settings() modules_by_name = {} for m in all_modules: - modules_by_name[m['module_name']] = m + modules_by_name[m["module_name"]] = m - module_categories = ['Modules', 'Domains', 'Places', 'Administration'] + module_categories = ["Modules", "Domains", "Places", "Administration"] user_modules_by_category = {} user_saved_modules_by_category = home_settings.modules_by_category or {} @@ -340,7 +374,7 @@ def get_desktop_settings(): all_links = get_links(module.app, module.module_name) module_links_by_name = {} for link in all_links: - module_links_by_name[link['name']] = link + module_links_by_name[link["name"]] = link if module.module_name in user_saved_links_by_module: user_links = frappe.parse_json(user_saved_links_by_module[module.module_name]) @@ -351,21 +385,26 @@ def get_desktop_settings(): for category in module_categories: if category in user_saved_modules_by_category: user_modules = user_saved_modules_by_category[category] - user_modules_by_category[category] = [apply_user_saved_links(modules_by_name[m]) \ - for m in user_modules if modules_by_name.get(m)] + user_modules_by_category[category] = [ + apply_user_saved_links(modules_by_name[m]) for m in user_modules if modules_by_name.get(m) + ] else: - user_modules_by_category[category] = [apply_user_saved_links(m) \ - for m in all_modules if m.get('category') == category] + user_modules_by_category[category] = [ + apply_user_saved_links(m) for m in all_modules if m.get("category") == category + ] # filter out hidden modules if home_settings.hidden_modules: for category in user_modules_by_category: hidden_modules = home_settings.hidden_modules or [] modules = user_modules_by_category[category] - user_modules_by_category[category] = [module for module in modules if module.module_name not in hidden_modules] + user_modules_by_category[category] = [ + module for module in modules if module.module_name not in hidden_modules + ] return user_modules_by_category + @frappe.whitelist() def update_hidden_modules(category_map): category_map = frappe.parse_json(category_map) @@ -378,8 +417,10 @@ def update_hidden_modules(category_map): saved_hidden_modules += config.removed or [] saved_hidden_modules = [d for d in saved_hidden_modules if d not in (config.added or [])] - if home_settings.get('modules_by_category') and home_settings.modules_by_category.get(category): - module_placement = [d for d in (config.added or []) if d not in home_settings.modules_by_category[category]] + if home_settings.get("modules_by_category") and home_settings.modules_by_category.get(category): + module_placement = [ + d for d in (config.added or []) if d not in home_settings.modules_by_category[category] + ] home_settings.modules_by_category[category] += module_placement home_settings.hidden_modules = saved_hidden_modules @@ -387,17 +428,16 @@ def update_hidden_modules(category_map): return get_desktop_settings() + @frappe.whitelist() def update_global_hidden_modules(modules): modules = frappe.parse_json(modules) - frappe.only_for('System Manager') + frappe.only_for("System Manager") - doc = frappe.get_doc('User', 'Administrator') - doc.set('block_modules', []) + doc = frappe.get_doc("User", "Administrator") + doc.set("block_modules", []) for module in modules: - doc.append('block_modules', { - 'module': module - }) + doc.append("block_modules", {"module": module}) doc.save(ignore_permissions=True) @@ -414,53 +454,58 @@ def update_modules_order(module_category, modules): set_home_settings(home_settings) + @frappe.whitelist() def update_links_for_module(module_name, links): links = frappe.parse_json(links) home_settings = get_home_settings() - home_settings.setdefault('links_by_module', {}) - home_settings['links_by_module'].setdefault(module_name, None) - home_settings['links_by_module'][module_name] = links + home_settings.setdefault("links_by_module", {}) + home_settings["links_by_module"].setdefault(module_name, None) + home_settings["links_by_module"][module_name] = links set_home_settings(home_settings) return get_desktop_settings() + @frappe.whitelist() def get_options_for_show_hide_cards(): global_options = [] - if 'System Manager' in frappe.get_roles(): + if "System Manager" in frappe.get_roles(): global_options = get_options_for_global_modules() - return { - 'user_options': get_options_for_user_blocked_modules(), - 'global_options': global_options - } + return {"user_options": get_options_for_user_blocked_modules(), "global_options": global_options} + @frappe.whitelist() def get_options_for_global_modules(): from frappe.config import get_modules_from_all_apps + all_modules = get_modules_from_all_apps() - blocked_modules = frappe.get_doc('User', 'Administrator').get_blocked_modules() + blocked_modules = frappe.get_doc("User", "Administrator").get_blocked_modules() options = [] for module in all_modules: module = frappe._dict(module) - options.append({ - 'category': module.category, - 'label': module.label, - 'value': module.module_name, - 'checked': module.module_name not in blocked_modules - }) + options.append( + { + "category": module.category, + "label": module.label, + "value": module.module_name, + "checked": module.module_name not in blocked_modules, + } + ) return options + @frappe.whitelist() def get_options_for_user_blocked_modules(): from frappe.config import get_modules_from_all_apps_for_user + all_modules = get_modules_from_all_apps_for_user() home_settings = get_home_settings() @@ -469,26 +514,30 @@ def get_options_for_user_blocked_modules(): options = [] for module in all_modules: module = frappe._dict(module) - options.append({ - 'category': module.category, - 'label': module.label, - 'value': module.module_name, - 'checked': module.module_name not in hidden_modules - }) + options.append( + { + "category": module.category, + "label": module.label, + "value": module.module_name, + "checked": module.module_name not in hidden_modules, + } + ) return options + def set_home_settings(home_settings): - frappe.cache().hset('home_settings', frappe.session.user, home_settings) - frappe.db.set_value('User', frappe.session.user, 'home_settings', json.dumps(home_settings)) + frappe.cache().hset("home_settings", frappe.session.user, home_settings) + frappe.db.set_value("User", frappe.session.user, "home_settings", json.dumps(home_settings)) + @frappe.whitelist() def get_home_settings(): def get_from_db(): - settings = frappe.db.get_value("User", frappe.session.user, 'home_settings') - return frappe.parse_json(settings or '{}') + settings = frappe.db.get_value("User", frappe.session.user, "home_settings") + return frappe.parse_json(settings or "{}") - home_settings = frappe.cache().hget('home_settings', frappe.session.user, get_from_db) + home_settings = frappe.cache().hget("home_settings", frappe.session.user, get_from_db) return home_settings @@ -513,10 +562,13 @@ def set_last_modified(data): if item["type"] == "doctype": item["last_modified"] = get_last_modified(item["name"]) + def get_last_modified(doctype): def _get(): try: - last_modified = frappe.get_all(doctype, fields=["max(modified)"], as_list=True, limit_page_length=1)[0][0] + last_modified = frappe.get_all( + doctype, fields=["max(modified)"], as_list=True, limit_page_length=1 + )[0][0] except Exception as e: if frappe.db.is_table_missing(e): last_modified = None @@ -531,25 +583,33 @@ def get_last_modified(doctype): last_modified = frappe.cache().hget("last_modified", doctype, _get) - if last_modified==-1: + if last_modified == -1: last_modified = None return last_modified + def get_report_list(module, is_standard="No"): """Returns list on new style reports for modules.""" - reports = frappe.get_list("Report", fields=["name", "ref_doctype", "report_type"], filters= - {"is_standard": is_standard, "disabled": 0, "module": module}, - order_by="name") + reports = frappe.get_list( + "Report", + fields=["name", "ref_doctype", "report_type"], + filters={"is_standard": is_standard, "disabled": 0, "module": module}, + order_by="name", + ) out = [] for r in reports: - out.append({ - "type": "report", - "doctype": r.ref_doctype, - "is_query_report": 1 if r.report_type in ("Query Report", "Script Report", "Custom Report") else 0, - "label": _(r.name), - "name": r.name - }) - - return out \ No newline at end of file + out.append( + { + "type": "report", + "doctype": r.ref_doctype, + "is_query_report": 1 + if r.report_type in ("Query Report", "Script Report", "Custom Report") + else 0, + "label": _(r.name), + "name": r.name, + } + ) + + return out diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py index 3fa41790b4..e92a7492ce 100644 --- a/frappe/desk/notifications.py +++ b/frappe/desk/notifications.py @@ -1,10 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -from frappe.desk.doctype.notification_settings.notification_settings import get_subscribed_documents import json +import frappe +from frappe.desk.doctype.notification_settings.notification_settings import ( + get_subscribed_documents, +) + + @frappe.whitelist() @frappe.read_only() def get_notifications(): @@ -12,8 +16,7 @@ def get_notifications(): "open_count_doctype": {}, "targets": {}, } - if (frappe.flags.in_install or - not frappe.db.get_single_value('System Settings', 'setup_complete')): + if frappe.flags.in_install or not frappe.db.get_single_value("System Settings", "setup_complete"): return out config = get_notification_config() @@ -32,11 +35,12 @@ def get_notifications(): if count is not None: notification_count[name] = count - out['open_count_doctype'] = get_notifications_for_doctypes(config, notification_count) - out['targets'] = get_notifications_for_targets(config, notification_percent) + out["open_count_doctype"] = get_notifications_for_doctypes(config, notification_count) + out["targets"] = get_notifications_for_targets(config, notification_percent) return out + def get_notifications_for_doctypes(config, notification_count): """Notifications for DocTypes""" can_read = frappe.get_user().get_can_read() @@ -51,7 +55,9 @@ def get_notifications_for_doctypes(config, notification_count): else: try: if isinstance(condition, dict): - result = frappe.get_list(d, fields=["count(*) as count"], filters=condition, ignore_ifnull=True)[0].count + result = frappe.get_list( + d, fields=["count(*) as count"], filters=condition, ignore_ifnull=True + )[0].count else: result = frappe.get_attr(condition)() @@ -72,6 +78,7 @@ def get_notifications_for_doctypes(config, notification_count): return open_count_doctype + def get_notifications_for_targets(config, notification_percent): """Notifications for doc targets""" can_read = frappe.get_user().get_can_read() @@ -96,8 +103,13 @@ def get_notifications_for_targets(config, notification_percent): value_field = d["value_field"] try: if isinstance(condition, dict): - doc_list = frappe.get_list(doctype, fields=["name", target_field, value_field], - filters=condition, limit_page_length = 100, ignore_ifnull=True) + doc_list = frappe.get_list( + doctype, + fields=["name", target_field, value_field], + filters=condition, + limit_page_length=100, + ignore_ifnull=True, + ) except frappe.PermissionError: frappe.clear_messages() @@ -110,10 +122,11 @@ def get_notifications_for_targets(config, notification_percent): for doc in doc_list: value = doc[value_field] target = doc[target_field] - doc_target_percents[doctype][doc.name] = (value/target * 100) if value < target else 100 + doc_target_percents[doctype][doc.name] = (value / target * 100) if value < target else 100 return doc_target_percents + def clear_notifications(user=None): if frappe.flags.in_install: return @@ -123,8 +136,8 @@ def clear_notifications(user=None): if not config: return - for_doctype = list(config.get('for_doctype')) if config.get('for_doctype') else [] - for_module = list(config.get('for_module')) if config.get('for_module') else [] + for_doctype = list(config.get("for_doctype")) if config.get("for_doctype") else [] + for_module = list(config.get("for_module")) if config.get("for_module") else [] groups = for_doctype + for_module for name in groups: @@ -133,21 +146,24 @@ def clear_notifications(user=None): else: cache.delete_key("notification_count:" + name) - frappe.publish_realtime('clear_notifications') + frappe.publish_realtime("clear_notifications") + def clear_notification_config(user): - frappe.cache().hdel('notification_config', user) + frappe.cache().hdel("notification_config", user) + def delete_notification_count_for(doctype): frappe.cache().delete_key("notification_count:" + doctype) - frappe.publish_realtime('clear_notifications') + frappe.publish_realtime("clear_notifications") + def clear_doctype_notifications(doc, method=None, *args, **kwargs): config = get_notification_config() if not config: return if isinstance(doc, str): - doctype = doc # assuming doctype name was passed directly + doctype = doc # assuming doctype name was passed directly else: doctype = doc.doctype @@ -155,6 +171,7 @@ def clear_doctype_notifications(doc, method=None, *args, **kwargs): delete_notification_count_for(doctype) return + @frappe.whitelist() def get_notification_info(): config = get_notification_config() @@ -171,15 +188,18 @@ def get_notification_info(): if d in doctype_info: module_doctypes.setdefault(doctype_info[d], []).append(d) - out.update({ - "conditions": conditions, - "module_doctypes": module_doctypes, - }) + out.update( + { + "conditions": conditions, + "module_doctypes": module_doctypes, + } + ) return out + def get_notification_config(): - user = frappe.session.user or 'Guest' + user = frappe.session.user or "Guest" def _get(): subscribed_documents = get_subscribed_documents() @@ -206,28 +226,28 @@ def get_notification_config(): return frappe.cache().hget("notification_config", user, _get) + def get_filters_for(doctype): - '''get open filters for doctype''' + """get open filters for doctype""" config = get_notification_config() doctype_config = config.get("for_doctype").get(doctype, {}) filters = doctype_config if not isinstance(doctype_config, str) else None return filters + @frappe.whitelist() @frappe.read_only() def get_open_count(doctype, name, items=None): - '''Get open count for given transactions and filters + """Get open count for given transactions and filters :param doctype: Reference DocType :param name: Reference Name :param transactions: List of transactions (json/dict) - :param filters: optional filters (json/list)''' + :param filters: optional filters (json/list)""" if frappe.flags.in_migrate or frappe.flags.in_install: - return { - "count": [] - } + return {"count": []} frappe.has_permission(doc=frappe.get_doc(doctype, name), throw=True) @@ -250,18 +270,22 @@ def get_open_count(doctype, name, items=None): continue filters = get_filters_for(d) - fieldname = links.get("non_standard_fieldnames", {}).get(d, links.get('fieldname')) + fieldname = links.get("non_standard_fieldnames", {}).get(d, links.get("fieldname")) data = {"name": d} if filters: # get the fieldname for the current document # we only need open documents related to the current document filters[fieldname] = name - total = len(frappe.get_all(d, fields="name", - filters=filters, limit=100, distinct=True, ignore_ifnull=True)) + total = len( + frappe.get_all(d, fields="name", filters=filters, limit=100, distinct=True, ignore_ifnull=True) + ) data["open_count"] = total - total = len(frappe.get_all(d, fields="name", - filters={fieldname: name}, limit=100, distinct=True, ignore_ifnull=True)) + total = len( + frappe.get_all( + d, fields="name", filters={fieldname: name}, limit=100, distinct=True, ignore_ifnull=True + ) + ) data["count"] = total out.append(data) diff --git a/frappe/desk/page/activity/activity.py b/frappe/desk/page/activity/activity.py index 71130f2304..d22fa006a4 100644 --- a/frappe/desk/page/activity/activity.py +++ b/frappe/desk/page/activity/activity.py @@ -2,16 +2,18 @@ # License: MIT. See LICENSE import frappe -from frappe.utils import cint from frappe.core.doctype.activity_log.feed import get_feed_match_conditions +from frappe.utils import cint + @frappe.whitelist() def get_feed(start, page_length): """get feed""" - match_conditions_communication = get_feed_match_conditions(frappe.session.user, 'Communication') - match_conditions_comment = get_feed_match_conditions(frappe.session.user, 'Comment') + match_conditions_communication = get_feed_match_conditions(frappe.session.user, "Communication") + match_conditions_comment = get_feed_match_conditions(frappe.session.user, "Comment") - result = frappe.db.sql("""select X.* + result = frappe.db.sql( + """select X.* from (select name, owner, modified, creation, seen, comment_type, reference_doctype, reference_name, '' as link_doctype, '' as link_name, subject, communication_type, communication_medium, content @@ -38,21 +40,26 @@ def get_feed(start, page_length): ) X order by X.creation DESC LIMIT %(page_length)s - OFFSET %(start)s""" - .format(match_conditions_comment = match_conditions_comment, - match_conditions_communication = match_conditions_communication), { - "user": frappe.session.user, - "start": cint(start), - "page_length": cint(page_length) - }, as_dict=True) + OFFSET %(start)s""".format( + match_conditions_comment=match_conditions_comment, + match_conditions_communication=match_conditions_communication, + ), + {"user": frappe.session.user, "start": cint(start), "page_length": cint(page_length)}, + as_dict=True, + ) return result + @frappe.whitelist() def get_heatmap_data(): - return dict(frappe.db.sql("""select unix_timestamp(date(creation)), count(name) + return dict( + frappe.db.sql( + """select unix_timestamp(date(creation)), count(name) from `tabActivity Log` where date(creation) > subdate(curdate(), interval 1 year) group by date(creation) - order by creation asc""")) + order by creation asc""" + ) + ) diff --git a/frappe/desk/page/backups/backups.py b/frappe/desk/page/backups/backups.py index 14ed025e08..ec5bea3c4b 100644 --- a/frappe/desk/page/backups/backups.py +++ b/frappe/desk/page/backups/backups.py @@ -1,15 +1,18 @@ - +import datetime import os + import frappe from frappe import _ -from frappe.utils import get_site_path, cint, get_url +from frappe.utils import cint, get_site_path, get_url from frappe.utils.data import convert_utc_to_user_timezone -import datetime + def get_context(context): def get_time(path): dt = os.path.getmtime(path) - return convert_utc_to_user_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime('%a %b %d %H:%M %Y') + return convert_utc_to_user_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime( + "%a %b %d %H:%M %Y" + ) def get_encrytion_status(path): if "-enc" in path: @@ -22,29 +25,37 @@ def get_context(context): else: return "{0:.1f}K".format(float(size) / 1024) - path = get_site_path('private', 'backups') + path = get_site_path("private", "backups") files = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))] backup_limit = get_scheduled_backup_limit() if len(files) > backup_limit: cleanup_old_backups(path, files, backup_limit) - files = [('/backups/' + _file, + files = [ + ( + "/backups/" + _file, get_time(os.path.join(path, _file)), get_encrytion_status(os.path.join(path, _file)), - get_size(os.path.join(path, _file))) for _file in files if _file.endswith('sql.gz')] + get_size(os.path.join(path, _file)), + ) + for _file in files + if _file.endswith("sql.gz") + ] files.sort(key=lambda x: x[1], reverse=True) return {"files": files[:backup_limit]} + def get_scheduled_backup_limit(): - backup_limit = frappe.db.get_singles_value('System Settings', 'backup_limit') + backup_limit = frappe.db.get_singles_value("System Settings", "backup_limit") return cint(backup_limit) + def cleanup_old_backups(site_path, files, limit): backup_paths = [] for f in files: - if f.endswith('sql.gz'): + if f.endswith("sql.gz"): _path = os.path.abspath(os.path.join(site_path, f)) backup_paths.append(_path) @@ -57,28 +68,39 @@ def cleanup_old_backups(site_path, files, limit): os.remove(backup_paths[idx]) + def delete_downloadable_backups(): - path = get_site_path('private', 'backups') + path = get_site_path("private", "backups") files = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))] backup_limit = get_scheduled_backup_limit() if len(files) > backup_limit: cleanup_old_backups(path, files, backup_limit) + @frappe.whitelist() def schedule_files_backup(user_email): from frappe.utils.background_jobs import enqueue, get_jobs + queued_jobs = get_jobs(site=frappe.local.site, queue="long") - method = 'frappe.desk.page.backups.backups.backup_files_and_notify_user' + method = "frappe.desk.page.backups.backups.backup_files_and_notify_user" if method not in queued_jobs[frappe.local.site]: - enqueue("frappe.desk.page.backups.backups.backup_files_and_notify_user", queue='long', user_email=user_email) + enqueue( + "frappe.desk.page.backups.backups.backup_files_and_notify_user", + queue="long", + user_email=user_email, + ) frappe.msgprint(_("Queued for backup. You will receive an email with the download link")) else: - frappe.msgprint(_("Backup job is already queued. You will receive an email with the download link")) + frappe.msgprint( + _("Backup job is already queued. You will receive an email with the download link") + ) + def backup_files_and_notify_user(user_email=None): from frappe.utils.backups import backup + backup_files = backup(with_files=True) get_downloadable_links(backup_files) @@ -88,10 +110,11 @@ def backup_files_and_notify_user(user_email=None): subject=subject, template="file_backup_notification", args=backup_files, - header=[subject, 'green'] + header=[subject, "green"], ) + def get_downloadable_links(backup_files): - for key in ['backup_path_files', 'backup_path_private_files']: + for key in ["backup_path_files", "backup_path_private_files"]: path = backup_files[key] - backup_files[key] = get_url('/'.join(path.split('/')[-2:])) + backup_files[key] = get_url("/".join(path.split("/")[-2:])) diff --git a/frappe/desk/page/leaderboard/leaderboard.py b/frappe/desk/page/leaderboard/leaderboard.py index ad22eb9199..0ad4538a81 100644 --- a/frappe/desk/page/leaderboard/leaderboard.py +++ b/frappe/desk/page/leaderboard/leaderboard.py @@ -2,11 +2,12 @@ # License: MIT. See LICENSE import frappe + @frappe.whitelist() def get_leaderboard_config(): leaderboard_config = frappe._dict() - leaderboard_hooks = frappe.get_hooks('leaderboards') + leaderboard_hooks = frappe.get_hooks("leaderboards") for hook in leaderboard_hooks: leaderboard_config.update(frappe.get_attr(hook)()) - return leaderboard_config \ No newline at end of file + return leaderboard_config diff --git a/frappe/desk/page/setup_wizard/install_fixtures.py b/frappe/desk/page/setup_wizard/install_fixtures.py index 1ef83f7ba0..18a519f87f 100644 --- a/frappe/desk/page/setup_wizard/install_fixtures.py +++ b/frappe/desk/page/setup_wizard/install_fixtures.py @@ -3,9 +3,12 @@ import frappe from frappe import _ -from frappe.desk.doctype.global_search_settings.global_search_settings import update_global_search_doctypes +from frappe.desk.doctype.global_search_settings.global_search_settings import ( + update_global_search_doctypes, +) from frappe.utils.dashboard import sync_dashboards + def install(): update_genders() update_salutations() @@ -14,33 +17,47 @@ def install(): sync_dashboards() add_unsubscribe() + @frappe.whitelist() def update_genders(): - default_genders = ["Male", "Female", "Other","Transgender", "Genderqueer", "Non-Conforming","Prefer not to say"] - records = [{'doctype': 'Gender', 'gender': d} for d in default_genders] + default_genders = [ + "Male", + "Female", + "Other", + "Transgender", + "Genderqueer", + "Non-Conforming", + "Prefer not to say", + ] + records = [{"doctype": "Gender", "gender": d} for d in default_genders] for record in records: frappe.get_doc(record).insert(ignore_permissions=True, ignore_if_duplicate=True) + @frappe.whitelist() def update_salutations(): - default_salutations = ["Mr", "Ms", 'Mx', "Dr", "Mrs", "Madam", "Miss", "Master", "Prof"] - records = [{'doctype': 'Salutation', 'salutation': d} for d in default_salutations] + default_salutations = ["Mr", "Ms", "Mx", "Dr", "Mrs", "Madam", "Miss", "Master", "Prof"] + records = [{"doctype": "Salutation", "salutation": d} for d in default_salutations] for record in records: doc = frappe.new_doc(record.get("doctype")) doc.update(record) doc.insert(ignore_permissions=True, ignore_if_duplicate=True) + def setup_email_linking(): - doc = frappe.get_doc({ - "doctype": "Email Account", - "email_id": "email_linking@example.com", - }) + doc = frappe.get_doc( + { + "doctype": "Email Account", + "email_id": "email_linking@example.com", + } + ) doc.insert(ignore_permissions=True, ignore_if_duplicate=True) + def add_unsubscribe(): email_unsubscribe = [ {"email": "admin@example.com", "global_unsubscribe": 1}, - {"email": "guest@example.com", "global_unsubscribe": 1} + {"email": "guest@example.com", "global_unsubscribe": 1}, ] for unsubscribe in email_unsubscribe: diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index 0c32e886f4..f85d24704f 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -1,93 +1,101 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json, os -from frappe.utils import strip, cint -from frappe.translate import (set_default_language, get_dict, send_translations) +import json +import os + +import frappe from frappe.geo.country_info import get_country_info +from frappe.translate import get_dict, send_translations, set_default_language +from frappe.utils import cint, strip from frappe.utils.password import update_password + from . import install_fixtures + def get_setup_stages(args): # App setup stage functions should not include frappe.db.commit # That is done by frappe after successful completion of all stages stages = [ { - 'status': 'Updating global settings', - 'fail_msg': 'Failed to update global settings', - 'tasks': [ - { - 'fn': update_global_settings, - 'args': args, - 'fail_msg': 'Failed to update global settings' - } - ] + "status": "Updating global settings", + "fail_msg": "Failed to update global settings", + "tasks": [ + {"fn": update_global_settings, "args": args, "fail_msg": "Failed to update global settings"} + ], } ] stages += get_stages_hooks(args) + get_setup_complete_hooks(args) - stages.append({ - # post executing hooks - 'status': 'Wrapping up', - 'fail_msg': 'Failed to complete setup', - 'tasks': [ - { - 'fn': run_post_setup_complete, - 'args': args, - 'fail_msg': 'Failed to complete setup' - } - ] - }) + stages.append( + { + # post executing hooks + "status": "Wrapping up", + "fail_msg": "Failed to complete setup", + "tasks": [ + {"fn": run_post_setup_complete, "args": args, "fail_msg": "Failed to complete setup"} + ], + } + ) return stages + @frappe.whitelist() def setup_complete(args): """Calls hooks for `setup_wizard_complete`, sets home page as `desktop` and clears cache. If wizard breaks, calls `setup_wizard_exception` hook""" # Setup complete: do not throw an exception, let the user continue to desk - if cint(frappe.db.get_single_value('System Settings', 'setup_complete')): - return {'status': 'ok'} + if cint(frappe.db.get_single_value("System Settings", "setup_complete")): + return {"status": "ok"} args = parse_args(args) stages = get_setup_stages(args) - is_background_task = frappe.conf.get('trigger_site_setup_in_background') + is_background_task = frappe.conf.get("trigger_site_setup_in_background") if is_background_task: process_setup_stages.enqueue(stages=stages, user_input=args, is_background_task=True) - return {'status': 'registered'} + return {"status": "registered"} else: return process_setup_stages(stages, args) + @frappe.task() def process_setup_stages(stages, user_input, is_background_task=False): try: frappe.flags.in_setup_wizard = True current_task = None for idx, stage in enumerate(stages): - frappe.publish_realtime('setup_task', {"progress": [idx, len(stages)], - "stage_status": stage.get('status')}, user=frappe.session.user) + frappe.publish_realtime( + "setup_task", + {"progress": [idx, len(stages)], "stage_status": stage.get("status")}, + user=frappe.session.user, + ) - for task in stage.get('tasks'): + for task in stage.get("tasks"): current_task = task - task.get('fn')(task.get('args')) + task.get("fn")(task.get("args")) except Exception: handle_setup_exception(user_input) if not is_background_task: - return {'status': 'fail', 'fail': current_task.get('fail_msg')} - frappe.publish_realtime('setup_task', - {'status': 'fail', "fail_msg": current_task.get('fail_msg')}, user=frappe.session.user) + return {"status": "fail", "fail": current_task.get("fail_msg")} + frappe.publish_realtime( + "setup_task", + {"status": "fail", "fail_msg": current_task.get("fail_msg")}, + user=frappe.session.user, + ) else: run_setup_success(user_input) if not is_background_task: - return {'status': 'ok'} - frappe.publish_realtime('setup_task', {"status": 'ok'}, user=frappe.session.user) + return {"status": "ok"} + frappe.publish_realtime("setup_task", {"status": "ok"}, user=frappe.session.user) finally: frappe.flags.in_setup_wizard = False + def update_global_settings(args): if args.language and args.language != "English": set_default_language(get_language_code(args.lang)) @@ -97,38 +105,41 @@ def update_global_settings(args): update_system_settings(args) update_user_name(args) + def run_post_setup_complete(args): disable_future_access() frappe.db.commit() frappe.clear_cache() + def run_setup_success(args): for hook in frappe.get_hooks("setup_wizard_success"): frappe.get_attr(hook)(args) install_fixtures.install() + def get_stages_hooks(args): stages = [] for method in frappe.get_hooks("setup_wizard_stages"): stages += frappe.get_attr(method)(args) return stages + def get_setup_complete_hooks(args): stages = [] for method in frappe.get_hooks("setup_wizard_complete"): - stages.append({ - 'status': 'Executing method', - 'fail_msg': 'Failed to execute method', - 'tasks': [ - { - 'fn': frappe.get_attr(method), - 'args': args, - 'fail_msg': 'Failed to execute method' - } - ] - }) + stages.append( + { + "status": "Executing method", + "fail_msg": "Failed to execute method", + "tasks": [ + {"fn": frappe.get_attr(method), "args": args, "fail_msg": "Failed to execute method"} + ], + } + ) return stages + def handle_setup_exception(args): frappe.db.rollback() if args: @@ -137,83 +148,91 @@ def handle_setup_exception(args): for hook in frappe.get_hooks("setup_wizard_exception"): frappe.get_attr(hook)(traceback, args) + def update_system_settings(args): number_format = get_country_info(args.get("country")).get("number_format", "#,###.##") # replace these as float number formats, as they have 0 precision # and are currency number formats and not for floats - if number_format=="#.###": + if number_format == "#.###": number_format = "#.###,##" - elif number_format=="#,###": + elif number_format == "#,###": number_format = "#,###.##" system_settings = frappe.get_doc("System Settings", "System Settings") - system_settings.update({ - "country": args.get("country"), - "language": get_language_code(args.get("language")) or 'en', - "time_zone": args.get("timezone"), - "float_precision": 3, - 'date_format': frappe.db.get_value("Country", args.get("country"), "date_format"), - 'time_format': frappe.db.get_value("Country", args.get("country"), "time_format"), - 'number_format': number_format, - 'enable_scheduler': 1 if not frappe.flags.in_test else 0, - 'backup_limit': 3 # Default for downloadable backups - }) + system_settings.update( + { + "country": args.get("country"), + "language": get_language_code(args.get("language")) or "en", + "time_zone": args.get("timezone"), + "float_precision": 3, + "date_format": frappe.db.get_value("Country", args.get("country"), "date_format"), + "time_format": frappe.db.get_value("Country", args.get("country"), "time_format"), + "number_format": number_format, + "enable_scheduler": 1 if not frappe.flags.in_test else 0, + "backup_limit": 3, # Default for downloadable backups + } + ) system_settings.save() + def update_user_name(args): - first_name, last_name = args.get('full_name', ''), '' - if ' ' in first_name: - first_name, last_name = first_name.split(' ', 1) + first_name, last_name = args.get("full_name", ""), "" + if " " in first_name: + first_name, last_name = first_name.split(" ", 1) if args.get("email"): - if frappe.db.exists('User', args.get('email')): + if frappe.db.exists("User", args.get("email")): # running again return - - args['name'] = args.get("email") + args["name"] = args.get("email") _mute_emails, frappe.flags.mute_emails = frappe.flags.mute_emails, True - doc = frappe.get_doc({ - "doctype":"User", - "email": args.get("email"), - "first_name": first_name, - "last_name": last_name - }) + doc = frappe.get_doc( + { + "doctype": "User", + "email": args.get("email"), + "first_name": first_name, + "last_name": last_name, + } + ) doc.flags.no_welcome_mail = True doc.insert() frappe.flags.mute_emails = _mute_emails update_password(args.get("email"), args.get("password")) elif first_name: - args.update({ - "name": frappe.session.user, - "first_name": first_name, - "last_name": last_name - }) + args.update({"name": frappe.session.user, "first_name": first_name, "last_name": last_name}) - frappe.db.sql("""update `tabUser` SET first_name=%(first_name)s, - last_name=%(last_name)s WHERE name=%(name)s""", args) + frappe.db.sql( + """update `tabUser` SET first_name=%(first_name)s, + last_name=%(last_name)s WHERE name=%(name)s""", + args, + ) if args.get("attach_user"): attach_user = args.get("attach_user").split(",") - if len(attach_user)==3: + if len(attach_user) == 3: filename, filetype, content = attach_user - _file = frappe.get_doc({ - "doctype": "File", - "file_name": filename, - "attached_to_doctype": "User", - "attached_to_name": args.get("name"), - "content": content, - "decode": True}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": filename, + "attached_to_doctype": "User", + "attached_to_name": args.get("name"), + "content": content, + "decode": True, + } + ) _file.save() fileurl = _file.file_url frappe.db.set_value("User", args.get("name"), "user_image", fileurl) - if args.get('name'): + if args.get("name"): add_all_roles_to(args.get("name")) + def parse_args(args): if not args: args = frappe.local.form_dict @@ -229,31 +248,42 @@ def parse_args(args): return args + def add_all_roles_to(name): user = frappe.get_doc("User", name) for role in frappe.db.sql("""select name from tabRole"""): - if role[0] not in ["Administrator", "Guest", "All", "Customer", "Supplier", "Partner", "Employee"]: + if role[0] not in [ + "Administrator", + "Guest", + "All", + "Customer", + "Supplier", + "Partner", + "Employee", + ]: d = user.append("roles") d.role = role[0] user.save() + def disable_future_access(): - frappe.db.set_default('desktop:home_page', 'workspace') - frappe.db.set_value('System Settings', 'System Settings', 'setup_complete', 1) - frappe.db.set_value('System Settings', 'System Settings', 'is_first_startup', 1) + frappe.db.set_default("desktop:home_page", "workspace") + frappe.db.set_value("System Settings", "System Settings", "setup_complete", 1) + frappe.db.set_value("System Settings", "System Settings", "is_first_startup", 1) # Enable onboarding after install - frappe.db.set_value('System Settings', 'System Settings', 'enable_onboarding', 1) + frappe.db.set_value("System Settings", "System Settings", "enable_onboarding", 1) if not frappe.flags.in_test: # remove all roles and add 'Administrator' to prevent future access - page = frappe.get_doc('Page', 'setup-wizard') + page = frappe.get_doc("Page", "setup-wizard") page.roles = [] - page.append('roles', {'role': 'Administrator'}) + page.append("roles", {"role": "Administrator"}) page.flags.do_not_update_json = True page.flags.ignore_permissions = True page.save() + @frappe.whitelist() def load_messages(language): """Load translation messages for given language from all `setup_wizard_requires` @@ -272,33 +302,42 @@ def load_messages(language): send_translations(m) return frappe.local.lang + @frappe.whitelist() def load_languages(): - language_codes = frappe.db.sql('select language_code, language_name from tabLanguage order by name', as_dict=True) + language_codes = frappe.db.sql( + "select language_code, language_name from tabLanguage order by name", as_dict=True + ) codes_to_names = {} for d in language_codes: codes_to_names[d.language_code] = d.language_name return { - "default_language": frappe.db.get_value('Language', frappe.local.lang, 'language_name') or frappe.local.lang, - "languages": sorted(frappe.db.sql_list('select language_name from tabLanguage order by name')), - "codes_to_names": codes_to_names + "default_language": frappe.db.get_value("Language", frappe.local.lang, "language_name") + or frappe.local.lang, + "languages": sorted(frappe.db.sql_list("select language_name from tabLanguage order by name")), + "codes_to_names": codes_to_names, } + @frappe.whitelist() def load_country(): from frappe.sessions import get_geo_ip_country + return get_geo_ip_country(frappe.local.request_ip) if frappe.local.request_ip else None + @frappe.whitelist() def load_user_details(): return { "full_name": frappe.cache().hget("full_name", "signup"), - "email": frappe.cache().hget("email", "signup") + "email": frappe.cache().hget("email", "signup"), } + @frappe.whitelist() def reset_is_first_startup(): - frappe.db.set_value('System Settings', 'System Settings', 'is_first_startup', 0) + frappe.db.set_value("System Settings", "System Settings", "is_first_startup", 0) + def prettify_args(args): # remove attachments @@ -313,6 +352,7 @@ def prettify_args(args): pretty_args.append("{} = {}".format(key, args[key])) return pretty_args + def email_setup_wizard_exception(traceback, args): if not frappe.conf.setup_wizard_exception_email: return @@ -349,25 +389,31 @@ def email_setup_wizard_exception(traceback, args): headers=frappe.request.headers, ) - frappe.sendmail(recipients=frappe.conf.setup_wizard_exception_email, + frappe.sendmail( + recipients=frappe.conf.setup_wizard_exception_email, sender=frappe.session.user, subject="Setup failed: {}".format(frappe.local.site), message=message, - delayed=False) + delayed=False, + ) + def log_setup_wizard_exception(traceback, args): - with open('../logs/setup-wizard.log', 'w+') as setup_log: + with open("../logs/setup-wizard.log", "w+") as setup_log: setup_log.write(traceback) setup_log.write(json.dumps(args)) + def get_language_code(lang): - return frappe.db.get_value('Language', {'language_name':lang}) + return frappe.db.get_value("Language", {"language_name": lang}) + def enable_twofactor_all_roles(): - all_role = frappe.get_doc('Role',{'role_name':'All'}) + all_role = frappe.get_doc("Role", {"role_name": "All"}) all_role.two_factor_auth = True all_role.save(ignore_permissions=True) + def make_records(records, debug=False): from frappe import _dict from frappe.modules import scrub @@ -378,7 +424,7 @@ def make_records(records, debug=False): # LOG every success and failure for record in records: doctype = record.get("doctype") - condition = record.get('__condition') + condition = record.get("__condition") if condition and not condition(): continue @@ -387,7 +433,7 @@ def make_records(records, debug=False): doc.update(record) # ignore mandatory for root - parent_link_field = ("parent_" + scrub(doc.doctype)) + parent_link_field = "parent_" + scrub(doc.doctype) if doc.meta.get_field(parent_link_field) and not doc.get(parent_link_field): doc.flags.ignore_mandatory = True @@ -399,7 +445,7 @@ def make_records(records, debug=False): # print("Failed to insert duplicate {0} {1}".format(doctype, doc.name)) # pass DuplicateEntryError and continue - if e.args and e.args[0]==doc.doctype and e.args[1]==doc.name: + if e.args and e.args[0] == doc.doctype and e.args[1] == doc.name: # make sure DuplicateEntryError is for the exact same doc and not a related doc frappe.clear_messages() else: @@ -407,7 +453,7 @@ def make_records(records, debug=False): except Exception as e: frappe.db.rollback() - exception = record.get('__exception') + exception = record.get("__exception") if exception: config = _dict(exception) if isinstance(e, config.exception): diff --git a/frappe/desk/page/user_profile/user_profile.py b/frappe/desk/page/user_profile/user_profile.py index 0d91fd0d91..f9c8d98869 100644 --- a/frappe/desk/page/user_profile/user_profile.py +++ b/frappe/desk/page/user_profile/user_profile.py @@ -1,7 +1,9 @@ -import frappe from datetime import datetime + +import frappe from frappe.utils import getdate + @frappe.whitelist() def get_energy_points_heatmap_data(user, date): try: @@ -9,7 +11,9 @@ def get_energy_points_heatmap_data(user, date): except Exception: date = getdate() - return dict(frappe.db.sql("""select unix_timestamp(date(creation)), sum(points) + return dict( + frappe.db.sql( + """select unix_timestamp(date(creation)), sum(points) from `tabEnergy Point Log` where date(creation) > subdate('{date}', interval 1 year) and @@ -17,68 +21,93 @@ def get_energy_points_heatmap_data(user, date): user = %s and type != 'Review' group by date(creation) - order by creation asc""".format(date = date), user)) + order by creation asc""".format( + date=date + ), + user, + ) + ) @frappe.whitelist() def get_energy_points_percentage_chart_data(user, field): - result = frappe.db.get_all('Energy Point Log', - filters = {'user': user, 'type': ['!=', 'Review']}, - group_by = field, - order_by = field, - fields = [field, 'ABS(sum(points)) as points'], - as_list = True) + result = frappe.db.get_all( + "Energy Point Log", + filters={"user": user, "type": ["!=", "Review"]}, + group_by=field, + order_by=field, + fields=[field, "ABS(sum(points)) as points"], + as_list=True, + ) return { "labels": [r[0] for r in result if r[0] is not None], - "datasets": [{ - "values": [r[1] for r in result] - }] + "datasets": [{"values": [r[1] for r in result]}], } + @frappe.whitelist() def get_user_rank(user): month_start = datetime.today().replace(day=1) - monthly_rank = frappe.db.get_all('Energy Point Log', - group_by = 'user', - filters = {'creation': ['>', month_start], 'type' : ['!=', 'Review']}, - fields = ['user', 'sum(points)'], - order_by = 'sum(points) desc', - as_list = True) - - all_time_rank = frappe.db.get_all('Energy Point Log', - group_by = 'user', - filters = {'type' : ['!=', 'Review']}, - fields = ['user', 'sum(points)'], - order_by = 'sum(points) desc', - as_list = True) + monthly_rank = frappe.db.get_all( + "Energy Point Log", + group_by="user", + filters={"creation": [">", month_start], "type": ["!=", "Review"]}, + fields=["user", "sum(points)"], + order_by="sum(points) desc", + as_list=True, + ) + + all_time_rank = frappe.db.get_all( + "Energy Point Log", + group_by="user", + filters={"type": ["!=", "Review"]}, + fields=["user", "sum(points)"], + order_by="sum(points) desc", + as_list=True, + ) return { - 'monthly_rank': [i+1 for i, r in enumerate(monthly_rank) if r[0] == user], - 'all_time_rank': [i+1 for i, r in enumerate(all_time_rank) if r[0] == user] + "monthly_rank": [i + 1 for i, r in enumerate(monthly_rank) if r[0] == user], + "all_time_rank": [i + 1 for i, r in enumerate(all_time_rank) if r[0] == user], } @frappe.whitelist() def update_profile_info(profile_info): profile_info = frappe.parse_json(profile_info) - keys = ['location', 'interest', 'user_image', 'bio'] + keys = ["location", "interest", "user_image", "bio"] for key in keys: if key not in profile_info: profile_info[key] = None - user = frappe.get_doc('User', frappe.session.user) + user = frappe.get_doc("User", frappe.session.user) user.update(profile_info) user.save() return user + @frappe.whitelist() def get_energy_points_list(start, limit, user): - return frappe.db.get_list('Energy Point Log', - filters = {'user': user, 'type': ['!=', 'Review']}, - fields = ['name','user', 'points', 'reference_doctype', 'reference_name', 'reason', - 'type', 'seen', 'rule', 'owner', 'creation', 'revert_of'], - start = start, - limit = limit, - order_by = 'creation desc') + return frappe.db.get_list( + "Energy Point Log", + filters={"user": user, "type": ["!=", "Review"]}, + fields=[ + "name", + "user", + "points", + "reference_doctype", + "reference_name", + "reason", + "type", + "seen", + "rule", + "owner", + "creation", + "revert_of", + ], + start=start, + limit=limit, + order_by="creation desc", + ) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index f5f50b14fe..7de8ccabbf 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -1,27 +1,27 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import os import json +import os +from datetime import timedelta +import frappe +import frappe.desk.reportview from frappe import _ -from frappe.modules import scrub, get_module_path +from frappe.core.utils import ljust_list +from frappe.model.utils import render_include +from frappe.modules import get_module_path, scrub +from frappe.permissions import get_role_permissions +from frappe.translate import send_translations from frappe.utils import ( - flt, cint, cstr, + flt, + format_duration, get_html_format, get_url_to_form, gzip_decompress, - format_duration, ) -from frappe.model.utils import render_include -from frappe.translate import send_translations -import frappe.desk.reportview -from frappe.permissions import get_role_permissions -from datetime import timedelta -from frappe.core.utils import ljust_list def get_report_doc(report_name): @@ -47,9 +47,7 @@ def get_report_doc(report_name): if not frappe.has_permission(doc.ref_doctype, "report"): frappe.throw( - _("You don't have permission to get a report on: {0}").format( - doc.ref_doctype - ), + _("You don't have permission to get a report on: {0}").format(doc.ref_doctype), frappe.PermissionError, ) @@ -72,8 +70,11 @@ def get_report_result(report, filters): return res + @frappe.read_only() -def generate_report_result(report, filters=None, user=None, custom_columns=None, is_tree=False, parent_field=None): +def generate_report_result( + report, filters=None, user=None, custom_columns=None, is_tree=False, parent_field=None +): user = user or frappe.session.user filters = filters or [] @@ -99,7 +100,9 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None, columns.insert(custom_column["insert_after_index"] + 1, custom_column) # all columns which are not in original report - report_custom_columns = [column for column in columns if column["fieldname"] not in report_column_names] + report_custom_columns = [ + column for column in columns if column["fieldname"] not in report_column_names + ] if report_custom_columns: result = add_custom_column_data(report_custom_columns, result) @@ -118,10 +121,10 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None, "report_summary": report_summary, "skip_total_row": skip_total_row or 0, "status": None, - "execution_time": frappe.cache().hget("report_execution_time", report.name) - or 0, + "execution_time": frappe.cache().hget("report_execution_time", report.name) or 0, } + def normalize_result(result, columns): # Converts to list of dicts from list of lists/tuples data = [] @@ -137,6 +140,7 @@ def normalize_result(result, columns): return data + @frappe.whitelist() def background_enqueue_run(report_name, filters=None, user=None): """run reports in background""" @@ -169,14 +173,12 @@ def background_enqueue_run(report_name, filters=None, user=None): @frappe.whitelist() def get_script(report_name): report = get_report_doc(report_name) - module = report.module or frappe.db.get_value( - "DocType", report.ref_doctype, "module" - ) + module = report.module or frappe.db.get_value("DocType", report.ref_doctype, "module") is_custom_module = frappe.get_cached_value("Module Def", module, "custom") # custom modules are virtual modules those exists in DB but not in disk. - module_path = '' if is_custom_module else get_module_path(module) + module_path = "" if is_custom_module else get_module_path(module) report_folder = module_path and os.path.join(module_path, "report", scrub(report.name)) script_path = report_folder and os.path.join(report_folder, scrub(report.name) + ".js") print_path = report_folder and os.path.join(report_folder, scrub(report.name) + ".html") @@ -203,14 +205,21 @@ def get_script(report_name): return { "script": render_include(script), "html_format": html_format, - "execution_time": frappe.cache().hget("report_execution_time", report_name) - or 0, + "execution_time": frappe.cache().hget("report_execution_time", report_name) or 0, } @frappe.whitelist() @frappe.read_only() -def run(report_name, filters=None, user=None, ignore_prepared_report=False, custom_columns=None, is_tree=False, parent_field=None): +def run( + report_name, + filters=None, + user=None, + ignore_prepared_report=False, + custom_columns=None, + is_tree=False, + parent_field=None, +): report = get_report_doc(report_name) if not user: user = frappe.session.user @@ -240,9 +249,7 @@ def run(report_name, filters=None, user=None, ignore_prepared_report=False, cust else: result = generate_report_result(report, filters, user, custom_columns, is_tree, parent_field) - result["add_total_row"] = report.add_total_row and not result.get( - "skip_total_row", False - ) + result["add_total_row"] = report.add_total_row and not result.get("skip_total_row", False) return result @@ -251,14 +258,14 @@ def add_custom_column_data(custom_columns, result): custom_column_data = get_data_for_custom_report(custom_columns) for column in custom_columns: - key = (column.get('doctype'), column.get('fieldname')) + key = (column.get("doctype"), column.get("fieldname")) if key in custom_column_data: for row in result: - row_reference = row.get(column.get('link_field')) + row_reference = row.get(column.get("link_field")) # possible if the row is empty if not row_reference: continue - row[column.get('fieldname')] = custom_column_data.get(key).get(row_reference) + row[column.get("fieldname")] = custom_column_data.get(key).get(row_reference) return result @@ -382,7 +389,7 @@ def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=F if column.get("hidden"): continue result[0].append(_(column.get("label"))) - column_width = cint(column.get('width', 0)) + column_width = cint(column.get("width", 0)) # to convert into scale accepted by openpyxl column_width /= 10 column_widths.append(column_width) @@ -441,9 +448,7 @@ def add_total_row(result, columns, meta=None, is_tree=False, parent_field=None): if i >= len(row): continue cell = row.get(fieldname) if isinstance(row, dict) else row[i] - if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt( - cell - ): + if fieldtype in ["Currency", "Int", "Float", "Percent", "Duration"] and flt(cell): if not (is_tree and row.get(parent_field)): total_row[i] = flt(total_row[i]) + flt(cell) @@ -456,11 +461,7 @@ def add_total_row(result, columns, meta=None, is_tree=False, parent_field=None): total_row[i] = total_row[i] + cell if fieldtype == "Link" and options == "Currency": - total_row[i] = ( - result[0].get(fieldname) - if isinstance(result[0], dict) - else result[0][i] - ) + total_row[i] = result[0].get(fieldname) if isinstance(result[0], dict) else result[0][i] for i in has_percent: total_row[i] = flt(total_row[i]) / len(result) @@ -498,9 +499,7 @@ def get_data_for_custom_report(columns): if column.get("link_field"): fieldname = column.get("fieldname") doctype = column.get("doctype") - doc_field_value_map[(doctype, fieldname)] = get_data_for_custom_field( - doctype, fieldname - ) + doc_field_value_map[(doctype, fieldname)] = get_data_for_custom_field(doctype, fieldname) return doc_field_value_map @@ -522,7 +521,7 @@ def save_report(reference_report, report_name, columns): report = frappe.get_doc("Report", docname) existing_jd = json.loads(report.json) existing_jd["columns"] = json.loads(columns) - report.update({"json": json.dumps(existing_jd, separators=(',', ':'))}) + report.update({"json": json.dumps(existing_jd, separators=(",", ":"))}) report.save() frappe.msgprint(_("Report updated successfully")) @@ -556,11 +555,7 @@ def get_filtered_data(ref_doctype, columns, data, user): if match_filters_per_doctype: for row in data: # Why linked_doctypes.get(ref_doctype)? because if column is empty, linked_doctypes[ref_doctype] is removed - if ( - linked_doctypes.get(ref_doctype) - and shared - and row[linked_doctypes[ref_doctype]] in shared - ): + if linked_doctypes.get(ref_doctype) and shared and row[linked_doctypes[ref_doctype]] in shared: result.append(row) elif has_match( @@ -610,11 +605,7 @@ def has_match( if doctype == ref_doctype and if_owner: idx = linked_doctypes.get("User") - if ( - idx is not None - and row[idx] == user - and columns_dict[idx] == columns_dict.get("owner") - ): + if idx is not None and row[idx] == user and columns_dict[idx] == columns_dict.get("owner"): # owner match is true matched_for_doctype = True diff --git a/frappe/desk/report/todo/todo.py b/frappe/desk/report/todo/todo.py index b1e49bc95d..7c566c74ea 100644 --- a/frappe/desk/report/todo/todo.py +++ b/frappe/desk/report/todo/todo.py @@ -5,29 +5,65 @@ import frappe from frappe import _ from frappe.utils import getdate + def execute(filters=None): priority_map = {"High": 3, "Medium": 2, "Low": 1} - todo_list = frappe.get_list('ToDo', fields=["name", "date", "description", - "priority", "reference_type", "reference_name", "assigned_by", "owner"], - filters={'status': 'Open'}) + todo_list = frappe.get_list( + "ToDo", + fields=[ + "name", + "date", + "description", + "priority", + "reference_type", + "reference_name", + "assigned_by", + "owner", + ], + filters={"status": "Open"}, + ) - todo_list.sort(key=lambda todo: (priority_map.get(todo.priority, 0), - todo.date and getdate(todo.date) or getdate("1900-01-01")), reverse=True) + todo_list.sort( + key=lambda todo: ( + priority_map.get(todo.priority, 0), + todo.date and getdate(todo.date) or getdate("1900-01-01"), + ), + reverse=True, + ) - columns = [_("ID")+":Link/ToDo:90", _("Priority")+"::60", _("Date")+ ":Date", - _("Description")+"::150", _("Assigned To/Owner") + ":Data:120", - _("Assigned By")+":Data:120", _("Reference")+"::200"] + columns = [ + _("ID") + ":Link/ToDo:90", + _("Priority") + "::60", + _("Date") + ":Date", + _("Description") + "::150", + _("Assigned To/Owner") + ":Data:120", + _("Assigned By") + ":Data:120", + _("Reference") + "::200", + ] result = [] for todo in todo_list: - if todo.owner==frappe.session.user or todo.assigned_by==frappe.session.user: + if todo.owner == frappe.session.user or todo.assigned_by == frappe.session.user: if todo.reference_type: - todo.reference = """%s: %s""" % (todo.reference_type, - todo.reference_name, todo.reference_type, todo.reference_name) + todo.reference = """%s: %s""" % ( + todo.reference_type, + todo.reference_name, + todo.reference_type, + todo.reference_name, + ) else: todo.reference = None - result.append([todo.name, todo.priority, todo.date, todo.description, - todo.owner, todo.assigned_by, todo.reference]) + result.append( + [ + todo.name, + todo.priority, + todo.date, + todo.description, + todo.owner, + todo.assigned_by, + todo.reference, + ] + ) - return columns, result \ No newline at end of file + return columns, result diff --git a/frappe/desk/report_dump.py b/frappe/desk/report_dump.py index f57ed97fa5..ac01cf892f 100644 --- a/frappe/desk/report_dump.py +++ b/frappe/desk/report_dump.py @@ -2,9 +2,11 @@ # License: MIT. See LICENSE -import frappe -import json import copy +import json + +import frappe + @frappe.whitelist() def get_data(doctypes, last_modified): @@ -19,7 +21,7 @@ def get_data(doctypes, last_modified): for d in doctypes: args = copy.deepcopy(data_map[d]) - dt = d.find("[") != -1 and d[:d.find("[")] or d + dt = d.find("[") != -1 and d[: d.find("[")] or d out[dt] = {} if args.get("from"): @@ -32,10 +34,14 @@ def get_data(doctypes, last_modified): if d in last_modified: if not args.get("conditions"): - args['conditions'] = [] - args['conditions'].append(modified_table + "modified > '" + last_modified[d] + "'") - out[dt]["modified_names"] = frappe.db.sql_list("""select %sname from %s - where %smodified > %s""" % (modified_table, table, modified_table, "%s"), last_modified[d]) + args["conditions"] = [] + args["conditions"].append(modified_table + "modified > '" + last_modified[d] + "'") + out[dt]["modified_names"] = frappe.db.sql_list( + """select %sname from %s + where %smodified > %s""" + % (modified_table, table, modified_table, "%s"), + last_modified[d], + ) if args.get("force_index"): conditions = " force index (%s) " % args["force_index"] @@ -44,16 +50,23 @@ def get_data(doctypes, last_modified): if args.get("order_by"): order_by = " order by " + args["order_by"] - out[dt]["data"] = [list(t) for t in frappe.db.sql("""select %s from %s %s %s""" \ - % (",".join(args["columns"]), table, conditions, order_by))] + out[dt]["data"] = [ + list(t) + for t in frappe.db.sql( + """select %s from %s %s %s""" % (",".join(args["columns"]), table, conditions, order_by) + ) + ] # last modified modified_table = table if "," in table: modified_table = " ".join(table.split(",")[0].split(" ")[:-1]) - tmp = frappe.db.sql("""select `modified` - from %s order by modified desc limit 1""" % modified_table) + tmp = frappe.db.sql( + """select `modified` + from %s order by modified desc limit 1""" + % modified_table + ) out[dt]["last_modified"] = tmp and tmp[0][0] or "" out[dt]["columns"] = list(map(lambda c: c.split(" as ")[-1], args["columns"])) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 1ec8ede62e..b45f80f6ff 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -3,16 +3,18 @@ """build query for doclistview and return results""" -import frappe, json +import json +from io import StringIO + +import frappe import frappe.permissions -from frappe.model.db_query import DatabaseQuery -from frappe.model import default_fields, optional_fields, child_table_fields from frappe import _ -from io import StringIO from frappe.core.doctype.access_log.access_log import make_access_log -from frappe.utils import cstr, format_duration +from frappe.model import child_table_fields, default_fields, optional_fields from frappe.model.base_document import get_controller -from frappe.utils import add_user_info +from frappe.model.db_query import DatabaseQuery +from frappe.utils import add_user_info, cstr, format_duration + @frappe.whitelist() @frappe.read_only() @@ -26,6 +28,7 @@ def get(): data = compress(execute(**args), args=args) return data + @frappe.whitelist() @frappe.read_only() def get_list(): @@ -40,6 +43,7 @@ def get_list(): return data + @frappe.whitelist() @frappe.read_only() def get_count(): @@ -49,15 +53,17 @@ def get_count(): controller = get_controller(args.doctype) data = controller(args.doctype).get_count(args) else: - distinct = 'distinct ' if args.distinct=='true' else '' + distinct = "distinct " if args.distinct == "true" else "" args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"] - data = execute(**args)[0].get('total_count') + data = execute(**args)[0].get("total_count") return data + def execute(doctype, *args, **kwargs): return DatabaseQuery(doctype).execute(*args, **kwargs) + def get_form_params(): """Stringify GET request parameters.""" data = frappe._dict(frappe.local.form_dict) @@ -65,6 +71,7 @@ def get_form_params(): validate_args(data) return data + def validate_args(data): parse_json(data) setup_group_by(data) @@ -79,6 +86,7 @@ def validate_args(data): return data + def validate_fields(data): wildcard = update_wildcard_field_param(data) @@ -96,19 +104,20 @@ def validate_fields(data): raise_invalid_field(fieldname) # remove the field from the query if the report hide flag is set and current view is Report - if df.report_hide and data.view == 'Report': + if df.report_hide and data.view == "Report": data.fields.remove(field) continue if df.fieldname in [_df.fieldname for _df in meta.get_high_permlevel_fields()]: - if df.get('permlevel') not in meta.get_permlevel_access(parenttype=data.doctype): + if df.get("permlevel") not in meta.get_permlevel_access(parenttype=data.doctype): data.fields.remove(field) + def validate_filters(data, filters): if isinstance(filters, list): # filters as list for condition in filters: - if len(condition)==3: + if len(condition) == 3: # [fieldname, condition, value] fieldname = condition[0] if is_standard(fieldname): @@ -133,60 +142,71 @@ def validate_filters(data, filters): if not df: raise_invalid_field(fieldname) + def setup_group_by(data): - '''Add columns for aggregated values e.g. count(name)''' + """Add columns for aggregated values e.g. count(name)""" if data.group_by and data.aggregate_function: - if data.aggregate_function.lower() not in ('count', 'sum', 'avg'): - frappe.throw(_('Invalid aggregate function')) + if data.aggregate_function.lower() not in ("count", "sum", "avg"): + frappe.throw(_("Invalid aggregate function")) if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field): - data.fields.append('{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column'.format(**data)) + data.fields.append( + "{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column".format( + **data + ) + ) if data.aggregate_on_field: data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`") else: raise_invalid_field(data.aggregate_on_field) - data.pop('aggregate_on_doctype') - data.pop('aggregate_on_field') - data.pop('aggregate_function') + data.pop("aggregate_on_doctype") + data.pop("aggregate_on_field") + data.pop("aggregate_function") + def raise_invalid_field(fieldname): - frappe.throw(_('Field not permitted in query') + ': {0}'.format(fieldname), frappe.DataError) + frappe.throw(_("Field not permitted in query") + ": {0}".format(fieldname), frappe.DataError) + def is_standard(fieldname): - if '.' in fieldname: + if "." in fieldname: parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None) - return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields + return ( + fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields + ) + def extract_fieldname(field): - for text in (',', '/*', '#'): + for text in (",", "/*", "#"): if text in field: raise_invalid_field(field) fieldname = field - for sep in (' as ', ' AS '): + for sep in (" as ", " AS "): if sep in fieldname: fieldname = fieldname.split(sep)[0] # certain functions allowed, extract the fieldname from the function - if (fieldname.startswith('count(') - or fieldname.startswith('sum(') - or fieldname.startswith('avg(')): - if not fieldname.strip().endswith(')'): + if fieldname.startswith("count(") or fieldname.startswith("sum(") or fieldname.startswith("avg("): + if not fieldname.strip().endswith(")"): raise_invalid_field(field) - fieldname = fieldname.split('(', 1)[1][:-1] + fieldname = fieldname.split("(", 1)[1][:-1] return fieldname + def get_meta_and_docfield(fieldname, data): parenttype, fieldname = get_parenttype_and_fieldname(fieldname, data) meta = frappe.get_meta(parenttype) df = meta.get_field(fieldname) return meta, df + def update_wildcard_field_param(data): - if ((isinstance(data.fields, str) and data.fields == "*") - or (isinstance(data.fields, (list, tuple)) and len(data.fields) == 1 and data.fields[0] == "*")): + if (isinstance(data.fields, str) and data.fields == "*") or ( + isinstance(data.fields, (list, tuple)) and len(data.fields) == 1 and data.fields[0] == "*" + ): data.fields = frappe.db.get_table_columns(data.doctype) return True @@ -194,17 +214,10 @@ def update_wildcard_field_param(data): def clean_params(data): - for param in ( - "cmd", - "data", - "ignore_permissions", - "view", - "user", - "csrf_token", - "join" - ): + for param in ("cmd", "data", "ignore_permissions", "view", "user", "csrf_token", "join"): data.pop(param, None) + def parse_json(data): if isinstance(data.get("filters"), str): data["filters"] = json.loads(data["filters"]) @@ -229,13 +242,15 @@ def get_parenttype_and_fieldname(field, data): return parenttype, fieldname + def compress(data, args=None): """separate keys and values""" from frappe.desk.query_report import add_total_row user_info = {} - if not data: return data + if not data: + return data if args is None: args = {} values = [] @@ -255,34 +270,25 @@ def compress(data, args=None): meta = frappe.get_meta(args.doctype) values = add_total_row(values, keys, meta) - return { - "keys": keys, - "values": values, - "user_info": user_info - } + return {"keys": keys, "values": values, "user_info": user_info} + @frappe.whitelist() def save_report(name, doctype, report_settings): """Save reports of type Report Builder from Report View""" - if frappe.db.exists('Report', name): - report = frappe.get_doc('Report', name) + if frappe.db.exists("Report", name): + report = frappe.get_doc("Report", name) if report.is_standard == "Yes": frappe.throw(_("Standard Reports cannot be edited")) if report.report_type != "Report Builder": frappe.throw(_("Only reports of type Report Builder can be edited")) - if ( - report.owner != frappe.session.user - and not frappe.has_permission("Report", "write") - ): - frappe.throw( - _("Insufficient Permissions for editing Report"), - frappe.PermissionError - ) + if report.owner != frappe.session.user and not frappe.has_permission("Report", "write"): + frappe.throw(_("Insufficient Permissions for editing Report"), frappe.PermissionError) else: - report = frappe.new_doc('Report') + report = frappe.new_doc("Report") report.report_name = name report.ref_doctype = doctype @@ -296,6 +302,7 @@ def save_report(name, doctype, report_settings): ) return report.name + @frappe.whitelist() def delete_report(name): """Delete reports of type Report Builder from Report View""" @@ -307,14 +314,8 @@ def delete_report(name): if report.report_type != "Report Builder": frappe.throw(_("Only reports of type Report Builder can be deleted")) - if ( - report.owner != frappe.session.user - and not frappe.has_permission("Report", "delete") - ): - frappe.throw( - _("Insufficient Permissions for deleting Report"), - frappe.PermissionError - ) + if report.owner != frappe.session.user and not frappe.has_permission("Report", "delete"): + frappe.throw(_("Insufficient Permissions for deleting Report"), frappe.PermissionError) report.delete(ignore_permissions=True) frappe.msgprint( @@ -323,12 +324,13 @@ def delete_report(name): alert=True, ) + @frappe.whitelist() @frappe.read_only() def export_query(): """export from report builder""" title = frappe.form_dict.title - frappe.form_dict.pop('title', None) + frappe.form_dict.pop("title", None) form_params = get_form_params() form_params["limit_page_length"] = None @@ -341,21 +343,23 @@ def export_query(): del form_params["doctype"] del form_params["file_format_type"] - if 'add_totals_row' in form_params and form_params['add_totals_row']=='1': + if "add_totals_row" in form_params and form_params["add_totals_row"] == "1": add_totals_row = 1 del form_params["add_totals_row"] frappe.permissions.can_export(doctype, raise_exception=True) - if 'selected_items' in form_params: - si = json.loads(frappe.form_dict.get('selected_items')) + if "selected_items" in form_params: + si = json.loads(frappe.form_dict.get("selected_items")) form_params["filters"] = {"name": ("in", si)} del form_params["selected_items"] - make_access_log(doctype=doctype, + make_access_log( + doctype=doctype, file_type=file_format_type, report_name=form_params.report_name, - filters=form_params.filters) + filters=form_params.filters, + ) db_query = DatabaseQuery(doctype) ret = db_query.execute(**form_params) @@ -363,9 +367,9 @@ def export_query(): if add_totals_row: ret = append_totals_row(ret) - data = [[_('Sr')] + get_labels(db_query.fields, doctype)] + data = [[_("Sr")] + get_labels(db_query.fields, doctype)] for i, row in enumerate(ret): - data.append([i+1] + list(row)) + data.append([i + 1] + list(row)) data = handle_duration_fieldtype_values(doctype, data, db_query.fields) @@ -373,28 +377,29 @@ def export_query(): # convert to csv import csv + from frappe.utils.xlsxutils import handle_html f = StringIO() writer = csv.writer(f) for r in data: # encode only unicode type strings and not int, floats etc. - writer.writerow([handle_html(frappe.as_unicode(v)) \ - if isinstance(v, str) else v for v in r]) + writer.writerow([handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r]) f.seek(0) - frappe.response['result'] = cstr(f.read()) - frappe.response['type'] = 'csv' - frappe.response['doctype'] = title + frappe.response["result"] = cstr(f.read()) + frappe.response["type"] = "csv" + frappe.response["doctype"] = title elif file_format_type == "Excel": from frappe.utils.xlsxutils import make_xlsx + xlsx_file = make_xlsx(data, doctype) - frappe.response['filename'] = title + '.xlsx' - frappe.response['filecontent'] = xlsx_file.getvalue() - frappe.response['type'] = 'binary' + frappe.response["filename"] = title + ".xlsx" + frappe.response["filecontent"] = xlsx_file.getvalue() + frappe.response["type"] = "binary" def append_totals_row(data): @@ -402,7 +407,7 @@ def append_totals_row(data): return data data = list(data) totals = [] - totals.extend([""]*len(data[0])) + totals.extend([""] * len(data[0])) for row in data: for i in range(len(row)): @@ -410,19 +415,20 @@ def append_totals_row(data): totals[i] = (totals[i] or 0) + row[i] if not isinstance(totals[0], (int, float)): - totals[0] = 'Total' + totals[0] = "Total" data.append(totals) return data + def get_labels(fields, doctype): """get column labels based on column names""" labels = [] for key in fields: key = key.split(" as ")[0] - if key.startswith(('count(', 'sum(', 'avg(')): + if key.startswith(("count(", "sum(", "avg(")): continue if "." in key: @@ -445,11 +451,13 @@ def get_labels(fields, doctype): return labels + def handle_duration_fieldtype_values(doctype, data, fields): for field in fields: key = field.split(" as ")[0] - if key.startswith(('count(', 'sum(', 'avg(')): continue + if key.startswith(("count(", "sum(", "avg(")): + continue if "." in key: parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`") @@ -459,7 +467,7 @@ def handle_duration_fieldtype_values(doctype, data, fields): df = frappe.get_meta(parenttype).get_field(fieldname) - if df and df.fieldtype == 'Duration': + if df and df.fieldtype == "Duration": index = fields.index(field) + 1 for i in range(1, len(data)): val_in_seconds = data[i][index] @@ -468,28 +476,31 @@ def handle_duration_fieldtype_values(doctype, data, fields): data[i][index] = duration_val return data + @frappe.whitelist() def delete_items(): """delete selected items""" import json - items = sorted(json.loads(frappe.form_dict.get('items')), reverse=True) - doctype = frappe.form_dict.get('doctype') + items = sorted(json.loads(frappe.form_dict.get("items")), reverse=True) + doctype = frappe.form_dict.get("doctype") if len(items) > 10: - frappe.enqueue('frappe.desk.reportview.delete_bulk', - doctype=doctype, items=items) + frappe.enqueue("frappe.desk.reportview.delete_bulk", doctype=doctype, items=items) else: delete_bulk(doctype, items) + def delete_bulk(doctype, items): for i, d in enumerate(items): try: frappe.delete_doc(doctype, d) if len(items) >= 5: - frappe.publish_realtime("progress", - dict(progress=[i+1, len(items)], title=_('Deleting {0}').format(doctype), description=d), - user=frappe.session.user) + frappe.publish_realtime( + "progress", + dict(progress=[i + 1, len(items)], title=_("Deleting {0}").format(doctype), description=d), + user=frappe.session.user, + ) # Commit after successful deletion frappe.db.commit() except Exception: @@ -497,6 +508,7 @@ def delete_bulk(doctype, items): # if not rollbacked, queries get committed on after_request method in app.py frappe.db.rollback() + @frappe.whitelist() @frappe.read_only() def get_sidebar_stats(stats, doctype, filters=None): @@ -512,6 +524,7 @@ def get_sidebar_stats(stats, doctype, filters=None): return {"stats": data} + @frappe.whitelist() @frappe.read_only() def get_stats(stats, doctype, filters=None): @@ -536,19 +549,21 @@ def get_stats(stats, doctype, filters=None): if tag not in columns: continue try: - tag_count = frappe.get_list(doctype, + tag_count = frappe.get_list( + doctype, fields=[tag, "count(*)"], - filters=filters + [[tag, '!=', '']], + filters=filters + [[tag, "!=", ""]], group_by=tag, as_list=True, distinct=1, ) - if tag == '_user_tags': + if tag == "_user_tags": stats[tag] = scrub_user_tags(tag_count) - no_tag_count = frappe.get_list(doctype, + no_tag_count = frappe.get_list( + doctype, fields=[tag, "count(*)"], - filters=filters + [[tag, "in", ('', ',')]], + filters=filters + [[tag, "in", ("", ",")]], as_list=True, group_by=tag, order_by=tag, @@ -568,34 +583,52 @@ def get_stats(stats, doctype, filters=None): return stats + @frappe.whitelist() def get_filter_dashboard_data(stats, doctype, filters=None): """get tags info""" import json + tags = json.loads(stats) filters = json.loads(filters or []) stats = {} columns = frappe.db.get_table_columns(doctype) for tag in tags: - if not tag["name"] in columns: continue + if not tag["name"] in columns: + continue tagcount = [] - if tag["type"] not in ['Date', 'Datetime']: - tagcount = frappe.get_list(doctype, + if tag["type"] not in ["Date", "Datetime"]: + tagcount = frappe.get_list( + doctype, fields=[tag["name"], "count(*)"], - filters = filters + ["ifnull(`%s`,'')!=''" % tag["name"]], - group_by = tag["name"], - as_list = True) + filters=filters + ["ifnull(`%s`,'')!=''" % tag["name"]], + group_by=tag["name"], + as_list=True, + ) - if tag["type"] not in ['Check','Select','Date','Datetime','Int', - 'Float','Currency','Percent'] and tag['name'] not in ['docstatus']: + if tag["type"] not in [ + "Check", + "Select", + "Date", + "Datetime", + "Int", + "Float", + "Currency", + "Percent", + ] and tag["name"] not in ["docstatus"]: stats[tag["name"]] = list(tagcount) if stats[tag["name"]]: - data =["No Data", frappe.get_list(doctype, - fields=[tag["name"], "count(*)"], - filters=filters + ["({0} = '' or {0} is null)".format(tag["name"])], - as_list=True)[0][1]] - if data and data[1]!=0: + data = [ + "No Data", + frappe.get_list( + doctype, + fields=[tag["name"], "count(*)"], + filters=filters + ["({0} = '' or {0} is null)".format(tag["name"])], + as_list=True, + )[0][1], + ] + if data and data[1] != 0: stats[tag["name"]].append(data) else: @@ -603,6 +636,7 @@ def get_filter_dashboard_data(stats, doctype, filters=None): return stats + def scrub_user_tags(tagcount): """rebuild tag list for tags""" rdict = {} @@ -610,7 +644,7 @@ def scrub_user_tags(tagcount): for t in tagdict: if not t: continue - alltags = t.split(',') + alltags = t.split(",") for tag in alltags: if tag: if tag not in rdict: @@ -624,22 +658,29 @@ def scrub_user_tags(tagcount): return rlist + # used in building query in queries.py def get_match_cond(doctype, as_condition=True): cond = DatabaseQuery(doctype).build_match_conditions(as_condition=as_condition) if not as_condition: return cond - return ((' and ' + cond) if cond else "").replace("%", "%%") + return ((" and " + cond) if cond else "").replace("%", "%%") + def build_match_conditions(doctype, user=None, as_condition=True): - match_conditions = DatabaseQuery(doctype, user=user).build_match_conditions(as_condition=as_condition) + match_conditions = DatabaseQuery(doctype, user=user).build_match_conditions( + as_condition=as_condition + ) if as_condition: return match_conditions.replace("%", "%%") else: return match_conditions -def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with_match_conditions=False): + +def get_filters_cond( + doctype, filters, conditions, ignore_permissions=None, with_match_conditions=False +): if isinstance(filters, str): filters = json.loads(filters) @@ -649,14 +690,24 @@ def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with filters = filters.items() flt = [] for f in filters: - if isinstance(f[1], str) and f[1][0] == '!': - flt.append([doctype, f[0], '!=', f[1][1:]]) - elif isinstance(f[1], (list, tuple)) and \ - f[1][0] in (">", "<", ">=", "<=", "!=", "like", "not like", "in", "not in", "between"): + if isinstance(f[1], str) and f[1][0] == "!": + flt.append([doctype, f[0], "!=", f[1][1:]]) + elif isinstance(f[1], (list, tuple)) and f[1][0] in ( + ">", + "<", + ">=", + "<=", + "!=", + "like", + "not like", + "in", + "not in", + "between", + ): flt.append([doctype, f[0], f[1][0], f[1][1]]) else: - flt.append([doctype, f[0], '=', f[1]]) + flt.append([doctype, f[0], "=", f[1]]) query = DatabaseQuery(doctype) query.filters = flt @@ -667,11 +718,11 @@ def get_filters_cond(doctype, filters, conditions, ignore_permissions=None, with query.build_filter_conditions(flt, conditions, ignore_permissions) - cond = ' and ' + ' and '.join(query.conditions) + cond = " and " + " and ".join(query.conditions) else: - cond = '' + cond = "" return cond + def is_virtual_doctype(doctype): return frappe.db.get_value("DocType", doctype, "is_virtual") - diff --git a/frappe/desk/search.py b/frappe/desk/search.py index dd22f821cf..ba4c5fb4fb 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -1,20 +1,23 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -# Search -import frappe, json -from frappe.utils import cstr, unique, cint -from frappe.permissions import has_permission -from frappe import _, is_whitelisted +import json import re + import wrapt +# Search +import frappe +from frappe import _, is_whitelisted +from frappe.permissions import has_permission +from frappe.utils import cint, cstr, unique + def sanitize_searchfield(searchfield): - blacklisted_keywords = ['select', 'delete', 'drop', 'update', 'case', 'and', 'or', 'like'] + blacklisted_keywords = ["select", "delete", "drop", "update", "case", "and", "or", "like"] def _raise_exception(searchfield): - frappe.throw(_('Invalid Search Field {0}').format(searchfield), frappe.DataError) + frappe.throw(_("Invalid Search Field {0}").format(searchfield), frappe.DataError) if len(searchfield) == 1: # do not allow special characters to pass as searchfields @@ -25,15 +28,15 @@ def sanitize_searchfield(searchfield): if len(searchfield) >= 3: # to avoid 1=1 - if '=' in searchfield: + if "=" in searchfield: _raise_exception(searchfield) # in mysql -- is used for commenting the query - elif ' --' in searchfield: + elif " --" in searchfield: _raise_exception(searchfield) # to avoid and, or and like - elif any(' {0} '.format(keyword) in searchfield.split() for keyword in blacklisted_keywords): + elif any(" {0} ".format(keyword) in searchfield.split() for keyword in blacklisted_keywords): _raise_exception(searchfield) # to avoid select, delete, drop, update and case @@ -45,19 +48,49 @@ def sanitize_searchfield(searchfield): if any(regex.match(f) for f in searchfield.split()): _raise_exception(searchfield) + # this is called by the Link Field @frappe.whitelist() -def search_link(doctype, txt, query=None, filters=None, page_length=20, searchfield=None, reference_doctype=None, ignore_user_permissions=False): - search_widget(doctype, txt.strip(), query, searchfield=searchfield, page_length=page_length, filters=filters, - reference_doctype=reference_doctype, ignore_user_permissions=ignore_user_permissions) +def search_link( + doctype, + txt, + query=None, + filters=None, + page_length=20, + searchfield=None, + reference_doctype=None, + ignore_user_permissions=False, +): + search_widget( + doctype, + txt.strip(), + query, + searchfield=searchfield, + page_length=page_length, + filters=filters, + reference_doctype=reference_doctype, + ignore_user_permissions=ignore_user_permissions, + ) frappe.response["results"] = build_for_autosuggest(frappe.response["values"], doctype=doctype) del frappe.response["values"] + # this is called by the search box @frappe.whitelist() -def search_widget(doctype, txt, query=None, searchfield=None, start=0, - page_length=20, filters=None, filter_fields=None, as_dict=False, reference_doctype=None, ignore_user_permissions=False): +def search_widget( + doctype, + txt, + query=None, + searchfield=None, + start=0, + page_length=20, + filters=None, + filter_fields=None, + as_dict=False, + reference_doctype=None, + ignore_user_permissions=False, +): start = cint(start) @@ -72,25 +105,28 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, standard_queries = frappe.get_hooks().standard_queries or {} - if query and query.split()[0].lower()!="select": + if query and query.split()[0].lower() != "select": # by method try: is_whitelisted(frappe.get_attr(query)) - frappe.response["values"] = frappe.call(query, doctype, txt, - searchfield, start, page_length, filters, as_dict=as_dict) + frappe.response["values"] = frappe.call( + query, doctype, txt, searchfield, start, page_length, filters, as_dict=as_dict + ) except frappe.exceptions.PermissionError as e: if frappe.local.conf.developer_mode: raise e else: - frappe.respond_as_web_page(title='Invalid Method', html='Method not found', - indicator_color='red', http_status_code=404) + frappe.respond_as_web_page( + title="Invalid Method", html="Method not found", indicator_color="red", http_status_code=404 + ) return except Exception as e: raise e elif not query and doctype in standard_queries: # from standard queries - search_widget(doctype, txt, standard_queries[doctype][0], - searchfield, start, page_length, filters) + search_widget( + doctype, txt, standard_queries[doctype][0], searchfield, start, page_length, filters + ) else: meta = frappe.get_meta(doctype) @@ -112,7 +148,6 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, filters = [] or_filters = [] - translated_search_doctypes = frappe.get_hooks("translated_search_doctypes") # build from doctype if txt: @@ -125,20 +160,26 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, for f in search_fields: fmeta = meta.get_field(f.strip()) - if (doctype not in translated_search_doctypes) and (f == "name" or (fmeta and fmeta.fieldtype in ["Data", "Text", "Small Text", "Long Text", - "Link", "Select", "Read Only", "Text Editor"])): - or_filters.append([doctype, f.strip(), "like", "%{0}%".format(txt)]) - - if meta.get("fields", {"fieldname":"enabled", "fieldtype":"Check"}): + if (doctype not in translated_search_doctypes) and ( + f == "name" + or ( + fmeta + and fmeta.fieldtype + in ["Data", "Text", "Small Text", "Long Text", "Link", "Select", "Read Only", "Text Editor"] + ) + ): + or_filters.append([doctype, f.strip(), "like", "%{0}%".format(txt)]) + + if meta.get("fields", {"fieldname": "enabled", "fieldtype": "Check"}): filters.append([doctype, "enabled", "=", 1]) - if meta.get("fields", {"fieldname":"disabled", "fieldtype":"Check"}): + if meta.get("fields", {"fieldname": "disabled", "fieldtype": "Check"}): filters.append([doctype, "disabled", "!=", 1]) # format a list of fields combining search fields and filter fields fields = get_std_fields_list(meta, searchfield or "name") if filter_fields: fields = list(set(fields + json.loads(filter_fields))) - formatted_fields = ['`tab%s`.`%s`' % (meta.name, f.strip()) for f in fields] + formatted_fields = ["`tab%s`.`%s`" % (meta.name, f.strip()) for f in fields] title_field_query = get_title_field_query(meta) @@ -147,23 +188,31 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, formatted_fields.insert(1, title_field_query) # find relevance as location of search term from the beginning of string `name`. used for sorting results. - formatted_fields.append("""locate({_txt}, `tab{doctype}`.`name`) as `_relevance`""".format( - _txt=frappe.db.escape((txt or "").replace("%", "").replace("@", "")), doctype=doctype)) - + formatted_fields.append( + """locate({_txt}, `tab{doctype}`.`name`) as `_relevance`""".format( + _txt=frappe.db.escape((txt or "").replace("%", "").replace("@", "")), doctype=doctype + ) + ) # In order_by, `idx` gets second priority, because it stores link count from frappe.model.db_query import get_order_by + order_by_based_on_meta = get_order_by(doctype, meta) # 2 is the index of _relevance column order_by = "_relevance, {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype) - ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read' - ignore_permissions = True if doctype == "DocType" else (cint(ignore_user_permissions) and has_permission(doctype, ptype=ptype)) + ptype = "select" if frappe.only_has_select_perm(doctype) else "read" + ignore_permissions = ( + True + if doctype == "DocType" + else (cint(ignore_user_permissions) and has_permission(doctype, ptype=ptype)) + ) if doctype in translated_search_doctypes: page_length = None - values = frappe.get_list(doctype, + values = frappe.get_list( + doctype, filters=filters, fields=formatted_fields, or_filters=or_filters, @@ -173,15 +222,15 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, ignore_permissions=ignore_permissions, reference_doctype=reference_doctype, as_list=not as_dict, - strict=False) + strict=False, + ) if doctype in translated_search_doctypes: # Filtering the values array so that query is included in very element values = ( - v for v in values - if re.search( - f"{re.escape(txt)}.*", _(v.name if as_dict else v[0]), re.IGNORECASE - ) + v + for v in values + if re.search(f"{re.escape(txt)}.*", _(v.name if as_dict else v[0]), re.IGNORECASE) ) # Sorting the values array so that relevant results always come first @@ -197,6 +246,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, else: frappe.response["values"] = [r[:-1] for r in values] + def get_std_fields_list(meta, key): # get additional search fields sflist = ["name"] @@ -213,9 +263,12 @@ def get_std_fields_list(meta, key): return sflist + def get_title_field_query(meta): title_field = meta.title_field if meta.title_field else None - show_title_field_in_link = meta.show_title_field_in_link if meta.show_title_field_in_link else None + show_title_field_in_link = ( + meta.show_title_field_in_link if meta.show_title_field_in_link else None + ) field = None if title_field and show_title_field_in_link: @@ -223,52 +276,52 @@ def get_title_field_query(meta): return field + def build_for_autosuggest(res, doctype): results = [] meta = frappe.get_meta(doctype) if not (meta.title_field and meta.show_title_field_in_link): for r in res: r = list(r) - results.append({ - "value": r[0], - "description": ", ".join(unique(cstr(d) for d in r[1:] if d)) - }) + results.append({"value": r[0], "description": ", ".join(unique(cstr(d) for d in r[1:] if d))}) else: title_field_exists = meta.title_field and meta.show_title_field_in_link - _from = 2 if title_field_exists else 1 # to exclude title from description if title_field_exists + _from = 2 if title_field_exists else 1 # to exclude title from description if title_field_exists for r in res: r = list(r) - results.append({ - "value": r[0], - "label": r[1] if title_field_exists else None, - "description": ", ".join(unique(cstr(d) for d in r[_from:] if d)) - }) + results.append( + { + "value": r[0], + "label": r[1] if title_field_exists else None, + "description": ", ".join(unique(cstr(d) for d in r[_from:] if d)), + } + ) return results + def scrub_custom_query(query, key, txt): - if '%(key)s' in query: - query = query.replace('%(key)s', key) - if '%s' in query: - query = query.replace('%s', ((txt or '') + '%')) + if "%(key)s" in query: + query = query.replace("%(key)s", key) + if "%s" in query: + query = query.replace("%s", ((txt or "") + "%")) return query + def relevance_sorter(key, query, as_dict): value = _(key.name if as_dict else key[0]) - return ( - cstr(value).lower().startswith(query.lower()) is not True, - value - ) + return (cstr(value).lower().startswith(query.lower()) is not True, value) + @wrapt.decorator def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): kwargs.update(dict(zip(fn.__code__.co_varnames, args))) - sanitize_searchfield(kwargs['searchfield']) - kwargs['start'] = cint(kwargs['start']) - kwargs['page_len'] = cint(kwargs['page_len']) + sanitize_searchfield(kwargs["searchfield"]) + kwargs["start"] = cint(kwargs["start"]) + kwargs["page_len"] = cint(kwargs["page_len"]) - if kwargs['doctype'] and not frappe.db.exists('DocType', kwargs['doctype']): + if kwargs["doctype"] and not frappe.db.exists("DocType", kwargs["doctype"]): return [] return fn(**kwargs) @@ -276,37 +329,41 @@ def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): @frappe.whitelist() def get_names_for_mentions(search_term): - users_for_mentions = frappe.cache().get_value('users_for_mentions', get_users_for_mentions) - user_groups = frappe.cache().get_value('user_groups', get_user_groups) + users_for_mentions = frappe.cache().get_value("users_for_mentions", get_users_for_mentions) + user_groups = frappe.cache().get_value("user_groups", get_user_groups) filtered_mentions = [] for mention_data in users_for_mentions + user_groups: if search_term.lower() not in mention_data.value.lower(): continue - mention_data['link'] = frappe.utils.get_url_to_form( - 'User Group' if mention_data.get('is_group') else 'User Profile', - mention_data['id'] + mention_data["link"] = frappe.utils.get_url_to_form( + "User Group" if mention_data.get("is_group") else "User Profile", mention_data["id"] ) filtered_mentions.append(mention_data) - return sorted(filtered_mentions, key=lambda d: d['value']) + return sorted(filtered_mentions, key=lambda d: d["value"]) + def get_users_for_mentions(): - return frappe.get_all('User', - fields=['name as id', 'full_name as value'], + return frappe.get_all( + "User", + fields=["name as id", "full_name as value"], filters={ - 'name': ['not in', ('Administrator', 'Guest')], - 'allowed_in_mentions': True, - 'user_type': 'System User', - 'enabled': True, - }) + "name": ["not in", ("Administrator", "Guest")], + "allowed_in_mentions": True, + "user_type": "System User", + "enabled": True, + }, + ) + def get_user_groups(): - return frappe.get_all('User Group', fields=['name as id', 'name as value'], update={ - 'is_group': True - }) + return frappe.get_all( + "User Group", fields=["name as id", "name as value"], update={"is_group": True} + ) + @frappe.whitelist() def get_link_title(doctype, docname): @@ -315,4 +372,4 @@ def get_link_title(doctype, docname): if meta.title_field and meta.show_title_field_in_link: return frappe.db.get_value(doctype, docname, meta.title_field) - return docname \ No newline at end of file + return docname diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py index 5e8fb18fe4..d7a8db8792 100644 --- a/frappe/desk/treeview.py +++ b/frappe/desk/treeview.py @@ -7,11 +7,11 @@ from frappe import _ @frappe.whitelist() def get_all_nodes(doctype, label, parent, tree_method, **filters): - '''Recursively gets all data from tree nodes''' + """Recursively gets all data from tree nodes""" - if 'cmd' in filters: - del filters['cmd'] - filters.pop('data', None) + if "cmd" in filters: + del filters["cmd"] + filters.pop("data", None) tree_method = frappe.get_attr(tree_method) @@ -21,43 +21,45 @@ def get_all_nodes(doctype, label, parent, tree_method, **filters): data = tree_method(doctype, parent, **filters) out = [dict(parent=label, data=data)] - if 'is_root' in filters: - del filters['is_root'] - to_check = [d.get('value') for d in data if d.get('expandable')] + if "is_root" in filters: + del filters["is_root"] + to_check = [d.get("value") for d in data if d.get("expandable")] while to_check: parent = to_check.pop() data = tree_method(doctype, parent, is_root=False, **filters) out.append(dict(parent=parent, data=data)) for d in data: - if d.get('expandable'): - to_check.append(d.get('value')) + if d.get("expandable"): + to_check.append(d.get("value")) return out + @frappe.whitelist() -def get_children(doctype, parent='', **filters): +def get_children(doctype, parent="", **filters): return _get_children(doctype, parent) -def _get_children(doctype, parent='', ignore_permissions=False): - parent_field = 'parent_' + doctype.lower().replace(' ', '_') - filters = [["ifnull(`{0}`,'')".format(parent_field), '=', parent], - ['docstatus', '<' ,2]] + +def _get_children(doctype, parent="", ignore_permissions=False): + parent_field = "parent_" + doctype.lower().replace(" ", "_") + filters = [["ifnull(`{0}`,'')".format(parent_field), "=", parent], ["docstatus", "<", 2]] meta = frappe.get_meta(doctype) return frappe.get_list( doctype, fields=[ - 'name as value', - '{0} as title'.format(meta.get('title_field') or 'name'), - 'is_group as expandable' + "name as value", + "{0} as title".format(meta.get("title_field") or "name"), + "is_group as expandable", ], filters=filters, - order_by='name', - ignore_permissions=ignore_permissions + order_by="name", + ignore_permissions=ignore_permissions, ) + @frappe.whitelist() def add_node(): args = make_tree_args(**frappe.form_dict) @@ -65,17 +67,18 @@ def add_node(): doc.save() + def make_tree_args(**kwarg): - kwarg.pop('cmd', None) + kwarg.pop("cmd", None) - doctype = kwarg['doctype'] - parent_field = 'parent_' + doctype.lower().replace(' ', '_') + doctype = kwarg["doctype"] + parent_field = "parent_" + doctype.lower().replace(" ", "_") - if kwarg['is_root'] == 'false': kwarg['is_root'] = False - if kwarg['is_root'] == 'true': kwarg['is_root'] = True + if kwarg["is_root"] == "false": + kwarg["is_root"] = False + if kwarg["is_root"] == "true": + kwarg["is_root"] = True - kwarg.update({ - parent_field: kwarg.get("parent") or kwarg.get(parent_field) - }) + kwarg.update({parent_field: kwarg.get("parent") or kwarg.get(parent_field)}) return frappe._dict(kwarg) diff --git a/frappe/desk/utils.py b/frappe/desk/utils.py index 3328d47318..72cba79963 100644 --- a/frappe/desk/utils.py +++ b/frappe/desk/utils.py @@ -3,21 +3,27 @@ import frappe + def validate_route_conflict(doctype, name): - ''' + """ Raises exception if name clashes with routes from other documents for /app routing - ''' + """ all_names = [] - for _doctype in ['Page', 'Workspace', 'DocType']: + for _doctype in ["Page", "Workspace", "DocType"]: try: - all_names.extend([slug(d) for d in frappe.get_all(_doctype, pluck='name') if (doctype != _doctype and d != name)]) + all_names.extend( + [ + slug(d) for d in frappe.get_all(_doctype, pluck="name") if (doctype != _doctype and d != name) + ] + ) except frappe.db.TableMissingError: pass if slug(name) in all_names: - frappe.msgprint(frappe._('Name already taken, please set a new name')) + frappe.msgprint(frappe._("Name already taken, please set a new name")) raise frappe.NameError + def slug(name): - return name.lower().replace(' ', '-') + return name.lower().replace(" ", "-") diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index 79dec977b7..fae60baebf 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -4,9 +4,11 @@ import frappe from frappe.desk.reportview import build_match_conditions + def sendmail_to_system_managers(subject, content): frappe.sendmail(recipients=get_system_managers(), subject=subject, content=content) + @frappe.whitelist() def get_contact_list(txt, page_length=20): """Returns contacts (from autosuggest)""" @@ -16,19 +18,19 @@ def get_contact_list(txt, page_length=20): return cached_contacts[:page_length] try: - match_conditions = build_match_conditions('Contact') + match_conditions = build_match_conditions("Contact") match_conditions = "and {0}".format(match_conditions) if match_conditions else "" - out = frappe.db.sql("""select email_id as value, + out = frappe.db.sql( + """select email_id as value, concat(first_name, ifnull(concat(' ',last_name), '' )) as description from tabContact where name like %(txt)s or email_id like %(txt)s %(condition)s - limit %(page_length)s""", { - 'txt': '%' + txt + '%', - 'condition': match_conditions, - 'page_length': page_length - }, as_dict=True) + limit %(page_length)s""", + {"txt": "%" + txt + "%", "condition": match_conditions, "page_length": page_length}, + as_dict=True, + ) out = filter(None, out) except: @@ -38,15 +40,20 @@ def get_contact_list(txt, page_length=20): return out + def get_system_managers(): - return frappe.db.sql_list("""select parent FROM `tabHas Role` + return frappe.db.sql_list( + """select parent FROM `tabHas Role` WHERE role='System Manager' AND parent!='Administrator' - AND parent IN (SELECT email FROM tabUser WHERE enabled=1)""") + AND parent IN (SELECT email FROM tabUser WHERE enabled=1)""" + ) + @frappe.whitelist() def relink(name, reference_doctype=None, reference_name=None): - frappe.db.sql("""update + frappe.db.sql( + """update `tabCommunication` set reference_doctype = %s, @@ -54,7 +61,10 @@ def relink(name, reference_doctype=None, reference_name=None): status = "Linked" where communication_type = "Communication" and - name = %s""", (reference_doctype, reference_name, name)) + name = %s""", + (reference_doctype, reference_name, name), + ) + @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs @@ -63,19 +73,22 @@ def get_communication_doctype(doctype, txt, searchfield, start, page_len, filter user_perms.build_permissions() can_read = user_perms.can_read from frappe.modules import load_doctype_module + com_doctypes = [] - if len(txt)<2: + if len(txt) < 2: for name in frappe.get_hooks("communication_doctypes"): try: - module = load_doctype_module(name, suffix='_dashboard') - if hasattr(module, 'get_data'): - for i in module.get_data()['transactions']: + module = load_doctype_module(name, suffix="_dashboard") + if hasattr(module, "get_data"): + for i in module.get_data()["transactions"]: com_doctypes += i["items"] except ImportError: pass else: - com_doctypes = [d[0] for d in frappe.db.get_values("DocType", {"issingle": 0, "istable": 0, "hide_toolbar": 0})] + com_doctypes = [ + d[0] for d in frappe.db.get_values("DocType", {"issingle": 0, "istable": 0, "hide_toolbar": 0}) + ] out = [] for dt in com_doctypes: @@ -83,6 +96,7 @@ def get_communication_doctype(doctype, txt, searchfield, start, page_len, filter out.append([dt]) return out + def get_cached_contacts(txt): contacts = frappe.cache().hget("contacts", frappe.session.user) or [] @@ -92,9 +106,14 @@ def get_cached_contacts(txt): if not txt: return contacts - match = [d for d in contacts if (d.value and ((d.value and txt in d.value) or (d.description and txt in d.description)))] + match = [ + d + for d in contacts + if (d.value and ((d.value and txt in d.value) or (d.description and txt in d.description))) + ] return match + def update_contact_cache(contacts): cached_contacts = frappe.cache().hget("contacts", frappe.session.user) or [] diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index abeb681a25..f4fdcf4275 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -7,20 +7,29 @@ from datetime import timedelta import frappe from frappe import _ +from frappe.desk.query_report import build_xlsx_data from frappe.model.document import Document -from frappe.utils import (format_time, get_link_to_form, get_url_to_report, - global_date_format, now, now_datetime, validate_email_address, today, add_to_date) from frappe.model.naming import append_number_if_name_exists +from frappe.utils import ( + add_to_date, + format_time, + get_link_to_form, + get_url_to_report, + global_date_format, + now, + now_datetime, + today, + validate_email_address, +) from frappe.utils.csvutils import to_csv from frappe.utils.xlsxutils import make_xlsx -from frappe.desk.query_report import build_xlsx_data class AutoEmailReport(Document): def autoname(self): self.name = _(self.report) - if frappe.db.exists('Auto Email Report', self.name): - self.name = append_number_if_name_exists('Auto Email Report', self.name) + if frappe.db.exists("Auto Email Report", self.name): + self.name = append_number_if_name_exists("Auto Email Report", self.name) def validate(self): self.validate_report_count() @@ -29,9 +38,9 @@ class AutoEmailReport(Document): self.validate_mandatory_fields() def validate_emails(self): - '''Cleanup list of emails''' - if ',' in self.email_to: - self.email_to.replace(',', '\n') + """Cleanup list of emails""" + if "," in self.email_to: + self.email_to.replace(",", "\n") valid = [] for email in self.email_to.split(): @@ -39,22 +48,27 @@ class AutoEmailReport(Document): validate_email_address(email, True) valid.append(email) - self.email_to = '\n'.join(valid) + self.email_to = "\n".join(valid) def validate_report_count(self): - '''check that there are only 3 enabled reports per user''' - count = frappe.db.sql('select count(*) from `tabAuto Email Report` where user=%s and enabled=1', self.user)[0][0] + """check that there are only 3 enabled reports per user""" + count = frappe.db.sql( + "select count(*) from `tabAuto Email Report` where user=%s and enabled=1", self.user + )[0][0] max_reports_per_user = frappe.local.conf.max_reports_per_user or 3 if count > max_reports_per_user + (-1 if self.flags.in_insert else 0): - frappe.throw(_('Only {0} emailed reports are allowed per user').format(max_reports_per_user)) + frappe.throw(_("Only {0} emailed reports are allowed per user").format(max_reports_per_user)) def validate_report_format(self): - """ check if user has select correct report format """ + """check if user has select correct report format""" valid_report_formats = ["HTML", "XLSX", "CSV"] if self.format not in valid_report_formats: - frappe.throw(_("{0} is not a valid report format. Report format should one of the following {1}") - .format(frappe.bold(self.format), frappe.bold(", ".join(valid_report_formats)))) + frappe.throw( + _("{0} is not a valid report format. Report format should one of the following {1}").format( + frappe.bold(self.format), frappe.bold(", ".join(valid_report_formats)) + ) + ) def validate_mandatory_fields(self): # Check if all Mandatory Report Filters are filled by the User @@ -63,77 +77,87 @@ class AutoEmailReport(Document): throw_list = [] for meta in filter_meta: if meta.get("reqd") and not filters.get(meta["fieldname"]): - throw_list.append(meta['label']) + throw_list.append(meta["label"]) if throw_list: frappe.throw( - title= _('Missing Filters Required'), - msg= _('Following Report Filters have missing values:') + - '

  • ' + '
  • '.join(throw_list) + '
', + title=_("Missing Filters Required"), + msg=_("Following Report Filters have missing values:") + + "

  • " + + "
  • ".join(throw_list) + + "
", ) def get_report_content(self): - '''Returns file in for the report in given format''' - report = frappe.get_doc('Report', self.report) + """Returns file in for the report in given format""" + report = frappe.get_doc("Report", self.report) self.filters = frappe.parse_json(self.filters) if self.filters else {} - if self.report_type=='Report Builder' and self.data_modified_till: - self.filters['modified'] = ('>', now_datetime() - timedelta(hours=self.data_modified_till)) + if self.report_type == "Report Builder" and self.data_modified_till: + self.filters["modified"] = (">", now_datetime() - timedelta(hours=self.data_modified_till)) - if self.report_type != 'Report Builder' and self.dynamic_date_filters_set(): + if self.report_type != "Report Builder" and self.dynamic_date_filters_set(): self.prepare_dynamic_filters() - columns, data = report.get_data(limit=self.no_of_rows or 100, user = self.user, - filters = self.filters, as_dict=True, ignore_prepared_report=True) + columns, data = report.get_data( + limit=self.no_of_rows or 100, + user=self.user, + filters=self.filters, + as_dict=True, + ignore_prepared_report=True, + ) # add serial numbers - columns.insert(0, frappe._dict(fieldname='idx', label='', width='30px')) + columns.insert(0, frappe._dict(fieldname="idx", label="", width="30px")) for i in range(len(data)): - data[i]['idx'] = i+1 + data[i]["idx"] = i + 1 - if len(data)==0 and self.send_if_data: + if len(data) == 0 and self.send_if_data: return None - if self.format == 'HTML': + if self.format == "HTML": columns, data = make_links(columns, data) columns = update_field_types(columns) return self.get_html_table(columns, data) - elif self.format == 'XLSX': + elif self.format == "XLSX": report_data = frappe._dict() - report_data['columns'] = columns - report_data['result'] = data + report_data["columns"] = columns + report_data["result"] = data xlsx_data, column_widths = build_xlsx_data(report_data, [], 1, ignore_visible_idx=True) xlsx_file = make_xlsx(xlsx_data, "Auto Email Report", column_widths=column_widths) return xlsx_file.getvalue() - elif self.format == 'CSV': + elif self.format == "CSV": report_data = frappe._dict() - report_data['columns'] = columns - report_data['result'] = data + report_data["columns"] = columns + report_data["result"] = data xlsx_data, column_widths = build_xlsx_data(report_data, [], 1, ignore_visible_idx=True) return to_csv(xlsx_data) else: - frappe.throw(_('Invalid Output Format')) + frappe.throw(_("Invalid Output Format")) def get_html_table(self, columns=None, data=None): - date_time = global_date_format(now()) + ' ' + format_time(now()) - report_doctype = frappe.db.get_value('Report', self.report, 'ref_doctype') - - return frappe.render_template('frappe/templates/emails/auto_email_report.html', { - 'title': self.name, - 'description': self.description, - 'date_time': date_time, - 'columns': columns, - 'data': data, - 'report_url': get_url_to_report(self.report, self.report_type, report_doctype), - 'report_name': self.report, - 'edit_report_settings': get_link_to_form('Auto Email Report', self.name) - }) + date_time = global_date_format(now()) + " " + format_time(now()) + report_doctype = frappe.db.get_value("Report", self.report, "ref_doctype") + + return frappe.render_template( + "frappe/templates/emails/auto_email_report.html", + { + "title": self.name, + "description": self.description, + "date_time": date_time, + "columns": columns, + "data": data, + "report_url": get_url_to_report(self.report, self.report_type, report_doctype), + "report_name": self.report, + "edit_report_settings": get_link_to_form("Auto Email Report", self.name), + }, + ) def get_file_name(self): return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower()) @@ -143,12 +167,12 @@ class AutoEmailReport(Document): to_date = today() from_date_value = { - 'Daily': ('days', -1), - 'Weekly': ('weeks', -1), - 'Monthly': ('months', -1), - 'Quarterly': ('months', -3), - 'Half Yearly': ('months', -6), - 'Yearly': ('years', -1) + "Daily": ("days", -1), + "Weekly": ("weeks", -1), + "Monthly": ("months", -1), + "Quarterly": ("months", -3), + "Half Yearly": ("months", -6), + "Yearly": ("years", -1), }[self.dynamic_date_period] from_date = add_to_date(to_date, **{from_date_value[0]: from_date_value[1]}) @@ -170,77 +194,79 @@ class AutoEmailReport(Document): else: message = self.get_html_table() - if not self.format=='HTML': - attachments = [{ - 'fname': self.get_file_name(), - 'fcontent': data - }] + if not self.format == "HTML": + attachments = [{"fname": self.get_file_name(), "fcontent": data}] frappe.sendmail( - recipients = self.email_to.split(), - subject = self.name, - message = message, - attachments = attachments, - reference_doctype = self.doctype, - reference_name = self.name + recipients=self.email_to.split(), + subject=self.name, + message=message, + attachments=attachments, + reference_doctype=self.doctype, + reference_name=self.name, ) def dynamic_date_filters_set(self): return self.dynamic_date_period and self.from_date_field and self.to_date_field + @frappe.whitelist() def download(name): - '''Download report locally''' - auto_email_report = frappe.get_doc('Auto Email Report', name) + """Download report locally""" + auto_email_report = frappe.get_doc("Auto Email Report", name) auto_email_report.check_permission() data = auto_email_report.get_report_content() if not data: - frappe.msgprint(_('No Data')) + frappe.msgprint(_("No Data")) return frappe.local.response.filecontent = data frappe.local.response.type = "download" frappe.local.response.filename = auto_email_report.get_file_name() + @frappe.whitelist() def send_now(name): - '''Send Auto Email report now''' - auto_email_report = frappe.get_doc('Auto Email Report', name) + """Send Auto Email report now""" + auto_email_report = frappe.get_doc("Auto Email Report", name) auto_email_report.check_permission() auto_email_report.send() + def send_daily(): - '''Check reports to be sent daily''' + """Check reports to be sent daily""" current_day = calendar.day_name[now_datetime().weekday()] - enabled_reports = frappe.get_all('Auto Email Report', - filters={'enabled': 1, 'frequency': ('in', ('Daily', 'Weekdays', 'Weekly'))}) + enabled_reports = frappe.get_all( + "Auto Email Report", filters={"enabled": 1, "frequency": ("in", ("Daily", "Weekdays", "Weekly"))} + ) for report in enabled_reports: - auto_email_report = frappe.get_doc('Auto Email Report', report.name) + auto_email_report = frappe.get_doc("Auto Email Report", report.name) # if not correct weekday, skip if auto_email_report.frequency == "Weekdays": if current_day in ("Saturday", "Sunday"): continue - elif auto_email_report.frequency == 'Weekly': + elif auto_email_report.frequency == "Weekly": if auto_email_report.day_of_week != current_day: continue try: auto_email_report.send() except Exception as e: - frappe.log_error(e, _('Failed to send {0} Auto Email Report').format(auto_email_report.name)) + frappe.log_error(e, _("Failed to send {0} Auto Email Report").format(auto_email_report.name)) def send_monthly(): - '''Check reports to be sent monthly''' - for report in frappe.get_all('Auto Email Report', {'enabled': 1, 'frequency': 'Monthly'}): - frappe.get_doc('Auto Email Report', report.name).send() + """Check reports to be sent monthly""" + for report in frappe.get_all("Auto Email Report", {"enabled": 1, "frequency": "Monthly"}): + frappe.get_doc("Auto Email Report", report.name).send() + def make_links(columns, data): for row in data: - doc_name = row.get('name') + doc_name = row.get("name") for col in columns: if not row.get(col.fieldname): continue @@ -257,9 +283,10 @@ def make_links(columns, data): row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc) return columns, data + def update_field_types(columns): for col in columns: - if col.fieldtype in ("Link", "Dynamic Link", "Currency") and col.options != "Currency": + if col.fieldtype in ("Link", "Dynamic Link", "Currency") and col.options != "Currency": col.fieldtype = "Data" col.options = "" return columns diff --git a/frappe/email/doctype/auto_email_report/test_auto_email_report.py b/frappe/email/doctype/auto_email_report/test_auto_email_report.py index 559adfbe1a..aa8dffb9e0 100644 --- a/frappe/email/doctype/auto_email_report/test_auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/test_auto_email_report.py @@ -10,54 +10,56 @@ from frappe.utils.data import is_html # test_records = frappe.get_test_records('Auto Email Report') + class TestAutoEmailReport(unittest.TestCase): def test_auto_email(self): - frappe.delete_doc('Auto Email Report', 'Permitted Documents For User') + frappe.delete_doc("Auto Email Report", "Permitted Documents For User") auto_email_report = get_auto_email_report() data = auto_email_report.get_report_content() self.assertTrue(is_html(data)) - self.assertTrue(str(get_link_to_form('Module Def', 'Core')) in data) + self.assertTrue(str(get_link_to_form("Module Def", "Core")) in data) - auto_email_report.format = 'CSV' + auto_email_report.format = "CSV" data = auto_email_report.get_report_content() self.assertTrue('"Language","Core"' in data) - auto_email_report.format = 'XLSX' + auto_email_report.format = "XLSX" data = auto_email_report.get_report_content() - def test_dynamic_date_filters(self): auto_email_report = get_auto_email_report() - auto_email_report.dynamic_date_period = 'Weekly' - auto_email_report.from_date_field = 'from_date' - auto_email_report.to_date_field = 'to_date' + auto_email_report.dynamic_date_period = "Weekly" + auto_email_report.from_date_field = "from_date" + auto_email_report.to_date_field = "to_date" auto_email_report.prepare_dynamic_filters() - self.assertEqual(auto_email_report.filters['from_date'], add_to_date(today(), weeks=-1)) - self.assertEqual(auto_email_report.filters['to_date'], today()) + self.assertEqual(auto_email_report.filters["from_date"], add_to_date(today(), weeks=-1)) + self.assertEqual(auto_email_report.filters["to_date"], today()) def get_auto_email_report(): - if not frappe.db.exists('Auto Email Report', 'Permitted Documents For User'): - auto_email_report = frappe.get_doc(dict( - doctype='Auto Email Report', - report='Permitted Documents For User', - report_type='Script Report', - user='Administrator', - enabled=1, - email_to='test@example.com', - format='HTML', - frequency='Daily', - filters=json.dumps(dict(user='Administrator', doctype='DocType')) - )).insert() + if not frappe.db.exists("Auto Email Report", "Permitted Documents For User"): + auto_email_report = frappe.get_doc( + dict( + doctype="Auto Email Report", + report="Permitted Documents For User", + report_type="Script Report", + user="Administrator", + enabled=1, + email_to="test@example.com", + format="HTML", + frequency="Daily", + filters=json.dumps(dict(user="Administrator", doctype="DocType")), + ) + ).insert() else: - auto_email_report = frappe.get_doc('Auto Email Report', 'Permitted Documents For User') + auto_email_report = frappe.get_doc("Auto Email Report", "Permitted Documents For User") return auto_email_report diff --git a/frappe/email/doctype/document_follow/document_follow.py b/frappe/email/doctype/document_follow/document_follow.py index 97f8237736..3b7d411fb5 100644 --- a/frappe/email/doctype/document_follow/document_follow.py +++ b/frappe/email/doctype/document_follow/document_follow.py @@ -4,6 +4,6 @@ from frappe.model.document import Document + class DocumentFollow(Document): pass - diff --git a/frappe/email/doctype/document_follow/test_document_follow.py b/frappe/email/doctype/document_follow/test_document_follow.py index 0f6ff6c114..1f31338351 100644 --- a/frappe/email/doctype/document_follow/test_document_follow.py +++ b/frappe/email/doctype/document_follow/test_document_follow.py @@ -1,17 +1,19 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest from dataclasses import dataclass + +import frappe import frappe.desk.form.document_follow as document_follow -from frappe.query_builder import DocType -from frappe.desk.form.utils import add_comment +from frappe.desk.form.assign_to import add from frappe.desk.form.document_follow import get_document_followed_by_user +from frappe.desk.form.utils import add_comment from frappe.desk.like import toggle_like -from frappe.desk.form.assign_to import add -from frappe.share import add as share +from frappe.query_builder import DocType from frappe.query_builder.functions import Cast_ +from frappe.share import add as share + class TestDocumentFollow(unittest.TestCase): def test_document_follow_version(self): @@ -26,25 +28,25 @@ class TestDocumentFollow(unittest.TestCase): self.assertEqual(doc.user, user.name) document_follow.send_hourly_updates() - emails = get_emails(event_doc, '%This is a test description for sending mail%') + emails = get_emails(event_doc, "%This is a test description for sending mail%") self.assertIsNotNone(emails) - def test_document_follow_comment(self): user = get_user() event_doc = get_event() - add_comment(event_doc.doctype, event_doc.name, "This is a test comment", 'Administrator@example.com', 'Bosh') + add_comment( + event_doc.doctype, event_doc.name, "This is a test comment", "Administrator@example.com", "Bosh" + ) document_follow.unfollow_document("Event", event_doc.name, user.name) doc = document_follow.follow_document("Event", event_doc.name, user.name) self.assertEqual(doc.user, user.name) document_follow.send_hourly_updates() - emails = get_emails(event_doc, '%This is a test comment%') + emails = get_emails(event_doc, "%This is a test comment%") self.assertIsNotNone(emails) - def test_follow_limit(self): user = get_user() for _ in range(25): @@ -90,7 +92,9 @@ class TestDocumentFollow(unittest.TestCase): frappe.set_user(user.name) event = get_event() - add_comment(event.doctype, event.name, "This is a test comment", 'Administrator@example.com', 'Bosh') + add_comment( + event.doctype, event.name, "This is a test comment", "Administrator@example.com", "Bosh" + ) documents_followed = get_events_followed_by_user(event.name, user.name) self.assertTrue(documents_followed) @@ -100,7 +104,9 @@ class TestDocumentFollow(unittest.TestCase): frappe.set_user(user.name) event = get_event() - add_comment(event.doctype, event.name, "This is a test comment", 'Administrator@example.com', 'Bosh') + add_comment( + event.doctype, event.name, "This is a test comment", "Administrator@example.com", "Bosh" + ) documents_followed = get_events_followed_by_user(event.name, user.name) self.assertFalse(documents_followed) @@ -129,11 +135,7 @@ class TestDocumentFollow(unittest.TestCase): user = get_user(DocumentFollowConditions(0, 0, 0, 1)) event = get_event() - add({ - 'assign_to': [user.name], - 'doctype': event.doctype, - 'name': event.name - }) + add({"assign_to": [user.name], "doctype": event.doctype, "name": event.name}) documents_followed = get_events_followed_by_user(event.name, user.name) self.assertTrue(documents_followed) @@ -143,11 +145,7 @@ class TestDocumentFollow(unittest.TestCase): frappe.set_user(user.name) event = get_event() - add({ - 'assign_to': [user.name], - 'doctype': event.doctype, - 'name': event.name - }) + add({"assign_to": [user.name], "doctype": event.doctype, "name": event.name}) documents_followed = get_events_followed_by_user(event.name, user.name) self.assertFalse(documents_followed) @@ -156,11 +154,7 @@ class TestDocumentFollow(unittest.TestCase): user = get_user(DocumentFollowConditions(0, 0, 0, 0, 1)) event = get_event() - share( - user= user.name, - doctype= event.doctype, - name= event.name - ) + share(user=user.name, doctype=event.doctype, name=event.name) documents_followed = get_events_followed_by_user(event.name, user.name) self.assertTrue(documents_followed) @@ -169,46 +163,48 @@ class TestDocumentFollow(unittest.TestCase): user = get_user() event = get_event() - share( - user = user.name, - doctype = event.doctype, - name = event.name - ) + share(user=user.name, doctype=event.doctype, name=event.name) documents_followed = get_events_followed_by_user(event.name, user.name) self.assertFalse(documents_followed) - def tearDown(self): frappe.db.rollback() - frappe.db.delete('Email Queue') - frappe.db.delete('Email Queue Recipient') - frappe.db.delete('Document Follow') - frappe.db.delete('Event') + frappe.db.delete("Email Queue") + frappe.db.delete("Email Queue Recipient") + frappe.db.delete("Document Follow") + frappe.db.delete("Event") + def get_events_followed_by_user(event_name, user_name): - DocumentFollow = DocType('Document Follow') - return (frappe.qb.from_(DocumentFollow) - .where(DocumentFollow.ref_doctype == 'Event') + DocumentFollow = DocType("Document Follow") + return ( + frappe.qb.from_(DocumentFollow) + .where(DocumentFollow.ref_doctype == "Event") .where(DocumentFollow.ref_docname == event_name) .where(DocumentFollow.user == user_name) - .select(DocumentFollow.name)).run() + .select(DocumentFollow.name) + ).run() + def get_event(): - doc = frappe.get_doc({ - 'doctype': 'Event', - 'subject': "_Test_Doc_Follow", - 'doc.starts_on': frappe.utils.now(), - 'doc.ends_on': frappe.utils.add_days(frappe.utils.now(),5), - 'doc.description': "Hello" - }) + doc = frappe.get_doc( + { + "doctype": "Event", + "subject": "_Test_Doc_Follow", + "doc.starts_on": frappe.utils.now(), + "doc.ends_on": frappe.utils.add_days(frappe.utils.now(), 5), + "doc.description": "Hello", + } + ) doc.insert() return doc + def get_user(document_follow=None): frappe.set_user("Administrator") - if frappe.db.exists('User', 'test@docsub.com'): - doc = frappe.delete_doc('User', 'test@docsub.com') + if frappe.db.exists("User", "test@docsub.com"): + doc = frappe.delete_doc("User", "test@docsub.com") doc = frappe.new_doc("User") doc.email = "test@docsub.com" doc.first_name = "Test" @@ -218,22 +214,28 @@ def get_user(document_follow=None): doc.document_follow_frequency = "Hourly" doc.__dict__.update(document_follow.__dict__ if document_follow else {}) doc.insert() - doc.add_roles('System Manager') + doc.add_roles("System Manager") return doc + def get_emails(event_doc, search_string): - EmailQueue = DocType('Email Queue') - EmailQueueRecipient = DocType('Email Queue Recipient') + EmailQueue = DocType("Email Queue") + EmailQueueRecipient = DocType("Email Queue Recipient") - return (frappe.qb.from_(EmailQueue) + return ( + frappe.qb.from_(EmailQueue) .join(EmailQueueRecipient) .on(EmailQueueRecipient.parent == Cast_(EmailQueue.name, "varchar")) - .where(EmailQueueRecipient.recipient == 'test@docsub.com',) - .where(EmailQueue.message.like(f'%{event_doc.doctype}%')) - .where(EmailQueue.message.like(f'%{event_doc.name}%')) + .where( + EmailQueueRecipient.recipient == "test@docsub.com", + ) + .where(EmailQueue.message.like(f"%{event_doc.doctype}%")) + .where(EmailQueue.message.like(f"%{event_doc.name}%")) .where(EmailQueue.message.like(search_string)) .select(EmailQueue.message) - .limit(1)).run() + .limit(1) + ).run() + @dataclass class DocumentFollowConditions: diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 3a1b683398..e60be0d965 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -23,11 +23,15 @@ from frappe.utils.error import raise_error_on_no_output from frappe.utils.jinja import render_template from frappe.utils.user import get_system_managers -OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account") +OUTGOING_EMAIL_ACCOUNT_MISSING = _( + "Please setup default Email Account from Setup > Email > Email Account" +) + class SentEmailInInbox(Exception): pass + def cache_email_account(cache_name): def decorator_cache_email_account(func): @functools.wraps(func) @@ -36,7 +40,7 @@ def cache_email_account(cache_name): setattr(frappe.local, cache_name, {}) cached_accounts = getattr(frappe.local, cache_name) - match_by = list(kwargs.values()) + ['default'] + match_by = list(kwargs.values()) + ["default"] matched_accounts = list(filter(None, [cached_accounts.get(key) for key in match_by])) if matched_accounts: return matched_accounts[0] @@ -44,17 +48,21 @@ def cache_email_account(cache_name): matched_accounts = func(*args, **kwargs) cached_accounts.update(matched_accounts or {}) return matched_accounts and list(matched_accounts.values())[0] + return wrapper_cache_email_account + return decorator_cache_email_account + class EmailAccount(Document): - DOCTYPE = 'Email Account' + DOCTYPE = "Email Account" def autoname(self): """Set name as `email_account_name` or make title from Email Address.""" if not self.email_account_name: - self.email_account_name = self.email_id.split("@", 1)[0]\ - .replace("_", " ").replace(".", " ").replace("-", " ").title() + self.email_account_name = ( + self.email_id.split("@", 1)[0].replace("_", " ").replace(".", " ").replace("-", " ").title() + ) self.name = self.email_account_name @@ -73,20 +81,25 @@ class EmailAccount(Document): if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0: frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id))) - duplicate_email_account = frappe.get_all("Email Account", filters={ - "email_id": self.email_id, - "name": ("!=", self.name) - }) + duplicate_email_account = frappe.get_all( + "Email Account", filters={"email_id": self.email_id, "name": ("!=", self.name)} + ) if duplicate_email_account: - frappe.throw(_("Email ID must be unique, Email Account already exists for {0}") \ - .format(frappe.bold(self.email_id))) + frappe.throw( + _("Email ID must be unique, Email Account already exists for {0}").format( + frappe.bold(self.email_id) + ) + ) if frappe.local.flags.in_patch or frappe.local.flags.in_test: return - if (not self.awaiting_password and not frappe.local.flags.in_install - and not frappe.local.flags.in_patch): - if self.password or self.smtp_server in ('127.0.0.1', 'localhost'): + if ( + not self.awaiting_password + and not frappe.local.flags.in_install + and not frappe.local.flags.in_patch + ): + if self.password or self.smtp_server in ("127.0.0.1", "localhost"): if self.enable_incoming: self.get_incoming_server() self.no_failed = 0 @@ -121,30 +134,33 @@ class EmailAccount(Document): as_list = 1 if not self.enable_incoming and self.default_incoming: self.default_incoming = False - messages.append(_("{} has been disabled. It can only be enabled if {} is checked.") - .format( - frappe.bold(_('Default Incoming')), - frappe.bold(_('Enable Incoming')) + messages.append( + _("{} has been disabled. It can only be enabled if {} is checked.").format( + frappe.bold(_("Default Incoming")), frappe.bold(_("Enable Incoming")) ) ) if not self.enable_outgoing and self.default_outgoing: self.default_outgoing = False - messages.append(_("{} has been disabled. It can only be enabled if {} is checked.") - .format( - frappe.bold(_('Default Outgoing')), - frappe.bold(_('Enable Outgoing')) - ) + messages.append( + _("{} has been disabled. It can only be enabled if {} is checked.").format( + frappe.bold(_("Default Outgoing")), frappe.bold(_("Enable Outgoing")) ) + ) if messages: - if len(messages) == 1: (as_list, messages) = (0, messages[0]) - frappe.msgprint(messages, as_list= as_list, indicator='orange', title=_("Defaults Updated")) + if len(messages) == 1: + (as_list, messages) = (0, messages[0]) + frappe.msgprint(messages, as_list=as_list, indicator="orange", title=_("Defaults Updated")) def on_update(self): """Check there is only one default of each type.""" self.check_automatic_linking_email_account() self.there_must_be_only_one_default() - setup_user_email_inbox(email_account=self.name, awaiting_password=self.awaiting_password, - email_id=self.email_id, enable_outgoing=self.enable_outgoing) + setup_user_email_inbox( + email_account=self.name, + awaiting_password=self.awaiting_password, + email_id=self.email_id, + enable_outgoing=self.enable_outgoing, + ) def there_must_be_only_one_default(self): """If current Email Account is default, un-default all other accounts.""" @@ -152,8 +168,8 @@ class EmailAccount(Document): if not self.get(field): continue - for email_account in frappe.get_all("Email Account", filters={ field: 1 }): - if email_account.name==self.name: + for email_account in frappe.get_all("Email Account", filters={field: 1}): + if email_account.name == self.name: continue email_account = frappe.get_doc("Email Account", email_account.name) @@ -166,10 +182,16 @@ class EmailAccount(Document): try: domain = email_id.split("@") fields = [ - "name as domain", "use_imap", "email_server", - "use_ssl", "smtp_server", "use_tls", - "smtp_port", "incoming_port", "append_emails_to_sent_folder", - "use_ssl_for_outgoing" + "name as domain", + "use_imap", + "email_server", + "use_ssl", + "smtp_server", + "use_tls", + "smtp_port", + "incoming_port", + "append_emails_to_sent_folder", + "use_ssl_for_outgoing", ] return frappe.db.get_value("Email Domain", domain[1], fields, as_dict=True) except Exception: @@ -180,17 +202,19 @@ class EmailAccount(Document): if frappe.cache().get_value("workers:no-internet") == True: return None - args = frappe._dict({ - "email_account_name": self.email_account_name, - "email_account": self.name, - "host": self.email_server, - "use_ssl": self.use_ssl, - "username": getattr(self, "login_id", None) or self.email_id, - "use_imap": self.use_imap, - "email_sync_rule": email_sync_rule, - "incoming_port": get_port(self), - "initial_sync_count": self.initial_sync_count or 100 - }) + args = frappe._dict( + { + "email_account_name": self.email_account_name, + "email_account": self.name, + "host": self.email_server, + "use_ssl": self.use_ssl, + "username": getattr(self, "login_id", None) or self.email_id, + "use_imap": self.use_imap, + "email_sync_rule": email_sync_rule, + "incoming_port": get_port(self), + "initial_sync_count": self.initial_sync_count or 100, + } + ) if self.password: args.password = self.get_password() @@ -214,24 +238,22 @@ class EmailAccount(Document): try: email_server.connect() except (error_proto, imaplib.IMAP4.error) as e: - message = cstr(e).lower().replace(" ","") + message = cstr(e).lower().replace(" ", "") auth_error_codes = [ - 'authenticationfailed', - 'loginfailed', + "authenticationfailed", + "loginfailed", ] - other_error_codes = [ - 'err[auth]', - 'errtemporaryerror', - 'loginviayourwebbrowser' - ] + other_error_codes = ["err[auth]", "errtemporaryerror", "loginviayourwebbrowser"] all_error_codes = auth_error_codes + other_error_codes if in_receive and any(map(lambda t: t in message, all_error_codes)): # if called via self.receive and it leads to authentication error, # disable incoming and send email to System Manager - error_message = _("Authentication failed while receiving emails from Email Account: {0}.").format(self.name) + error_message = _( + "Authentication failed while receiving emails from Email Account: {0}." + ).format(self.name) error_message += "
" + _("Message from server: {0}").format(cstr(e)) self.handle_incoming_connect_error(description=error_message) return None @@ -296,9 +318,11 @@ class EmailAccount(Document): @classmethod @raise_error_on_no_output( - keep_quiet = lambda: not cint(frappe.get_system_settings('setup_complete')), - error_message = OUTGOING_EMAIL_ACCOUNT_MISSING, error_type = frappe.OutgoingEmailError) # noqa - @cache_email_account('outgoing_email_account') + keep_quiet=lambda: not cint(frappe.get_system_settings("setup_complete")), + error_message=OUTGOING_EMAIL_ACCOUNT_MISSING, + error_type=frappe.OutgoingEmailError, + ) # noqa + @cache_email_account("outgoing_email_account") def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False): """Find the outgoing Email account to use. @@ -319,12 +343,11 @@ class EmailAccount(Document): doc = cls.find_default_outgoing() if doc: - return {'default': doc} + return {"default": doc} @classmethod def find_default_outgoing(cls): - """ Find default outgoing account. - """ + """Find default outgoing account.""" doc = cls.find_one_by_filters(enable_outgoing=1, default_outgoing=1) doc = doc or cls.find_from_config() return doc or (are_emails_muted() and cls.create_dummy()) @@ -357,35 +380,42 @@ class EmailAccount(Document): return {} field_to_conf_name_map = { - 'smtp_server': {'conf_names': ('mail_server',)}, - 'smtp_port': {'conf_names': ('mail_port',)}, - 'use_tls': {'conf_names': ('use_tls', 'mail_login')}, - 'login_id': {'conf_names': ('mail_login',)}, - 'email_id': {'conf_names': ('auto_email_id', 'mail_login'), 'default': 'notifications@example.com'}, - 'password': {'conf_names': ('mail_password',)}, - 'always_use_account_email_id_as_sender': - {'conf_names': ('always_use_account_email_id_as_sender',), 'default': 0}, - 'always_use_account_name_as_sender_name': - {'conf_names': ('always_use_account_name_as_sender_name',), 'default': 0}, - 'name': {'conf_names': ('email_sender_name',), 'default': 'Frappe'}, - 'from_site_config': {'default': True} + "smtp_server": {"conf_names": ("mail_server",)}, + "smtp_port": {"conf_names": ("mail_port",)}, + "use_tls": {"conf_names": ("use_tls", "mail_login")}, + "login_id": {"conf_names": ("mail_login",)}, + "email_id": { + "conf_names": ("auto_email_id", "mail_login"), + "default": "notifications@example.com", + }, + "password": {"conf_names": ("mail_password",)}, + "always_use_account_email_id_as_sender": { + "conf_names": ("always_use_account_email_id_as_sender",), + "default": 0, + }, + "always_use_account_name_as_sender_name": { + "conf_names": ("always_use_account_name_as_sender_name",), + "default": 0, + }, + "name": {"conf_names": ("email_sender_name",), "default": "Frappe"}, + "from_site_config": {"default": True}, } account_details = {} for doc_field_name, d in field_to_conf_name_map.items(): - conf_names, default = d.get('conf_names') or [], d.get('default') + conf_names, default = d.get("conf_names") or [], d.get("default") value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)] account_details[doc_field_name] = (value and value[0]) or default return account_details def sendmail_config(self): return { - 'server': self.smtp_server, - 'port': cint(self.smtp_port), - 'login': getattr(self, "login_id", None) or self.email_id, - 'password': self._password, - 'use_ssl': cint(self.use_ssl_for_outgoing), - 'use_tls': cint(self.use_tls) + "server": self.smtp_server, + "port": cint(self.smtp_port), + "login": getattr(self, "login_id", None) or self.email_id, + "password": self._password, + "use_ssl": cint(self.use_ssl_for_outgoing), + "use_tls": cint(self.use_tls), } def get_smtp_server(self): @@ -399,14 +429,16 @@ class EmailAccount(Document): for user in get_system_managers(only_name=True): try: - assign_to.add({ - 'assign_to': user, - 'doctype': self.doctype, - 'name': self.name, - 'description': description, - 'priority': 'High', - 'notify': 1 - }) + assign_to.add( + { + "assign_to": user, + "doctype": self.doctype, + "name": self.name, + "description": description, + "priority": "High", + "notify": 1, + } + ) except assign_to.DuplicateToDoError: frappe.message_log.pop() pass @@ -416,10 +448,10 @@ class EmailAccount(Document): frappe.cache().set_value("workers:no-internet", True) def set_failed_attempts_count(self, value): - frappe.cache().set('{0}:email-account-failed-attempts'.format(self.name), value) + frappe.cache().set("{0}:email-account-failed-attempts".format(self.name), value) def get_failed_attempts_count(self): - return cint(frappe.cache().get('{0}:email-account-failed-attempts'.format(self.name))) + return cint(frappe.cache().get("{0}:email-account-failed-attempts".format(self.name))) def receive(self): """Called by scheduler to receive emails from this EMail account using POP3/IMAP.""" @@ -448,26 +480,24 @@ class EmailAccount(Document): else: frappe.db.commit() - #notify if user is linked to account - if len(inbound_mails)>0 and not frappe.local.flags.in_test: - frappe.publish_realtime('new_email', - {"account":self.email_account_name, "number":len(inbound_mails)} + # notify if user is linked to account + if len(inbound_mails) > 0 and not frappe.local.flags.in_test: + frappe.publish_realtime( + "new_email", {"account": self.email_account_name, "number": len(inbound_mails)} ) if exceptions: raise Exception(frappe.as_json(exceptions)) def get_inbound_mails(self) -> List[InboundMail]: - """retrive and return inbound mails. - - """ + """retrive and return inbound mails.""" mails = [] def process_mail(messages, append_to=None): for index, message in enumerate(messages.get("latest_messages", [])): - uid = messages['uid_list'][index] if messages.get('uid_list') else None - seen_status = messages.get('seen_status', {}).get(uid) - if self.email_sync_option != 'UNSEEN' or seen_status != "SEEN": + uid = messages["uid_list"][index] if messages.get("uid_list") else None + seen_status = messages.get("seen_status", {}).get(uid) + if self.email_sync_option != "UNSEEN" or seen_status != "SEEN": # only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN' mails.append(InboundMail(message, self, uid, seen_status, append_to)) @@ -481,7 +511,7 @@ class EmailAccount(Document): # process all given imap folder for folder in self.imap_folder: if email_server.select_imap_folder(folder.folder_name): - email_server.settings['uid_validity'] = folder.uidvalidity + email_server.settings["uid_validity"] = folder.uidvalidity messages = email_server.get_messages(folder=f'"{folder.folder_name}"') or {} process_mail(messages, folder.append_to) else: @@ -498,45 +528,51 @@ class EmailAccount(Document): def handle_bad_emails(self, uid, raw, reason): if cint(self.use_imap): import email + try: if isinstance(raw, bytes): mail = email.message_from_bytes(raw) else: mail = email.message_from_string(raw) - message_id = mail.get('Message-ID') + message_id = mail.get("Message-ID") except Exception: message_id = "can't be parsed" - unhandled_email = frappe.get_doc({ - "raw": raw, - "uid": uid, - "reason":reason, - "message_id": message_id, - "doctype": "Unhandled Email", - "email_account": self.name - }) + unhandled_email = frappe.get_doc( + { + "raw": raw, + "uid": uid, + "reason": reason, + "message_id": message_id, + "doctype": "Unhandled Email", + "email_account": self.name, + } + ) unhandled_email.insert(ignore_permissions=True) frappe.db.commit() def send_auto_reply(self, communication, email): """Send auto reply if set.""" from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts + if self.enable_auto_reply: set_incoming_outgoing_accounts(communication) unsubscribe_message = (self.send_unsubscribe_message and _("Leave this conversation")) or "" - frappe.sendmail(recipients = [email.from_email], - sender = self.email_id, - reply_to = communication.incoming_email_account, - subject = " ".join([_("Re:"), communication.subject]), - content = render_template(self.auto_reply_message or "", communication.as_dict()) or \ - frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()), - reference_doctype = communication.reference_doctype, - reference_name = communication.reference_name, - in_reply_to = email.mail.get("Message-Id"), # send back the Message-Id as In-Reply-To - unsubscribe_message = unsubscribe_message) + frappe.sendmail( + recipients=[email.from_email], + sender=self.email_id, + reply_to=communication.incoming_email_account, + subject=" ".join([_("Re:"), communication.subject]), + content=render_template(self.auto_reply_message or "", communication.as_dict()) + or frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()), + reference_doctype=communication.reference_doctype, + reference_name=communication.reference_name, + in_reply_to=email.mail.get("Message-Id"), # send back the Message-Id as In-Reply-To + unsubscribe_message=unsubscribe_message, + ) def get_unreplied_notification_emails(self): """Return list of emails listed""" @@ -547,9 +583,9 @@ class EmailAccount(Document): def on_trash(self): """Clear communications where email account is linked""" Communication = frappe.qb.DocType("Communication") - frappe.qb.update(Communication) \ - .set(Communication.email_account, "") \ - .where(Communication.email_account == self.name).run() + frappe.qb.update(Communication).set(Communication.email_account, "").where( + Communication.email_account == self.name + ).run() remove_user_email_inbox(email_account=self.name) @@ -568,7 +604,7 @@ class EmailAccount(Document): return self.email_sync_option or "UNSEEN" def mark_emails_as_read_unread(self, email_server=None, folder_name="INBOX"): - """ mark Email Flag Queue of self.email_account mails as read""" + """mark Email Flag Queue of self.email_account mails as read""" if not self.use_imap: return @@ -580,7 +616,7 @@ class EmailAccount(Document): .where(EmailFlagQ.email_account == frappe.db.escape(self.name)) ).run(as_dict=True) - uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags } + uid_list = {flag.get("uid", None): flag.get("action", "Read") for flag in flags} if flags and uid_list: if not email_server: email_server = self.get_incoming_server() @@ -589,37 +625,41 @@ class EmailAccount(Document): email_server.update_flag(folder_name, uid_list=uid_list) # mark communication as read - docnames = ",".join("'%s'"%flag.get("communication") for flag in flags \ - if flag.get("action") == "Read") + docnames = ",".join( + "'%s'" % flag.get("communication") for flag in flags if flag.get("action") == "Read" + ) self.set_communication_seen_status(docnames, seen=1) # mark communication as unread - docnames = ",".join([ "'%s'"%flag.get("communication") for flag in flags \ - if flag.get("action") == "Unread" ]) + docnames = ",".join( + ["'%s'" % flag.get("communication") for flag in flags if flag.get("action") == "Unread"] + ) self.set_communication_seen_status(docnames, seen=0) - docnames = ",".join([ "'%s'"%flag.get("name") for flag in flags ]) + docnames = ",".join(["'%s'" % flag.get("name") for flag in flags]) EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") - frappe.qb.update(EmailFlagQueue) \ - .set(EmailFlagQueue.is_completed, 1) \ - .where(EmailFlagQueue.name.isin(docnames)).run() + frappe.qb.update(EmailFlagQueue).set(EmailFlagQueue.is_completed, 1).where( + EmailFlagQueue.name.isin(docnames) + ).run() def set_communication_seen_status(self, docnames, seen=0): - """ mark Email Flag Queue of self.email_account mails as read""" + """mark Email Flag Queue of self.email_account mails as read""" if not docnames: return Communication = frappe.qb.from_("Communication") - frappe.qb.update(Communication) \ - .set(Communication.seen == seen) \ - .where(Communication.name.isin(docnames)).run() + frappe.qb.update(Communication).set(Communication.seen == seen).where( + Communication.name.isin(docnames) + ).run() def check_automatic_linking_email_account(self): if self.enable_automatic_linking: if not self.enable_incoming: frappe.throw(_("Automatic Linking can be activated only if Incoming is enabled.")) - if frappe.db.exists("Email Account", {"enable_automatic_linking": 1, "name": ('!=', self.name)}): + if frappe.db.exists( + "Email Account", {"enable_automatic_linking": 1, "name": ("!=", self.name)} + ): frappe.throw(_("Automatic Linking can be activated only for one Email Account.")) def append_email_to_sent_folder(self, message): @@ -643,7 +683,9 @@ class EmailAccount(Document): @frappe.whitelist() -def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None): +def get_append_to( + doctype=None, txt=None, searchfield=None, start=None, page_len=None, filters=None +): txt = txt if txt else "" email_append_to_list = [] @@ -653,13 +695,16 @@ def get_append_to(doctype=None, txt=None, searchfield=None, start=None, page_len email_append_to_list.append(dt.name) # Set Email Append To DocTypes set via Customize Form - for dt in frappe.get_list("Property Setter", filters={"property": "email_append_to", "value": 1}, fields=["doc_type"]): + for dt in frappe.get_list( + "Property Setter", filters={"property": "email_append_to", "value": 1}, fields=["doc_type"] + ): email_append_to_list.append(dt.doc_type) email_append_to = [[d] for d in set(email_append_to_list) if txt in d] return email_append_to + def test_internet(host="8.8.8.8", port=53, timeout=3): """Returns True if internet is connected @@ -675,10 +720,13 @@ def test_internet(host="8.8.8.8", port=53, timeout=3): print(ex.message) return False + def notify_unreplied(): """Sends email notifications if there are unreplied Communications - and `notify_if_unreplied` is set as true.""" - for email_account in frappe.get_all("Email Account", "name", filters={"enable_incoming": 1, "notify_if_unreplied": 1}): + and `notify_if_unreplied` is set as true.""" + for email_account in frappe.get_all( + "Email Account", "name", filters={"enable_incoming": 1, "notify_if_unreplied": 1} + ): email_account = frappe.get_doc("Email Account", email_account.name) if email_account.use_imap: @@ -688,25 +736,44 @@ def notify_unreplied(): if append_to: # get open communications younger than x mins, for given doctype - for comm in frappe.get_all("Communication", "name", filters=[ + for comm in frappe.get_all( + "Communication", + "name", + filters=[ {"sent_or_received": "Received"}, {"reference_doctype": ("in", append_to)}, {"unread_notification_sent": 0}, - {"email_account":email_account.name}, - {"creation": ("<", datetime.now() - timedelta(seconds = (email_account.unreplied_for_mins or 30) * 60))}, - {"creation": (">", datetime.now() - timedelta(seconds = (email_account.unreplied_for_mins or 30) * 60 * 3))} - ]): + {"email_account": email_account.name}, + { + "creation": ( + "<", + datetime.now() - timedelta(seconds=(email_account.unreplied_for_mins or 30) * 60), + ) + }, + { + "creation": ( + ">", + datetime.now() - timedelta(seconds=(email_account.unreplied_for_mins or 30) * 60 * 3), + ) + }, + ], + ): comm = frappe.get_doc("Communication", comm.name) - if frappe.db.get_value(comm.reference_doctype, comm.reference_name, "status")=="Open": + if frappe.db.get_value(comm.reference_doctype, comm.reference_name, "status") == "Open": # if status is still open - frappe.sendmail(recipients=email_account.get_unreplied_notification_emails(), - content=comm.content, subject=comm.subject, doctype= comm.reference_doctype, - name=comm.reference_name) + frappe.sendmail( + recipients=email_account.get_unreplied_notification_emails(), + content=comm.content, + subject=comm.subject, + doctype=comm.reference_doctype, + name=comm.reference_name, + ) # update flag comm.db_set("unread_notification_sent", 1) + def pull(now=False): """Will be called via scheduler, pull emails from all enabled Email accounts.""" if frappe.cache().get_value("workers:no-internet") == True: @@ -714,34 +781,46 @@ def pull(now=False): frappe.cache().set_value("workers:no-internet", False) else: return - queued_jobs = get_jobs(site=frappe.local.site, key='job_name')[frappe.local.site] - for email_account in frappe.get_list("Email Account", - filters={"enable_incoming": 1, "awaiting_password": 0}): + queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site] + for email_account in frappe.get_list( + "Email Account", filters={"enable_incoming": 1, "awaiting_password": 0} + ): if now: pull_from_email_account(email_account.name) else: # job_name is used to prevent duplicates in queue - job_name = 'pull_from_email_account|{0}'.format(email_account.name) + job_name = "pull_from_email_account|{0}".format(email_account.name) if job_name not in queued_jobs: - enqueue(pull_from_email_account, 'short', event='all', job_name=job_name, - email_account=email_account.name) + enqueue( + pull_from_email_account, + "short", + event="all", + job_name=job_name, + email_account=email_account.name, + ) + def pull_from_email_account(email_account): - '''Runs within a worker process''' + """Runs within a worker process""" email_account = frappe.get_doc("Email Account", email_account) email_account.receive() + def get_max_email_uid(email_account): # get maximum uid of emails max_uid = 1 - result = frappe.db.get_all("Communication", filters={ - "communication_medium": "Email", - "sent_or_received": "Received", - "email_account": email_account - }, fields=["max(uid) as uid"]) + result = frappe.db.get_all( + "Communication", + filters={ + "communication_medium": "Email", + "sent_or_received": "Received", + "email_account": email_account, + }, + fields=["max(uid) as uid"], + ) if not result: return 1 @@ -751,7 +830,7 @@ def get_max_email_uid(email_account): def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing): - """ setup email inbox for user """ + """setup email inbox for user""" from frappe.core.doctype.user.user import ask_pass_update def add_user_email(user): @@ -777,10 +856,12 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou user_name = user.get("name") # check if inbox is alreay configured - user_inbox = frappe.db.get_value("User Email", { - "email_account": email_account, - "parent": user_name - }, ["name"]) or None + user_inbox = ( + frappe.db.get_value( + "User Email", {"email_account": email_account, "parent": user_name}, ["name"] + ) + or None + ) if not user_inbox: add_user_email(user_name) @@ -790,24 +871,24 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou if update_user_email_settings: UserEmail = frappe.qb.DocType("User Email") - frappe.qb.update(UserEmail) \ - .set(UserEmail.awaiting_password, (awaiting_password or 0)) \ - .set(UserEmail.enable_outgoing, enable_outgoing) \ - .where(UserEmail.email_account == email_account).run() + frappe.qb.update(UserEmail).set(UserEmail.awaiting_password, (awaiting_password or 0)).set( + UserEmail.enable_outgoing, enable_outgoing + ).where(UserEmail.email_account == email_account).run() else: users = " and ".join([frappe.bold(user.get("name")) for user in user_names]) frappe.msgprint(_("Enabled email inbox for user {0}").format(users)) ask_pass_update() + def remove_user_email_inbox(email_account): - """ remove user email inbox settings if email account is deleted """ + """remove user email inbox settings if email account is deleted""" if not email_account: return - users = frappe.get_all("User Email", filters={ - "email_account": email_account - }, fields=["parent as name"]) + users = frappe.get_all( + "User Email", filters={"email_account": email_account}, fields=["parent as name"] + ) for user in users: doc = frappe.get_doc("User", user.get("name")) @@ -816,6 +897,7 @@ def remove_user_email_inbox(email_account): doc.save(ignore_permissions=True) + @frappe.whitelist(allow_guest=False) def set_email_password(email_account, user, password): account = frappe.get_doc("Email Account", email_account) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index f609c2947d..a027a81bd7 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -1,24 +1,24 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import os import email +import os import unittest from datetime import datetime, timedelta +from unittest.mock import patch -from frappe.email.receive import InboundMail, SentEmailInInboxError, Email -from frappe.email.email_body import get_message_id import frappe -from frappe.test_runner import make_test_records from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments from frappe.email.doctype.email_account.email_account import notify_unreplied - -from unittest.mock import patch +from frappe.email.email_body import get_message_id +from frappe.email.receive import Email, InboundMail, SentEmailInInboxError +from frappe.test_runner import make_test_records make_test_records("User") make_test_records("Email Account") + class TestEmailAccount(unittest.TestCase): @classmethod def setUpClass(cls): @@ -48,13 +48,9 @@ class TestEmailAccount(unittest.TestCase): messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - self.get_test_mail('incoming-1.raw') - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] + "latest_messages": [self.get_test_mail("incoming-1.raw")], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], } } @@ -70,29 +66,33 @@ class TestEmailAccount(unittest.TestCase): self.test_incoming() comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - comm.db_set("creation", datetime.now() - timedelta(seconds = 30 * 60)) + comm.db_set("creation", datetime.now() - timedelta(seconds=30 * 60)) frappe.db.delete("Email Queue") notify_unreplied() - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, - "reference_name": comm.reference_name, "status":"Not Sent"})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + { + "reference_doctype": comm.reference_doctype, + "reference_name": comm.reference_name, + "status": "Not Sent", + }, + ) + ) def test_incoming_with_attach(self): cleanup("test_sender@example.com") - existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) + existing_file = frappe.get_doc({"doctype": "File", "file_name": "erpnext-conf-14.png"}) frappe.delete_doc("File", existing_file.name) messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - self.get_test_mail('incoming-2.raw') - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] + "latest_messages": [self.get_test_mail("incoming-2.raw")], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], } } @@ -107,23 +107,18 @@ class TestEmailAccount(unittest.TestCase): self.assertTrue("erpnext-conf-14.png" in [f.file_name for f in attachments]) # cleanup - existing_file = frappe.get_doc({'doctype': 'File', 'file_name': 'erpnext-conf-14.png'}) + existing_file = frappe.get_doc({"doctype": "File", "file_name": "erpnext-conf-14.png"}) frappe.delete_doc("File", existing_file.name) - def test_incoming_attached_email_from_outlook_plain_text_only(self): cleanup("test_sender@example.com") messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - self.get_test_mail('incoming-3.raw') - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] + "latest_messages": [self.get_test_mail("incoming-3.raw")], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], } } @@ -131,8 +126,12 @@ class TestEmailAccount(unittest.TestCase): TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) - self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) + self.assertTrue( + "From: "Microsoft Outlook" <test_sender@example.com>" in comm.content + ) + self.assertTrue( + "This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content + ) def test_incoming_attached_email_from_outlook_layers(self): cleanup("test_sender@example.com") @@ -140,13 +139,9 @@ class TestEmailAccount(unittest.TestCase): messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - self.get_test_mail('incoming-4.raw') - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] + "latest_messages": [self.get_test_mail("incoming-4.raw")], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], } } @@ -154,38 +149,65 @@ class TestEmailAccount(unittest.TestCase): TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue("From: "Microsoft Outlook" <test_sender@example.com>" in comm.content) - self.assertTrue("This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content) + self.assertTrue( + "From: "Microsoft Outlook" <test_sender@example.com>" in comm.content + ) + self.assertTrue( + "This is an e-mail message sent automatically by Microsoft Outlook while" in comm.content + ) def test_outgoing(self): - make(subject = "test-mail-000", content="test mail 000", recipients="test_receiver@example.com", - send_email=True, sender="test_sender@example.com") + make( + subject="test-mail-000", + content="test mail 000", + recipients="test_receiver@example.com", + send_email=True, + sender="test_sender@example.com", + ) mail = email.message_from_string(frappe.get_last_doc("Email Queue").message) self.assertTrue("test-mail-000" in mail.get("Subject")) def test_sendmail(self): - frappe.sendmail(sender="test_sender@example.com", recipients="test_recipient@example.com", - content="test mail 001", subject="test-mail-001", delayed=False) + frappe.sendmail( + sender="test_sender@example.com", + recipients="test_recipient@example.com", + content="test mail 001", + subject="test-mail-001", + delayed=False, + ) sent_mail = email.message_from_string(frappe.safe_decode(frappe.flags.sent_mail)) self.assertTrue("test-mail-001" in sent_mail.get("Subject")) def test_print_format(self): - make(sender="test_sender@example.com", recipients="test_recipient@example.com", - content="test mail 001", subject="test-mail-002", doctype="Email Account", - name="_Test Email Account 1", print_format="Standard", send_email=True) + make( + sender="test_sender@example.com", + recipients="test_recipient@example.com", + content="test mail 001", + subject="test-mail-002", + doctype="Email Account", + name="_Test Email Account 1", + print_format="Standard", + send_email=True, + ) sent_mail = email.message_from_string(frappe.get_last_doc("Email Queue").message) self.assertTrue("test-mail-002" in sent_mail.get("Subject")) def test_threading(self): - cleanup(["in", ['test_sender@example.com', 'test@example.com']]) + cleanup(["in", ["test_sender@example.com", "test@example.com"]]) # send - sent_name = make(subject = "Test", content="test content", - recipients="test_receiver@example.com", sender="test@example.com",doctype="ToDo",name=frappe.get_last_doc("ToDo").name, - send_email=True)["name"] + sent_name = make( + subject="Test", + content="test content", + recipients="test_receiver@example.com", + sender="test@example.com", + doctype="ToDo", + name=frappe.get_last_doc("ToDo").name, + send_email=True, + )["name"] sent_mail = email.message_from_string(frappe.get_last_doc("Email Queue").message) @@ -196,15 +218,7 @@ class TestEmailAccount(unittest.TestCase): # parse reply messages = { # append_to = ToDo - '"INBOX"': { - 'latest_messages': [ - raw - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] - } + '"INBOX"': {"latest_messages": [raw], "seen_status": {2: "UNSEEN"}, "uid_list": [2]} } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -217,7 +231,7 @@ class TestEmailAccount(unittest.TestCase): self.assertEqual(comm.reference_name, sent.reference_name) def test_threading_by_subject(self): - cleanup(["in", ['test_sender@example.com', 'test@example.com']]) + cleanup(["in", ["test_sender@example.com", "test@example.com"]]) with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-2.raw"), "r") as f: test_mails = [f.read()] @@ -229,20 +243,20 @@ class TestEmailAccount(unittest.TestCase): messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': test_mails, - 'seen_status': { - 2: 'UNSEEN', - 3: 'UNSEEN' - }, - 'uid_list': [2, 3] + "latest_messages": test_mails, + "seen_status": {2: "UNSEEN", 3: "UNSEEN"}, + "uid_list": [2, 3], } } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") TestEmailAccount.mocked_email_receive(email_account, messages) - comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, - fields=["name", "reference_doctype", "reference_name"]) + comm_list = frappe.get_all( + "Communication", + filters={"sender": "test_sender@example.com"}, + fields=["name", "reference_doctype", "reference_name"], + ) # both communications attached to the same reference self.assertEqual(comm_list[0].reference_doctype, comm_list[1].reference_doctype) self.assertEqual(comm_list[0].reference_name, comm_list[1].reference_name) @@ -252,26 +266,27 @@ class TestEmailAccount(unittest.TestCase): frappe.db.delete("Email Queue") # reference document for testing - event = frappe.get_doc(dict(doctype='Event', subject='test-message')).insert() + event = frappe.get_doc(dict(doctype="Event", subject="test-message")).insert() # send a mail against this - frappe.sendmail(recipients='test@example.com', subject='test message for threading', - message='testing', reference_doctype=event.doctype, reference_name=event.name) + frappe.sendmail( + recipients="test@example.com", + subject="test message for threading", + message="testing", + reference_doctype=event.doctype, + reference_name=event.name, + ) - last_mail = frappe.get_doc('Email Queue', dict(reference_name=event.name)) + last_mail = frappe.get_doc("Email Queue", dict(reference_name=event.name)) # get test mail with message-id as in-reply-to with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f: messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - f.read().replace('{{ message_id }}', last_mail.message_id) - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] + "latest_messages": [f.read().replace("{{ message_id }}", last_mail.message_id)], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], } } @@ -279,8 +294,11 @@ class TestEmailAccount(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") TestEmailAccount.mocked_email_receive(email_account, messages) - comm_list = frappe.get_all("Communication", filters={"sender":"test_sender@example.com"}, - fields=["name", "reference_doctype", "reference_name"]) + comm_list = frappe.get_all( + "Communication", + filters={"sender": "test_sender@example.com"}, + fields=["name", "reference_doctype", "reference_name"], + ) # check if threaded correctly self.assertEqual(comm_list[0].reference_doctype, event.doctype) @@ -292,13 +310,9 @@ class TestEmailAccount(unittest.TestCase): messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - self.get_test_mail('incoming-1.raw') - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] + "latest_messages": [self.get_test_mail("incoming-1.raw")], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], } } @@ -306,16 +320,20 @@ class TestEmailAccount(unittest.TestCase): TestEmailAccount.mocked_email_receive(email_account, messages) comm = frappe.get_doc("Communication", {"sender": "test_sender@example.com"}) - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": comm.reference_doctype, - "reference_name": comm.reference_name})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": comm.reference_doctype, "reference_name": comm.reference_name}, + ) + ) def test_handle_bad_emails(self): mail_content = self.get_test_mail(fname="incoming-1.raw") - message_id = Email(mail_content).mail.get('Message-ID') + message_id = Email(mail_content).mail.get("Message-ID") email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing") - self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id})) + self.assertTrue(frappe.db.get_value("Unhandled Email", {"message_id": message_id})) def test_imap_folder(self): # assert tests if imap_folder >= 1 and imap is checked @@ -337,10 +355,10 @@ class TestEmailAccount(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") mail_content = self.get_test_mail(fname="incoming-2.raw") - inbound_mail = InboundMail(mail_content, email_account, 12345, 1, 'ToDo') + inbound_mail = InboundMail(mail_content, email_account, 12345, 1, "ToDo") communication = inbound_mail.process() # the append_to for the email is set to ToDO in "_Test Email Account 1" - self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertEqual(communication.reference_doctype, "ToDo") self.assertTrue(communication.reference_name) self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) @@ -352,26 +370,16 @@ class TestEmailAccount(unittest.TestCase): messages = { # append_to = ToDo '"INBOX"': { - 'latest_messages': [ - mail_content_1, - mail_content_2 - ], - 'seen_status': { - 0: 'UNSEEN', - 1: 'UNSEEN' - }, - 'uid_list': [0,1] + "latest_messages": [mail_content_1, mail_content_2], + "seen_status": {0: "UNSEEN", 1: "UNSEEN"}, + "uid_list": [0, 1], }, # append_to = Communication '"Test Folder"': { - 'latest_messages': [ - mail_content_3 - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] - } + "latest_messages": [mail_content_3], + "seen_status": {2: "UNSEEN"}, + "uid_list": [2], + }, } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -383,11 +391,13 @@ class TestEmailAccount(unittest.TestCase): for mail in mails: communication = mail.process() - if mail.append_to == 'ToDo': + if mail.append_to == "ToDo": inbox_mails += 1 - self.assertEqual(communication.reference_doctype, 'ToDo') + self.assertEqual(communication.reference_doctype, "ToDo") self.assertTrue(communication.reference_name) - self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name)) + self.assertTrue( + frappe.db.exists(communication.reference_doctype, communication.reference_name) + ) else: test_folder_mails += 1 self.assertEqual(communication.reference_doctype, None) @@ -397,7 +407,9 @@ class TestEmailAccount(unittest.TestCase): @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) - def mocked_get_inbound_mails(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + def mocked_get_inbound_mails( + email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None + ): from frappe.email.receive import EmailServer def get_mocked_messages(**kwargs): @@ -410,14 +422,18 @@ class TestEmailAccount(unittest.TestCase): @patch("frappe.email.receive.EmailServer.select_imap_folder", return_value=True) @patch("frappe.email.receive.EmailServer.logout", side_effect=lambda: None) - def mocked_email_receive(email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None): + def mocked_email_receive( + email_account, messages={}, mocked_logout=None, mocked_select_imap_folder=None + ): def get_mocked_messages(**kwargs): return messages.get(kwargs["folder"], {}) from frappe.email.receive import EmailServer + with patch.object(EmailServer, "get_messages", side_effect=get_mocked_messages): email_account.receive() + class TestInboundMail(unittest.TestCase): @classmethod def setUpClass(cls): @@ -446,29 +462,22 @@ class TestInboundMail(unittest.TestCase): return doc def new_communication(self, **kwargs): - defaults = { - 'subject': "Test Subject" - } + defaults = {"subject": "Test Subject"} d = {**defaults, **kwargs} - return self.new_doc('Communication', **d) + return self.new_doc("Communication", **d) def new_email_queue(self, **kwargs): - defaults = { - 'message_id': get_message_id().strip(" <>") - } + defaults = {"message_id": get_message_id().strip(" <>")} d = {**defaults, **kwargs} - return self.new_doc('Email Queue', **d) + return self.new_doc("Email Queue", **d) def new_todo(self, **kwargs): - defaults = { - 'description': "Description" - } + defaults = {"description": "Description"} d = {**defaults, **kwargs} - return self.new_doc('ToDo', **d) + return self.new_doc("ToDo", **d) def test_self_sent_mail(self): - """Check that we raise SentEmailInInboxError if the inbound mail is self sent mail. - """ + """Check that we raise SentEmailInInboxError if the inbound mail is self sent mail.""" mail_content = self.get_test_mail(fname="incoming-self-sent.raw") email_account = frappe.get_doc("Email Account", "_Test Email Account 1") inbound_mail = InboundMail(mail_content, email_account, 1, 1) @@ -476,8 +485,7 @@ class TestInboundMail(unittest.TestCase): inbound_mail.process() def test_mail_exist_validation(self): - """Do not create communication record if the mail is already downloaded into the system. - """ + """Do not create communication record if the mail is already downloaded into the system.""" mail_content = self.get_test_mail(fname="incoming-1.raw") message_id = Email(mail_content).message_id # Create new communication record in DB @@ -492,8 +500,7 @@ class TestInboundMail(unittest.TestCase): self.assertEqual(communication.name, new_communication.name) def test_find_parent_email_queue(self): - """If the mail is reply to the already sent mail, there will be a email queue record. - """ + """If the mail is reply to the already sent mail, there will be a email queue record.""" # Create email queue record queue_record = self.new_email_queue() @@ -532,9 +539,7 @@ class TestInboundMail(unittest.TestCase): Ex: User replied to his/her mail. """ message_id = "new-message-id" - mail_content = self.get_test_mail(fname="reply-4.raw").replace( - "{{ message_id }}", message_id - ) + mail_content = self.get_test_mail(fname="reply-4.raw").replace("{{ message_id }}", message_id) email_account = frappe.get_doc("Email Account", "_Test Email Account 1") inbound_mail = InboundMail(mail_content, email_account, 12345, 1) @@ -547,8 +552,7 @@ class TestInboundMail(unittest.TestCase): self.assertEqual(parent_communication.name, communication.name) def test_find_parent_communication_from_header(self): - """Incase of header contains parent communication name - """ + """Incase of header contains parent communication name""" communication = self.new_communication() mail_content = self.get_test_mail(fname="reply-4.raw").replace( "{{ message_id }}", f"<{communication.name}@{frappe.local.site}>" @@ -563,7 +567,7 @@ class TestInboundMail(unittest.TestCase): # Create email queue record todo = self.new_todo() # communication = self.new_communication(reference_doctype='ToDo', reference_name=todo.name) - queue_record = self.new_email_queue(reference_doctype='ToDo', reference_name=todo.name) + queue_record = self.new_email_queue(reference_doctype="ToDo", reference_name=todo.name) mail_content = self.get_test_mail(fname="reply-4.raw").replace( "{{ message_id }}", queue_record.message_id ) @@ -588,7 +592,7 @@ class TestInboundMail(unittest.TestCase): def test_reference_document_by_subject_match(self): subject = "New todo" - todo = self.new_todo(sender='test_sender@example.com', description=subject) + todo = self.new_todo(sender="test_sender@example.com", description=subject) mail_content = self.get_test_mail(fname="incoming-subject-placeholder.raw").replace( "{{ subject }}", f"RE: {subject}" @@ -607,6 +611,7 @@ class TestInboundMail(unittest.TestCase): self.assertTrue(communication.is_first) self.assertTrue(communication._attachments) + def cleanup(sender=None): filters = {} if sender: diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py index 1611d32351..ff59725fd7 100644 --- a/frappe/email/doctype/email_domain/email_domain.py +++ b/frappe/email/doctype/email_domain/email_domain.py @@ -2,12 +2,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE +import imaplib +import poplib +import smtplib + import frappe from frappe import _ -from frappe.model.document import Document -from frappe.utils import validate_email_address ,cint, cstr -import imaplib,poplib,smtplib from frappe.email.utils import get_port +from frappe.model.document import Document +from frappe.utils import cint, cstr, validate_email_address + class EmailDomain(Document): def autoname(self): @@ -27,25 +31,35 @@ class EmailDomain(Document): if not frappe.local.flags.in_install and not frappe.local.flags.in_patch: try: if self.use_imap: - logger.info('Checking incoming IMAP email server {host}:{port} ssl={ssl}...'.format( - host=self.email_server, port=get_port(self), ssl=self.use_ssl)) + logger.info( + "Checking incoming IMAP email server {host}:{port} ssl={ssl}...".format( + host=self.email_server, port=get_port(self), ssl=self.use_ssl + ) + ) if self.use_ssl: test = imaplib.IMAP4_SSL(self.email_server, port=get_port(self)) else: test = imaplib.IMAP4(self.email_server, port=get_port(self)) else: - logger.info('Checking incoming POP3 email server {host}:{port} ssl={ssl}...'.format( - host=self.email_server, port=get_port(self), ssl=self.use_ssl)) + logger.info( + "Checking incoming POP3 email server {host}:{port} ssl={ssl}...".format( + host=self.email_server, port=get_port(self), ssl=self.use_ssl + ) + ) if self.use_ssl: test = poplib.POP3_SSL(self.email_server, port=get_port(self)) else: test = poplib.POP3(self.email_server, port=get_port(self)) except Exception as e: - logger.warn('Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e) - frappe.throw(title=_("Incoming email account not correct"), - msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e)) + logger.warn( + 'Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e + ) + frappe.throw( + title=_("Incoming email account not correct"), + msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e), + ) finally: try: @@ -57,34 +71,58 @@ class EmailDomain(Document): pass try: - if self.get('use_ssl_for_outgoing'): - if not self.get('smtp_port'): + if self.get("use_ssl_for_outgoing"): + if not self.get("smtp_port"): self.smtp_port = 465 - logger.info('Checking outgoing SMTPS email server {host}:{port}...'.format( - host=self.smtp_server, port=self.smtp_port)) - sess = smtplib.SMTP_SSL((self.smtp_server or "").encode('utf-8'), - cint(self.smtp_port) or None) + logger.info( + "Checking outgoing SMTPS email server {host}:{port}...".format( + host=self.smtp_server, port=self.smtp_port + ) + ) + sess = smtplib.SMTP_SSL( + (self.smtp_server or "").encode("utf-8"), cint(self.smtp_port) or None + ) else: if self.use_tls and not self.smtp_port: self.smtp_port = 587 - logger.info('Checking outgoing SMTP email server {host}:{port} STARTTLS={tls}...'.format( - host=self.smtp_server, port=self.get('smtp_port'), tls=self.use_tls)) + logger.info( + "Checking outgoing SMTP email server {host}:{port} STARTTLS={tls}...".format( + host=self.smtp_server, port=self.get("smtp_port"), tls=self.use_tls + ) + ) sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None) sess.quit() except Exception as e: - logger.warn('Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e) - frappe.throw(title=_("Outgoing email account not correct"), - msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e)) + logger.warn( + 'Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e + ) + frappe.throw( + title=_("Outgoing email account not correct"), + msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e), + ) def on_update(self): """update all email accounts using this domain""" for email_account in frappe.get_all("Email Account", filters={"domain": self.name}): try: email_account = frappe.get_doc("Email Account", email_account.name) - for attr in ["email_server", "use_imap", "use_ssl", "use_tls", "attachment_limit", "smtp_server", "smtp_port", "use_ssl_for_outgoing", "append_emails_to_sent_folder", "incoming_port"]: + for attr in [ + "email_server", + "use_imap", + "use_ssl", + "use_tls", + "attachment_limit", + "smtp_server", + "smtp_port", + "use_ssl_for_outgoing", + "append_emails_to_sent_folder", + "incoming_port", + ]: email_account.set(attr, self.get(attr, default=0)) email_account.save() except Exception as e: - frappe.msgprint(_("Error has occurred in {0}").format(email_account.name), raise_exception=e.__class__) + frappe.msgprint( + _("Error has occurred in {0}").format(email_account.name), raise_exception=e.__class__ + ) diff --git a/frappe/email/doctype/email_domain/test_email_domain.py b/frappe/email/doctype/email_domain/test_email_domain.py index 7522dd5282..0162e4efe0 100644 --- a/frappe/email/doctype/email_domain/test_email_domain.py +++ b/frappe/email/doctype/email_domain/test_email_domain.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest + +import frappe from frappe.test_runner import make_test_objects -test_records = frappe.get_test_records('Email Domain') +test_records = frappe.get_test_records("Email Domain") -class TestDomain(unittest.TestCase): +class TestDomain(unittest.TestCase): def setUp(self): - make_test_objects('Email Domain', reset=True) + make_test_objects("Email Domain", reset=True) def tearDown(self): frappe.delete_doc("Email Account", "Test") diff --git a/frappe/email/doctype/email_flag_queue/email_flag_queue.py b/frappe/email/doctype/email_flag_queue/email_flag_queue.py index 886cf3c24b..1b29d2f9d8 100644 --- a/frappe/email/doctype/email_flag_queue/email_flag_queue.py +++ b/frappe/email/doctype/email_flag_queue/email_flag_queue.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class EmailFlagQueue(Document): pass diff --git a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py index b0e17b3b85..f52aeb61fa 100644 --- a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py +++ b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Email Flag Queue') + class TestEmailFlagQueue(unittest.TestCase): pass diff --git a/frappe/email/doctype/email_group/email_group.py b/frappe/email/doctype/email_group/email_group.py index ad52d9a9ec..6489e82341 100755 --- a/frappe/email/doctype/email_group/email_group.py +++ b/frappe/email/doctype/email_group/email_group.py @@ -4,22 +4,27 @@ import frappe from frappe import _ -from frappe.utils import validate_email_address from frappe.model.document import Document -from frappe.utils import parse_addr +from frappe.utils import parse_addr, validate_email_address + class EmailGroup(Document): def onload(self): - singles = [d.name for d in frappe.db.get_all("DocType", "name", {"issingle": 1})] - self.get("__onload").import_types = [{"value": d.parent, "label": "{0} ({1})".format(d.parent, d.label)} \ - for d in frappe.db.get_all("DocField", ("parent", "label"), {"options": "Email"}) - if d.parent not in singles] + singles = [d.name for d in frappe.db.get_all("DocType", "name", {"issingle": 1})] + self.get("__onload").import_types = [ + {"value": d.parent, "label": "{0} ({1})".format(d.parent, d.label)} + for d in frappe.db.get_all("DocField", ("parent", "label"), {"options": "Email"}) + if d.parent not in singles + ] def import_from(self, doctype): """Extract Email Addresses from given doctype and add them to the current list""" meta = frappe.get_meta(doctype) - email_field = [d.fieldname for d in meta.fields - if d.fieldtype in ("Data", "Small Text", "Text", "Code") and d.options=="Email"][0] + email_field = [ + d.fieldname + for d in meta.fields + if d.fieldtype in ("Data", "Small Text", "Text", "Code") and d.options == "Email" + ][0] unsubscribed_field = "unsubscribed" if meta.get_field("unsubscribed") else None added = 0 @@ -27,12 +32,14 @@ class EmailGroup(Document): try: email = parse_addr(user.get(email_field))[1] if user.get(email_field) else None if email: - frappe.get_doc({ - "doctype": "Email Group Member", - "email_group": self.name, - "email": email, - "unsubscribed": user.get(unsubscribed_field) if unsubscribed_field else 0 - }).insert(ignore_permissions=True) + frappe.get_doc( + { + "doctype": "Email Group Member", + "email_group": self.name, + "email": email, + "unsubscribed": user.get(unsubscribed_field) if unsubscribed_field else 0, + } + ).insert(ignore_permissions=True) added += 1 except frappe.UniqueValidationError: @@ -48,25 +55,30 @@ class EmailGroup(Document): return self.total_subscribers def get_total_subscribers(self): - return frappe.db.sql("""select count(*) from `tabEmail Group Member` - where email_group=%s""", self.name)[0][0] + return frappe.db.sql( + """select count(*) from `tabEmail Group Member` + where email_group=%s""", + self.name, + )[0][0] def on_trash(self): for d in frappe.get_all("Email Group Member", "name", {"email_group": self.name}): frappe.delete_doc("Email Group Member", d.name) + @frappe.whitelist() def import_from(name, doctype): nlist = frappe.get_doc("Email Group", name) if nlist.has_permission("write"): return nlist.import_from(doctype) + @frappe.whitelist() def add_subscribers(name, email_list): if not isinstance(email_list, (list, tuple)): email_list = email_list.replace(",", "\n").split("\n") - template = frappe.db.get_value('Email Group', name, 'welcome_email_template') + template = frappe.db.get_value("Email Group", name, "welcome_email_template") welcome_email = frappe.get_doc("Email Template", template) if template else None count = 0 @@ -75,13 +87,10 @@ def add_subscribers(name, email_list): parsed_email = validate_email_address(email, False) if parsed_email: - if not frappe.db.get_value("Email Group Member", - {"email_group": name, "email": parsed_email}): - frappe.get_doc({ - "doctype": "Email Group Member", - "email_group": name, - "email": parsed_email - }).insert(ignore_permissions=frappe.flags.ignore_permissions) + if not frappe.db.get_value("Email Group Member", {"email_group": name, "email": parsed_email}): + frappe.get_doc( + {"doctype": "Email Group Member", "email_group": name, "email": parsed_email} + ).insert(ignore_permissions=frappe.flags.ignore_permissions) send_welcome_email(welcome_email, parsed_email, name) @@ -95,15 +104,13 @@ def add_subscribers(name, email_list): return frappe.get_doc("Email Group", name).update_total_subscribers() + def send_welcome_email(welcome_email, email, email_group): """Send welcome email for the subscribers of a given email group.""" if not welcome_email: return - args = dict( - email=email, - email_group=email_group - ) + args = dict(email=email, email_group=email_group) email_message = welcome_email.response or welcome_email.response_html message = frappe.render_template(email_message, args) frappe.sendmail(email, subject=welcome_email.subject, message=message) diff --git a/frappe/email/doctype/email_group/test_email_group.py b/frappe/email/doctype/email_group/test_email_group.py index 06341c128e..4013c17d93 100644 --- a/frappe/email/doctype/email_group/test_email_group.py +++ b/frappe/email/doctype/email_group/test_email_group.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Email Group') + class TestEmailGroup(unittest.TestCase): pass diff --git a/frappe/email/doctype/email_group_member/email_group_member.py b/frappe/email/doctype/email_group_member/email_group_member.py index a9fd26f710..76990b18fb 100644 --- a/frappe/email/doctype/email_group_member/email_group_member.py +++ b/frappe/email/doctype/email_group_member/email_group_member.py @@ -5,14 +5,16 @@ import frappe from frappe.model.document import Document + class EmailGroupMember(Document): def after_delete(self): - email_group = frappe.get_doc('Email Group', self.email_group) + email_group = frappe.get_doc("Email Group", self.email_group) email_group.update_total_subscribers() def after_insert(self): - email_group = frappe.get_doc('Email Group', self.email_group) + email_group = frappe.get_doc("Email Group", self.email_group) email_group.update_total_subscribers() + def after_doctype_insert(): frappe.db.add_unique("Email Group Member", ("email_group", "email")) diff --git a/frappe/email/doctype/email_group_member/test_email_group_member.py b/frappe/email/doctype/email_group_member/test_email_group_member.py index de006dccb9..562b0327f2 100644 --- a/frappe/email/doctype/email_group_member/test_email_group_member.py +++ b/frappe/email/doctype/email_group_member/test_email_group_member.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Email Group Member') + class TestEmailGroupMember(unittest.TestCase): pass diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 9003158a85..38577eeb97 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -2,45 +2,46 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import traceback import json - -from rq.timeouts import JobTimeoutException -import smtplib import quopri +import smtplib +import traceback from email.parser import Parser from email.policy import SMTPUTF8 + from html2text import html2text +from rq.timeouts import JobTimeoutException import frappe from frappe import _, safe_encode, task -from frappe.model.document import Document -from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message -from frappe.email.email_body import add_attachment, get_formatted_html, get_email -from frappe.utils import cint, split_emails, add_days, nowdate, cstr, get_hook_method from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.email.email_body import add_attachment, get_email, get_formatted_html +from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message +from frappe.model.document import Document from frappe.query_builder.utils import DocType - +from frappe.utils import add_days, cint, cstr, get_hook_method, nowdate, split_emails MAX_RETRY_COUNT = 3 + + class EmailQueue(Document): - DOCTYPE = 'Email Queue' + DOCTYPE = "Email Queue" def set_recipients(self, recipients): self.set("recipients", []) for r in recipients: - self.append("recipients", {"recipient":r, "status":"Not Sent"}) + self.append("recipients", {"recipient": r, "status": "Not Sent"}) def on_trash(self): self.prevent_email_queue_delete() def prevent_email_queue_delete(self): - if frappe.session.user != 'Administrator': - frappe.throw(_('Only Administrator can delete Email Queue')) + if frappe.session.user != "Administrator": + frappe.throw(_("Only Administrator can delete Email Queue")) def get_duplicate(self, recipients): values = self.as_dict() - del values['name'] + del values["name"] duplicate = frappe.get_doc(values) duplicate.set_recipients(recipients) return duplicate @@ -48,10 +49,10 @@ class EmailQueue(Document): @classmethod def new(cls, doc_data, ignore_permissions=False): data = doc_data.copy() - if not data.get('recipients'): + if not data.get("recipients"): return - recipients = data.pop('recipients') + recipients = data.pop("recipients") doc = frappe.new_doc(cls.DOCTYPE) doc.update(data) doc.set_recipients(recipients) @@ -73,9 +74,9 @@ class EmailQueue(Document): frappe.db.commit() def update_status(self, status, commit=False, **kwargs): - self.update_db(status = status, commit = commit, **kwargs) + self.update_db(status=status, commit=commit, **kwargs) if self.communication: - communication_doc = frappe.get_doc('Communication', self.communication) + communication_doc = frappe.get_doc("Communication", self.communication) communication_doc.set_delivery_status(commit=commit) @property @@ -92,24 +93,24 @@ class EmailQueue(Document): def get_email_account(self): if self.email_account: - return frappe.get_doc('Email Account', self.email_account) + return frappe.get_doc("Email Account", self.email_account) return EmailAccount.find_outgoing( - match_by_email = self.sender, match_by_doctype = self.reference_doctype) + match_by_email=self.sender, match_by_doctype=self.reference_doctype + ) def is_to_be_sent(self): - return self.status in ['Not Sent','Partially Sent'] + return self.status in ["Not Sent", "Partially Sent"] def can_send_now(self): - hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1) + hold_queue = cint(frappe.defaults.get_defaults().get("hold_queue")) == 1 if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue: return False return True def send(self, is_background_task=False): - """ Send emails to recipients. - """ + """Send emails to recipients.""" if not self.can_send_now(): return @@ -120,7 +121,7 @@ class EmailQueue(Document): continue message = ctx.build_message(recipient.recipient) - method = get_hook_method('override_email_send') + method = get_hook_method("override_email_send") if method: method(self, self.sender, recipient.recipient, message) else: @@ -136,7 +137,7 @@ class EmailQueue(Document): ctx.email_account_doc.append_email_to_sent_folder(message) -@task(queue = 'short') +@task(queue="short") def send_mail(email_queue_name, is_background_task=False): """This is equalent to EmqilQueue.send. @@ -145,6 +146,7 @@ def send_mail(email_queue_name, is_background_task=False): record = EmailQueue.find(email_queue_name) record.send(is_background_task=is_background_task) + class SendMailContext: def __init__(self, queue_doc: Document, is_background_task: bool = False): self.queue_doc = queue_doc @@ -154,7 +156,7 @@ class SendMailContext: self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()] def __enter__(self): - self.queue_doc.update_status(status='Sending', commit=True) + self.queue_doc.update_status(status="Sending", commit=True) return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -164,32 +166,32 @@ class SendMailContext: smtplib.SMTPRecipientsRefused, smtplib.SMTPConnectError, smtplib.SMTPHeloError, - JobTimeoutException + JobTimeoutException, ] self.smtp_server.quit() self.log_exception(exc_type, exc_val, exc_tb) if exc_type in exceptions: - email_status = (self.sent_to and 'Partially Sent') or 'Not Sent' - self.queue_doc.update_status(status = email_status, commit = True) + email_status = (self.sent_to and "Partially Sent") or "Not Sent" + self.queue_doc.update_status(status=email_status, commit=True) elif exc_type: if self.queue_doc.retry < MAX_RETRY_COUNT: - update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1} + update_fields = {"status": "Not Sent", "retry": self.queue_doc.retry + 1} else: - update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'} - self.queue_doc.update_status(**update_fields, commit = True) + update_fields = {"status": (self.sent_to and "Partially Errored") or "Error"} + self.queue_doc.update_status(**update_fields, commit=True) else: - email_status = self.is_mail_sent_to_all() and 'Sent' - email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent' + email_status = self.is_mail_sent_to_all() and "Sent" + email_status = email_status or (self.sent_to and "Partially Sent") or "Not Sent" - update_fields = {'status': email_status} + update_fields = {"status": email_status} if self.email_account_doc.is_exists_in_db(): - update_fields['email_account'] = self.email_account_doc.name + update_fields["email_account"] = self.email_account_doc.name else: - update_fields['email_account'] = None + update_fields["email_account"] = None - self.queue_doc.update_status(**update_fields, commit = True) + self.queue_doc.update_status(**update_fields, commit=True) def log_exception(self, exc_type, exc_val, exc_tb): if exc_type: @@ -197,9 +199,9 @@ class SendMailContext: traceback_string += f"\n Queue Name: {self.queue_doc.name}" if self.is_background_task: - frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string) + frappe.log_error(title="frappe.email.queue.flush", message=traceback_string) else: - frappe.log_error(message = traceback_string) + frappe.log_error(message=traceback_string) @property def smtp_session(self): @@ -209,7 +211,7 @@ class SendMailContext: def add_to_sent_list(self, recipient): # Update recipient status - recipient.update_db(status='Sent', commit=True) + recipient.update_db(status="Sent", commit=True) self.sent_to.append(recipient.recipient) def is_mail_sent_to_all(self): @@ -220,34 +222,34 @@ class SendMailContext: def message_placeholder(self, placeholder_key): map = { - 'tracker': '', - 'unsubscribe_url': '', - 'cc': '', - 'recipient': '', + "tracker": "", + "unsubscribe_url": "", + "cc": "", + "recipient": "", } return map.get(placeholder_key) def build_message(self, recipient_email): - """Build message specific to the recipient. - """ + """Build message specific to the recipient.""" message = self.queue_doc.message if not message: return "" - message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str()) - message = message.replace(self.message_placeholder('unsubscribe_url'), - self.get_unsubscribe_str(recipient_email)) - message = message.replace(self.message_placeholder('cc'), self.get_receivers_str()) - message = message.replace(self.message_placeholder('recipient'), - self.get_receipient_str(recipient_email)) + message = message.replace(self.message_placeholder("tracker"), self.get_tracker_str()) + message = message.replace( + self.message_placeholder("unsubscribe_url"), self.get_unsubscribe_str(recipient_email) + ) + message = message.replace(self.message_placeholder("cc"), self.get_receivers_str()) + message = message.replace( + self.message_placeholder("recipient"), self.get_receipient_str(recipient_email) + ) message = self.include_attachments(message) return message def get_tracker_str(self): - tracker_url_html = \ - '' + tracker_url_html = '' - message = '' + message = "" if frappe.conf.use_ssl and self.email_account_doc.track_email_status: message = quopri.encodestring( tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode() @@ -255,25 +257,30 @@ class SendMailContext: return message def get_unsubscribe_str(self, recipient_email): - unsubscribe_url = '' + unsubscribe_url = "" if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype: doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name - unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email, - self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param) + unsubscribe_url = get_unsubcribed_url( + doctype, + doc_name, + recipient_email, + self.queue_doc.unsubscribe_method, + self.queue_doc.unsubscribe_param, + ) return quopri.encodestring(unsubscribe_url.encode()).decode() def get_receivers_str(self): - message = '' + message = "" if self.queue_doc.expose_recipients == "footer": - to_str = ', '.join(self.queue_doc.to) - cc_str = ', '.join(self.queue_doc.cc) + to_str = ", ".join(self.queue_doc.to) + cc_str = ", ".join(self.queue_doc.cc) message = f"This email was sent to {to_str}" message = message + f" and copied to {cc_str}" if cc_str else message return message def get_receipient_str(self, recipient_email): - message = '' + message = "" if self.queue_doc.expose_recipients != "header": message = recipient_email return message @@ -283,23 +290,19 @@ class SendMailContext: attachments = self.queue_doc.attachments_list for attachment in attachments: - if attachment.get('fcontent'): + if attachment.get("fcontent"): continue file_filters = {} - if attachment.get('fid'): - file_filters['name'] = attachment.get('fid') - elif attachment.get('file_url'): - file_filters['file_url'] = attachment.get('file_url') + if attachment.get("fid"): + file_filters["name"] = attachment.get("fid") + elif attachment.get("file_url"): + file_filters["file_url"] = attachment.get("file_url") if file_filters: _file = frappe.get_doc("File", file_filters) fcontent = _file.get_content() - attachment.update({ - 'fname': _file.file_name, - 'fcontent': fcontent, - 'parent': message_obj - }) + attachment.update({"fname": _file.file_name, "fcontent": fcontent, "parent": message_obj}) attachment.pop("fid", None) attachment.pop("file_url", None) add_attachment(**attachment) @@ -312,37 +315,66 @@ class SendMailContext: return safe_encode(message_obj.as_string()) + @frappe.whitelist() def retry_sending(name): doc = frappe.get_doc("Email Queue", name) if doc and (doc.status == "Error" or doc.status == "Partially Errored"): doc.status = "Not Sent" for d in doc.recipients: - if d.status != 'Sent': - d.status = 'Not Sent' + if d.status != "Sent": + d.status = "Not Sent" doc.save(ignore_permissions=True) + @frappe.whitelist() def send_now(name): record = EmailQueue.find(name) if record: record.send() + def on_doctype_update(): """Add index in `tabCommunication` for `(reference_doctype, reference_name)`""" - frappe.db.add_index('Email Queue', ('status', 'send_after', 'priority', 'creation'), 'index_bulk_flush') + frappe.db.add_index( + "Email Queue", ("status", "send_after", "priority", "creation"), "index_bulk_flush" + ) + class QueueBuilder: - """Builds Email Queue from the given data - """ - def __init__(self, recipients=None, sender=None, subject=None, message=None, - text_content=None, reference_doctype=None, reference_name=None, - unsubscribe_method=None, unsubscribe_params=None, unsubscribe_message=None, - attachments=None, reply_to=None, cc=None, bcc=None, message_id=None, in_reply_to=None, - send_after=None, expose_recipients=None, send_priority=1, communication=None, - read_receipt=None, queue_separately=False, is_notification=False, - add_unsubscribe_link=1, inline_images=None, header=None, - print_letterhead=False, with_container=False): + """Builds Email Queue from the given data""" + + def __init__( + self, + recipients=None, + sender=None, + subject=None, + message=None, + text_content=None, + reference_doctype=None, + reference_name=None, + unsubscribe_method=None, + unsubscribe_params=None, + unsubscribe_message=None, + attachments=None, + reply_to=None, + cc=None, + bcc=None, + message_id=None, + in_reply_to=None, + send_after=None, + expose_recipients=None, + send_priority=1, + communication=None, + read_receipt=None, + queue_separately=False, + is_notification=False, + add_unsubscribe_link=1, + inline_images=None, + header=None, + print_letterhead=False, + with_container=False, + ): """Add email to sending queue (Email Queue) :param recipients: List of recipients. @@ -403,7 +435,7 @@ class QueueBuilder: @property def unsubscribe_method(self): - return self._unsubscribe_method or '/api/method/frappe.email.queue.unsubscribe' + return self._unsubscribe_method or "/api/method/frappe.email.queue.unsubscribe" def _get_emails_list(self, emails=None): emails = split_emails(emails) if isinstance(emails, str) else (emails or []) @@ -436,7 +468,7 @@ class QueueBuilder: def email_text_content(self): unsubscribe_msg = self.unsubscribe_message() - unsubscribe_text_message = (unsubscribe_msg and unsubscribe_msg.text) or '' + unsubscribe_text_message = (unsubscribe_msg and unsubscribe_msg.text) or "" if self._text_content: return self._text_content + unsubscribe_text_message @@ -449,14 +481,21 @@ class QueueBuilder: def email_html_content(self): email_account = self.get_outgoing_email_account() - return get_formatted_html(self.subject, self._message, header=self.header, - email_account=email_account, unsubscribe_link=self.unsubscribe_message(), - with_container=self.with_container) + return get_formatted_html( + self.subject, + self._message, + header=self.header, + email_account=email_account, + unsubscribe_link=self.unsubscribe_message(), + with_container=self.with_container, + ) def should_include_unsubscribe_link(self): - return (self._add_unsubscribe_link == 1 + return ( + self._add_unsubscribe_link == 1 and self.reference_doctype - and (self._unsubscribe_message or self.reference_doctype=="Newsletter")) + and (self._unsubscribe_message or self.reference_doctype == "Newsletter") + ) def unsubscribe_message(self): if self.should_include_unsubscribe_link(): @@ -467,7 +506,8 @@ class QueueBuilder: return self._email_account self._email_account = EmailAccount.find_outgoing( - match_by_doctype=self.reference_doctype, match_by_email=self._sender, _raise_error=True) + match_by_doctype=self.reference_doctype, match_by_email=self._sender, _raise_error=True + ) return self._email_account def get_unsubscribed_user_emails(self): @@ -480,19 +520,19 @@ class QueueBuilder: if len(all_ids) > 0: unsubscribed = ( - frappe.qb.from_(EmailUnsubscribe).select( - EmailUnsubscribe.email - ).where( + frappe.qb.from_(EmailUnsubscribe) + .select(EmailUnsubscribe.email) + .where( EmailUnsubscribe.email.isin(all_ids) & ( ( (EmailUnsubscribe.reference_doctype == self.reference_doctype) & (EmailUnsubscribe.reference_name == self.reference_name) - ) | ( - EmailUnsubscribe.global_unsubscribe == 1 ) + | (EmailUnsubscribe.global_unsubscribe == 1) ) - ).distinct() + ) + .distinct() ).run(pluck=True) else: unsubscribed = None @@ -513,17 +553,18 @@ class QueueBuilder: if self._attachments: # store attachments with fid or print format details, to be attached on-demand later for att in self._attachments: - if att.get('fid') or att.get('file_url'): + if att.get("fid") or att.get("file_url"): attachments.append(att) elif att.get("print_format_attachment") == 1: - if not att.get('lang', None): - att['lang'] = frappe.local.lang - att['print_letterhead'] = self.print_letterhead + if not att.get("lang", None): + att["lang"] = frappe.local.lang + att["print_letterhead"] = self.print_letterhead attachments.append(att) return attachments def prepare_email_content(self): - mail = get_email(recipients=self.final_recipients(), + mail = get_email( + recipients=self.final_recipients(), sender=self.sender, subject=self.subject, formatted=self.email_html_content(), @@ -535,7 +576,8 @@ class QueueBuilder: email_account=self.get_outgoing_email_account(), expose_recipients=self.expose_recipients, inline_images=self.inline_images, - header=self.header) + header=self.header, + ) mail.set_message_id(self.message_id, self.is_notification) if self.read_receipt: @@ -561,12 +603,12 @@ class QueueBuilder: if not queue_separately: recipients = list(set(final_recipients + self.final_cc() + self.bcc)) - q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True) email_queues.append(q) else: for r in final_recipients: recipients = [r] if email_queues else list(set([r] + self.final_cc() + self.bcc)) - q = EmailQueue.new({**queue_data, **{'recipients': recipients}}, ignore_permissions=True) + q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True) email_queues.append(q) if send_now: @@ -583,32 +625,34 @@ class QueueBuilder: mail_to_string = cstr(mail.as_string()) except frappe.InvalidEmailAddressError: # bad Email Address - don't add to queue - frappe.log_error('Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} ' - .format(self.sender, ', '.join(self.final_recipients()), traceback.format_exc()), - 'Email Not Sent' + frappe.log_error( + "Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} ".format( + self.sender, ", ".join(self.final_recipients()), traceback.format_exc() + ), + "Email Not Sent", ) return d = { - 'priority': self.send_priority, - 'attachments': json.dumps(self.get_attachments()), - 'message_id': mail.msg_root["Message-Id"].strip(" <>"), - 'message': mail_to_string, - 'sender': self.sender, - 'reference_doctype': self.reference_doctype, - 'reference_name': self.reference_name, - 'add_unsubscribe_link': self._add_unsubscribe_link, - 'unsubscribe_method': self.unsubscribe_method, - 'unsubscribe_params': self.unsubscribe_params, - 'expose_recipients': self.expose_recipients, - 'communication': self.communication, - 'send_after': self.send_after, - 'show_as_cc': ",".join(self.final_cc()), - 'show_as_bcc': ','.join(self.bcc), - 'email_account': email_account_name or None + "priority": self.send_priority, + "attachments": json.dumps(self.get_attachments()), + "message_id": mail.msg_root["Message-Id"].strip(" <>"), + "message": mail_to_string, + "sender": self.sender, + "reference_doctype": self.reference_doctype, + "reference_name": self.reference_name, + "add_unsubscribe_link": self._add_unsubscribe_link, + "unsubscribe_method": self.unsubscribe_method, + "unsubscribe_params": self.unsubscribe_params, + "expose_recipients": self.expose_recipients, + "communication": self.communication, + "send_after": self.send_after, + "show_as_cc": ",".join(self.final_cc()), + "show_as_bcc": ",".join(self.bcc), + "email_account": email_account_name or None, } if include_recipients: - d['recipients'] = self.final_recipients() + d["recipients"] = self.final_recipients() return d diff --git a/frappe/email/doctype/email_queue/test_email_queue.py b/frappe/email/doctype/email_queue/test_email_queue.py index 8ebcb68a38..b90390916a 100644 --- a/frappe/email/doctype/email_queue/test_email_queue.py +++ b/frappe/email/doctype/email_queue/test_email_queue.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Email Queue') + class TestEmailQueue(unittest.TestCase): pass diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py index 95b8593c4c..d2e4753ddf 100644 --- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py +++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py @@ -5,17 +5,17 @@ import frappe from frappe.model.document import Document + class EmailQueueRecipient(Document): - DOCTYPE = 'Email Queue Recipient' + DOCTYPE = "Email Queue Recipient" def is_mail_to_be_sent(self): - return self.status == 'Not Sent' + return self.status == "Not Sent" def is_main_sent(self): - return self.status == 'Sent' + return self.status == "Sent" def update_db(self, commit=False, **kwargs): frappe.db.set_value(self.DOCTYPE, self.name, kwargs) if commit: frappe.db.commit() - diff --git a/frappe/email/doctype/email_rule/email_rule.py b/frappe/email/doctype/email_rule/email_rule.py index b2a4be5421..f603aed77e 100644 --- a/frappe/email/doctype/email_rule/email_rule.py +++ b/frappe/email/doctype/email_rule/email_rule.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class EmailRule(Document): pass diff --git a/frappe/email/doctype/email_rule/test_email_rule.py b/frappe/email/doctype/email_rule/test_email_rule.py index eef5448e57..2cea421bd6 100644 --- a/frappe/email/doctype/email_rule/test_email_rule.py +++ b/frappe/email/doctype/email_rule/test_email_rule.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestEmailRule(unittest.TestCase): pass diff --git a/frappe/email/doctype/email_template/email_template.py b/frappe/email/doctype/email_template/email_template.py index c51c46d72d..fcc6ce5010 100644 --- a/frappe/email/doctype/email_template/email_template.py +++ b/frappe/email/doctype/email_template/email_template.py @@ -1,10 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe from frappe.model.document import Document from frappe.utils.jinja import validate_template + class EmailTemplate(Document): def validate(self): if self.use_html: @@ -25,17 +28,14 @@ class EmailTemplate(Document): if isinstance(doc, str): doc = json.loads(doc) - return { - "subject" : self.get_formatted_subject(doc), - "message" : self.get_formatted_response(doc) - } + return {"subject": self.get_formatted_subject(doc), "message": self.get_formatted_response(doc)} @frappe.whitelist() def get_email_template(template_name, doc): - '''Returns the processed HTML of a email template with the given doc''' + """Returns the processed HTML of a email template with the given doc""" if isinstance(doc, str): doc = json.loads(doc) email_template = frappe.get_doc("Email Template", template_name) - return email_template.get_formatted_email(doc) \ No newline at end of file + return email_template.get_formatted_email(doc) diff --git a/frappe/email/doctype/email_template/test_email_template.py b/frappe/email/doctype/email_template/test_email_template.py index a92ee9f9c3..d37b5497fb 100644 --- a/frappe/email/doctype/email_template/test_email_template.py +++ b/frappe/email/doctype/email_template/test_email_template.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestEmailTemplate(unittest.TestCase): pass diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py index d2ee828a55..131546a3c2 100644 --- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py +++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py @@ -3,36 +3,45 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document + class EmailUnsubscribe(Document): def validate(self): if not self.global_unsubscribe and not (self.reference_doctype and self.reference_name): frappe.throw(_("Reference DocType and Reference Name are required"), frappe.MandatoryError) - if not self.global_unsubscribe and frappe.db.get_value(self.doctype, self.name, "global_unsubscribe"): + if not self.global_unsubscribe and frappe.db.get_value( + self.doctype, self.name, "global_unsubscribe" + ): frappe.throw(_("Delete this record to allow sending to this email address")) if self.global_unsubscribe: - if frappe.get_all("Email Unsubscribe", - filters={"email": self.email, "global_unsubscribe": 1, "name": ["!=", self.name]}): + if frappe.get_all( + "Email Unsubscribe", + filters={"email": self.email, "global_unsubscribe": 1, "name": ["!=", self.name]}, + ): frappe.throw(_("{0} already unsubscribed").format(self.email), frappe.DuplicateEntryError) else: - if frappe.get_all("Email Unsubscribe", + if frappe.get_all( + "Email Unsubscribe", filters={ "email": self.email, "reference_doctype": self.reference_doctype, "reference_name": self.reference_name, - "name": ["!=", self.name] - }): - frappe.throw(_("{0} already unsubscribed for {1} {2}").format( - self.email, self.reference_doctype, self.reference_name), - frappe.DuplicateEntryError) + "name": ["!=", self.name], + }, + ): + frappe.throw( + _("{0} already unsubscribed for {1} {2}").format( + self.email, self.reference_doctype, self.reference_name + ), + frappe.DuplicateEntryError, + ) def on_update(self): if self.reference_doctype and self.reference_name: doc = frappe.get_doc(self.reference_doctype, self.reference_name) doc.add_comment("Label", _("Left this conversation"), comment_email=self.email) - diff --git a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py index fdea802fdf..7f9173d0b0 100644 --- a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py +++ b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Email Unsubscribe') + class TestEmailUnsubscribe(unittest.TestCase): pass diff --git a/frappe/email/doctype/imap_folder/imap_folder.py b/frappe/email/doctype/imap_folder/imap_folder.py index b0bb36b677..222d67189c 100644 --- a/frappe/email/doctype/imap_folder/imap_folder.py +++ b/frappe/email/doctype/imap_folder/imap_folder.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class IMAPFolder(Document): pass diff --git a/frappe/email/doctype/newsletter/exceptions.py b/frappe/email/doctype/newsletter/exceptions.py index a6c688dbe8..ccce54c957 100644 --- a/frappe/email/doctype/newsletter/exceptions.py +++ b/frappe/email/doctype/newsletter/exceptions.py @@ -3,11 +3,14 @@ from frappe.exceptions import ValidationError + class NewsletterAlreadySentError(ValidationError): pass + class NoRecipientFoundError(ValidationError): pass + class NewsletterNotSavedError(ValidationError): pass diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index aa6fa2c40a..45a4539866 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -5,13 +5,12 @@ from typing import Dict, List import frappe import frappe.utils - from frappe import _ -from frappe.website.website_generator import WebsiteGenerator -from frappe.utils.verified_command import get_signed_params, verify_request from frappe.email.doctype.email_group.email_group import add_subscribers +from frappe.utils.verified_command import get_signed_params, verify_request +from frappe.website.website_generator import WebsiteGenerator -from .exceptions import NewsletterAlreadySentError, NoRecipientFoundError, NewsletterNotSavedError +from .exceptions import NewsletterAlreadySentError, NewsletterNotSavedError, NoRecipientFoundError class Newsletter(WebsiteGenerator): @@ -29,11 +28,12 @@ class Newsletter(WebsiteGenerator): @frappe.whitelist() def get_sending_status(self): - count_by_status = frappe.get_all("Email Queue", + count_by_status = frappe.get_all( + "Email Queue", filters={"reference_doctype": self.doctype, "reference_name": self.name}, fields=["status", "count(name) as count"], group_by="status", - order_by="status" + order_by="status", ) sent = 0 total = 0 @@ -42,7 +42,7 @@ class Newsletter(WebsiteGenerator): sent = row.count total += row.count - return {'sent': sent, 'total': total} + return {"sent": sent, "total": total} @frappe.whitelist() def send_test_email(self, email): @@ -52,8 +52,8 @@ class Newsletter(WebsiteGenerator): @frappe.whitelist() def find_broken_links(self): - from bs4 import BeautifulSoup import requests + from bs4 import BeautifulSoup html = self.get_message() soup = BeautifulSoup(html, "html.parser") @@ -79,8 +79,7 @@ class Newsletter(WebsiteGenerator): frappe.msgprint(_("Email queued to {0} recipients").format(self.total_recipients)) def validate_send(self): - """Validate if Newsletter can be sent. - """ + """Validate if Newsletter can be sent.""" self.validate_newsletter_status() self.validate_newsletter_recipients() @@ -97,15 +96,15 @@ class Newsletter(WebsiteGenerator): self.validate_recipient_address() def validate_sender_address(self): - """Validate self.send_from is a valid email address or not. - """ + """Validate self.send_from is a valid email address or not.""" if self.sender_email: frappe.utils.validate_email_address(self.sender_email, throw=True) - self.send_from = f"{self.sender_name} <{self.sender_email}>" if self.sender_name else self.sender_email + self.send_from = ( + f"{self.sender_name} <{self.sender_email}>" if self.sender_name else self.sender_email + ) def validate_recipient_address(self): - """Validate if self.newsletter_recipients are all valid email addresses or not. - """ + """Validate if self.newsletter_recipients are all valid email addresses or not.""" for recipient in self.newsletter_recipients: frappe.utils.validate_email_address(recipient, throw=True) @@ -114,9 +113,9 @@ class Newsletter(WebsiteGenerator): frappe.throw(_("Newsletter must be published to send webview link in email")) def get_linked_email_queue(self) -> List[str]: - """Get list of email queue linked to this newsletter. - """ - return frappe.get_all("Email Queue", + """Get list of email queue linked to this newsletter.""" + return frappe.get_all( + "Email Queue", filters={ "reference_doctype": self.doctype, "reference_name": self.name, @@ -129,7 +128,8 @@ class Newsletter(WebsiteGenerator): Couldn't think of a better name ;) """ - return frappe.get_all("Email Queue Recipient", + return frappe.get_all( + "Email Queue Recipient", filters={ "status": ("in", ["Not Sent", "Sending", "Sent"]), "parentfield": ("in", self.get_linked_email_queue()), @@ -141,13 +141,10 @@ class Newsletter(WebsiteGenerator): """Get list of pending recipients of the newsletter. These recipients may not have receive the newsletter in the previous iteration. """ - return [ - x for x in self.newsletter_recipients if x not in self.get_success_recipients() - ] + return [x for x in self.newsletter_recipients if x not in self.get_success_recipients()] def queue_all(self): - """Queue Newsletter to all the recipients generated from the `Email Group` table - """ + """Queue Newsletter to all the recipients generated from the `Email Group` table""" self.validate() self.validate_send() @@ -160,13 +157,11 @@ class Newsletter(WebsiteGenerator): self.save() def get_newsletter_attachments(self) -> List[Dict[str, str]]: - """Get list of attachments on current Newsletter - """ + """Get list of attachments on current Newsletter""" return [{"file_url": row.attachment} for row in self.attachments] def send_newsletter(self, emails: List[str]): - """Trigger email generation for `emails` and add it in Email Queue. - """ + """Trigger email generation for `emails` and add it in Email Queue.""" attachments = self.get_newsletter_attachments() sender = self.send_from or frappe.utils.get_formatted_email(self.owner) args = self.as_dict() @@ -213,9 +208,7 @@ class Newsletter(WebsiteGenerator): def get_email_groups(self) -> List[str]: # wondering why the 'or'? i can't figure out why both aren't equivalent - @gavin - return [ - x.email_group for x in self.email_group - ] or frappe.get_all( + return [x.email_group for x in self.email_group] or frappe.get_all( "Newsletter Email Group", filters={"parent": self.name, "parenttype": "Newsletter"}, pluck="email_group", @@ -235,7 +228,7 @@ class Newsletter(WebsiteGenerator): @frappe.whitelist(allow_guest=True) def confirmed_unsubscribe(email, group): - """ unsubscribe the email(user) from the mailing list(email_group) """ + """unsubscribe the email(user) from the mailing list(email_group)""" frappe.flags.ignore_permissions = True doc = frappe.get_doc("Email Group Member", {"email": email, "email_group": group}) if not doc.unsubscribed: @@ -245,8 +238,7 @@ def confirmed_unsubscribe(email, group): @frappe.whitelist(allow_guest=True) def subscribe(email, email_group=_("Website")): - """API endpoint to subscribe an email to a particular email group. Triggers a confirmation email. - """ + """API endpoint to subscribe an email to a particular email group. Triggers a confirmation email.""" # build subscription confirmation URL api_endpoint = frappe.utils.get_url( @@ -277,7 +269,9 @@ def subscribe(email, email_group=_("Website")): content = """

{0}. {1}.

{3}

- """.format(*translatable_content) + """.format( + *translatable_content + ) frappe.sendmail( email, @@ -296,9 +290,7 @@ def confirm_subscription(email, email_group=_("Website")): return if not frappe.db.exists("Email Group", email_group): - frappe.get_doc({"doctype": "Email Group", "title": email_group}).insert( - ignore_permissions=True - ) + frappe.get_doc({"doctype": "Email Group", "title": email_group}).insert(ignore_permissions=True) frappe.flags.ignore_permissions = True @@ -313,13 +305,15 @@ def confirm_subscription(email, email_group=_("Website")): def get_list_context(context=None): - context.update({ - "show_search": True, - "no_breadcrumbs": True, - "title": _("Newsletters"), - "filters": {"published": 1}, - "row_template": "email/doctype/newsletter/templates/newsletter_row.html", - }) + context.update( + { + "show_search": True, + "no_breadcrumbs": True, + "title": _("Newsletters"), + "filters": {"published": 1}, + "row_template": "email/doctype/newsletter/templates/newsletter_row.html", + } + ) def send_scheduled_email(): @@ -345,9 +339,7 @@ def send_scheduled_email(): # wasn't able to send emails :( frappe.db.set_value("Newsletter", newsletter, "email_sent", 0) message = ( - f"Newsletter {newsletter} failed to send" - "\n\n" - f"Traceback: {frappe.get_traceback()}" + f"Newsletter {newsletter} failed to send" "\n\n" f"Traceback: {frappe.get_traceback()}" ) frappe.log_error(title="Send Newsletter", message=message) diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index b091c31c74..c62b7e84aa 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -9,17 +9,17 @@ from unittest.mock import MagicMock, PropertyMock, patch import frappe from frappe.desk.form.load import run_onload from frappe.email.doctype.newsletter.exceptions import ( - NewsletterAlreadySentError, NoRecipientFoundError + NewsletterAlreadySentError, + NoRecipientFoundError, ) from frappe.email.doctype.newsletter.newsletter import ( Newsletter, confirmed_unsubscribe, - send_scheduled_email + send_scheduled_email, ) from frappe.email.queue import flush from frappe.utils import add_days, getdate - test_dependencies = ["Email Group"] emails = [ "test_subscriber1@example.com", @@ -33,8 +33,8 @@ newsletters = [] def get_dotted_path(obj: type) -> str: klass = obj.__class__ module = klass.__module__ - if module == 'builtins': - return klass.__qualname__ # avoid outputs like 'builtins.str' + if module == "builtins": + return klass.__qualname__ # avoid outputs like 'builtins.str' return f"{module}.{klass.__qualname__}" @@ -46,32 +46,31 @@ class TestNewsletterMixin: def tearDown(self): frappe.set_user("Administrator") for newsletter in newsletters: - frappe.db.delete("Email Queue", { - "reference_doctype": "Newsletter", - "reference_name": newsletter, - }) + frappe.db.delete( + "Email Queue", + { + "reference_doctype": "Newsletter", + "reference_name": newsletter, + }, + ) frappe.delete_doc("Newsletter", newsletter) frappe.db.delete("Newsletter Email Group", {"parent": newsletter}) newsletters.remove(newsletter) def setup_email_group(self): if not frappe.db.exists("Email Group", "_Test Email Group"): - frappe.get_doc({ - "doctype": "Email Group", - "title": "_Test Email Group" - }).insert() + frappe.get_doc({"doctype": "Email Group", "title": "_Test Email Group"}).insert() for email in emails: doctype = "Email Group Member" - email_filters = { - "email": email, - "email_group": "_Test Email Group" - } + email_filters = {"email": email, "email_group": "_Test Email Group"} try: - frappe.get_doc({ - "doctype": doctype, - **email_filters, - }).insert() + frappe.get_doc( + { + "doctype": doctype, + **email_filters, + } + ).insert() except Exception: frappe.db.update(doctype, email_filters, "unsubscribed", 0) @@ -83,7 +82,7 @@ class TestNewsletterMixin: newsletter_options = { "published": published, "schedule_sending": bool(schedule_send), - "schedule_send": schedule_send + "schedule_send": schedule_send, } newsletter = self.get_newsletter(**newsletter_options) @@ -95,8 +94,7 @@ class TestNewsletterMixin: @staticmethod def get_newsletter(**kwargs) -> "Newsletter": - """Generate and return Newsletter object - """ + """Generate and return Newsletter object""" doctype = "Newsletter" newsletter_content = { "subject": "_Test Newsletter", @@ -116,7 +114,9 @@ class TestNewsletterMixin: newsletter.reload() newsletters.append(newsletter.name) - attached_files = frappe.get_all("File", { + attached_files = frappe.get_all( + "File", + { "attached_to_doctype": newsletter.doctype, "attached_to_name": newsletter.name, }, @@ -141,15 +141,15 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase): def test_unsubscribe(self): name = self.send_newsletter() to_unsubscribe = choice(emails) - group = frappe.get_all("Newsletter Email Group", filters={"parent": name}, fields=["email_group"]) + group = frappe.get_all( + "Newsletter Email Group", filters={"parent": name}, fields=["email_group"] + ) flush(from_test=True) confirmed_unsubscribe(to_unsubscribe, group[0].email_group) name = self.send_newsletter() - email_queue_list = [ - frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue") - ] + email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")] self.assertEqual(len(email_queue_list), 3) recipients = [e.recipients[0].recipient for e in email_queue_list] @@ -160,15 +160,14 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase): def test_schedule_send(self): self.send_newsletter(schedule_send=add_days(getdate(), -1)) - email_queue_list = [frappe.get_doc('Email Queue', e.name) for e in frappe.get_all("Email Queue")] + email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")] self.assertEqual(len(email_queue_list), 4) recipients = [e.recipients[0].recipient for e in email_queue_list] for email in emails: self.assertTrue(email in recipients) def test_newsletter_send_test_email(self): - """Test "Send Test Email" functionality of Newsletter - """ + """Test "Send Test Email" functionality of Newsletter""" newsletter = self.get_newsletter() test_email = choice(emails) newsletter.send_test_email(test_email) @@ -177,21 +176,23 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase): newsletter.save = MagicMock() self.assertFalse(newsletter.save.called) # check if the test email is in the queue - email_queue = frappe.db.get_all('Email Queue', filters=[ - ['reference_doctype', '=', 'Newsletter'], - ['reference_name', '=', newsletter.name], - ['Email Queue Recipient', 'recipient', '=', test_email] - ]) + email_queue = frappe.db.get_all( + "Email Queue", + filters=[ + ["reference_doctype", "=", "Newsletter"], + ["reference_name", "=", newsletter.name], + ["Email Queue Recipient", "recipient", "=", test_email], + ], + ) self.assertTrue(email_queue) def test_newsletter_status(self): - """Test for Newsletter's stats on onload event - """ + """Test for Newsletter's stats on onload event""" newsletter = self.get_newsletter() newsletter.email_sent = True result = newsletter.get_sending_status() - self.assertTrue('total' in result) - self.assertTrue('sent' in result) + self.assertTrue("total" in result) + self.assertTrue("sent" in result) def test_already_sent_newsletter(self): newsletter = self.get_newsletter() diff --git a/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py b/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py index 7842badbe1..01d194fb8d 100644 --- a/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py +++ b/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class NewsletterAttachment(Document): pass diff --git a/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py b/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py index 89476c4d53..b7a00ac7d2 100644 --- a/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py +++ b/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class NewsletterEmailGroup(Document): pass diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index bad32fb68f..5c27eb95eb 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -2,22 +2,25 @@ # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE +import json +import os + import frappe -import json, os from frappe import _ -from frappe.model.document import Document from frappe.core.doctype.role.role import get_info_based_on_role, get_user_info -from frappe.utils import validate_email_address, nowdate, parse_val, is_html, add_to_date -from frappe.utils.jinja import validate_template -from frappe.utils.safe_exec import get_safe_globals -from frappe.modules.utils import export_module_json, get_doc_module -from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message from frappe.core.doctype.sms_settings.sms_settings import send_sms from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification +from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message +from frappe.model.document import Document +from frappe.modules.utils import export_module_json, get_doc_module +from frappe.utils import add_to_date, is_html, nowdate, parse_val, validate_email_address +from frappe.utils.jinja import validate_template +from frappe.utils.safe_exec import get_safe_globals + class Notification(Document): def onload(self): - '''load message''' + """load message""" if self.is_standard: self.message = self.get_template() @@ -34,35 +37,39 @@ class Notification(Document): if self.event in ("Days Before", "Days After") and not self.date_changed: frappe.throw(_("Please specify which date field must be checked")) - if self.event=="Value Change" and not self.value_changed: + if self.event == "Value Change" and not self.value_changed: frappe.throw(_("Please specify which value field must be checked")) self.validate_forbidden_types() self.validate_condition() self.validate_standard() - frappe.cache().hdel('notifications', self.document_type) + frappe.cache().hdel("notifications", self.document_type) def on_update(self): path = export_module_json(self, self.is_standard, self.module) if path: # js - if not os.path.exists(path + '.md') and not os.path.exists(path + '.html'): - with open(path + '.md', 'w') as f: + if not os.path.exists(path + ".md") and not os.path.exists(path + ".html"): + with open(path + ".md", "w") as f: f.write(self.message) # py - if not os.path.exists(path + '.py'): - with open(path + '.py', 'w') as f: - f.write("""import frappe + if not os.path.exists(path + ".py"): + with open(path + ".py", "w") as f: + f.write( + """import frappe def get_context(context): # do your magic here pass -""") +""" + ) def validate_standard(self): if self.is_standard and self.enabled and not frappe.conf.developer_mode: - frappe.throw(_('Cannot edit Standard Notification. To edit, please disable this and duplicate it')) + frappe.throw( + _("Cannot edit Standard Notification. To edit, please disable this and duplicate it") + ) def validate_condition(self): temp_doc = frappe.new_doc(self.document_type) @@ -74,30 +81,31 @@ def get_context(context): def validate_forbidden_types(self): forbidden_document_types = ("Email Queue",) - if (self.document_type in forbidden_document_types - or frappe.get_meta(self.document_type).istable): + if self.document_type in forbidden_document_types or frappe.get_meta(self.document_type).istable: # currently notifications don't work on child tables as events are not fired for each record of child table frappe.throw(_("Cannot set Notification on Document Type {0}").format(self.document_type)) def get_documents_for_today(self): - '''get list of documents that will be triggered today''' + """get list of documents that will be triggered today""" docs = [] diff_days = self.days_in_advance - if self.event=="Days After": + if self.event == "Days After": diff_days = -diff_days reference_date = add_to_date(nowdate(), days=diff_days) - reference_date_start = reference_date + ' 00:00:00.000000' - reference_date_end = reference_date + ' 23:59:59.000000' + reference_date_start = reference_date + " 00:00:00.000000" + reference_date_end = reference_date + " 23:59:59.000000" - doc_list = frappe.get_all(self.document_type, - fields='name', + doc_list = frappe.get_all( + self.document_type, + fields="name", filters=[ - { self.date_changed: ('>=', reference_date_start) }, - { self.date_changed: ('<=', reference_date_end) } - ]) + {self.date_changed: (">=", reference_date_start)}, + {self.date_changed: ("<=", reference_date_end)}, + ], + ) for d in doc_list: doc = frappe.get_doc(self.document_type, d.name) @@ -110,7 +118,7 @@ def get_context(context): return docs def send(self, doc): - '''Build recipients and send Notification''' + """Build recipients and send Notification""" context = get_context(doc) context = {"doc": doc, "alert": self, "comments": None} @@ -120,24 +128,27 @@ def get_context(context): if self.is_standard: self.load_standard_properties(context) try: - if self.channel == 'Email': + if self.channel == "Email": self.send_an_email(doc, context) - if self.channel == 'Slack': + if self.channel == "Slack": self.send_a_slack_msg(doc, context) - if self.channel == 'SMS': + if self.channel == "SMS": self.send_sms(doc, context) - if self.channel == 'System Notification' or self.send_system_notification: + if self.channel == "System Notification" or self.send_system_notification: self.create_system_notification(doc, context) except: - frappe.log_error(title='Failed to send notification', message=frappe.get_traceback()) + frappe.log_error(title="Failed to send notification", message=frappe.get_traceback()) if self.set_property_after_alert: allow_update = True - if doc.docstatus.is_submitted() and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit: + if ( + doc.docstatus.is_submitted() + and not doc.meta.get_field(self.set_property_after_alert).allow_on_submit + ): allow_update = False try: if allow_update and not doc.flags.in_notification_update: @@ -149,15 +160,15 @@ def get_context(context): doc.reload() doc.set(fieldname, value) doc.flags.updater_reference = { - 'doctype': self.doctype, - 'docname': self.name, - 'label': _('via Notification') + "doctype": self.doctype, + "docname": self.name, + "label": _("via Notification"), } doc.flags.in_notification_update = True doc.save(ignore_permissions=True) doc.flags.in_notification_update = False except Exception: - frappe.log_error(title='Document update failed', message=frappe.get_traceback()) + frappe.log_error(title="Document update failed", message=frappe.get_traceback()) def create_system_notification(self, doc, context): subject = self.subject @@ -174,19 +185,21 @@ def get_context(context): return notification_doc = { - 'type': 'Alert', - 'document_type': doc.doctype, - 'document_name': doc.name, - 'subject': subject, - 'from_user': doc.modified_by or doc.owner, - 'email_content': frappe.render_template(self.message, context), - 'attached_file': attachments and json.dumps(attachments[0]) + "type": "Alert", + "document_type": doc.doctype, + "document_name": doc.name, + "subject": subject, + "from_user": doc.modified_by or doc.owner, + "email_content": frappe.render_template(self.message, context), + "attached_file": attachments and json.dumps(attachments[0]), } enqueue_create_notification(users, notification_doc) def send_an_email(self, doc, context): from email.utils import formataddr + from frappe.core.doctype.communication.email import _make as make_communication + subject = self.subject if "{" in subject: subject = frappe.render_template(self.subject, context) @@ -200,22 +213,23 @@ def get_context(context): message = frappe.render_template(self.message, context) if self.sender and self.sender_email: sender = formataddr((self.sender, self.sender_email)) - frappe.sendmail(recipients = recipients, - subject = subject, - sender = sender, - cc = cc, - bcc = bcc, - message = message, - reference_doctype = doc.doctype, - reference_name = doc.name, - attachments = attachments, + frappe.sendmail( + recipients=recipients, + subject=subject, + sender=sender, + cc=cc, + bcc=bcc, + message=message, + reference_doctype=doc.doctype, + reference_name=doc.name, + attachments=attachments, expose_recipients="header", - print_letterhead = ((attachments - and attachments[0].get('print_letterhead')) or False)) + print_letterhead=((attachments and attachments[0].get("print_letterhead")) or False), + ) # Add mail notification to communication list # No need to add if it is already a communication. - if doc.doctype != 'Communication': + if doc.doctype != "Communication": make_communication( doctype=doc.doctype, name=doc.name, @@ -228,7 +242,7 @@ def get_context(context): attachments=attachments, cc=cc, bcc=bcc, - communication_type='Automated Message', + communication_type="Automated Message", ) def send_a_slack_msg(self, doc, context): @@ -236,12 +250,13 @@ def get_context(context): webhook_url=self.slack_webhook_url, message=frappe.render_template(self.message, context), reference_doctype=doc.doctype, - reference_name=doc.name) + reference_name=doc.name, + ) def send_sms(self, doc, context): send_sms( receiver_list=self.get_receiver_list(doc, context), - msg=frappe.render_template(self.message, context) + msg=frappe.render_template(self.message, context), ) def get_list_of_recipients(self, doc, context): @@ -253,7 +268,7 @@ def get_context(context): if not frappe.safe_eval(recipient.condition, None, context): continue if recipient.receiver_by_document_field: - fields = recipient.receiver_by_document_field.split(',') + fields = recipient.receiver_by_document_field.split(",") # fields from child table if len(fields) > 1: for d in doc.get(fields[1]): @@ -281,9 +296,9 @@ def get_context(context): recipient.bcc = recipient.bcc.replace(",", "\n") bcc = bcc + recipient.bcc.split("\n") - #For sending emails to specified role + # For sending emails to specified role if recipient.receiver_by_role: - emails = get_info_based_on_role(recipient.receiver_by_role, 'email') + emails = get_info_based_on_role(recipient.receiver_by_role, "email") for email in emails: recipients = recipients + email.split("\n") @@ -294,7 +309,7 @@ def get_context(context): return list(set(recipients)), list(set(cc)), list(set(bcc)) def get_receiver_list(self, doc, context): - ''' return receiver list based on the doc field and role specified ''' + """return receiver list based on the doc field and role specified""" receiver_list = [] for recipient in self.recipients: if recipient.condition: @@ -302,89 +317,98 @@ def get_context(context): continue # For sending messages to the owner's mobile phone number - if recipient.receiver_by_document_field == 'owner': - receiver_list += get_user_info([dict(user_name=doc.get('owner'))], 'mobile_no') + if recipient.receiver_by_document_field == "owner": + receiver_list += get_user_info([dict(user_name=doc.get("owner"))], "mobile_no") # For sending messages to the number specified in the receiver field elif recipient.receiver_by_document_field: receiver_list.append(doc.get(recipient.receiver_by_document_field)) - #For sending messages to specified role + # For sending messages to specified role if recipient.receiver_by_role: - receiver_list += get_info_based_on_role(recipient.receiver_by_role, 'mobile_no') + receiver_list += get_info_based_on_role(recipient.receiver_by_role, "mobile_no") return receiver_list def get_attachment(self, doc): - """ check print settings are attach the pdf """ + """check print settings are attach the pdf""" if not self.attach_print: return None print_settings = frappe.get_doc("Print Settings", "Print Settings") - if (doc.docstatus == 0 and not print_settings.allow_print_for_draft) or \ - (doc.docstatus == 2 and not print_settings.allow_print_for_cancelled): + if (doc.docstatus == 0 and not print_settings.allow_print_for_draft) or ( + doc.docstatus == 2 and not print_settings.allow_print_for_cancelled + ): # ignoring attachment as draft and cancelled documents are not allowed to print status = "Draft" if doc.docstatus == 0 else "Cancelled" - frappe.throw(_("""Not allowed to attach {0} document, please enable Allow Print For {0} in Print Settings""").format(status), - title=_("Error in Notification")) + frappe.throw( + _( + """Not allowed to attach {0} document, please enable Allow Print For {0} in Print Settings""" + ).format(status), + title=_("Error in Notification"), + ) else: - return [{ - "print_format_attachment": 1, - "doctype": doc.doctype, - "name": doc.name, - "print_format": self.print_format, - "print_letterhead": print_settings.with_letterhead, - "lang": frappe.db.get_value('Print Format', self.print_format, 'default_print_language') - if self.print_format else 'en' - }] - + return [ + { + "print_format_attachment": 1, + "doctype": doc.doctype, + "name": doc.name, + "print_format": self.print_format, + "print_letterhead": print_settings.with_letterhead, + "lang": frappe.db.get_value("Print Format", self.print_format, "default_print_language") + if self.print_format + else "en", + } + ] def get_template(self): module = get_doc_module(self.module, self.doctype, self.name) + def load_template(extn): - template = '' - template_path = os.path.join(os.path.dirname(module.__file__), - frappe.scrub(self.name) + extn) + template = "" + template_path = os.path.join(os.path.dirname(module.__file__), frappe.scrub(self.name) + extn) if os.path.exists(template_path): - with open(template_path, 'r') as f: + with open(template_path, "r") as f: template = f.read() return template - return load_template('.html') or load_template('.md') + return load_template(".html") or load_template(".md") def load_standard_properties(self, context): - '''load templates and run get_context''' + """load templates and run get_context""" module = get_doc_module(self.module, self.doctype, self.name) if module: - if hasattr(module, 'get_context'): + if hasattr(module, "get_context"): out = module.get_context(context) - if out: context.update(out) + if out: + context.update(out) self.message = self.get_template() if not is_html(self.message): self.message = frappe.utils.md_to_html(self.message) + @frappe.whitelist() def get_documents_for_today(notification): - notification = frappe.get_doc('Notification', notification) - notification.check_permission('read') + notification = frappe.get_doc("Notification", notification) + notification.check_permission("read") return [d.name for d in notification.get_documents_for_today()] + def trigger_daily_alerts(): trigger_notifications(None, "daily") + def trigger_notifications(doc, method=None): if frappe.flags.in_import or frappe.flags.in_patch: # don't send notifications while syncing or patching return if method == "daily": - doc_list = frappe.get_all('Notification', - filters={ - 'event': ('in', ('Days Before', 'Days After')), - 'enabled': 1 - }) + doc_list = frappe.get_all( + "Notification", filters={"event": ("in", ("Days Before", "Days After")), "enabled": 1} + ) for d in doc_list: alert = frappe.get_doc("Notification", d.name) @@ -392,8 +416,10 @@ def trigger_notifications(doc, method=None): evaluate_alert(doc, alert, alert.event) frappe.db.commit() + def evaluate_alert(doc, alert, event): from jinja2 import TemplateError + try: if isinstance(alert, str): alert = frappe.get_doc("Notification", alert) @@ -404,17 +430,17 @@ def evaluate_alert(doc, alert, event): if not frappe.safe_eval(alert.condition, None, context): return - if event=="Value Change" and not doc.is_new(): + if event == "Value Change" and not doc.is_new(): if not frappe.db.has_column(doc.doctype, alert.value_changed): - alert.db_set('enabled', 0) - frappe.log_error('Notification {0} has been disabled due to missing field'.format(alert.name)) + alert.db_set("enabled", 0) + frappe.log_error("Notification {0} has been disabled due to missing field".format(alert.name)) return doc_before_save = doc.get_doc_before_save() field_value_before_save = doc_before_save.get(alert.value_changed) if doc_before_save else None field_value_before_save = parse_val(field_value_before_save) - if (doc.get(alert.value_changed) == field_value_before_save): + if doc.get(alert.value_changed) == field_value_before_save: # value not changed return @@ -424,19 +450,33 @@ def evaluate_alert(doc, alert, event): doc.reload() alert.send(doc) except TemplateError: - frappe.throw(_("Error while evaluating Notification {0}. Please fix your template.").format(alert)) + frappe.throw( + _("Error while evaluating Notification {0}. Please fix your template.").format(alert) + ) except Exception as e: error_log = frappe.log_error(message=frappe.get_traceback(), title=str(e)) - frappe.throw(_("Error in Notification: {}").format( - frappe.utils.get_link_to_form('Error Log', error_log.name))) + frappe.throw( + _("Error in Notification: {}").format( + frappe.utils.get_link_to_form("Error Log", error_log.name) + ) + ) + def get_context(doc): - return {"doc": doc, "nowdate": nowdate, "frappe": frappe._dict(utils=get_safe_globals().get("frappe").get("utils"))} + return { + "doc": doc, + "nowdate": nowdate, + "frappe": frappe._dict(utils=get_safe_globals().get("frappe").get("utils")), + } + def get_assignees(doc): assignees = [] - assignees = frappe.get_all('ToDo', filters={'status': 'Open', 'reference_name': doc.name, - 'reference_type': doc.doctype}, fields=['allocated_to']) + assignees = frappe.get_all( + "ToDo", + filters={"status": "Open", "reference_name": doc.name, "reference_type": doc.doctype}, + fields=["allocated_to"], + ) recipients = [d.allocated_to for d in assignees] diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index f6f216ada2..4d8b26c559 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -1,91 +1,122 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe, frappe.utils, frappe.utils.scheduler -from frappe.desk.form import assign_to import unittest +import frappe +import frappe.utils +import frappe.utils.scheduler +from frappe.desk.form import assign_to + test_dependencies = ["User", "Notification"] + class TestNotification(unittest.TestCase): def setUp(self): frappe.db.delete("Email Queue") frappe.set_user("test@example.com") - if not frappe.db.exists('Notification', {'name': 'ToDo Status Update'}, 'name'): - notification = frappe.new_doc('Notification') - notification.name = 'ToDo Status Update' - notification.subject = 'ToDo Status Update' - notification.document_type = 'ToDo' - notification.event = 'Value Change' - notification.value_changed = 'status' + if not frappe.db.exists("Notification", {"name": "ToDo Status Update"}, "name"): + notification = frappe.new_doc("Notification") + notification.name = "ToDo Status Update" + notification.subject = "ToDo Status Update" + notification.document_type = "ToDo" + notification.event = "Value Change" + notification.value_changed = "status" notification.send_to_all_assignees = 1 - notification.set_property_after_alert = 'description' - notification.property_value = 'Changed by Notification' + notification.set_property_after_alert = "description" + notification.property_value = "Changed by Notification" notification.save() - if not frappe.db.exists('Notification', {'name': 'Contact Status Update'}, 'name'): - notification = frappe.new_doc('Notification') - notification.name = 'Contact Status Update' - notification.subject = 'Contact Status Update' - notification.document_type = 'Contact' - notification.event = 'Value Change' - notification.value_changed = 'status' - notification.message = 'Test Contact Update' - notification.append('recipients', { - 'receiver_by_document_field': 'email_id,email_ids' - }) + if not frappe.db.exists("Notification", {"name": "Contact Status Update"}, "name"): + notification = frappe.new_doc("Notification") + notification.name = "Contact Status Update" + notification.subject = "Contact Status Update" + notification.document_type = "Contact" + notification.event = "Value Change" + notification.value_changed = "status" + notification.message = "Test Contact Update" + notification.append("recipients", {"receiver_by_document_field": "email_id,email_ids"}) notification.save() - def tearDown(self): frappe.set_user("Administrator") def test_new_and_save(self): - """Check creating a new communication triggers a notification. - """ + """Check creating a new communication triggers a notification.""" communication = frappe.new_doc("Communication") - communication.communication_type = 'Comment' + communication.communication_type = "Comment" communication.subject = "test" communication.content = "test" communication.insert(ignore_permissions=True) - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Communication", - "reference_name": communication.name, "status":"Not Sent"})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + { + "reference_doctype": "Communication", + "reference_name": communication.name, + "status": "Not Sent", + }, + ) + ) frappe.db.delete("Email Queue") communication.reload() communication.content = "test 2" communication.save() - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Communication", - "reference_name": communication.name, "status":"Not Sent"})) - - self.assertEqual(frappe.db.get_value('Communication', - communication.name, 'subject'), '__testing__') + self.assertTrue( + frappe.db.get_value( + "Email Queue", + { + "reference_doctype": "Communication", + "reference_name": communication.name, + "status": "Not Sent", + }, + ) + ) + + self.assertEqual( + frappe.db.get_value("Communication", communication.name, "subject"), "__testing__" + ) def test_condition(self): - """Check notification is triggered based on a condition. - """ + """Check notification is triggered based on a condition.""" event = frappe.new_doc("Event") - event.subject = "test", + event.subject = ("test",) event.event_type = "Private" - event.starts_on = "2014-06-06 12:00:00" + event.starts_on = "2014-06-06 12:00:00" event.insert() - self.assertFalse(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status":"Not Sent"})) + self.assertFalse( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) event.event_type = "Public" event.save() - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status":"Not Sent"})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) # Make sure that we track the triggered notifications in communication doctype. - self.assertTrue(frappe.db.get_value("Communication", {"reference_doctype": "Event", - "reference_name": event.name, "communication_type": 'Automated Message'})) - + self.assertTrue( + frappe.db.get_value( + "Communication", + { + "reference_doctype": "Event", + "reference_name": event.name, + "communication_type": "Automated Message", + }, + ) + ) def test_invalid_condition(self): frappe.set_user("Administrator") @@ -104,49 +135,60 @@ class TestNotification(unittest.TestCase): self.assertRaises(frappe.ValidationError, notification.save) notification.delete() - def test_value_changed(self): event = frappe.new_doc("Event") - event.subject = "test", + event.subject = ("test",) event.event_type = "Private" - event.starts_on = "2014-06-06 12:00:00" + event.starts_on = "2014-06-06 12:00:00" event.insert() - self.assertFalse(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status":"Not Sent"})) + self.assertFalse( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) event.subject = "test 1" event.save() - self.assertFalse(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status":"Not Sent"})) + self.assertFalse( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) event.description = "test" event.save() - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status":"Not Sent"})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) def test_alert_disabled_on_wrong_field(self): - frappe.set_user('Administrator') - notification = frappe.get_doc({ - "doctype": "Notification", - "subject":"_Test Notification for wrong field", - "document_type": "Event", - "event": "Value Change", - "attach_print": 0, - "value_changed": "description1", - "message": "Description changed", - "recipients": [ - { "receiver_by_document_field": "owner" } - ] - }).insert() + frappe.set_user("Administrator") + notification = frappe.get_doc( + { + "doctype": "Notification", + "subject": "_Test Notification for wrong field", + "document_type": "Event", + "event": "Value Change", + "attach_print": 0, + "value_changed": "description1", + "message": "Description changed", + "recipients": [{"receiver_by_document_field": "owner"}], + } + ).insert() frappe.db.commit() event = frappe.new_doc("Event") - event.subject = "test-2", + event.subject = ("test-2",) event.event_type = "Private" - event.starts_on = "2014-06-06 12:00:00" + event.starts_on = "2014-06-06 12:00:00" event.insert() event.subject = "test 1" event.save() @@ -160,34 +202,56 @@ class TestNotification(unittest.TestCase): def test_date_changed(self): event = frappe.new_doc("Event") - event.subject = "test", + event.subject = ("test",) event.event_type = "Private" event.starts_on = "2014-01-01 12:00:00" event.insert() - self.assertFalse(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status": "Not Sent"})) + self.assertFalse( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) - frappe.set_user('Administrator') - frappe.get_doc('Scheduled Job Type', dict(method='frappe.email.doctype.notification.notification.trigger_daily_alerts')).execute() + frappe.set_user("Administrator") + frappe.get_doc( + "Scheduled Job Type", + dict(method="frappe.email.doctype.notification.notification.trigger_daily_alerts"), + ).execute() # not today, so no alert - self.assertFalse(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status": "Not Sent"})) - - event.starts_on = frappe.utils.add_days(frappe.utils.nowdate(), 2) + " 12:00:00" + self.assertFalse( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) + + event.starts_on = frappe.utils.add_days(frappe.utils.nowdate(), 2) + " 12:00:00" event.save() # Value Change notification alert will be trigger as description is not changed # mail will not be sent - self.assertFalse(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status": "Not Sent"})) - - frappe.get_doc('Scheduled Job Type', dict(method='frappe.email.doctype.notification.notification.trigger_daily_alerts')).execute() + self.assertFalse( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) + + frappe.get_doc( + "Scheduled Job Type", + dict(method="frappe.email.doctype.notification.notification.trigger_daily_alerts"), + ).execute() # today so show alert - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "Event", - "reference_name": event.name, "status":"Not Sent"})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "Event", "reference_name": event.name, "status": "Not Sent"}, + ) + ) def test_cc_jinja(self): @@ -196,85 +260,92 @@ class TestNotification(unittest.TestCase): frappe.db.delete("Email Queue Recipient") test_user = frappe.new_doc("User") - test_user.name = 'test_jinja' - test_user.first_name = 'test_jinja' - test_user.email = 'test_jinja@example.com' + test_user.name = "test_jinja" + test_user.first_name = "test_jinja" + test_user.email = "test_jinja@example.com" test_user.insert(ignore_permissions=True) - self.assertTrue(frappe.db.get_value("Email Queue", {"reference_doctype": "User", - "reference_name": test_user.name, "status":"Not Sent"})) + self.assertTrue( + frappe.db.get_value( + "Email Queue", + {"reference_doctype": "User", "reference_name": test_user.name, "status": "Not Sent"}, + ) + ) - self.assertTrue(frappe.db.get_value("Email Queue Recipient", {"recipient": "test_jinja@example.com"})) + self.assertTrue( + frappe.db.get_value("Email Queue Recipient", {"recipient": "test_jinja@example.com"}) + ) frappe.db.delete("User", {"email": "test_jinja@example.com"}) frappe.db.delete("Email Queue") frappe.db.delete("Email Queue Recipient") def test_notification_to_assignee(self): - todo = frappe.new_doc('ToDo') - todo.description = 'Test Notification' + todo = frappe.new_doc("ToDo") + todo.description = "Test Notification" todo.save() - assign_to.add({ - "assign_to": ["test2@example.com"], - "doctype": todo.doctype, - "name": todo.name, - "description": "Close this Todo" - }) - - assign_to.add({ - "assign_to": ["test1@example.com"], - "doctype": todo.doctype, - "name": todo.name, - "description": "Close this Todo" - }) - - #change status of todo - todo.status = 'Closed' + assign_to.add( + { + "assign_to": ["test2@example.com"], + "doctype": todo.doctype, + "name": todo.name, + "description": "Close this Todo", + } + ) + + assign_to.add( + { + "assign_to": ["test1@example.com"], + "doctype": todo.doctype, + "name": todo.name, + "description": "Close this Todo", + } + ) + + # change status of todo + todo.status = "Closed" todo.save() - email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'ToDo', - 'reference_name': todo.name}) + email_queue = frappe.get_doc( + "Email Queue", {"reference_doctype": "ToDo", "reference_name": todo.name} + ) self.assertTrue(email_queue) # check if description is changed after alert since set_property_after_alert is set - self.assertEqual(todo.description, 'Changed by Notification') + self.assertEqual(todo.description, "Changed by Notification") recipients = [d.recipient for d in email_queue.recipients] - self.assertTrue('test2@example.com' in recipients) - self.assertTrue('test1@example.com' in recipients) + self.assertTrue("test2@example.com" in recipients) + self.assertTrue("test1@example.com" in recipients) def test_notification_by_child_table_field(self): - contact = frappe.new_doc('Contact') - contact.first_name = 'John Doe' - contact.status = 'Open' - contact.append('email_ids', { - 'email_id': 'test2@example.com', - 'is_primary': 1 - }) - - contact.append('email_ids', { - 'email_id': 'test1@example.com' - }) + contact = frappe.new_doc("Contact") + contact.first_name = "John Doe" + contact.status = "Open" + contact.append("email_ids", {"email_id": "test2@example.com", "is_primary": 1}) + + contact.append("email_ids", {"email_id": "test1@example.com"}) contact.save() - #change status of contact - contact.status = 'Replied' + # change status of contact + contact.status = "Replied" contact.save() - email_queue = frappe.get_doc('Email Queue', {'reference_doctype': 'Contact', - 'reference_name': contact.name}) + email_queue = frappe.get_doc( + "Email Queue", {"reference_doctype": "Contact", "reference_name": contact.name} + ) self.assertTrue(email_queue) recipients = [d.recipient for d in email_queue.recipients] - self.assertTrue('test2@example.com' in recipients) - self.assertTrue('test1@example.com' in recipients) + self.assertTrue("test2@example.com" in recipients) + self.assertTrue("test1@example.com" in recipients) @classmethod def tearDownClass(cls): frappe.delete_doc_if_exists("Notification", "ToDo Status Update") - frappe.delete_doc_if_exists("Notification", "Contact Status Update") \ No newline at end of file + frappe.delete_doc_if_exists("Notification", "Contact Status Update") diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.py b/frappe/email/doctype/notification_recipient/notification_recipient.py index 68871e5047..9de15f46c0 100644 --- a/frappe/email/doctype/notification_recipient/notification_recipient.py +++ b/frappe/email/doctype/notification_recipient/notification_recipient.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class NotificationRecipient(Document): pass diff --git a/frappe/email/doctype/unhandled_email/test_unhandled_email.py b/frappe/email/doctype/unhandled_email/test_unhandled_email.py index 37c65584e0..694b3e03a6 100644 --- a/frappe/email/doctype/unhandled_email/test_unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/test_unhandled_email.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Unhandled Emails') + class TestUnhandledEmail(unittest.TestCase): pass diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index db14a50d09..e703f1ec97 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -5,11 +5,12 @@ import frappe from frappe.model.document import Document + class UnhandledEmail(Document): pass def remove_old_unhandled_emails(): - frappe.db.delete("Unhandled Email", { - "creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30)) - }) \ No newline at end of file + frappe.db.delete( + "Unhandled Email", {"creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30))} + ) diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 0f45e42aac..07f698f740 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -1,28 +1,57 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, re, os -from frappe.utils.pdf import get_pdf -from frappe.email.doctype.email_account.email_account import EmailAccount -from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint, - split_emails, to_markdown, markdown, random_string, parse_addr) import email.utils -from email.mime.multipart import MIMEMultipart -from email.header import Header +import os +import re from email import policy +from email.header import Header +from email.mime.multipart import MIMEMultipart + +import frappe +from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.utils import ( + cint, + expand_relative_urls, + get_url, + markdown, + parse_addr, + random_string, + scrub_urls, + split_emails, + strip, + to_markdown, +) +from frappe.utils.pdf import get_pdf -def get_email(recipients, sender='', msg='', subject='[No Subject]', - text_content = None, footer=None, print_html=None, formatted=None, attachments=None, - content=None, reply_to=None, cc=None, bcc=None, email_account=None, expose_recipients=None, - inline_images=None, header=None): - """ Prepare an email with the following format: - - multipart/mixed - - multipart/alternative - - text/plain - - multipart/related - - text/html - - inline image - - attachment + +def get_email( + recipients, + sender="", + msg="", + subject="[No Subject]", + text_content=None, + footer=None, + print_html=None, + formatted=None, + attachments=None, + content=None, + reply_to=None, + cc=None, + bcc=None, + email_account=None, + expose_recipients=None, + inline_images=None, + header=None, +): + """Prepare an email with the following format: + - multipart/mixed + - multipart/alternative + - text/plain + - multipart/related + - text/html + - inline image + - attachment """ content = content or msg @@ -33,36 +62,67 @@ def get_email(recipients, sender='', msg='', subject='[No Subject]', if inline_images is None: inline_images = [] - emailobj = EMail(sender, recipients, subject, reply_to=reply_to, cc=cc, bcc=bcc, email_account=email_account, expose_recipients=expose_recipients) + emailobj = EMail( + sender, + recipients, + subject, + reply_to=reply_to, + cc=cc, + bcc=bcc, + email_account=email_account, + expose_recipients=expose_recipients, + ) if not content.strip().startswith("<"): content = markdown(content) - emailobj.set_html(content, text_content, footer=footer, header=header, - print_html=print_html, formatted=formatted, inline_images=inline_images) + emailobj.set_html( + content, + text_content, + footer=footer, + header=header, + print_html=print_html, + formatted=formatted, + inline_images=inline_images, + ) if isinstance(attachments, dict): attachments = [attachments] - for attach in (attachments or []): + for attach in attachments or []: # cannot attach if no filecontent - if attach.get('fcontent') is None: continue + if attach.get("fcontent") is None: + continue emailobj.add_attachment(**attach) return emailobj + class EMail: """ Wrapper on the email module. Email object represents emails to be sent to the client. Also provides a clean way to add binary `FileData` attachments Also sets all messages as multipart/alternative for cleaner reading in text-only clients """ - def __init__(self, sender='', recipients=(), subject='', alternative=0, reply_to=None, cc=(), bcc=(), email_account=None, expose_recipients=None): + + def __init__( + self, + sender="", + recipients=(), + subject="", + alternative=0, + reply_to=None, + cc=(), + bcc=(), + email_account=None, + expose_recipients=None, + ): from email import charset as Charset - Charset.add_charset('utf-8', Charset.QP, Charset.QP, 'utf-8') + + Charset.add_charset("utf-8", Charset.QP, Charset.QP, "utf-8") if isinstance(recipients, str): - recipients = recipients.replace(';', ',').replace('\n', '') + recipients = recipients.replace(";", ",").replace("\n", "") recipients = split_emails(recipients) # remove null @@ -74,22 +134,38 @@ class EMail: self.subject = subject self.expose_recipients = expose_recipients - self.msg_root = MIMEMultipart('mixed', policy=policy.SMTPUTF8) - self.msg_alternative = MIMEMultipart('alternative', policy=policy.SMTPUTF8) + self.msg_root = MIMEMultipart("mixed", policy=policy.SMTPUTF8) + self.msg_alternative = MIMEMultipart("alternative", policy=policy.SMTPUTF8) self.msg_root.attach(self.msg_alternative) self.cc = cc or [] self.bcc = bcc or [] self.html_set = False - self.email_account = email_account or \ - EmailAccount.find_outgoing(match_by_email=sender, _raise_error=True) - - def set_html(self, message, text_content = None, footer=None, print_html=None, - formatted=None, inline_images=None, header=None): + self.email_account = email_account or EmailAccount.find_outgoing( + match_by_email=sender, _raise_error=True + ) + + def set_html( + self, + message, + text_content=None, + footer=None, + print_html=None, + formatted=None, + inline_images=None, + header=None, + ): """Attach message in the html portion of multipart/alternative""" if not formatted: - formatted = get_formatted_html(self.subject, message, footer, print_html, - email_account=self.email_account, header=header, sender=self.sender) + formatted = get_formatted_html( + self.subject, + message, + footer, + print_html, + email_account=self.email_account, + header=header, + sender=self.sender, + ) # this is the first html part of a multi-part message, # convert to text well @@ -104,48 +180,56 @@ class EMail: def set_text(self, message): """ - Attach message in the text portion of multipart/alternative + Attach message in the text portion of multipart/alternative """ from email.mime.text import MIMEText - part = MIMEText(message, 'plain', 'utf-8', policy=policy.SMTPUTF8) + + part = MIMEText(message, "plain", "utf-8", policy=policy.SMTPUTF8) self.msg_alternative.attach(part) def set_part_html(self, message, inline_images): from email.mime.text import MIMEText - has_inline_images = re.search('''embed=['"].*?['"]''', message) + has_inline_images = re.search("""embed=['"].*?['"]""", message) if has_inline_images: # process inline images message, _inline_images = replace_filename_with_cid(message) # prepare parts - msg_related = MIMEMultipart('related', policy=policy.SMTPUTF8) + msg_related = MIMEMultipart("related", policy=policy.SMTPUTF8) - html_part = MIMEText(message, 'html', 'utf-8', policy=policy.SMTPUTF8) + html_part = MIMEText(message, "html", "utf-8", policy=policy.SMTPUTF8) msg_related.attach(html_part) for image in _inline_images: - self.add_attachment(image.get('filename'), image.get('filecontent'), - content_id=image.get('content_id'), parent=msg_related, inline=True) + self.add_attachment( + image.get("filename"), + image.get("filecontent"), + content_id=image.get("content_id"), + parent=msg_related, + inline=True, + ) self.msg_alternative.attach(msg_related) else: - self.msg_alternative.attach(MIMEText(message, 'html', 'utf-8', policy=policy.SMTPUTF8)) + self.msg_alternative.attach(MIMEText(message, "html", "utf-8", policy=policy.SMTPUTF8)) def set_html_as_text(self, html): """Set plain text from HTML""" self.set_text(to_markdown(html)) - def set_message(self, message, mime_type='text/html', as_attachment=0, filename='attachment.html'): + def set_message( + self, message, mime_type="text/html", as_attachment=0, filename="attachment.html" + ): """Append the message with MIME content to the root node (as attachment)""" from email.mime.text import MIMEText - maintype, subtype = mime_type.split('/') - part = MIMEText(message, _subtype = subtype, policy=policy.SMTPUTF8) + maintype, subtype = mime_type.split("/") + part = MIMEText(message, _subtype=subtype, policy=policy.SMTPUTF8) if as_attachment: - part.add_header('Content-Disposition', 'attachment', filename=filename) + part.add_header("Content-Disposition", "attachment", filename=filename) self.msg_root.attach(part) @@ -158,8 +242,9 @@ class EMail: self.add_attachment(_file.file_name, content) - def add_attachment(self, fname, fcontent, content_type=None, - parent=None, content_id=None, inline=False): + def add_attachment( + self, fname, fcontent, content_type=None, parent=None, content_id=None, inline=False + ): """add attachment""" if not parent: @@ -168,7 +253,7 @@ class EMail: add_attachment(fname, fcontent, content_type, parent, content_id, inline) def add_pdf_attachment(self, name, html, options=None): - self.add_attachment(name, get_pdf(html, options), 'application/octet-stream') + self.add_attachment(name, get_pdf(html, options), "application/octet-stream") def validate(self): """validate the Email Addresses""" @@ -192,42 +277,46 @@ class EMail: def replace_sender(self): if cint(self.email_account.always_use_account_email_id_as_sender): - self.set_header('X-Original-From', self.sender) + self.set_header("X-Original-From", self.sender) sender_name, sender_email = parse_addr(self.sender) - self.sender = email.utils.formataddr((str(Header(sender_name or self.email_account.name, 'utf-8')), self.email_account.email_id)) + self.sender = email.utils.formataddr( + (str(Header(sender_name or self.email_account.name, "utf-8")), self.email_account.email_id) + ) def replace_sender_name(self): if cint(self.email_account.always_use_account_name_as_sender_name): - self.set_header('X-Original-From', self.sender) + self.set_header("X-Original-From", self.sender) sender_name, sender_email = parse_addr(self.sender) - self.sender = email.utils.formataddr((str(Header(self.email_account.name, 'utf-8')), sender_email)) + self.sender = email.utils.formataddr( + (str(Header(self.email_account.name, "utf-8")), sender_email) + ) def set_message_id(self, message_id, is_notification=False): if message_id: - message_id = '<' + message_id + '>' + message_id = "<" + message_id + ">" else: message_id = get_message_id() - self.set_header('isnotification', '') + self.set_header("isnotification", "") if is_notification: - self.set_header('isnotification', '') + self.set_header("isnotification", "") - self.set_header('Message-Id', message_id) + self.set_header("Message-Id", message_id) def set_in_reply_to(self, in_reply_to): """Used to send the Message-Id of a received email back as In-Reply-To""" - self.set_header('In-Reply-To', in_reply_to) + self.set_header("In-Reply-To", in_reply_to) def make(self): """build into msg_root""" headers = { - "Subject": strip(self.subject), - "From": self.sender, - "To": ', '.join(self.recipients) if self.expose_recipients=="header" else "", - "Date": email.utils.formatdate(), - "Reply-To": self.reply_to if self.reply_to else None, - "CC": ', '.join(self.cc) if self.cc and self.expose_recipients=="header" else None, - 'X-Frappe-Site': get_url(), + "Subject": strip(self.subject), + "From": self.sender, + "To": ", ".join(self.recipients) if self.expose_recipients == "header" else "", + "Date": email.utils.formatdate(), + "Reply-To": self.reply_to if self.reply_to else None, + "CC": ", ".join(self.cc) if self.cc and self.expose_recipients == "header" else None, + "X-Frappe-Site": get_url(), } # reset headers as values may be changed. @@ -254,22 +343,34 @@ class EMail: self.make() return self.msg_root.as_string(policy=policy.SMTPUTF8) -def get_formatted_html(subject, message, footer=None, print_html=None, - email_account=None, header=None, unsubscribe_link=None, sender=None, with_container=False): + +def get_formatted_html( + subject, + message, + footer=None, + print_html=None, + email_account=None, + header=None, + unsubscribe_link=None, + sender=None, + with_container=False, +): email_account = email_account or EmailAccount.find_outgoing(match_by_email=sender) - rendered_email = frappe.get_template("templates/emails/standard.html").render({ - "brand_logo": get_brand_logo(email_account) if with_container or header else None, - "with_container": with_container, - "site_url": get_url(), - "header": get_header(header), - "content": message, - "footer": get_footer(email_account, footer), - "title": subject, - "print_html": print_html, - "subject": subject - }) + rendered_email = frappe.get_template("templates/emails/standard.html").render( + { + "brand_logo": get_brand_logo(email_account) if with_container or header else None, + "with_container": with_container, + "site_url": get_url(), + "header": get_header(header), + "content": message, + "footer": get_footer(email_account, footer), + "title": subject, + "print_html": print_html, + "subject": subject, + } + ) html = scrub_urls(rendered_email) @@ -278,26 +379,29 @@ def get_formatted_html(subject, message, footer=None, print_html=None, return inline_style_in_html(html) + @frappe.whitelist() def get_email_html(template, args, subject, header=None, with_container=False): import json + with_container = cint(with_container) args = json.loads(args) - if header and header.startswith('['): + if header and header.startswith("["): header = json.loads(header) email = frappe.utils.jinja.get_email_from_template(template, args) return get_formatted_html(subject, email[0], header=header, with_container=with_container) + def inline_style_in_html(html): - ''' Convert email.css and html to inline-styled html - ''' + """Convert email.css and html to inline-styled html""" from premailer import Premailer + from frappe.utils.jinja_globals import bundled_asset # get email css files from hooks - css_files = frappe.get_hooks('email_css') + css_files = frappe.get_hooks("email_css") css_files = [bundled_asset(path) for path in css_files] - css_files = [path.lstrip('/') for path in css_files] + css_files = [path.lstrip("/") for path in css_files] css_files = [css_file for css_file in css_files if os.path.exists(os.path.abspath(css_file))] p = Premailer(html=html, external_styles=css_files, strip_important=False) @@ -305,15 +409,14 @@ def inline_style_in_html(html): return p.transform() -def add_attachment(fname, fcontent, content_type=None, - parent=None, content_id=None, inline=False): +def add_attachment(fname, fcontent, content_type=None, parent=None, content_id=None, inline=False): """Add attachment to parent which must an email object""" + import mimetypes from email.mime.audio import MIMEAudio from email.mime.base import MIMEBase from email.mime.image import MIMEImage from email.mime.text import MIMEText - import mimetypes if not content_type: content_type, encoding = mimetypes.guess_type(fname) @@ -323,44 +426,48 @@ def add_attachment(fname, fcontent, content_type=None, if content_type is None: # No guess could be made, or the file is encoded (compressed), so # use a generic bag-of-bits type. - content_type = 'application/octet-stream' + content_type = "application/octet-stream" - maintype, subtype = content_type.split('/', 1) - if maintype == 'text': + maintype, subtype = content_type.split("/", 1) + if maintype == "text": # Note: we should handle calculating the charset if isinstance(fcontent, str): fcontent = fcontent.encode("utf-8") part = MIMEText(fcontent, _subtype=subtype, _charset="utf-8") - elif maintype == 'image': + elif maintype == "image": part = MIMEImage(fcontent, _subtype=subtype) - elif maintype == 'audio': + elif maintype == "audio": part = MIMEAudio(fcontent, _subtype=subtype) else: part = MIMEBase(maintype, subtype) part.set_payload(fcontent) # Encode the payload using Base64 from email import encoders + encoders.encode_base64(part) # Set the filename parameter if fname: - attachment_type = 'inline' if inline else 'attachment' - part.add_header('Content-Disposition', attachment_type, filename=str(fname)) + attachment_type = "inline" if inline else "attachment" + part.add_header("Content-Disposition", attachment_type, filename=str(fname)) if content_id: - part.add_header('Content-ID', '<{0}>'.format(content_id)) + part.add_header("Content-ID", "<{0}>".format(content_id)) parent.attach(part) + def get_message_id(): - '''Returns Message ID created from doctype and name''' + """Returns Message ID created from doctype and name""" return email.utils.make_msgid(domain=frappe.local.site) + def get_signature(email_account): if email_account and email_account.add_signature and email_account.signature: return "
" + email_account.signature else: return "" + def get_footer(email_account, footer=None): """append a footer (signature)""" footer = footer or "" @@ -368,75 +475,78 @@ def get_footer(email_account, footer=None): args = {} if email_account and email_account.footer: - args.update({'email_account_footer': email_account.footer}) + args.update({"email_account_footer": email_account.footer}) sender_address = frappe.db.get_default("email_footer_address") if sender_address: - args.update({'sender_address': sender_address}) + args.update({"sender_address": sender_address}) if not cint(frappe.db.get_default("disable_standard_email_footer")): - args.update({'default_mail_footer': frappe.get_hooks('default_mail_footer')}) + args.update({"default_mail_footer": frappe.get_hooks("default_mail_footer")}) - footer += frappe.utils.jinja.get_email_from_template('email_footer', args)[0] + footer += frappe.utils.jinja.get_email_from_template("email_footer", args)[0] return footer + def replace_filename_with_cid(message): - """ Replaces with - and return the modified message and - a list of inline_images with {filename, filecontent, content_id} + """Replaces with + and return the modified message and + a list of inline_images with {filename, filecontent, content_id} """ inline_images = [] while True: - matches = re.search('''embed=["'](.*?)["']''', message) - if not matches: break + matches = re.search("""embed=["'](.*?)["']""", message) + if not matches: + break groups = matches.groups() # found match img_path = groups[0] - filename = img_path.rsplit('/')[-1] + filename = img_path.rsplit("/")[-1] filecontent = get_filecontent_from_path(img_path) if not filecontent: - message = re.sub('''embed=['"]{0}['"]'''.format(img_path), '', message) + message = re.sub("""embed=['"]{0}['"]""".format(img_path), "", message) continue content_id = random_string(10) - inline_images.append({ - 'filename': filename, - 'filecontent': filecontent, - 'content_id': content_id - }) + inline_images.append( + {"filename": filename, "filecontent": filecontent, "content_id": content_id} + ) - message = re.sub('''embed=['"]{0}['"]'''.format(img_path), - 'src="cid:{0}"'.format(content_id), message) + message = re.sub( + """embed=['"]{0}['"]""".format(img_path), 'src="cid:{0}"'.format(content_id), message + ) return (message, inline_images) + def get_filecontent_from_path(path): - if not path: return + if not path: + return - if path.startswith('/'): + if path.startswith("/"): path = path[1:] - if path.startswith('assets/'): + if path.startswith("assets/"): # from public folder full_path = os.path.abspath(path) - elif path.startswith('files/'): + elif path.startswith("files/"): # public file - full_path = frappe.get_site_path('public', path) - elif path.startswith('private/files/'): + full_path = frappe.get_site_path("public", path) + elif path.startswith("private/files/"): # private file full_path = frappe.get_site_path(path) else: full_path = path if os.path.exists(full_path): - with open(full_path, 'rb') as f: + with open(full_path, "rb") as f: filecontent = f.read() return filecontent @@ -445,10 +555,11 @@ def get_filecontent_from_path(path): def get_header(header=None): - """ Build header from template """ + """Build header from template""" from frappe.utils.jinja import get_email_from_template - if not header: return None + if not header: + return None if isinstance(header, str): # header = 'My Title' @@ -460,17 +571,18 @@ def get_header(header=None): title, indicator = header if not title: - title = frappe.get_hooks('app_title')[-1] + title = frappe.get_hooks("app_title")[-1] - email_header, text = get_email_from_template('email_header', { - 'header_title': title, - 'indicator': indicator - }) + email_header, text = get_email_from_template( + "email_header", {"header_title": title, "indicator": indicator} + ) return email_header + def sanitize_email_header(str): - return str.replace('\r', '').replace('\n', '') + return str.replace("\r", "").replace("\n", "") + def get_brand_logo(email_account): - return email_account.get('brand_logo') + return email_account.get("brand_logo") diff --git a/frappe/email/inbox.py b/frappe/email/inbox.py index c6020e14e4..b0673a3312 100644 --- a/frappe/email/inbox.py +++ b/frappe/email/inbox.py @@ -1,6 +1,7 @@ +import json import frappe -import json + def get_email_accounts(user=None): if not user: @@ -8,62 +9,53 @@ def get_email_accounts(user=None): email_accounts = [] - accounts = frappe.get_all("User Email", filters={ "parent": user }, + accounts = frappe.get_all( + "User Email", + filters={"parent": user}, fields=["email_account", "email_id", "enable_outgoing"], - distinct=True, order_by="idx") + distinct=True, + order_by="idx", + ) if not accounts: - return { - "email_accounts": [], - "all_accounts": "" - } + return {"email_accounts": [], "all_accounts": ""} all_accounts = ",".join(account.get("email_account") for account in accounts) if len(accounts) > 1: - email_accounts.append({ - "email_account": all_accounts, - "email_id": "All Accounts" - }) + email_accounts.append({"email_account": all_accounts, "email_id": "All Accounts"}) email_accounts.extend(accounts) - email_accounts.extend([ - { - "email_account": "Sent", - "email_id": "Sent Mail" - }, - { - "email_account": "Spam", - "email_id": "Spam" - }, - { - "email_account": "Trash", - "email_id": "Trash" - } - ]) - - return { - "email_accounts": email_accounts, - "all_accounts": all_accounts - } + email_accounts.extend( + [ + {"email_account": "Sent", "email_id": "Sent Mail"}, + {"email_account": "Spam", "email_id": "Spam"}, + {"email_account": "Trash", "email_id": "Trash"}, + ] + ) + + return {"email_accounts": email_accounts, "all_accounts": all_accounts} + @frappe.whitelist() def create_email_flag_queue(names, action): - """ create email flag queue to mark email either as read or unread """ + """create email flag queue to mark email either as read or unread""" + def mark_as_seen_unseen(name, action): doc = frappe.get_doc("Communication", name) if action == "Read": doc.add_seen() else: - _seen = json.loads(doc._seen or '[]') + _seen = json.loads(doc._seen or "[]") _seen = [user for user in _seen if frappe.session.user != user] - doc.db_set('_seen', json.dumps(_seen), update_modified=False) + doc.db_set("_seen", json.dumps(_seen), update_modified=False) if not all([names, action]): return for name in json.loads(names or []): - uid, seen_status, email_account = frappe.db.get_value("Communication", name, - ["ifnull(uid, -1)", "ifnull(seen, 0)", "email_account"]) + uid, seen_status, email_account = frappe.db.get_value( + "Communication", name, ["ifnull(uid, -1)", "ifnull(seen, 0)", "email_account"] + ) # can not mark email SEEN or UNSEEN without uid if not uid or uid == -1: @@ -71,10 +63,14 @@ def create_email_flag_queue(names, action): seen = 1 if action == "Read" else 0 # check if states are correct - if (action =='Read' and seen_status == 0) or (action =='Unread' and seen_status == 1): + if (action == "Read" and seen_status == 0) or (action == "Unread" and seen_status == 1): create_new = True - email_flag_queue = frappe.db.sql("""select name, action from `tabEmail Flag Queue` - where communication = %(name)s and is_completed=0""", {"name":name}, as_dict=True) + email_flag_queue = frappe.db.sql( + """select name, action from `tabEmail Flag Queue` + where communication = %(name)s and is_completed=0""", + {"name": name}, + as_dict=True, + ) for queue in email_flag_queue: if queue.action != action: @@ -84,46 +80,52 @@ def create_email_flag_queue(names, action): create_new = False if create_new: - flag_queue = frappe.get_doc({ - "uid": uid, - "action": action, - "communication": name, - "doctype": "Email Flag Queue", - "email_account": email_account - }) + flag_queue = frappe.get_doc( + { + "uid": uid, + "action": action, + "communication": name, + "doctype": "Email Flag Queue", + "email_account": email_account, + } + ) flag_queue.save(ignore_permissions=True) - frappe.db.set_value("Communication", name, "seen", seen, - update_modified=False) + frappe.db.set_value("Communication", name, "seen", seen, update_modified=False) mark_as_seen_unseen(name, action) + @frappe.whitelist() def mark_as_closed_open(communication, status): """Set status to open or close""" frappe.db.set_value("Communication", communication, "status", status) + @frappe.whitelist() def move_email(communication, email_account): """Move email to another email account.""" frappe.db.set_value("Communication", communication, "email_account", email_account) + @frappe.whitelist() def mark_as_trash(communication): """Set email status to trash.""" frappe.db.set_value("Communication", communication, "email_status", "Trash") + @frappe.whitelist() def mark_as_spam(communication, sender): """Set email status to spam.""" - email_rule = frappe.db.get_value("Email Rule", { "email_id": sender }) + email_rule = frappe.db.get_value("Email Rule", {"email_id": sender}) if not email_rule: - frappe.get_doc({ - "doctype": "Email Rule", - "email_id": sender, - "is_spam": 1 - }).insert(ignore_permissions=True) + frappe.get_doc({"doctype": "Email Rule", "email_id": sender, "is_spam": 1}).insert( + ignore_permissions=True + ) frappe.db.set_value("Communication", communication, "email_status", "Spam") -def link_communication_to_document(doc, reference_doctype, reference_name, ignore_communication_links): + +def link_communication_to_document( + doc, reference_doctype, reference_name, ignore_communication_links +): if not ignore_communication_links: doc.reference_doctype = reference_doctype doc.reference_name = reference_name diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 07a9c6552d..b0a3b0583b 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -2,11 +2,12 @@ # License: MIT. See LICENSE import frappe -from frappe import msgprint, _ -from frappe.utils.verified_command import get_signed_params, verify_request -from frappe.utils import get_url, now_datetime, cint +from frappe import _, msgprint from frappe.query_builder import DocType, Interval from frappe.query_builder.functions import Now +from frappe.utils import cint, get_url, now_datetime +from frappe.utils.verified_command import get_signed_params, verify_request + def get_emails_sent_this_month(email_account=None): """Get count of emails sent from a specific email account. @@ -30,12 +31,13 @@ def get_emails_sent_this_month(email_account=None): if email_account is not None: if email_account: q += " AND email_account = %(email_account)s" - q_args['email_account'] = email_account + q_args["email_account"] = email_account else: q += " AND (email_account is null OR email_account='')" return frappe.db.sql(q, q_args)[0][0] + def get_emails_sent_today(email_account=None): """Get count of emails sent from a specific email account. @@ -58,19 +60,24 @@ def get_emails_sent_today(email_account=None): if email_account is not None: if email_account: q += " AND email_account = %(email_account)s" - q_args['email_account'] = email_account + q_args["email_account"] = email_account else: q += " AND (email_account is null OR email_account='')" return frappe.db.sql(q, q_args)[0][0] + def get_unsubscribe_message(unsubscribe_message, expose_recipients): if unsubscribe_message: - unsubscribe_html = '''{0}'''.format(unsubscribe_message) + unsubscribe_html = """{0}""".format( + unsubscribe_message + ) else: - unsubscribe_link = '''{0}'''.format(_('Unsubscribe')) + unsubscribe_link = """{0}""".format( + _("Unsubscribe") + ) unsubscribe_html = _("{0} to stop receiving emails of this type").format(unsubscribe_link) html = """""".format(unsubscribe_html) +
""".format( + unsubscribe_html + ) if expose_recipients == "footer": text = "\n" else: text = "" - text += "\n\n{unsubscribe_message}: \n".format(unsubscribe_message=unsubscribe_message) + text += "\n\n{unsubscribe_message}: \n".format( + unsubscribe_message=unsubscribe_message + ) + + return frappe._dict({"html": html, "text": text}) - return frappe._dict({ - "html": html, - "text": text - }) -def get_unsubcribed_url(reference_doctype, reference_name, email, unsubscribe_method, unsubscribe_params): - params = {"email": email.encode("utf-8"), +def get_unsubcribed_url( + reference_doctype, reference_name, email, unsubscribe_method, unsubscribe_params +): + params = { + "email": email.encode("utf-8"), "doctype": reference_doctype.encode("utf-8"), - "name": reference_name.encode("utf-8")} + "name": reference_name.encode("utf-8"), + } if unsubscribe_params: params.update(unsubscribe_params) @@ -105,6 +118,7 @@ def get_unsubcribed_url(reference_doctype, reference_name, email, unsubscribe_me return get_url(unsubscribe_method + "?" + get_signed_params(params)) + @frappe.whitelist(allow_guest=True) def unsubscribe(doctype, name, email): # unsubsribe from comments and communications @@ -112,12 +126,14 @@ def unsubscribe(doctype, name, email): return try: - frappe.get_doc({ - "doctype": "Email Unsubscribe", - "email": email, - "reference_doctype": doctype, - "reference_name": name - }).insert(ignore_permissions=True) + frappe.get_doc( + { + "doctype": "Email Unsubscribe", + "email": email, + "reference_doctype": doctype, + "reference_name": name, + } + ).insert(ignore_permissions=True) except frappe.DuplicateEntryError: frappe.db.rollback() @@ -127,33 +143,39 @@ def unsubscribe(doctype, name, email): return_unsubscribed_page(email, doctype, name) + def return_unsubscribed_page(email, doctype, name): - frappe.respond_as_web_page(_("Unsubscribed"), + frappe.respond_as_web_page( + _("Unsubscribed"), _("{0} has left the conversation in {1} {2}").format(email, _(doctype), name), - indicator_color='green') + indicator_color="green", + ) + def flush(from_test=False): - """flush email queue, every time: called from scheduler - """ + """flush email queue, every time: called from scheduler""" from frappe.email.doctype.email_queue.email_queue import send_mail + # To avoid running jobs inside unit tests if frappe.are_emails_muted(): msgprint(_("Emails are muted")) from_test = True - if cint(frappe.defaults.get_defaults().get("hold_queue"))==1: + if cint(frappe.defaults.get_defaults().get("hold_queue")) == 1: return for row in get_queue(): try: func = send_mail if from_test else send_mail.enqueue is_background_task = not from_test - func(email_queue_name = row.name, is_background_task = is_background_task) + func(email_queue_name=row.name, is_background_task=is_background_task) except Exception: frappe.log_error() + def get_queue(): - return frappe.db.sql('''select + return frappe.db.sql( + """select name, sender from `tabEmail Queue` @@ -162,7 +184,11 @@ def get_queue(): (send_after is null or send_after < %(now)s) order by priority desc, creation asc - limit 500''', { 'now': now_datetime() }, as_dict=True) + limit 500""", + {"now": now_datetime()}, + as_dict=True, + ) + def clear_outbox(days: int = None) -> None: """Remove low priority older than 31 days in Outbox or configured in Log Settings. @@ -171,22 +197,29 @@ def clear_outbox(days: int = None) -> None: days = days or 31 email_queue = DocType("Email Queue") - email_queues = frappe.qb.from_(email_queue).select(email_queue.name).where( - email_queue.modified < (Now() - Interval(days=days)) - ).run(pluck=True) + email_queues = ( + frappe.qb.from_(email_queue) + .select(email_queue.name) + .where(email_queue.modified < (Now() - Interval(days=days))) + .run(pluck=True) + ) if email_queues: frappe.db.delete("Email Queue", {"name": ("in", email_queues)}) frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)}) + def set_expiry_for_email_queue(): - ''' Mark emails as expire that has not sent for 7 days. - Called daily via scheduler. - ''' + """Mark emails as expire that has not sent for 7 days. + Called daily via scheduler. + """ - frappe.db.sql(""" + frappe.db.sql( + """ UPDATE `tabEmail Queue` SET `status`='Expired' WHERE `modified` < (NOW() - INTERVAL '7' DAY) AND `status`='Not Sent' - AND (`send_after` IS NULL OR `send_after` < %(now)s)""", { 'now': now_datetime() }) + AND (`send_after` IS NULL OR `send_after` < %(now)s)""", + {"now": now_datetime()}, + ) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 8aa32fc1a5..4a6db65a84 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -5,10 +5,10 @@ import datetime import email import email.utils import imaplib +import json import poplib import re import time -import json from email.header import decode_header import _socket @@ -17,28 +17,50 @@ from email_reply_parser import EmailReplyParser import frappe from frappe import _, safe_decode, safe_encode -from frappe.core.doctype.file.file import (MaxFileSizeReachedError, - get_random_filename) -from frappe.utils import (cint, convert_utc_to_user_timezone, cstr, - extract_email_id, markdown, now, parse_addr, strip, get_datetime, - add_days, sanitize_html) -from frappe.utils.user import is_system_user +from frappe.core.doctype.file.file import MaxFileSizeReachedError, get_random_filename +from frappe.utils import ( + add_days, + cint, + convert_utc_to_user_timezone, + cstr, + extract_email_id, + get_datetime, + markdown, + now, + parse_addr, + sanitize_html, + strip, +) from frappe.utils.html_utils import clean_email_html +from frappe.utils.user import is_system_user # fix due to a python bug in poplib that limits it to 2048 poplib._MAXLINE = 20480 +class EmailSizeExceededError(frappe.ValidationError): + pass + + +class EmailTimeoutError(frappe.ValidationError): + pass + + +class TotalSizeExceededError(frappe.ValidationError): + pass + + +class LoginLimitExceeded(frappe.ValidationError): + pass + -class EmailSizeExceededError(frappe.ValidationError): pass -class EmailTimeoutError(frappe.ValidationError): pass -class TotalSizeExceededError(frappe.ValidationError): pass -class LoginLimitExceeded(frappe.ValidationError): pass class SentEmailInInboxError(Exception): pass + class EmailServer: """Wrapper for POP server to pull emails.""" + def __init__(self, args=None): self.setup(args) @@ -65,25 +87,33 @@ class EmailServer: """Connect to IMAP""" try: if cint(self.settings.use_ssl): - self.imap = Timed_IMAP4_SSL(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) + self.imap = Timed_IMAP4_SSL( + self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") + ) else: - self.imap = Timed_IMAP4(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) + self.imap = Timed_IMAP4( + self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") + ) self.imap.login(self.settings.username, self.settings.password) # connection established! return True except _socket.error: # Invalid mail server -- due to refusing connection - frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.')) + frappe.msgprint(_("Invalid Mail Server. Please rectify and try again.")) raise def connect_pop(self): - #this method return pop connection + # this method return pop connection try: if cint(self.settings.use_ssl): - self.pop = Timed_POP3_SSL(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) + self.pop = Timed_POP3_SSL( + self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") + ) else: - self.pop = Timed_POP3(self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")) + self.pop = Timed_POP3( + self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") + ) self.pop.user(self.settings.username) self.pop.pass_(self.settings.password) @@ -96,7 +126,7 @@ class EmailServer: frappe.log_error("receive.connect_pop") # Invalid mail server -- due to refusing connection - frappe.msgprint(_('Invalid Mail Server. Please rectify and try again.')) + frappe.msgprint(_("Invalid Mail Server. Please rectify and try again.")) raise except poplib.error_proto as e: @@ -104,12 +134,12 @@ class EmailServer: return False else: - frappe.msgprint(_('Invalid User Name or Support Password. Please rectify and try again.')) + frappe.msgprint(_("Invalid User Name or Support Password. Please rectify and try again.")) raise def select_imap_folder(self, folder): res = self.imap.select(f'"{folder}"') - return res[0] == 'OK' # The folder exsits TODO: handle other resoponses too + return res[0] == "OK" # The folder exsits TODO: handle other resoponses too def logout(self): if cint(self.settings.use_imap): @@ -142,7 +172,8 @@ class EmailServer: num = num_copy = len(email_list) # WARNING: Hard coded max no. of messages to be popped - if num > 50: num = 50 + if num > 50: + num = 50 # size limits self.total_size = 0 @@ -151,7 +182,7 @@ class EmailServer: for i, message_meta in enumerate(email_list[:num]): try: - self.retrieve_message(message_meta, i+1) + self.retrieve_message(message_meta, i + 1) except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded): break # WARNING: Mark as read - message number 101 onwards from the pop list @@ -159,7 +190,7 @@ class EmailServer: num = num_copy if not cint(self.settings.use_imap): if num > 100 and not self.errors: - for m in range(101, num+1): + for m in range(101, num + 1): self.pop.dele(m) except Exception as e: @@ -168,13 +199,11 @@ class EmailServer: else: raise - out = { "latest_messages": self.latest_messages } + out = {"latest_messages": self.latest_messages} if self.settings.use_imap: - out.update({ - "uid_list": uid_list, - "seen_status": self.seen_status, - "uid_reindexed": self.uid_reindexed - }) + out.update( + {"uid_list": uid_list, "seen_status": self.seen_status, "uid_reindexed": self.uid_reindexed} + ) return out @@ -187,9 +216,9 @@ class EmailServer: readonly = False if self.settings.email_sync_rule == "UNSEEN" else True self.imap.select(folder, readonly=readonly) - response, message = self.imap.uid('search', None, self.settings.email_sync_rule) + response, message = self.imap.uid("search", None, self.settings.email_sync_rule) if message[0]: - email_list = message[0].split() + email_list = message[0].split() else: email_list = self.pop.list()[1] @@ -208,25 +237,23 @@ class EmailServer: if not uid_validity or uid_validity != current_uid_validity: # uidvalidity changed & all email uids are reindexed by server Communication = frappe.qb.DocType("Communication") - frappe.qb.update(Communication) \ - .set(Communication.uid, -1) \ - .where(Communication.communication_medium == "Email") \ - .where(Communication.email_account == self.settings.email_account).run() + frappe.qb.update(Communication).set(Communication.uid, -1).where( + Communication.communication_medium == "Email" + ).where(Communication.email_account == self.settings.email_account).run() if self.settings.use_imap: # new update for the IMAP Folder DocType IMAPFolder = frappe.qb.DocType("IMAP Folder") - frappe.qb.update(IMAPFolder) \ - .set(IMAPFolder.uidvalidity, current_uid_validity) \ - .set(IMAPFolder.uidnext, uidnext) \ - .where(IMAPFolder.parent == self.settings.email_account_name) \ - .where(IMAPFolder.folder_name == folder).run() + frappe.qb.update(IMAPFolder).set(IMAPFolder.uidvalidity, current_uid_validity).set( + IMAPFolder.uidnext, uidnext + ).where(IMAPFolder.parent == self.settings.email_account_name).where( + IMAPFolder.folder_name == folder + ).run() else: EmailAccount = frappe.qb.DocType("Email Account") - frappe.qb.update(EmailAccount) \ - .set(EmailAccount.uidvalidity, current_uid_validity) \ - .set(EmailAccount.uidnext, uidnext) \ - .where(EmailAccount.name == self.settings.email_account_name).run() + frappe.qb.update(EmailAccount).set(EmailAccount.uidvalidity, current_uid_validity).set( + EmailAccount.uidnext, uidnext + ).where(EmailAccount.name == self.settings.email_account_name).run() # uid validity not found pulling emails for first time if not uid_validity: @@ -234,7 +261,9 @@ class EmailServer: return sync_count = 100 if uid_validity else int(self.settings.initial_sync_count) - from_uid = 1 if uidnext < (sync_count + 1) or (uidnext - sync_count) < 1 else uidnext - sync_count + from_uid = ( + 1 if uidnext < (sync_count + 1) or (uidnext - sync_count) < 1 else uidnext - sync_count + ) # sync last 100 email self.settings.email_sync_rule = "UID {}:{}".format(from_uid, uidnext) self.uid_reindexed = True @@ -244,7 +273,7 @@ class EmailServer: def parse_imap_response(self, cmd, response): pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd) - match = re.search(pattern, response.decode('utf-8'), re.U | re.I) + match = re.search(pattern, response.decode("utf-8"), re.U | re.I) if match: return match.group(0) @@ -257,14 +286,14 @@ class EmailServer: self.validate_message_limits(message_meta) if cint(self.settings.use_imap): - status, message = self.imap.uid('fetch', message_meta, '(BODY.PEEK[] BODY.PEEK[HEADER] FLAGS)') + status, message = self.imap.uid("fetch", message_meta, "(BODY.PEEK[] BODY.PEEK[HEADER] FLAGS)") raw = message[0] self.get_email_seen_status(message_meta, raw[0]) self.latest_messages.append(raw[1]) else: msg = self.pop.retr(msg_num) - self.latest_messages.append(b'\n'.join(msg[1])) + self.latest_messages.append(b"\n".join(msg[1])) except (TotalSizeExceededError, EmailTimeoutError): # propagate this error to break the loop self.errors = True @@ -286,17 +315,17 @@ class EmailServer: else: # mark as seen if email sync rule is UNSEEN (syncing only unseen mails) if self.settings.email_sync_rule == "UNSEEN": - self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') + self.imap.uid("STORE", message_meta, "+FLAGS", "(\\SEEN)") else: if not cint(self.settings.use_imap): self.pop.dele(msg_num) else: # mark as seen if email sync rule is UNSEEN (syncing only unseen mails) if self.settings.email_sync_rule == "UNSEEN": - self.imap.uid('STORE', message_meta, '+FLAGS', '(\\SEEN)') + self.imap.uid("STORE", message_meta, "+FLAGS", "(\\SEEN)") def get_email_seen_status(self, uid, flag_string): - """ parse the email FLAGS response """ + """parse the email FLAGS response""" if not flag_string: return None @@ -307,9 +336,9 @@ class EmailServer: flags.append(match.group(0)) if "Seen" in flags: - self.seen_status.update({ uid: "SEEN" }) + self.seen_status.update({uid: "SEEN"}) else: - self.seen_status.update({ uid: "UNSEEN" }) + self.seen_status.update({uid: "UNSEEN"}) def has_login_limit_exceeded(self, e): return "-ERR Exceeded the login limit" in strip(cstr(e.message)) @@ -320,7 +349,7 @@ class EmailServer: "Connection timed out", ) for message in messages: - if message in strip(cstr(e)) or message in strip(cstr(getattr(e, 'strerror', ''))): + if message in strip(cstr(e)) or message in strip(cstr(getattr(e, "strerror", ""))): return True return False @@ -344,18 +373,19 @@ class EmailServer: if not incoming_mail: try: # retrieve headers - incoming_mail = Email(b'\n'.join(self.pop.top(msg_num, 5)[1])) + incoming_mail = Email(b"\n".join(self.pop.top(msg_num, 5)[1])) except: pass if incoming_mail: error_msg += "\nDate: {date}\nFrom: {from_email}\nSubject: {subject}\n".format( - date=incoming_mail.date, from_email=incoming_mail.from_email, subject=incoming_mail.subject) + date=incoming_mail.date, from_email=incoming_mail.from_email, subject=incoming_mail.subject + ) return error_msg def update_flag(self, folder, uid_list=None): - """ set all uids mails the flag as seen """ + """set all uids mails the flag as seen""" if not uid_list: return @@ -364,16 +394,19 @@ class EmailServer: self.imap.select(folder) for uid, operation in uid_list.items(): - if not uid: continue + if not uid: + continue op = "+FLAGS" if operation == "Read" else "-FLAGS" try: - self.imap.uid('STORE', uid, op, '(\\SEEN)') + self.imap.uid("STORE", uid, op, "(\\SEEN)") except Exception: continue + class Email: """Wrapper for an email.""" + def __init__(self, content): """Parses headers, content, attachments from given raw message. @@ -384,21 +417,21 @@ class Email: self.mail = email.message_from_string(content) self.raw_message = content - self.text_content = '' - self.html_content = '' + self.text_content = "" + self.html_content = "" self.attachments = [] self.cid_map = {} self.parse() self.set_content_and_type() self.set_subject() self.set_from() - self.message_id = (self.mail.get('Message-ID') or "").strip(" <>") + self.message_id = (self.mail.get("Message-ID") or "").strip(" <>") if self.mail["Date"]: try: utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"])) utc_dt = datetime.datetime.utcfromtimestamp(utc) - self.date = convert_utc_to_user_timezone(utc_dt).strftime('%Y-%m-%d %H:%M:%S') + self.date = convert_utc_to_user_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S") except: self.date = now() else: @@ -434,7 +467,7 @@ class Email: _from_email = self.decode_email(self.mail.get("X-Original-From") or self.mail["From"]) _reply_to = self.decode_email(self.mail.get("Reply-To")) - if _reply_to and not frappe.db.get_value('Email Account', {"email_id":_reply_to}, 'email_id'): + if _reply_to and not frappe.db.get_value("Email Account", {"email_id": _reply_to}, "email_id"): self.from_email = extract_email_id(_reply_to) else: self.from_email = extract_email_id(_from_email) @@ -445,9 +478,12 @@ class Email: self.from_real_name = parse_addr(_from_email)[0] if "@" in _from_email else _from_email def decode_email(self, email): - if not email: return + if not email: + return decoded = "" - for part, encoding in decode_header(frappe.as_unicode(email).replace("\""," ").replace("\'"," ")): + for part, encoding in decode_header( + frappe.as_unicode(email).replace('"', " ").replace("'", " ") + ): if encoding: decoded += part.decode(encoding) else: @@ -455,26 +491,29 @@ class Email: return decoded def set_content_and_type(self): - self.content, self.content_type = '[Blank Email]', 'text/plain' + self.content, self.content_type = "[Blank Email]", "text/plain" if self.html_content: - self.content, self.content_type = self.html_content, 'text/html' + self.content, self.content_type = self.html_content, "text/html" else: - self.content, self.content_type = EmailReplyParser.read(self.text_content).text.replace("\n","\n\n"), 'text/plain' + self.content, self.content_type = ( + EmailReplyParser.read(self.text_content).text.replace("\n", "\n\n"), + "text/plain", + ) def process_part(self, part): """Parse email `part` and set it to `text_content`, `html_content` or `attachments`.""" content_type = part.get_content_type() - if content_type == 'text/plain': + if content_type == "text/plain": self.text_content += self.get_payload(part) - elif content_type == 'text/html': + elif content_type == "text/html": self.html_content += self.get_payload(part) - elif content_type == 'message/rfc822': + elif content_type == "message/rfc822": # sent by outlook when another email is sent as an attachment to this email self.show_attached_email_headers_in_content(part) - elif part.get_filename() or 'image' in content_type: + elif part.get_filename() or "image" in content_type: self.get_attachment(part) def show_attached_email_headers_in_content(self, part): @@ -486,15 +525,15 @@ class Email: message = list(part.walk())[1] headers = [] - for key in ('From', 'To', 'Subject', 'Date'): + for key in ("From", "To", "Subject", "Date"): value = cstr(message.get(key)) if value: - headers.append('{label}: {value}'.format(label=_(key), value=escape(value))) + headers.append("{label}: {value}".format(label=_(key), value=escape(value))) - self.text_content += '\n'.join(headers) - self.html_content += '
' + '\n'.join('

{0}

'.format(h) for h in headers) + self.text_content += "\n".join(headers) + self.html_content += "
" + "\n".join("

{0}

".format(h) for h in headers) - if not message.is_multipart() and message.get_content_type()=='text/plain': + if not message.is_multipart() and message.get_content_type() == "text/plain": # email.parser didn't parse it! text_content = self.get_payload(message) self.text_content += text_content @@ -504,7 +543,7 @@ class Email: """Detect charset.""" charset = part.get_content_charset() if not charset: - charset = chardet.detect(safe_encode(cstr(part)))['encoding'] + charset = chardet.detect(safe_encode(cstr(part)))["encoding"] return charset @@ -517,7 +556,7 @@ class Email: return part.get_payload() def get_attachment(self, part): - #charset = self.get_charset(part) + # charset = self.get_charset(part) fcontent = part.get_payload(decode=True) if fcontent: @@ -525,18 +564,20 @@ class Email: fname = part.get_filename() if fname: try: - fname = fname.replace('\n', ' ').replace('\r', '') + fname = fname.replace("\n", " ").replace("\r", "") fname = cstr(decode_header(fname)[0][0]) except: fname = get_random_filename(content_type=content_type) else: fname = get_random_filename(content_type=content_type) - self.attachments.append({ - 'content_type': content_type, - 'fname': fname, - 'fcontent': fcontent, - }) + self.attachments.append( + { + "content_type": content_type, + "fname": fname, + "fcontent": fcontent, + } + ) cid = (cstr(part.get("Content-Id")) or "").strip("><") if cid: @@ -548,18 +589,21 @@ class Email: for attachment in self.attachments: try: - _file = frappe.get_doc({ - "doctype": "File", - "file_name": attachment['fname'], - "attached_to_doctype": doc.doctype, - "attached_to_name": doc.name, - "is_private": 1, - "content": attachment['fcontent']}) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": attachment["fname"], + "attached_to_doctype": doc.doctype, + "attached_to_name": doc.name, + "is_private": 1, + "content": attachment["fcontent"], + } + ) _file.save() saved_attachments.append(_file) - if attachment['fname'] in self.cid_map: - self.cid_map[_file.name] = self.cid_map[attachment['fname']] + if attachment["fname"] in self.cid_map: + self.cid_map[_file.name] = self.cid_map[attachment["fname"]] except MaxFileSizeReachedError: # WARNING: bypass max file size exception @@ -574,15 +618,16 @@ class Email: def get_thread_id(self): """Extract thread ID from `[]`""" - l = re.findall(r'(?<=\[)[\w/-]+', self.subject) + l = re.findall(r"(?<=\[)[\w/-]+", self.subject) return l and l[0] or None def is_reply(self): return bool(self.in_reply_to) + class InboundMail(Email): - """Class representation of incoming mail along with mail handlers. - """ + """Class representation of incoming mail along with mail handlers.""" + def __init__(self, content, email_account, uid=None, seen_status=None, append_to=None): super().__init__(content) self.email_account = email_account @@ -598,15 +643,14 @@ class InboundMail(Email): self.flags = frappe._dict() def get_content(self): - if self.content_type == 'text/html': + if self.content_type == "text/html": return clean_email_html(self.content) def process(self): - """Create communication record from email. - """ + """Create communication record from email.""" if self.is_sender_same_as_receiver() and not self.is_reply(): if frappe.flags.in_test: - print('WARN: Cannot pull email. Sender same as recipient inbox') + print("WARN: Cannot pull email. Sender same as recipient inbox") raise SentEmailInInboxError communication = self.is_exist_in_system() @@ -620,30 +664,30 @@ class InboundMail(Email): def _build_communication_doc(self): data = self.as_dict() - data['doctype'] = "Communication" + data["doctype"] = "Communication" if self.parent_communication(): - data['in_reply_to'] = self.parent_communication().name + data["in_reply_to"] = self.parent_communication().name append_to = self.append_to if self.email_account.use_imap else self.email_account.append_to if self.reference_document(): - data['reference_doctype'] = self.reference_document().doctype - data['reference_name'] = self.reference_document().name + data["reference_doctype"] = self.reference_document().doctype + data["reference_name"] = self.reference_document().name else: - if append_to and append_to != 'Communication': + if append_to and append_to != "Communication": reference_doc = self._create_reference_document(append_to) if reference_doc: - data['reference_doctype'] = reference_doc.doctype - data['reference_name'] = reference_doc.name - data['is_first'] = True + data["reference_doctype"] = reference_doc.doctype + data["reference_name"] = reference_doc.name + data["is_first"] = True if self.is_notification(): # Disable notifications for notification. - data['unread_notification_sent'] = 1 + data["unread_notification_sent"] = 1 if self.seen_status: - data['_seen'] = json.dumps(self.get_users_linked_to_account(self.email_account)) + data["_seen"] = json.dumps(self.get_users_linked_to_account(self.email_account)) communication = frappe.get_doc(data) communication.flags.in_receive = True @@ -660,8 +704,7 @@ class InboundMail(Email): content = self.content for file in attachments: if file.name in self.cid_map and self.cid_map[file.name]: - content = content.replace("cid:{0}".format(self.cid_map[file.name]), - file.file_url) + content = content.replace("cid:{0}".format(self.cid_map[file.name]), file.file_url) return content def is_notification(self): @@ -669,21 +712,19 @@ class InboundMail(Email): return isnotification and ("notification" in isnotification) def is_exist_in_system(self): - """Check if this email already exists in the system(as communication document). - """ + """Check if this email already exists in the system(as communication document).""" from frappe.core.doctype.communication.communication import Communication + if not self.message_id: return - return Communication.find_one_by_filters(message_id = self.message_id, - order_by = 'creation DESC') + return Communication.find_one_by_filters(message_id=self.message_id, order_by="creation DESC") def is_sender_same_as_receiver(self): return self.from_email == self.email_account.email_id def is_reply_to_system_sent_mail(self): - """Is it a reply to already sent mail. - """ + """Is it a reply to already sent mail.""" return self.is_reply() and frappe.local.site in self.in_reply_to def parent_email_queue(self): @@ -696,11 +737,11 @@ class InboundMail(Email): if self._parent_email_queue is not None: return self._parent_email_queue - parent_email_queue = '' + parent_email_queue = "" if self.is_reply_to_system_sent_mail(): parent_email_queue = EmailQueue.find_one_by_filters(message_id=self.in_reply_to) - self._parent_email_queue = parent_email_queue or '' + self._parent_email_queue = parent_email_queue or "" return self._parent_email_queue def parent_communication(self): @@ -710,30 +751,32 @@ class InboundMail(Email): Here are the cases to handle: 1. If mail is a reply to already sent mail, then we can get parent communicaion from - Email Queue record. + Email Queue record. 2. Sometimes we send communication name in message-ID directly, use that to get parent communication. 3. Sender sent a reply but reply is on top of what (s)he sent before, - then parent record exists directly in communication. + then parent record exists directly in communication. """ from frappe.core.doctype.communication.communication import Communication + if self._parent_communication is not None: return self._parent_communication if not self.is_reply(): - return '' + return "" if not self.is_reply_to_system_sent_mail(): - communication = Communication.find_one_by_filters(message_id=self.in_reply_to, - creation = ['>=', self.get_relative_dt(-30)]) + communication = Communication.find_one_by_filters( + message_id=self.in_reply_to, creation=[">=", self.get_relative_dt(-30)] + ) elif self.parent_email_queue() and self.parent_email_queue().communication: communication = Communication.find(self.parent_email_queue().communication, ignore_error=True) else: reference = self.in_reply_to - if '@' in self.in_reply_to: + if "@" in self.in_reply_to: reference, _ = self.in_reply_to.split("@", 1) communication = Communication.find(reference, ignore_error=True) - self._parent_communication = communication or '' + self._parent_communication = communication or "" return self._parent_communication def reference_document(self): @@ -755,14 +798,14 @@ class InboundMail(Email): if not reference_document and self.email_account.append_to: reference_document = self.match_record_by_subject_and_sender(self.email_account.append_to) - self._reference_document = reference_document or '' + self._reference_document = reference_document or "" return self._reference_document def get_reference_name_from_subject(self): """ Ex: "Re: Your email (#OPP-2020-2334343)" """ - return self.subject.rsplit('#', 1)[-1].strip(' ()') + return self.subject.rsplit("#", 1)[-1].strip(" ()") def match_record_by_subject_and_sender(self, doctype): """Find a record in the given doctype that matches with email subject and sender. @@ -771,12 +814,12 @@ class InboundMail(Email): 1. Sometimes record name is part of subject. We can get document by parsing name from subject 2. Find by matching sender and subject 3. Find by matching subject alone (Special case) - Ex: when a System User is using Outlook and replies to an email from their own client, - it reaches the Email Account with the threading info lost and the (sender + subject match) - doesn't work because the sender in the first communication was someone different to whom - the system user is replying to via the common email account in Frappe. This fix bypasses - the sender match when the sender is a system user and subject is atleast 10 chars long - (for additional safety) + Ex: when a System User is using Outlook and replies to an email from their own client, + it reaches the Email Account with the threading info lost and the (sender + subject match) + doesn't work because the sender in the first communication was someone different to whom + the system user is replying to via the common email account in Frappe. This fix bypasses + the sender match when the sender is a system user and subject is atleast 10 chars long + (for additional safety) NOTE: We consider not to match by subject if match record is very old. """ @@ -789,20 +832,19 @@ class InboundMail(Email): subject = self.clean_subject(self.subject) filters = { email_fields.subject_field: ("like", f"%{subject}%"), - "creation": (">", self.get_relative_dt(days=-60)) + "creation": (">", self.get_relative_dt(days=-60)), } # Sender check is not needed incase mail is from system user. if not (len(subject) > 10 and is_system_user(self.from_email)): filters[email_fields.sender_field] = self.from_email - name = frappe.db.get_value(self.email_account.append_to, filters = filters) + name = frappe.db.get_value(self.email_account.append_to, filters=filters) record = self.get_doc(doctype, name, ignore_error=True) if name else None return record def _create_reference_document(self, doctype): - """ Create reference document if it does not exist in the system. - """ + """Create reference document if it does not exist in the system.""" parent = frappe.new_doc(doctype) email_fileds = self.get_email_fields(doctype) @@ -818,8 +860,8 @@ class InboundMail(Email): parent.insert(ignore_permissions=True) except frappe.DuplicateEntryError: # try and find matching parent - parent_name = frappe.db.get_value(self.email_account.append_to, - {email_fileds.sender_field: self.from_email} + parent_name = frappe.db.get_value( + self.email_account.append_to, {email_fileds.sender_field: self.from_email} ) if parent_name: parent.name = parent_name @@ -827,7 +869,6 @@ class InboundMail(Email): parent = None return parent - @staticmethod def get_doc(doctype, docname, ignore_error=False): try: @@ -839,33 +880,30 @@ class InboundMail(Email): @staticmethod def get_relative_dt(days): - """Get relative to current datetime. Only relative days are supported. - """ + """Get relative to current datetime. Only relative days are supported.""" return add_days(get_datetime(), days) @staticmethod def get_users_linked_to_account(email_account): - """Get list of users who linked to Email account. - """ - users = frappe.get_all("User Email", filters={"email_account": email_account.name}, - fields=["parent"]) + """Get list of users who linked to Email account.""" + users = frappe.get_all( + "User Email", filters={"email_account": email_account.name}, fields=["parent"] + ) return list(set([user.get("parent") for user in users])) @staticmethod def clean_subject(subject): - """Remove Prefixes like 'fw', FWD', 're' etc from subject. - """ + """Remove Prefixes like 'fw', FWD', 're' etc from subject.""" # Match strings like "fw:", "re :" etc. regex = r"(^\s*(fw|fwd|wg)[^:]*:|\s*(re|aw)[^:]*:\s*)*" return frappe.as_unicode(strip(re.sub(regex, "", subject, 0, flags=re.IGNORECASE))) @staticmethod def get_email_fields(doctype): - """Returns Email related fields of a doctype. - """ + """Returns Email related fields of a doctype.""" fields = frappe._dict() - email_fields = ['subject_field', 'sender_field'] + email_fields = ["subject_field", "sender_field"] meta = frappe.get_meta(doctype) for field in email_fields: @@ -875,20 +913,18 @@ class InboundMail(Email): @staticmethod def get_document(self, doctype, name): - """Is same as frappe.get_doc but suppresses the DoesNotExist error. - """ + """Is same as frappe.get_doc but suppresses the DoesNotExist error.""" try: return frappe.get_doc(doctype, name) except frappe.DoesNotExistError: return None def as_dict(self): - """ - """ + """ """ return { "subject": self.subject, "content": self.get_content(), - 'text_content': self.text_content, + "text_content": self.text_content, "sent_or_received": "Received", "sender_full_name": self.from_real_name, "sender": self.from_email, @@ -900,12 +936,13 @@ class InboundMail(Email): "message_id": self.message_id, "communication_date": self.date, "has_attachment": 1 if self.attachments else 0, - "seen": self.seen_status or 0 + "seen": self.seen_status or 0, } + class TimerMixin(object): def __init__(self, *args, **kwargs): - self.timeout = kwargs.pop('timeout', 0.0) + self.timeout = kwargs.pop("timeout", 0.0) self.elapsed_time = 0.0 self._super.__init__(self, *args, **kwargs) if self.timeout: @@ -926,14 +963,18 @@ class TimerMixin(object): self.elapsed_time = 0.0 return self._super.quit(self, *args, **kwargs) + class Timed_POP3(TimerMixin, poplib.POP3): _super = poplib.POP3 + class Timed_POP3_SSL(TimerMixin, poplib.POP3_SSL): _super = poplib.POP3_SSL + class Timed_IMAP4(TimerMixin, imaplib.IMAP4): _super = imaplib.IMAP4 + class Timed_IMAP4_SSL(TimerMixin, imaplib.IMAP4_SSL): _super = imaplib.IMAP4_SSL diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 6f73a73f11..1c91356506 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -1,28 +1,37 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import smtplib import email.utils -import _socket, sys +import smtplib +import sys + +import _socket + +import frappe from frappe import _ from frappe.utils import cint, cstr, parse_addr -CONNECTION_FAILED = _('Could not connect to outgoing email server') +CONNECTION_FAILED = _("Could not connect to outgoing email server") AUTH_ERROR_TITLE = _("Invalid Credentials") AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.") SOCKET_ERROR_TITLE = _("Incorrect Configuration") SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port") SEND_MAIL_FAILED = _("Unable to send emails at this time") -EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account') +EMAIL_ACCOUNT_MISSING = _( + "Email Account not setup. Please create a new Email Account from Setup > Email > Email Account" +) + class InvalidEmailCredentials(frappe.ValidationError): pass + def send(email, append_to=None, retry=1): """Deprecated: Send the message or add it to Outbox Email""" + def _send(retry): from frappe.email.doctype.email_account.email_account import EmailAccount + try: email_account = EmailAccount.find_outgoing(match_by_doctype=append_to) smtpserver = email_account.get_smtp_server() @@ -32,10 +41,10 @@ def send(email, append_to=None, retry=1): smtpserver.sess.sendmail(email.sender, email.recipients + (email.cc or []), email_body) except smtplib.SMTPSenderRefused: - frappe.throw(_("Invalid login or password"), title='Email Failed') + frappe.throw(_("Invalid login or password"), title="Email Failed") raise except smtplib.SMTPRecipientsRefused: - frappe.msgprint(_("Invalid recipient address"), title='Email Failed') + frappe.msgprint(_("Invalid recipient address"), title="Email Failed") raise except (smtplib.SMTPServerDisconnected, smtplib.SMTPAuthenticationError): if not retry: @@ -46,6 +55,7 @@ def send(email, append_to=None, retry=1): _send(retry) + class SMTPServer: def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None): self.login = login @@ -69,8 +79,7 @@ class SMTPServer: return cstr(self._server or "") def secure_session(self, conn): - """Secure the connection incase of TLS. - """ + """Secure the connection incase of TLS.""" if self.use_tls: conn.ehlo() conn.starttls() @@ -93,7 +102,7 @@ class SMTPServer: res = _session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly - if res[0]!=235: + if res[0] != 235: frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError) self._session = _session diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index c542bc2578..3de21f64ce 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -1,83 +1,96 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest, os, base64 +import base64 +import os +import unittest + from frappe import safe_decode +from frappe.email.doctype.email_queue.email_queue import QueueBuilder, SendMailContext +from frappe.email.email_body import ( + get_email, + get_header, + inline_style_in_html, + replace_filename_with_cid, +) from frappe.email.receive import Email -from frappe.email.email_body import (replace_filename_with_cid, - get_email, inline_style_in_html, get_header) -from frappe.email.doctype.email_queue.email_queue import SendMailContext, QueueBuilder class TestEmailBody(unittest.TestCase): def setUp(self): - email_html = ''' + email_html = """

Hey John Doe!

This is embedded image you asked for

-''' - email_text = ''' +""" + email_text = """ Hey John Doe! This is the text version of this email -''' +""" - img_path = os.path.abspath('assets/frappe/images/frappe-favicon.svg') - with open(img_path, 'rb') as f: + img_path = os.path.abspath("assets/frappe/images/frappe-favicon.svg") + with open(img_path, "rb") as f: img_content = f.read() img_base64 = base64.b64encode(img_content).decode() # email body keeps 76 characters on one line self.img_base64 = fixed_column_width(img_base64, 76) - self.email_string = get_email( - recipients=['test@example.com'], - sender='me@example.com', - subject='Test Subject', - content=email_html, - text_content=email_text - ).as_string().replace("\r\n", "\n") + self.email_string = ( + get_email( + recipients=["test@example.com"], + sender="me@example.com", + subject="Test Subject", + content=email_html, + text_content=email_text, + ) + .as_string() + .replace("\r\n", "\n") + ) def test_prepare_message_returns_already_encoded_string(self): uni_chr1 = chr(40960) uni_chr2 = chr(1972) queue_doc = QueueBuilder( - recipients=['test@example.com'], - sender='me@example.com', - subject='Test Subject', - message='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', - text_content='whatever').process()[0] - mail_ctx = SendMailContext(queue_doc = queue_doc) - result = mail_ctx.build_message(recipient_email = 'test@test.com') + recipients=["test@example.com"], + sender="me@example.com", + subject="Test Subject", + message="

" + uni_chr1 + "abcd" + uni_chr2 + "

", + text_content="whatever", + ).process()[0] + mail_ctx = SendMailContext(queue_doc=queue_doc) + result = mail_ctx.build_message(recipient_email="test@test.com") self.assertTrue(b"

=EA=80=80abcd=DE=B4

" in result) def test_prepare_message_returns_cr_lf(self): queue_doc = QueueBuilder( - recipients=['test@example.com'], - sender='me@example.com', - subject='Test Subject', - message='

\n this is a test of newlines\n' + '

', - text_content='whatever').process()[0] + recipients=["test@example.com"], + sender="me@example.com", + subject="Test Subject", + message="

\n this is a test of newlines\n" + "

", + text_content="whatever", + ).process()[0] - mail_ctx = SendMailContext(queue_doc = queue_doc) - result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com')) + mail_ctx = SendMailContext(queue_doc=queue_doc) + result = safe_decode(mail_ctx.build_message(recipient_email="test@test.com")) - self.assertTrue(result.count('\n') == result.count("\r")) + self.assertTrue(result.count("\n") == result.count("\r")) def test_image(self): - img_signature = ''' + img_signature = """ Content-Type: image/svg+xml MIME-Version: 1.0 Content-Transfer-Encoding: base64 Content-Disposition: inline; filename="frappe-favicon.svg" -''' +""" self.assertTrue(img_signature in self.email_string) self.assertTrue(self.img_base64 in self.email_string) def test_text_content(self): - text_content = ''' + text_content = """ Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable @@ -85,11 +98,11 @@ Content-Transfer-Encoding: quoted-printable Hey John Doe! This is the text version of this email -''' +""" self.assertTrue(text_content in self.email_string) def test_email_content(self): - html_head = ''' + html_head = """ Content-Type: text/html; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable @@ -97,84 +110,93 @@ Content-Transfer-Encoding: quoted-printable -''' +""" - html = '''

Hey John Doe!

''' + html = """

Hey John Doe!

""" self.assertTrue(html_head in self.email_string) self.assertTrue(html in self.email_string) def test_replace_filename_with_cid(self): - original_message = ''' + original_message = """
test
- ''' + """ message, inline_images = replace_filename_with_cid(original_message) - processed_message = ''' + processed_message = """
test
- '''.format(inline_images[0].get('content_id')) + """.format( + inline_images[0].get("content_id") + ) self.assertEqual(message, processed_message) def test_inline_styling(self): - html = ''' + html = """

Hi John

This is a test email

-''' - transformed_html = ''' +""" + transformed_html = """

Hi John

This is a test email

-''' +""" self.assertTrue(transformed_html in inline_style_in_html(html)) def test_email_header(self): - email_html = ''' + email_html = """

Hey John Doe!

This is embedded image you asked for

-''' - email_string = get_email( - recipients=['test@example.com'], - sender='me@example.com', - subject='Test Subject', - content=email_html, - header=['Email Title', 'green'] - ).as_string().replace("\r\n", "\n") +""" + email_string = ( + get_email( + recipients=["test@example.com"], + sender="me@example.com", + subject="Test Subject", + content=email_html, + header=["Email Title", "green"], + ) + .as_string() + .replace("\r\n", "\n") + ) # REDESIGN-TODO: Add style for indicators in email - self.assertTrue('''''' in email_string) - self.assertTrue('Email Title' in email_string) + self.assertTrue("""""" in email_string) + self.assertTrue("Email Title" in email_string) def test_get_email_header(self): - html = get_header(['This is test', 'orange']) + html = get_header(["This is test", "orange"]) self.assertTrue('' in html) - self.assertTrue('This is test' in html) + self.assertTrue("This is test" in html) - html = get_header(['This is another test']) - self.assertTrue('This is another test' in html) + html = get_header(["This is another test"]) + self.assertTrue("This is another test" in html) - html = get_header('This is string') - self.assertTrue('This is string' in html) + html = get_header("This is string") + self.assertTrue("This is string" in html) def test_8bit_utf_8_decoding(self): text_content_bytes = b"\xed\x95\x9c\xea\xb8\x80\xe1\xa5\xa1\xe2\x95\xa5\xe0\xba\xaa\xe0\xa4\x8f" - text_content = text_content_bytes.decode('utf-8') + text_content = text_content_bytes.decode("utf-8") - content_bytes = b"""MIME-Version: 1.0 + content_bytes = ( + b"""MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Disposition: inline Content-Transfer-Encoding: 8bit From: test1_@erpnext.com Reply-To: test2_@erpnext.com -""" + text_content_bytes +""" + + text_content_bytes + ) mail = Email(content_bytes) self.assertEqual(mail.text_content, text_content) def fixed_column_width(string, chunk_size): - parts = [string[0 + i:chunk_size + i] for i in range(0, len(string), chunk_size)] - return '\n'.join(parts) + parts = [string[0 + i : chunk_size + i] for i in range(0, len(string), chunk_size)] + return "\n".join(parts) diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py index 127bdd44ce..448f61925c 100644 --- a/frappe/email/test_smtp.py +++ b/frappe/email/test_smtp.py @@ -2,9 +2,11 @@ # License: The MIT License import unittest + import frappe -from frappe.email.smtp import SMTPServer from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.email.smtp import SMTPServer + class TestSMTP(unittest.TestCase): def test_smtp_ssl_session(self): @@ -16,65 +18,72 @@ class TestSMTP(unittest.TestCase): make_server(port, 0, 1) def test_get_email_account(self): - existing_email_accounts = frappe.get_all("Email Account", fields = ["name", "enable_outgoing", "default_outgoing","append_to", "use_imap"]) - unset_details = { - "enable_outgoing": 0, - "default_outgoing": 0, - "append_to": None, - "use_imap": 0 - } + existing_email_accounts = frappe.get_all( + "Email Account", fields=["name", "enable_outgoing", "default_outgoing", "append_to", "use_imap"] + ) + unset_details = {"enable_outgoing": 0, "default_outgoing": 0, "append_to": None, "use_imap": 0} for email_account in existing_email_accounts: - frappe.db.set_value('Email Account', email_account['name'], unset_details) + frappe.db.set_value("Email Account", email_account["name"], unset_details) # remove mail_server config so that test@example.com is not created - mail_server = frappe.conf.get('mail_server') - del frappe.conf['mail_server'] + mail_server = frappe.conf.get("mail_server") + del frappe.conf["mail_server"] frappe.local.outgoing_email_account = {} frappe.local.outgoing_email_account = {} # lowest preference given to email account with default incoming enabled - create_email_account(email_id="default_outgoing_enabled@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1) + create_email_account( + email_id="default_outgoing_enabled@gmail.com", + password="password", + enable_outgoing=1, + default_outgoing=1, + ) self.assertEqual(EmailAccount.find_outgoing().email_id, "default_outgoing_enabled@gmail.com") frappe.local.outgoing_email_account = {} # highest preference given to email account with append_to matching - create_email_account(email_id="append_to@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post") - self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com") + create_email_account( + email_id="append_to@gmail.com", + password="password", + enable_outgoing=1, + default_outgoing=1, + append_to="Blog Post", + ) + self.assertEqual( + EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com" + ) # add back the mail_server - frappe.conf['mail_server'] = mail_server + frappe.conf["mail_server"] = mail_server for email_account in existing_email_accounts: set_details = { - "enable_outgoing": email_account['enable_outgoing'], - "default_outgoing": email_account['default_outgoing'], - "append_to": email_account['append_to'] + "enable_outgoing": email_account["enable_outgoing"], + "default_outgoing": email_account["default_outgoing"], + "append_to": email_account["append_to"], } - frappe.db.set_value('Email Account', email_account['name'], set_details) + frappe.db.set_value("Email Account", email_account["name"], set_details) + def create_email_account(email_id, password, enable_outgoing, default_outgoing=0, append_to=None): email_dict = { "email_id": email_id, "passsword": password, - "enable_outgoing":enable_outgoing , - "default_outgoing":default_outgoing , + "enable_outgoing": enable_outgoing, + "default_outgoing": default_outgoing, "enable_incoming": 1, - "append_to":append_to, + "append_to": append_to, "is_dummy_password": 1, "smtp_server": "localhost", - "use_imap": 0 + "use_imap": 0, } - email_account = frappe.new_doc('Email Account') + email_account = frappe.new_doc("Email Account") email_account.update(email_dict) email_account.save() + def make_server(port, ssl, tls): - server = SMTPServer( - server = "smtp.gmail.com", - port = port, - use_ssl = ssl, - use_tls = tls - ) + server = SMTPServer(server="smtp.gmail.com", port=port, use_ssl=ssl, use_tls=tls) server.session diff --git a/frappe/email/utils.py b/frappe/email/utils.py index 1138698491..147284a625 100644 --- a/frappe/email/utils.py +++ b/frappe/email/utils.py @@ -1,15 +1,17 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import imaplib, poplib +import imaplib +import poplib from frappe.utils import cint + def get_port(doc): if not doc.incoming_port: if doc.use_imap: - doc.incoming_port = imaplib.IMAP4_SSL_PORT if doc.use_ssl else imaplib.IMAP4_PORT + doc.incoming_port = imaplib.IMAP4_SSL_PORT if doc.use_ssl else imaplib.IMAP4_PORT else: - doc.incoming_port = poplib.POP3_SSL_PORT if doc.use_ssl else poplib.POP3_PORT + doc.incoming_port = poplib.POP3_SSL_PORT if doc.use_ssl else poplib.POP3_PORT - return cint(doc.incoming_port) \ No newline at end of file + return cint(doc.incoming_port) diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py index 0565b3219d..bcd2b275d1 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import json + +import frappe from frappe import _ +from frappe.model import child_table_fields, default_fields from frappe.model.document import Document -from frappe.model import default_fields, child_table_fields + class DocumentTypeMapping(Document): def validate(self): @@ -17,19 +19,21 @@ class DocumentTypeMapping(Document): if field_map.local_fieldname not in (default_fields + child_table_fields): field = meta.get_field(field_map.local_fieldname) if not field: - frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx)) + frappe.throw(_("Row #{0}: Invalid Local Fieldname").format(field_map.idx)) - fieldtype = field.get('fieldtype') - if fieldtype in ['Link', 'Dynamic Link', 'Table']: + fieldtype = field.get("fieldtype") + if fieldtype in ["Link", "Dynamic Link", "Table"]: if not field_map.mapping and not field_map.default_value: - msg = _('Row #{0}: Please set Mapping or Default Value for the field {1} since its a dependency field').format( - field_map.idx, frappe.bold(field_map.local_fieldname)) - frappe.throw(msg, title='Inner Mapping Missing') + msg = _( + "Row #{0}: Please set Mapping or Default Value for the field {1} since its a dependency field" + ).format(field_map.idx, frappe.bold(field_map.local_fieldname)) + frappe.throw(msg, title="Inner Mapping Missing") - if field_map.mapping_type == 'Document' and not field_map.remote_value_filters: - msg = _('Row #{0}: Please set remote value filters for the field {1} to fetch the unique remote dependency document').format( - field_map.idx, frappe.bold(field_map.remote_fieldname)) - frappe.throw(msg, title='Remote Value Filters Missing') + if field_map.mapping_type == "Document" and not field_map.remote_value_filters: + msg = _( + "Row #{0}: Please set remote value filters for the field {1} to fetch the unique remote dependency document" + ).format(field_map.idx, frappe.bold(field_map.remote_fieldname)) + frappe.throw(msg, title="Remote Value Filters Missing") def get_mapping(self, doc, producer_site, update_type): remote_fields = [] @@ -38,7 +42,7 @@ class DocumentTypeMapping(Document): for mapping in self.field_mapping: if doc.get(mapping.remote_fieldname): - if mapping.mapping_type == 'Document': + if mapping.mapping_type == "Document": if not mapping.default_value: dependency = self.get_mapped_dependency(mapping, producer_site, doc) if dependency: @@ -46,8 +50,10 @@ class DocumentTypeMapping(Document): else: doc[mapping.local_fieldname] = mapping.default_value - if mapping.mapping_type == 'Child Table' and update_type != 'Update': - doc[mapping.local_fieldname] = get_mapped_child_table_docs(mapping.mapping, doc[mapping.remote_fieldname], producer_site) + if mapping.mapping_type == "Child Table" and update_type != "Update": + doc[mapping.local_fieldname] = get_mapped_child_table_docs( + mapping.mapping, doc[mapping.remote_fieldname], producer_site + ) else: # copy value into local fieldname key and remove remote fieldname key doc[mapping.local_fieldname] = doc[mapping.remote_fieldname] @@ -55,75 +61,75 @@ class DocumentTypeMapping(Document): if mapping.local_fieldname != mapping.remote_fieldname: remote_fields.append(mapping.remote_fieldname) - if not doc.get(mapping.remote_fieldname) and mapping.default_value and update_type != 'Update': + if not doc.get(mapping.remote_fieldname) and mapping.default_value and update_type != "Update": doc[mapping.local_fieldname] = mapping.default_value - #remove the remote fieldnames + # remove the remote fieldnames for field in remote_fields: doc.pop(field, None) - if update_type != 'Update': - doc['doctype'] = self.local_doctype + if update_type != "Update": + doc["doctype"] = self.local_doctype - mapping = {'doc': frappe.as_json(doc)} + mapping = {"doc": frappe.as_json(doc)} if len(dependencies): - mapping['dependencies'] = dependencies + mapping["dependencies"] = dependencies return mapping - def get_mapped_update(self, update, producer_site): update_diff = frappe._dict(json.loads(update.data)) mapping = update_diff dependencies = [] if update_diff.changed: - doc_map = self.get_mapping(update_diff.changed, producer_site, 'Update') - mapped_doc = doc_map.get('doc') + doc_map = self.get_mapping(update_diff.changed, producer_site, "Update") + mapped_doc = doc_map.get("doc") mapping.changed = json.loads(mapped_doc) - if doc_map.get('dependencies'): - dependencies += doc_map.get('dependencies') + if doc_map.get("dependencies"): + dependencies += doc_map.get("dependencies") if update_diff.removed: mapping = self.map_rows_removed(update_diff, mapping) if update_diff.added: - mapping = self.map_rows(update_diff, mapping, producer_site, operation='added') + mapping = self.map_rows(update_diff, mapping, producer_site, operation="added") if update_diff.row_changed: - mapping = self.map_rows(update_diff, mapping, producer_site, operation='row_changed') + mapping = self.map_rows(update_diff, mapping, producer_site, operation="row_changed") - update = {'doc': frappe.as_json(mapping)} + update = {"doc": frappe.as_json(mapping)} if len(dependencies): - update['dependencies'] = dependencies + update["dependencies"] = dependencies return update def get_mapped_dependency(self, mapping, producer_site, doc): - inner_mapping = frappe.get_doc('Document Type Mapping', mapping.mapping) + inner_mapping = frappe.get_doc("Document Type Mapping", mapping.mapping) filters = json.loads(mapping.remote_value_filters) for key, value in filters.items(): - if value.startswith('eval:'): + if value.startswith("eval:"): val = frappe.safe_eval(value[5:], None, dict(doc=doc)) filters[key] = val if doc.get(value): filters[key] = doc.get(value) matching_docs = producer_site.get_doc(inner_mapping.remote_doctype, filters=filters) if len(matching_docs): - remote_docname = matching_docs[0].get('name') + remote_docname = matching_docs[0].get("name") remote_doc = producer_site.get_doc(inner_mapping.remote_doctype, remote_docname) - doc = inner_mapping.get_mapping(remote_doc, producer_site, 'Insert').get('doc') + doc = inner_mapping.get_mapping(remote_doc, producer_site, "Insert").get("doc") return doc return def map_rows_removed(self, update_diff, mapping): removed = [] - mapping['removed'] = update_diff.removed + mapping["removed"] = update_diff.removed for key, value in update_diff.removed.copy().items(): - local_table_name = frappe.db.get_value('Document Type Field Mapping', { - 'remote_fieldname': key, - 'parent': self.name - },'local_fieldname') + local_table_name = frappe.db.get_value( + "Document Type Field Mapping", + {"remote_fieldname": key, "parent": self.name}, + "local_fieldname", + ) mapping.removed[local_table_name] = value if local_table_name != key: removed.append(key) - #remove the remote fieldnames + # remove the remote fieldnames for field in removed: mapping.removed.pop(field, None) return mapping @@ -131,12 +137,18 @@ class DocumentTypeMapping(Document): def map_rows(self, update_diff, mapping, producer_site, operation): remote_fields = [] for tablename, entries in update_diff.get(operation).copy().items(): - local_table_name = frappe.db.get_value('Document Type Field Mapping', {'remote_fieldname': tablename}, 'local_fieldname') - table_map = frappe.db.get_value('Document Type Field Mapping', {'local_fieldname': local_table_name, 'parent': self.name}, 'mapping') - table_map = frappe.get_doc('Document Type Mapping', table_map) + local_table_name = frappe.db.get_value( + "Document Type Field Mapping", {"remote_fieldname": tablename}, "local_fieldname" + ) + table_map = frappe.db.get_value( + "Document Type Field Mapping", + {"local_fieldname": local_table_name, "parent": self.name}, + "mapping", + ) + table_map = frappe.get_doc("Document Type Mapping", table_map) docs = [] for entry in entries: - mapped_doc = table_map.get_mapping(entry, producer_site, 'Update').get('doc') + mapped_doc = table_map.get_mapping(entry, producer_site, "Update").get("doc") docs.append(json.loads(mapped_doc)) mapping.get(operation)[local_table_name] = docs if local_table_name != tablename: @@ -148,9 +160,10 @@ class DocumentTypeMapping(Document): return mapping + def get_mapped_child_table_docs(child_map, table_entries, producer_site): """Get mapping for child doctypes""" - child_map = frappe.get_doc('Document Type Mapping', child_map) + child_map = frappe.get_doc("Document Type Mapping", child_map) mapped_entries = [] remote_fields = [] for child_doc in table_entries: @@ -161,9 +174,9 @@ def get_mapped_child_table_docs(child_map, table_entries, producer_site): child_doc.pop(mapping.remote_fieldname, None) mapped_entries.append(child_doc) - #remove the remote fieldnames + # remove the remote fieldnames for field in remote_fields: child_doc.pop(field, None) - child_doc['doctype'] = child_map.local_doctype + child_doc["doctype"] = child_map.local_doctype return mapped_entries diff --git a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py index a277139985..1d5c4862de 100644 --- a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py +++ b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDocumentTypeMapping(unittest.TestCase): pass diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py index e8b84d1345..287a1fca03 100644 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py +++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py @@ -2,24 +2,26 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import json -import requests import os + +import requests + +import frappe from frappe import _ -from frappe.model.document import Document from frappe.frappeclient import FrappeClient -from frappe.utils.data import get_url +from frappe.model.document import Document from frappe.utils.background_jobs import get_jobs +from frappe.utils.data import get_url class EventConsumer(Document): def validate(self): # approve subscribed doctypes for tests # frappe.flags.in_test won't work here as tests are running on the consumer site - if os.environ.get('CI'): + if os.environ.get("CI"): for entry in self.consumer_doctypes: - entry.status = 'Approved' + entry.status = "Approved" def on_update(self): if not self.incoming_change: @@ -29,28 +31,32 @@ class EventConsumer(Document): self.update_consumer_status() else: - frappe.db.set_value(self.doctype, self.name, 'incoming_change', 0) + frappe.db.set_value(self.doctype, self.name, "incoming_change", 0) - frappe.cache().delete_value('event_consumer_document_type_map') + frappe.cache().delete_value("event_consumer_document_type_map") def on_trash(self): - for i in frappe.get_all('Event Update Log Consumer', {'consumer': self.name}): - frappe.delete_doc('Event Update Log Consumer', i.name) - frappe.cache().delete_value('event_consumer_document_type_map') + for i in frappe.get_all("Event Update Log Consumer", {"consumer": self.name}): + frappe.delete_doc("Event Update Log Consumer", i.name) + frappe.cache().delete_value("event_consumer_document_type_map") def update_consumer_status(self): consumer_site = get_consumer_site(self.callback_url) - event_producer = consumer_site.get_doc('Event Producer', get_url()) + event_producer = consumer_site.get_doc("Event Producer", get_url()) event_producer = frappe._dict(event_producer) config = event_producer.producer_doctypes event_producer.producer_doctypes = [] for entry in config: - if entry.get('has_mapping'): - ref_doctype = consumer_site.get_value('Document Type Mapping', 'remote_doctype', entry.get('mapping')).get('remote_doctype') + if entry.get("has_mapping"): + ref_doctype = consumer_site.get_value( + "Document Type Mapping", "remote_doctype", entry.get("mapping") + ).get("remote_doctype") else: - ref_doctype = entry.get('ref_doctype') + ref_doctype = entry.get("ref_doctype") - entry['status'] = frappe.db.get_value('Event Consumer Document Type', {'parent': self.name, 'ref_doctype': ref_doctype}, 'status') + entry["status"] = frappe.db.get_value( + "Event Consumer Document Type", {"parent": self.name, "ref_doctype": ref_doctype}, "status" + ) event_producer.producer_doctypes = config # when producer doc is updated it updates the consumer doc @@ -61,38 +67,38 @@ class EventConsumer(Document): def get_consumer_status(self): response = requests.get(self.callback_url) if response.status_code != 200: - return 'offline' - return 'online' + return "offline" + return "online" + @frappe.whitelist() def register_consumer(data): """create an event consumer document for registering a consumer""" data = json.loads(data) # to ensure that consumer is created only once - if frappe.db.exists('Event Consumer', data['event_consumer']): + if frappe.db.exists("Event Consumer", data["event_consumer"]): return None - user = data['user'] - if not frappe.db.exists('User', user): - frappe.throw(_('User {0} not found on the producer site').format(user)) + user = data["user"] + if not frappe.db.exists("User", user): + frappe.throw(_("User {0} not found on the producer site").format(user)) if "System Manager" not in frappe.get_roles(user): frappe.throw(_("Event Subscriber has to be a System Manager.")) - consumer = frappe.new_doc('Event Consumer') - consumer.callback_url = data['event_consumer'] - consumer.user = data['user'] - consumer.api_key = data['api_key'] - consumer.api_secret = data['api_secret'] + consumer = frappe.new_doc("Event Consumer") + consumer.callback_url = data["event_consumer"] + consumer.user = data["user"] + consumer.api_key = data["api_key"] + consumer.api_secret = data["api_secret"] consumer.incoming_change = True - consumer_doctypes = json.loads(data['consumer_doctypes']) + consumer_doctypes = json.loads(data["consumer_doctypes"]) for entry in consumer_doctypes: - consumer.append('consumer_doctypes', { - 'ref_doctype': entry.get('doctype'), - 'status': 'Pending', - 'condition': entry.get('condition') - }) + consumer.append( + "consumer_doctypes", + {"ref_doctype": entry.get("doctype"), "status": "Pending", "condition": entry.get("condition")}, + ) consumer.insert() @@ -100,23 +106,25 @@ def register_consumer(data): # in producer's update log when subscribing # so that, updates after subscribing are consumed and not the old ones. last_update = str(get_last_update()) - return json.dumps({'last_update': last_update}) + return json.dumps({"last_update": last_update}) def get_consumer_site(consumer_url): """create a FrappeClient object for event consumer site""" - consumer_doc = frappe.get_doc('Event Consumer', consumer_url) + consumer_doc = frappe.get_doc("Event Consumer", consumer_url) consumer_site = FrappeClient( url=consumer_url, api_key=consumer_doc.api_key, - api_secret=consumer_doc.get_password('api_secret') + api_secret=consumer_doc.get_password("api_secret"), ) return consumer_site def get_last_update(): """get the creation timestamp of last update consumed""" - updates = frappe.get_list('Event Update Log', 'creation', ignore_permissions=True, limit=1, order_by='creation desc') + updates = frappe.get_list( + "Event Update Log", "creation", ignore_permissions=True, limit=1, order_by="creation desc" + ) if updates: return updates[0].creation return frappe.utils.now_datetime() @@ -125,9 +133,11 @@ def get_last_update(): @frappe.whitelist() def notify_event_consumers(doctype): """get all event consumers and set flag for notification status""" - event_consumers = frappe.get_all('Event Consumer Document Type', ['parent'], {'ref_doctype': doctype, 'status': 'Approved'}) + event_consumers = frappe.get_all( + "Event Consumer Document Type", ["parent"], {"ref_doctype": doctype, "status": "Approved"} + ) for entry in event_consumers: - consumer = frappe.get_doc('Event Consumer', entry.parent) + consumer = frappe.get_doc("Event Consumer", entry.parent) consumer.flags.notified = False notify(consumer) @@ -136,13 +146,15 @@ def notify_event_consumers(doctype): def notify(consumer): """notify individual event consumers about a new update""" consumer_status = consumer.get_consumer_status() - if consumer_status == 'online': + if consumer_status == "online": try: client = get_consumer_site(consumer.callback_url) - client.post_request({ - 'cmd': 'frappe.event_streaming.doctype.event_producer.event_producer.new_event_notification', - 'producer_url': get_url() - }) + client.post_request( + { + "cmd": "frappe.event_streaming.doctype.event_producer.event_producer.new_event_notification", + "producer_url": get_url(), + } + ) consumer.flags.notified = True except Exception: consumer.flags.notified = False @@ -151,35 +163,37 @@ def notify(consumer): # enqueue another job if the site was not notified if not consumer.flags.notified: - enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify' + enqueued_method = "frappe.event_streaming.doctype.event_consumer.event_consumer.notify" jobs = get_jobs() if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed: - frappe.enqueue(enqueued_method, queue='long', enqueue_after_commit=True, **{'consumer': consumer}) + frappe.enqueue( + enqueued_method, queue="long", enqueue_after_commit=True, **{"consumer": consumer} + ) def has_consumer_access(consumer, update_log): """Checks if consumer has completely satisfied all the conditions on the doc""" if isinstance(consumer, str): - consumer = frappe.get_doc('Event Consumer', consumer) + consumer = frappe.get_doc("Event Consumer", consumer) if not frappe.db.exists(update_log.ref_doctype, update_log.docname): # Delete Log # Check if the last Update Log of this document was read by this consumer last_update_log = frappe.get_all( - 'Event Update Log', + "Event Update Log", filters={ - 'ref_doctype': update_log.ref_doctype, - 'docname': update_log.docname, - 'creation': ['<', update_log.creation] + "ref_doctype": update_log.ref_doctype, + "docname": update_log.docname, + "creation": ["<", update_log.creation], }, - order_by='creation desc', - limit_page_length=1 + order_by="creation desc", + limit_page_length=1, ) if not len(last_update_log): return False - last_update_log = frappe.get_doc('Event Update Log', last_update_log[0].name) + last_update_log = frappe.get_doc("Event Update Log", last_update_log[0].name) return len([x for x in last_update_log.consumers if x.consumer == consumer.name]) doc = frappe.get_doc(update_log.ref_doctype, update_log.docname) @@ -192,16 +206,12 @@ def has_consumer_access(consumer, update_log): return True condition: str = dt_entry.condition - if condition.startswith('cmd:'): - cmd = condition.split('cmd:')[1].strip() - args = { - 'consumer': consumer, - 'doc': doc, - 'update_log': update_log - } + if condition.startswith("cmd:"): + cmd = condition.split("cmd:")[1].strip() + args = {"consumer": consumer, "doc": doc, "update_log": update_log} return frappe.call(cmd, **args) else: return frappe.safe_eval(condition, frappe._dict(doc=doc)) except Exception as e: - frappe.log_error(title='has_consumer_access error', message=e) - return False \ No newline at end of file + frappe.log_error(title="has_consumer_access error", message=e) + return False diff --git a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py index 11c69e7ba3..605fc7982a 100644 --- a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py +++ b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestEventConsumer(unittest.TestCase): pass diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index a6c2a257fa..f639e48b50 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -29,19 +29,21 @@ class EventProducer(Document): self.validate_event_subscriber() if frappe.flags.in_test: for entry in self.producer_doctypes: - entry.status = 'Approved' + entry.status = "Approved" def validate_event_subscriber(self): - if not frappe.db.get_value('User', self.user, 'api_key'): - frappe.throw(_('Please generate keys for the Event Subscriber User {0} first.').format( - frappe.bold(get_link_to_form('User', self.user)) - )) + if not frappe.db.get_value("User", self.user, "api_key"): + frappe.throw( + _("Please generate keys for the Event Subscriber User {0} first.").format( + frappe.bold(get_link_to_form("User", self.user)) + ) + ) def on_update(self): if not self.incoming_change: - if frappe.db.exists('Event Producer', self.name): + if frappe.db.exists("Event Producer", self.name): if not self.api_key or not self.api_secret: - frappe.throw(_('Please set API Key and Secret on the producer and consumer sites first.')) + frappe.throw(_("Please set API Key and Secret on the producer and consumer sites first.")) else: doc_before_save = self.get_doc_before_save() if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret: @@ -51,13 +53,13 @@ class EventProducer(Document): self.create_custom_fields() else: # when producer doc is updated it updates the consumer doc, set flag to avoid deadlock - self.db_set('incoming_change', 0) + self.db_set("incoming_change", 0) self.reload() def on_trash(self): - last_update = frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name)) + last_update = frappe.db.get_value("Event Producer Last Update", dict(event_producer=self.name)) if last_update: - frappe.delete_doc('Event Producer Last Update', last_update) + frappe.delete_doc("Event Producer Last Update", last_update) def check_url(self): valid_url_schemes = ("http", "https") @@ -72,73 +74,96 @@ class EventProducer(Document): """register event consumer on the producer site""" if self.is_producer_online(): producer_site = FrappeClient( - url=self.producer_url, - api_key=self.api_key, - api_secret=self.get_password('api_secret') + url=self.producer_url, api_key=self.api_key, api_secret=self.get_password("api_secret") ) response = producer_site.post_api( - 'frappe.event_streaming.doctype.event_consumer.event_consumer.register_consumer', - params={'data': json.dumps(self.get_request_data())} + "frappe.event_streaming.doctype.event_consumer.event_consumer.register_consumer", + params={"data": json.dumps(self.get_request_data())}, ) if response: response = json.loads(response) - self.set_last_update(response['last_update']) + self.set_last_update(response["last_update"]) else: - frappe.throw(_('Failed to create an Event Consumer or an Event Consumer for the current site is already registered.')) + frappe.throw( + _( + "Failed to create an Event Consumer or an Event Consumer for the current site is already registered." + ) + ) def set_last_update(self, last_update): - last_update_doc_name = frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name)) + last_update_doc_name = frappe.db.get_value( + "Event Producer Last Update", dict(event_producer=self.name) + ) if not last_update_doc_name: - frappe.get_doc(dict( - doctype = 'Event Producer Last Update', - event_producer = self.producer_url, - last_update = last_update - )).insert(ignore_permissions=True) + frappe.get_doc( + dict( + doctype="Event Producer Last Update", + event_producer=self.producer_url, + last_update=last_update, + ) + ).insert(ignore_permissions=True) else: - frappe.db.set_value('Event Producer Last Update', last_update_doc_name, 'last_update', last_update) + frappe.db.set_value( + "Event Producer Last Update", last_update_doc_name, "last_update", last_update + ) def get_last_update(self): - return frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name), 'last_update') + return frappe.db.get_value( + "Event Producer Last Update", dict(event_producer=self.name), "last_update" + ) def get_request_data(self): consumer_doctypes = [] for entry in self.producer_doctypes: if entry.has_mapping: # if mapping, subscribe to remote doctype on consumer's site - dt = frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype') + dt = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype") else: dt = entry.ref_doctype - consumer_doctypes.append({ - "doctype": dt, - "condition": entry.condition - }) + consumer_doctypes.append({"doctype": dt, "condition": entry.condition}) - user_key = frappe.db.get_value('User', self.user, 'api_key') - user_secret = get_decrypted_password('User', self.user, 'api_secret') + user_key = frappe.db.get_value("User", self.user, "api_key") + user_secret = get_decrypted_password("User", self.user, "api_secret") return { - 'event_consumer': get_url(), - 'consumer_doctypes': json.dumps(consumer_doctypes), - 'user': self.user, - 'api_key': user_key, - 'api_secret': user_secret + "event_consumer": get_url(), + "consumer_doctypes": json.dumps(consumer_doctypes), + "user": self.user, + "api_key": user_key, + "api_secret": user_secret, } def create_custom_fields(self): """create custom field to store remote docname and remote site url""" for entry in self.producer_doctypes: if not entry.use_same_name: - if not frappe.db.exists('Custom Field', {'fieldname': 'remote_docname', 'dt': entry.ref_doctype}): - df = dict(fieldname='remote_docname', label='Remote Document Name', fieldtype='Data', read_only=1, print_hide=1) + if not frappe.db.exists( + "Custom Field", {"fieldname": "remote_docname", "dt": entry.ref_doctype} + ): + df = dict( + fieldname="remote_docname", + label="Remote Document Name", + fieldtype="Data", + read_only=1, + print_hide=1, + ) create_custom_field(entry.ref_doctype, df) - if not frappe.db.exists('Custom Field', {'fieldname': 'remote_site_name', 'dt': entry.ref_doctype}): - df = dict(fieldname='remote_site_name', label='Remote Site', fieldtype='Data', read_only=1, print_hide=1) + if not frappe.db.exists( + "Custom Field", {"fieldname": "remote_site_name", "dt": entry.ref_doctype} + ): + df = dict( + fieldname="remote_site_name", + label="Remote Site", + fieldtype="Data", + read_only=1, + print_hide=1, + ) create_custom_field(entry.ref_doctype, df) def update_event_consumer(self): if self.is_producer_online(): producer_site = get_producer_site(self.producer_url) - event_consumer = producer_site.get_doc('Event Consumer', get_url()) + event_consumer = producer_site.get_doc("Event Consumer", get_url()) event_consumer = frappe._dict(event_consumer) if event_consumer: config = event_consumer.consumer_doctypes @@ -146,16 +171,18 @@ class EventProducer(Document): for entry in self.producer_doctypes: if entry.has_mapping: # if mapping, subscribe to remote doctype on consumer's site - ref_doctype = frappe.db.get_value('Document Type Mapping', entry.mapping, 'remote_doctype') + ref_doctype = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype") else: ref_doctype = entry.ref_doctype - event_consumer.consumer_doctypes.append({ - 'ref_doctype': ref_doctype, - 'status': get_approval_status(config, ref_doctype), - 'unsubscribed': entry.unsubscribe, - 'condition': entry.condition - }) + event_consumer.consumer_doctypes.append( + { + "ref_doctype": ref_doctype, + "status": get_approval_status(config, ref_doctype), + "unsubscribed": entry.unsubscribe, + "condition": entry.condition, + } + ) event_consumer.user = self.user event_consumer.incoming_change = True producer_site.update(event_consumer) @@ -169,16 +196,16 @@ class EventProducer(Document): return True retry -= 1 time.sleep(5) - frappe.throw(_('Failed to connect to the Event Producer site. Retry after some time.')) + frappe.throw(_("Failed to connect to the Event Producer site. Retry after some time.")) def get_producer_site(producer_url): """create a FrappeClient object for event producer site""" - producer_doc = frappe.get_doc('Event Producer', producer_url) + producer_doc = frappe.get_doc("Event Producer", producer_url) producer_site = FrappeClient( url=producer_url, api_key=producer_doc.api_key, - api_secret=producer_doc.get_password('api_secret') + api_secret=producer_doc.get_password("api_secret"), ) return producer_site @@ -186,9 +213,9 @@ def get_producer_site(producer_url): def get_approval_status(config, ref_doctype): """check the approval status for consumption""" for entry in config: - if entry.get('ref_doctype') == ref_doctype: - return entry.get('status') - return 'Pending' + if entry.get("ref_doctype") == ref_doctype: + return entry.get("status") + return "Pending" @frappe.whitelist() @@ -196,16 +223,16 @@ def pull_producer_data(): """Fetch data from producer node.""" response = requests.get(get_url()) if response.status_code == 200: - for event_producer in frappe.get_all('Event Producer'): + for event_producer in frappe.get_all("Event Producer"): pull_from_node(event_producer.name) - return 'success' + return "success" return None @frappe.whitelist() def pull_from_node(event_producer): """pull all updates after the last update timestamp from event producer site""" - event_producer = frappe.get_doc('Event Producer', event_producer) + event_producer = frappe.get_doc("Event Producer", event_producer) producer_site = get_producer_site(event_producer.producer_url) last_update = event_producer.get_last_update() @@ -219,7 +246,7 @@ def pull_from_node(event_producer): if mapping: update.mapping = mapping update = get_mapped_update(update, producer_site) - if not update.update_type == 'Delete': + if not update.update_type == "Delete": update.data = json.loads(update.data) sync(update, producer_site, event_producer) @@ -230,9 +257,11 @@ def get_config(event_config): doctypes, mapping_config, naming_config = [], {}, {} for entry in event_config: - if entry.status == 'Approved': + if entry.status == "Approved": if entry.has_mapping: - (mapped_doctype, mapping) = frappe.db.get_value('Document Type Mapping', entry.mapping, ['remote_doctype', 'name']) + (mapped_doctype, mapping) = frappe.db.get_value( + "Document Type Mapping", entry.mapping, ["remote_doctype", "name"] + ) mapping_config[mapped_doctype] = mapping naming_config[mapped_doctype] = entry.use_same_name doctypes.append(mapped_doctype) @@ -245,22 +274,22 @@ def get_config(event_config): def sync(update, producer_site, event_producer, in_retry=False): """Sync the individual update""" try: - if update.update_type == 'Create': + if update.update_type == "Create": set_insert(update, producer_site, event_producer.name) - if update.update_type == 'Update': + if update.update_type == "Update": set_update(update, producer_site) - if update.update_type == 'Delete': + if update.update_type == "Delete": set_delete(update) if in_retry: - return 'Synced' - log_event_sync(update, event_producer.name, 'Synced') + return "Synced" + log_event_sync(update, event_producer.name, "Synced") except Exception: if in_retry: if frappe.flags.in_test: print(frappe.get_traceback()) - return 'Failed' - log_event_sync(update, event_producer.name, 'Failed', frappe.get_traceback()) + return "Failed" + log_event_sync(update, event_producer.name, "Failed", frappe.get_traceback()) event_producer.set_last_update(update.creation) frappe.db.commit() @@ -274,7 +303,7 @@ def set_insert(update, producer_site, event_producer): doc = frappe.get_doc(update.data) if update.mapping: - if update.get('dependencies'): + if update.get("dependencies"): dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) for fieldname, value in dependencies_created.items(): doc.update({fieldname: value}) @@ -306,7 +335,7 @@ def set_update(update, producer_site): local_doc = update_row_added(local_doc, data.added) if update.mapping: - if update.get('dependencies'): + if update.get("dependencies"): dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) for fieldname, value in dependencies_created.items(): local_doc.update({fieldname: value}) @@ -331,7 +360,7 @@ def update_row_removed(local_doc, removed): def get_child_table_row(table_rows, row): for entry in table_rows: - if entry.get('name') == row: + if entry.get("name") == row: return entry @@ -341,7 +370,7 @@ def update_row_changed(local_doc, changed): old = local_doc.get(tablename) for doc in old: for row in rows: - if row['name'] == doc.get('name'): + if row["name"] == doc.get("name"): doc.update(row) @@ -366,12 +395,14 @@ def set_delete(update): def get_updates(producer_site, last_update, doctypes): """Get all updates generated after the last update timestamp""" - docs = producer_site.post_request({ - 'cmd': 'frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer', - 'event_consumer': get_url(), - 'doctypes': frappe.as_json(doctypes), - 'last_update': last_update - }) + docs = producer_site.post_request( + { + "cmd": "frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer", + "event_consumer": get_url(), + "doctypes": frappe.as_json(doctypes), + "last_update": last_update, + } + ) return [frappe._dict(d) for d in (docs or [])] @@ -379,7 +410,7 @@ def get_local_doc(update): """Get the local document if created with a different name""" try: if not update.use_same_name: - return frappe.get_doc(update.ref_doctype, {'remote_docname': update.docname}) + return frappe.get_doc(update.ref_doctype, {"remote_docname": update.docname}) return frappe.get_doc(update.ref_doctype, update.docname) except frappe.DoesNotExistError: return None @@ -479,9 +510,10 @@ def sync_mapped_dependencies(dependencies, producer_site): return dependencies_created + def log_event_sync(update, event_producer, sync_status, error=None): """Log event update received with the sync_status as Synced or Failed""" - doc = frappe.new_doc('Event Sync Log') + doc = frappe.new_doc("Event Sync Log") doc.update_type = update.update_type doc.ref_doctype = update.ref_doctype doc.status = sync_status @@ -493,7 +525,7 @@ def log_event_sync(update, event_producer, sync_status, error=None): if update.use_same_name: doc.docname = update.docname else: - doc.docname = frappe.db.get_value(update.ref_doctype, {'remote_docname': update.docname}, 'name') + doc.docname = frappe.db.get_value(update.ref_doctype, {"remote_docname": update.docname}, "name") if error: doc.error = error doc.insert() @@ -501,28 +533,28 @@ def log_event_sync(update, event_producer, sync_status, error=None): def get_mapped_update(update, producer_site): """get the new update document with mapped fields""" - mapping = frappe.get_doc('Document Type Mapping', update.mapping) - if update.update_type == 'Create': + mapping = frappe.get_doc("Document Type Mapping", update.mapping) + if update.update_type == "Create": doc = frappe._dict(json.loads(update.data)) mapped_update = mapping.get_mapping(doc, producer_site, update.update_type) - update.data = mapped_update.get('doc') - update.dependencies = mapped_update.get('dependencies', None) - elif update.update_type == 'Update': + update.data = mapped_update.get("doc") + update.dependencies = mapped_update.get("dependencies", None) + elif update.update_type == "Update": mapped_update = mapping.get_mapped_update(update, producer_site) - update.data = mapped_update.get('doc') - update.dependencies = mapped_update.get('dependencies', None) + update.data = mapped_update.get("doc") + update.dependencies = mapped_update.get("dependencies", None) - update['ref_doctype'] = mapping.local_doctype + update["ref_doctype"] = mapping.local_doctype return update @frappe.whitelist() def new_event_notification(producer_url): """Pull data from producer when notified""" - enqueued_method = 'frappe.event_streaming.doctype.event_producer.event_producer.pull_from_node' + enqueued_method = "frappe.event_streaming.doctype.event_producer.event_producer.pull_from_node" jobs = get_jobs() if not jobs or enqueued_method not in jobs[frappe.local.site]: - frappe.enqueue(enqueued_method, queue='default', **{'event_producer': producer_url}) + frappe.enqueue(enqueued_method, queue="default", **{"event_producer": producer_url}) @frappe.whitelist() @@ -530,7 +562,7 @@ def resync(update): """Retry syncing update if failed""" update = frappe._dict(json.loads(update)) producer_site = get_producer_site(update.event_producer) - event_producer = frappe.get_doc('Event Producer', update.event_producer) + event_producer = frappe.get_doc("Event Producer", update.event_producer) if update.mapping: update = get_mapped_update(update, producer_site) update.data = json.loads(update.data) @@ -539,5 +571,5 @@ def resync(update): def set_custom_fields(local_doc, remote_docname, remote_site_name): """sets custom field in doc for storing remote docname""" - frappe.db.set_value(local_doc.doctype, local_doc.name, 'remote_docname', remote_docname) - frappe.db.set_value(local_doc.doctype, local_doc.name, 'remote_site_name', remote_site_name) + frappe.db.set_value(local_doc.doctype, local_doc.name, "remote_docname", remote_docname) + frappe.db.set_value(local_doc.doctype, local_doc.name, "remote_site_name", remote_site_name) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index 3d697ceb3a..4464b0a434 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -import unittest import json -from frappe.frappeclient import FrappeClient -from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node +import unittest + +import frappe from frappe.core.doctype.user.user import generate_keys +from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node +from frappe.frappeclient import FrappeClient + +producer_url = "http://test_site_producer:8000" -producer_url = 'http://test_site_producer:8000' class TestEventProducer(unittest.TestCase): # @classmethod @@ -27,14 +29,14 @@ class TestEventProducer(unittest.TestCase): def test_insert(self): producer = get_remote_site() - producer_doc = insert_into_producer(producer, 'test creation 1 sync') + producer_doc = insert_into_producer(producer, "test creation 1 sync") self.pull_producer_data() - self.assertTrue(frappe.db.exists('ToDo', producer_doc.name)) + self.assertTrue(frappe.db.exists("ToDo", producer_doc.name)) def test_update(self): producer = get_remote_site() - producer_doc = insert_into_producer(producer, 'test update 1') - producer_doc['description'] = 'test update 2' + producer_doc = insert_into_producer(producer, "test update 1") + producer_doc["description"] = "test update 2" producer_doc = producer.update(producer_doc) self.pull_producer_data() local_doc = frappe.get_doc(producer_doc.doctype, producer_doc.name) @@ -42,111 +44,123 @@ class TestEventProducer(unittest.TestCase): def test_delete(self): producer = get_remote_site() - producer_doc = insert_into_producer(producer, 'test delete sync') + producer_doc = insert_into_producer(producer, "test delete sync") self.pull_producer_data() - self.assertTrue(frappe.db.exists('ToDo', producer_doc.name)) - producer.delete('ToDo', producer_doc.name) + self.assertTrue(frappe.db.exists("ToDo", producer_doc.name)) + producer.delete("ToDo", producer_doc.name) self.pull_producer_data() - self.assertFalse(frappe.db.exists('ToDo', producer_doc.name)) + self.assertFalse(frappe.db.exists("ToDo", producer_doc.name)) def test_multiple_doctypes_sync(self): producer = get_remote_site() - #insert todo and note in producer - producer_todo = insert_into_producer(producer, 'test multiple doc sync') - producer_note1 = frappe._dict(doctype='Note', title='test multiple doc sync 1') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']}) - frappe.db.delete('Note', {'title': producer_note1['title']}) + # insert todo and note in producer + producer_todo = insert_into_producer(producer, "test multiple doc sync") + producer_note1 = frappe._dict(doctype="Note", title="test multiple doc sync 1") + delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) + frappe.db.delete("Note", {"title": producer_note1["title"]}) producer_note1 = producer.insert(producer_note1) - producer_note2 = frappe._dict(doctype='Note', title='test multiple doc sync 2') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note2['title']}) - frappe.db.delete('Note', {'title': producer_note2['title']}) + producer_note2 = frappe._dict(doctype="Note", title="test multiple doc sync 2") + delete_on_remote_if_exists(producer, "Note", {"title": producer_note2["title"]}) + frappe.db.delete("Note", {"title": producer_note2["title"]}) producer_note2 = producer.insert(producer_note2) - #update in producer - producer_todo['description'] = 'test multiple doc update sync' + # update in producer + producer_todo["description"] = "test multiple doc update sync" producer_todo = producer.update(producer_todo) - producer_note1['content'] = 'testing update sync' + producer_note1["content"] = "testing update sync" producer_note1 = producer.update(producer_note1) - producer.delete('Note', producer_note2.name) + producer.delete("Note", producer_note2.name) self.pull_producer_data() - #check inserted - self.assertTrue(frappe.db.exists('ToDo', producer_todo.name)) + # check inserted + self.assertTrue(frappe.db.exists("ToDo", producer_todo.name)) - #check update - local_todo = frappe.get_doc('ToDo', producer_todo.name) + # check update + local_todo = frappe.get_doc("ToDo", producer_todo.name) self.assertEqual(local_todo.description, producer_todo.description) - local_note1 = frappe.get_doc('Note', producer_note1.name) + local_note1 = frappe.get_doc("Note", producer_note1.name) self.assertEqual(local_note1.content, producer_note1.content) - #check delete - self.assertFalse(frappe.db.exists('Note', producer_note2.name)) + # check delete + self.assertFalse(frappe.db.exists("Note", producer_note2.name)) def test_child_table_sync_with_dependencies(self): producer = get_remote_site() - producer_user = frappe._dict(doctype='User', email='test_user@sync.com', send_welcome_email=0, - first_name='Test Sync User', enabled=1, roles=[{'role': 'System Manager'}]) - delete_on_remote_if_exists(producer, 'User', {'email': producer_user.email}) - frappe.db.delete('User', {'email':producer_user.email}) + producer_user = frappe._dict( + doctype="User", + email="test_user@sync.com", + send_welcome_email=0, + first_name="Test Sync User", + enabled=1, + roles=[{"role": "System Manager"}], + ) + delete_on_remote_if_exists(producer, "User", {"email": producer_user.email}) + frappe.db.delete("User", {"email": producer_user.email}) producer_user = producer.insert(producer_user) - producer_note = frappe._dict(doctype='Note', title='test child table dependency sync', - seen_by=[{'user': producer_user.name}]) - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) - frappe.db.delete('Note', {'title': producer_note.title}) + producer_note = frappe._dict( + doctype="Note", title="test child table dependency sync", seen_by=[{"user": producer_user.name}] + ) + delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) + frappe.db.delete("Note", {"title": producer_note.title}) producer_note = producer.insert(producer_note) self.pull_producer_data() - self.assertTrue(frappe.db.exists('User', producer_user.name)) - if self.assertTrue(frappe.db.exists('Note', producer_note.name)): - local_note = frappe.get_doc('Note', producer_note.name) + self.assertTrue(frappe.db.exists("User", producer_user.name)) + if self.assertTrue(frappe.db.exists("Note", producer_note.name)): + local_note = frappe.get_doc("Note", producer_note.name) self.assertEqual(len(local_note.seen_by), 1) def test_dynamic_link_dependencies_synced(self): producer = get_remote_site() - #unsubscribe for Note to check whether dependency is fulfilled - event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) + # unsubscribe for Note to check whether dependency is fulfilled + event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) event_producer.producer_doctypes = [] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1 - }) + event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) event_producer.save() - producer_link_doc = frappe._dict(doctype='Note', title='Test Dynamic Link 1') + producer_link_doc = frappe._dict(doctype="Note", title="Test Dynamic Link 1") - delete_on_remote_if_exists(producer, 'Note', {'title': producer_link_doc.title}) - frappe.db.delete('Note', {'title': producer_link_doc.title}) + delete_on_remote_if_exists(producer, "Note", {"title": producer_link_doc.title}) + frappe.db.delete("Note", {"title": producer_link_doc.title}) producer_link_doc = producer.insert(producer_link_doc) - producer_doc = frappe._dict(doctype='ToDo', description='Test Dynamic Link 2', assigned_by='Administrator', - reference_type='Note', reference_name=producer_link_doc.name) + producer_doc = frappe._dict( + doctype="ToDo", + description="Test Dynamic Link 2", + assigned_by="Administrator", + reference_type="Note", + reference_name=producer_link_doc.name, + ) producer_doc = producer.insert(producer_doc) self.pull_producer_data() - #check dynamic link dependency created - self.assertTrue(frappe.db.exists('Note', producer_link_doc.name)) - self.assertEqual(producer_link_doc.name, frappe.db.get_value('ToDo', producer_doc.name, 'reference_name')) + # check dynamic link dependency created + self.assertTrue(frappe.db.exists("Note", producer_link_doc.name)) + self.assertEqual( + producer_link_doc.name, frappe.db.get_value("ToDo", producer_doc.name, "reference_name") + ) reset_configuration(producer_url) def test_naming_configuration(self): - #test with use_same_name = 0 + # test with use_same_name = 0 producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) + event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) event_producer.producer_doctypes = [] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 0 - }) + event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 0}) event_producer.save() - producer_doc = insert_into_producer(producer, 'test different name sync') + producer_doc = insert_into_producer(producer, "test different name sync") self.pull_producer_data() - self.assertTrue(frappe.db.exists('ToDo', {'remote_docname': producer_doc.name, 'remote_site_name': producer_url})) + self.assertTrue( + frappe.db.exists( + "ToDo", {"remote_docname": producer_doc.name, "remote_site_name": producer_url} + ) + ) reset_configuration(producer_url) @@ -154,36 +168,34 @@ class TestEventProducer(unittest.TestCase): producer = get_remote_site() # Add Condition - event_producer = frappe.get_doc('Event Producer', producer_url) - note_producer_entry = [ - x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note' - ][0] - note_producer_entry.condition = 'doc.public == 1' + event_producer = frappe.get_doc("Event Producer", producer_url) + note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0] + note_producer_entry.condition = "doc.public == 1" event_producer.save() # Make test doc - producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']}) + producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync") + delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) producer_note1 = producer.insert(producer_note1) # Make Update - producer_note1['content'] = 'Test Conditional Sync Content' + producer_note1["content"] = "Test Conditional Sync Content" producer_note1 = producer.update(producer_note1) self.pull_producer_data() # Check if synced here - self.assertFalse(frappe.db.exists('Note', producer_note1.name)) + self.assertFalse(frappe.db.exists("Note", producer_note1.name)) # Lets satisfy the condition - producer_note1['public'] = 1 + producer_note1["public"] = 1 producer_note1 = producer.update(producer_note1) self.pull_producer_data() # it should sync now - self.assertTrue(frappe.db.exists('Note', producer_note1.name)) - local_note = frappe.get_doc('Note', producer_note1.name) + self.assertTrue(frappe.db.exists("Note", producer_note1.name)) + local_note = frappe.get_doc("Note", producer_note1.name) self.assertEqual(local_note.content, producer_note1.content) reset_configuration(producer_url) @@ -192,90 +204,92 @@ class TestEventProducer(unittest.TestCase): producer = get_remote_site() # Add Condition - event_producer = frappe.get_doc('Event Producer', producer_url) - note_producer_entry = [ - x for x in event_producer.producer_doctypes if x.ref_doctype == 'Note' - ][0] - note_producer_entry.condition = 'cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note' + event_producer = frappe.get_doc("Event Producer", producer_url) + note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0] + note_producer_entry.condition = ( + "cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note" + ) event_producer.save() # Make test doc - producer_note1 = frappe._dict(doctype='Note', public=0, title='test conditional sync cmd') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note1['title']}) + producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync cmd") + delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) producer_note1 = producer.insert(producer_note1) # Make Update - producer_note1['content'] = 'Test Conditional Sync Content' + producer_note1["content"] = "Test Conditional Sync Content" producer_note1 = producer.update(producer_note1) self.pull_producer_data() # Check if synced here - self.assertFalse(frappe.db.exists('Note', producer_note1.name)) + self.assertFalse(frappe.db.exists("Note", producer_note1.name)) # Lets satisfy the condition - producer_note1['public'] = 1 + producer_note1["public"] = 1 producer_note1 = producer.update(producer_note1) self.pull_producer_data() # it should sync now - self.assertTrue(frappe.db.exists('Note', producer_note1.name)) - local_note = frappe.get_doc('Note', producer_note1.name) + self.assertTrue(frappe.db.exists("Note", producer_note1.name)) + local_note = frappe.get_doc("Note", producer_note1.name) self.assertEqual(local_note.content, producer_note1.content) reset_configuration(producer_url) def test_update_log(self): producer = get_remote_site() - producer_doc = insert_into_producer(producer, 'test update log') - update_log_doc = producer.get_value('Event Update Log', 'docname', {'docname': producer_doc.get('name')}) - self.assertEqual(update_log_doc.get('docname'), producer_doc.get('name')) + producer_doc = insert_into_producer(producer, "test update log") + update_log_doc = producer.get_value( + "Event Update Log", "docname", {"docname": producer_doc.get("name")} + ) + self.assertEqual(update_log_doc.get("docname"), producer_doc.get("name")) def test_event_sync_log(self): producer = get_remote_site() - producer_doc = insert_into_producer(producer, 'test event sync log') + producer_doc = insert_into_producer(producer, "test event sync log") self.pull_producer_data() - self.assertTrue(frappe.db.exists('Event Sync Log', {'docname': producer_doc.name})) + self.assertTrue(frappe.db.exists("Event Sync Log", {"docname": producer_doc.name})) def pull_producer_data(self): pull_from_node(producer_url) def test_mapping(self): producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) + event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) event_producer.producer_doctypes = [] - mapping = [{ - 'local_fieldname': 'description', - 'remote_fieldname': 'content' - }] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1, - 'has_mapping': 1, - 'mapping': get_mapping('ToDo to Note', 'ToDo', 'Note', mapping) - }) + mapping = [{"local_fieldname": "description", "remote_fieldname": "content"}] + event_producer.append( + "producer_doctypes", + { + "ref_doctype": "ToDo", + "use_same_name": 1, + "has_mapping": 1, + "mapping": get_mapping("ToDo to Note", "ToDo", "Note", mapping), + }, + ) event_producer.save() - producer_note = frappe._dict(doctype='Note', title='Test Mapping', content='Test Mapping') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) + producer_note = frappe._dict(doctype="Note", title="Test Mapping", content="Test Mapping") + delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) producer_note = producer.insert(producer_note) self.pull_producer_data() - #check inserted - self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) + # check inserted + self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content})) - #update in producer - producer_note['content'] = 'test mapped doc update sync' + # update in producer + producer_note["content"] = "test mapped doc update sync" producer_note = producer.update(producer_note) self.pull_producer_data() # check updated - self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note['content']})) + self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note["content"]})) - producer.delete('Note', producer_note.name) + producer.delete("Note", producer_note.name) self.pull_producer_data() - #check delete - self.assertFalse(frappe.db.exists('ToDo', {'description': producer_note.content})) + # check delete + self.assertFalse(frappe.db.exists("ToDo", {"description": producer_note.content})) reset_configuration(producer_url) @@ -283,158 +297,146 @@ class TestEventProducer(unittest.TestCase): producer = get_remote_site() setup_event_producer_for_inner_mapping() - producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) + producer_note = frappe._dict( + doctype="Note", title="Inner Mapping Tester", content="Test Inner Mapping" + ) + delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) producer_note = producer.insert(producer_note) self.pull_producer_data() - #check dependency inserted - self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) - #check doc inserted - self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) + # check dependency inserted + self.assertTrue(frappe.db.exists("Role", {"role_name": producer_note.title})) + # check doc inserted + self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content})) reset_configuration(producer_url) + def can_sync_note(consumer, doc, update_log): return doc.public == 1 + def setup_event_producer_for_inner_mapping(): - event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) + event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) event_producer.producer_doctypes = [] - inner_mapping = [ - { - 'local_fieldname':'role_name', - 'remote_fieldname':'title' - } - ] - inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) + inner_mapping = [{"local_fieldname": "role_name", "remote_fieldname": "title"}] + inner_map = get_mapping("Role to Note Dependency Creation", "Role", "Note", inner_mapping) mapping = [ { - 'local_fieldname':'description', - 'remote_fieldname':'content', + "local_fieldname": "description", + "remote_fieldname": "content", }, { - 'local_fieldname': 'role', - 'remote_fieldname': 'title', - 'mapping_type': 'Document', - 'mapping': inner_map, - 'remote_value_filters': json.dumps({'title': 'title'}) - } + "local_fieldname": "role", + "remote_fieldname": "title", + "mapping_type": "Document", + "mapping": inner_map, + "remote_value_filters": json.dumps({"title": "title"}), + }, ] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1, - 'has_mapping': 1, - 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) - }) + event_producer.append( + "producer_doctypes", + { + "ref_doctype": "ToDo", + "use_same_name": 1, + "has_mapping": 1, + "mapping": get_mapping("ToDo to Note Mapping", "ToDo", "Note", mapping), + }, + ) event_producer.save() return event_producer def insert_into_producer(producer, description): - #create and insert todo on remote site - todo = dict(doctype='ToDo', description=description, assigned_by='Administrator') + # create and insert todo on remote site + todo = dict(doctype="ToDo", description=description, assigned_by="Administrator") return producer.insert(todo) + def delete_on_remote_if_exists(producer, doctype, filters): - remote_doc = producer.get_value(doctype, 'name', filters) + remote_doc = producer.get_value(doctype, "name", filters) if remote_doc: - producer.delete(doctype, remote_doc.get('name')) + producer.delete(doctype, remote_doc.get("name")) + def get_mapping(mapping_name, local, remote, field_map): - name = frappe.db.exists('Document Type Mapping', mapping_name) + name = frappe.db.exists("Document Type Mapping", mapping_name) if name: - doc = frappe.get_doc('Document Type Mapping', name) + doc = frappe.get_doc("Document Type Mapping", name) else: - doc = frappe.new_doc('Document Type Mapping') + doc = frappe.new_doc("Document Type Mapping") doc.mapping_name = mapping_name doc.local_doctype = local doc.remote_doctype = remote for entry in field_map: - doc.append('field_mapping', entry) + doc.append("field_mapping", entry) doc.save() return doc.name def create_event_producer(producer_url): - if frappe.db.exists('Event Producer', producer_url): - event_producer = frappe.get_doc('Event Producer', producer_url) + if frappe.db.exists("Event Producer", producer_url): + event_producer = frappe.get_doc("Event Producer", producer_url) for entry in event_producer.producer_doctypes: entry.unsubscribe = 0 event_producer.save() return - generate_keys('Administrator') + generate_keys("Administrator") producer_site = connect() response = producer_site.post_api( - 'frappe.core.doctype.user.user.generate_keys', - params={'user': 'Administrator'} + "frappe.core.doctype.user.user.generate_keys", params={"user": "Administrator"} ) - api_secret = response.get('api_secret') + api_secret = response.get("api_secret") - response = producer_site.get_value('User', 'api_key', {'name': 'Administrator'}) - api_key = response.get('api_key') + response = producer_site.get_value("User", "api_key", {"name": "Administrator"}) + api_key = response.get("api_key") - event_producer = frappe.new_doc('Event Producer') + event_producer = frappe.new_doc("Event Producer") event_producer.producer_doctypes = [] event_producer.producer_url = producer_url - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1 - }) - event_producer.append('producer_doctypes', { - 'ref_doctype': 'Note', - 'use_same_name': 1 - }) - event_producer.user = 'Administrator' + event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) + event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1}) + event_producer.user = "Administrator" event_producer.api_key = api_key event_producer.api_secret = api_secret event_producer.save() + def reset_configuration(producer_url): - event_producer = frappe.get_doc('Event Producer', producer_url, for_update=True) + event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) event_producer.producer_doctypes = [] event_producer.conditions = [] event_producer.producer_url = producer_url - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1 - }) - event_producer.append('producer_doctypes', { - 'ref_doctype': 'Note', - 'use_same_name': 1 - }) - event_producer.user = 'Administrator' + event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) + event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1}) + event_producer.user = "Administrator" event_producer.save() + def get_remote_site(): - producer_doc = frappe.get_doc('Event Producer', producer_url) + producer_doc = frappe.get_doc("Event Producer", producer_url) producer_site = FrappeClient( - url=producer_doc.producer_url, - username='Administrator', - password='admin', - verify=False + url=producer_doc.producer_url, username="Administrator", password="admin", verify=False ) return producer_site + def unsubscribe_doctypes(producer_url): - event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer = frappe.get_doc("Event Producer", producer_url) for entry in event_producer.producer_doctypes: entry.unsubscribe = 1 event_producer.save() + def connect(): def _connect(): - return FrappeClient( - url=producer_url, - username='Administrator', - password='admin', - verify=False - ) + return FrappeClient(url=producer_url, username="Administrator", password="admin", verify=False) + try: return _connect() except Exception: diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py index 0868e86253..8e32e6fe6f 100644 --- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py +++ b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class EventProducerLastUpdate(Document): pass diff --git a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py index c2d943a463..6054ec873f 100644 --- a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py +++ b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestEventProducerLastUpdate(unittest.TestCase): pass diff --git a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py index b901f92ef8..da90c8e634 100644 --- a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py +++ b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestEventSyncLog(unittest.TestCase): pass diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py index cd5100623c..658a3b47cc 100644 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py +++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py @@ -3,60 +3,70 @@ # License: MIT. See LICENSE import frappe +from frappe.model import no_value_fields, table_fields from frappe.model.document import Document from frappe.utils.background_jobs import get_jobs -from frappe.model import no_value_fields, table_fields + class EventUpdateLog(Document): def after_insert(self): """Send update notification updates to event consumers whenever update log is generated""" - enqueued_method = 'frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers' + enqueued_method = ( + "frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers" + ) jobs = get_jobs() if not jobs or enqueued_method not in jobs[frappe.local.site]: - frappe.enqueue(enqueued_method, doctype=self.ref_doctype, queue='long', - enqueue_after_commit=True) + frappe.enqueue( + enqueued_method, doctype=self.ref_doctype, queue="long", enqueue_after_commit=True + ) + def notify_consumers(doc, event): - '''called via hooks''' + """called via hooks""" # make event update log for doctypes having event consumers if frappe.flags.in_install or frappe.flags.in_migrate: return consumers = check_doctype_has_consumers(doc.doctype) if consumers: - if event=='after_insert': - doc.flags.event_update_log = make_event_update_log(doc, update_type='Create') - elif event=='on_trash': - make_event_update_log(doc, update_type='Delete') + if event == "after_insert": + doc.flags.event_update_log = make_event_update_log(doc, update_type="Create") + elif event == "on_trash": + make_event_update_log(doc, update_type="Delete") else: # on_update # called after saving - if not doc.flags.event_update_log: # if not already inserted + if not doc.flags.event_update_log: # if not already inserted diff = get_update(doc.get_doc_before_save(), doc) if diff: doc.diff = diff - make_event_update_log(doc, update_type='Update') + make_event_update_log(doc, update_type="Update") + def check_doctype_has_consumers(doctype): """Check if doctype has event consumers for event streaming""" - return frappe.cache_manager.get_doctype_map('Event Consumer Document Type', doctype, - dict(ref_doctype=doctype, status='Approved', unsubscribed=0)) + return frappe.cache_manager.get_doctype_map( + "Event Consumer Document Type", + doctype, + dict(ref_doctype=doctype, status="Approved", unsubscribed=0), + ) + def get_update(old, new, for_child=False): """ Get document objects with updates only If there is a change, then returns a dict like: { - "changed" : {fieldname1: new_value1, fieldname2: new_value2, }, - "added" : {table_fieldname1: [{row_dict1}, {row_dict2}], }, - "removed" : {table_fieldname1: [row_name1, row_name2], }, - "row_changed" : {table_fieldname1: - { - child_fieldname1: new_val, - child_fieldname2: new_val - }, - }, + "changed" : {fieldname1: new_value1, fieldname2: new_value2, }, + "added" : {table_fieldname1: [{row_dict1}, {row_dict2}], }, + "removed" : {table_fieldname1: [row_name1, row_name2], }, + "row_changed" : {table_fieldname1: + { + child_fieldname1: new_val, + child_fieldname2: new_val + }, + }, } """ if not new: @@ -82,20 +92,24 @@ def get_update(old, new, for_child=False): return out return None + def make_event_update_log(doc, update_type): """Save update info for doctypes that have event consumers""" - if update_type != 'Delete': + if update_type != "Delete": # diff for update type, doc for create type - data = frappe.as_json(doc) if not doc.get('diff') else frappe.as_json(doc.diff) + data = frappe.as_json(doc) if not doc.get("diff") else frappe.as_json(doc.diff) else: data = None - return frappe.get_doc({ - 'doctype': 'Event Update Log', - 'update_type': update_type, - 'ref_doctype': doc.doctype, - 'docname': doc.name, - 'data': data - }).insert(ignore_permissions=True) + return frappe.get_doc( + { + "doctype": "Event Update Log", + "update_type": update_type, + "ref_doctype": doc.doctype, + "docname": doc.name, + "data": data, + } + ).insert(ignore_permissions=True) + def make_maps(old_value, new_value): """make maps""" @@ -115,7 +129,7 @@ def check_for_additions(out, df, new_value, old_row_by_name): if diff and diff.changed: if not out.row_changed.get(df.fieldname): out.row_changed[df.fieldname] = [] - diff.changed['name'] = d.name + diff.changed["name"] = d.name out.row_changed[df.fieldname].append(diff.changed) else: if not out.added.get(df.fieldname): @@ -137,7 +151,7 @@ def check_for_deletions(out, df, old_value, new_row_by_name): def check_docstatus(out, old, new, for_child): """docstatus changes""" if not for_child and old.docstatus != new.docstatus: - out.changed['docstatus'] = new.docstatus + out.changed["docstatus"] = new.docstatus return out @@ -147,32 +161,32 @@ def is_consumer_uptodate(update_log, consumer): :param update_log: The UpdateLog Doc in context :param consumer: The EventConsumer doc """ - if update_log.update_type == 'Create': + if update_log.update_type == "Create": # consumer is obviously up to date return True prev_logs = frappe.get_all( - 'Event Update Log', + "Event Update Log", filters={ - 'ref_doctype': update_log.ref_doctype, - 'docname': update_log.docname, - 'creation': ['<', update_log.creation] + "ref_doctype": update_log.ref_doctype, + "docname": update_log.docname, + "creation": ["<", update_log.creation], }, - order_by='creation desc', - limit_page_length=1 + order_by="creation desc", + limit_page_length=1, ) if not len(prev_logs): return False prev_log_consumers = frappe.get_all( - 'Event Update Log Consumer', - fields=['consumer'], + "Event Update Log Consumer", + fields=["consumer"], filters={ - 'parent': prev_logs[0].name, - 'parenttype': 'Event Update Log', - 'consumer': consumer.name - } + "parent": prev_logs[0].name, + "parenttype": "Event Update Log", + "consumer": consumer.name, + }, ) return len(prev_log_consumers) > 0 @@ -182,24 +196,29 @@ def mark_consumer_read(update_log_name, consumer_name): """ This function appends the Consumer to the list of Consumers that has 'read' an Update Log """ - update_log = frappe.get_doc('Event Update Log', update_log_name) + update_log = frappe.get_doc("Event Update Log", update_log_name) if len([x for x in update_log.consumers if x.consumer == consumer_name]): return - frappe.get_doc(frappe._dict( - doctype='Event Update Log Consumer', + frappe.get_doc( + frappe._dict( + doctype="Event Update Log Consumer", consumer=consumer_name, parent=update_log_name, - parenttype='Event Update Log', - parentfield='consumers' - )).insert(ignore_permissions=True) + parenttype="Event Update Log", + parentfield="consumers", + ) + ).insert(ignore_permissions=True) def get_unread_update_logs(consumer_name, dt, dn): """ Get old logs unread by the consumer on a particular document """ - already_consumed = [x[0] for x in frappe.db.sql(""" + already_consumed = [ + x[0] + for x in frappe.db.sql( + """ SELECT update_log.name FROM `tabEvent Update Log` update_log @@ -208,23 +227,24 @@ def get_unread_update_logs(consumer_name, dt, dn): consumer.consumer = %(consumer)s AND update_log.ref_doctype = %(dt)s AND update_log.docname = %(dn)s - """, { - "consumer": consumer_name, - "dt": dt, - "dn": dn, - "log_name": "update_log.name" if frappe.conf.db_type == "mariadb" else "CAST(update_log.name AS VARCHAR)" - }, as_dict=0)] + """, + { + "consumer": consumer_name, + "dt": dt, + "dn": dn, + "log_name": "update_log.name" + if frappe.conf.db_type == "mariadb" + else "CAST(update_log.name AS VARCHAR)", + }, + as_dict=0, + ) + ] logs = frappe.get_all( - 'Event Update Log', - fields=['update_type', 'ref_doctype', - 'docname', 'data', 'name', 'creation'], - filters={ - 'ref_doctype': dt, - 'docname': dn, - 'name': ['not in', already_consumed] - }, - order_by='creation' + "Event Update Log", + fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"], + filters={"ref_doctype": dt, "docname": dn, "name": ["not in", already_consumed]}, + order_by="creation", ) return logs @@ -242,14 +262,12 @@ def get_update_logs_for_consumer(event_consumer, doctypes, last_update): from frappe.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access - consumer = frappe.get_doc('Event Consumer', event_consumer) + consumer = frappe.get_doc("Event Consumer", event_consumer) docs = frappe.get_list( - doctype='Event Update Log', - filters={'ref_doctype': ('in', doctypes), - 'creation': ('>', last_update)}, - fields=['update_type', 'ref_doctype', - 'docname', 'data', 'name', 'creation'], - order_by='creation desc' + doctype="Event Update Log", + filters={"ref_doctype": ("in", doctypes), "creation": (">", last_update)}, + fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"], + order_by="creation desc", ) result = [] @@ -272,9 +290,8 @@ def get_update_logs_for_consumer(event_consumer, doctypes, last_update): else: result.append(d) - for d in result: mark_consumer_read(update_log_name=d.name, consumer_name=consumer.name) result.reverse() - return result \ No newline at end of file + return result diff --git a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py index 752f4bbb44..673164b8d7 100644 --- a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py +++ b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestEventUpdateLog(unittest.TestCase): pass diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py index 47180db74e..4f00504538 100644 --- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py +++ b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class EventUpdateLogConsumer(Document): pass diff --git a/frappe/exceptions.py b/frappe/exceptions.py index fcac349708..a8569481d3 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -4,47 +4,61 @@ # BEWARE don't put anything in this file except exceptions from werkzeug.exceptions import NotFound + class SiteNotSpecifiedError(Exception): def __init__(self, *args, **kwargs): self.message = "Please specify --site sitename" super(Exception, self).__init__(self.message) + class ValidationError(Exception): http_status_code = 417 + class AuthenticationError(Exception): http_status_code = 401 + class SessionExpired(Exception): http_status_code = 401 + class PermissionError(Exception): http_status_code = 403 + class DoesNotExistError(ValidationError): http_status_code = 404 + class PageDoesNotExistError(ValidationError): http_status_code = 404 + class NameError(Exception): http_status_code = 409 + class OutgoingEmailError(Exception): http_status_code = 501 + class SessionStopped(Exception): http_status_code = 503 + class UnsupportedMediaType(Exception): http_status_code = 415 + class RequestToken(Exception): http_status_code = 200 + class Redirect(Exception): http_status_code = 301 + class CSRFTokenError(Exception): http_status_code = 400 @@ -58,58 +72,194 @@ class ImproperDBConfigurationError(Exception): Used when frappe detects that database or tables are not properly configured """ + def __init__(self, reason, msg=None): if not msg: msg = "MariaDb is not properly configured" super(ImproperDBConfigurationError, self).__init__(msg) self.reason = reason -class DuplicateEntryError(NameError):pass -class DataError(ValidationError): pass -class UnknownDomainError(Exception): pass -class MappingMismatchError(ValidationError): pass -class InvalidStatusError(ValidationError): pass -class MandatoryError(ValidationError): pass -class NonNegativeError(ValidationError): pass -class InvalidSignatureError(ValidationError): pass -class RateLimitExceededError(ValidationError): pass -class CannotChangeConstantError(ValidationError): pass -class CharacterLengthExceededError(ValidationError): pass -class UpdateAfterSubmitError(ValidationError): pass -class LinkValidationError(ValidationError): pass -class CancelledLinkError(LinkValidationError): pass -class DocstatusTransitionError(ValidationError): pass -class TimestampMismatchError(ValidationError): pass -class EmptyTableError(ValidationError): pass -class LinkExistsError(ValidationError): pass -class InvalidEmailAddressError(ValidationError): pass -class InvalidNameError(ValidationError): pass -class InvalidPhoneNumberError(ValidationError): pass -class TemplateNotFoundError(ValidationError): pass -class UniqueValidationError(ValidationError): pass -class AppNotInstalledError(ValidationError): pass -class IncorrectSitePath(NotFound): pass -class ImplicitCommitError(ValidationError): pass -class RetryBackgroundJobError(Exception): pass -class DocumentLockedError(ValidationError): pass -class CircularLinkingError(ValidationError): pass -class SecurityException(Exception): pass -class InvalidColumnName(ValidationError): pass -class IncompatibleApp(ValidationError): pass -class InvalidDates(ValidationError): pass -class DataTooLongException(ValidationError): pass -class FileAlreadyAttachedException(Exception): pass -class DocumentAlreadyRestored(ValidationError): pass -class AttachmentLimitReached(ValidationError): pass -class QueryTimeoutError(Exception): pass -class QueryDeadlockError(Exception): pass -class TooManyWritesError(Exception): pass + +class DuplicateEntryError(NameError): + pass + + +class DataError(ValidationError): + pass + + +class UnknownDomainError(Exception): + pass + + +class MappingMismatchError(ValidationError): + pass + + +class InvalidStatusError(ValidationError): + pass + + +class MandatoryError(ValidationError): + pass + + +class NonNegativeError(ValidationError): + pass + + +class InvalidSignatureError(ValidationError): + pass + + +class RateLimitExceededError(ValidationError): + pass + + +class CannotChangeConstantError(ValidationError): + pass + + +class CharacterLengthExceededError(ValidationError): + pass + + +class UpdateAfterSubmitError(ValidationError): + pass + + +class LinkValidationError(ValidationError): + pass + + +class CancelledLinkError(LinkValidationError): + pass + + +class DocstatusTransitionError(ValidationError): + pass + + +class TimestampMismatchError(ValidationError): + pass + + +class EmptyTableError(ValidationError): + pass + + +class LinkExistsError(ValidationError): + pass + + +class InvalidEmailAddressError(ValidationError): + pass + + +class InvalidNameError(ValidationError): + pass + + +class InvalidPhoneNumberError(ValidationError): + pass + + +class TemplateNotFoundError(ValidationError): + pass + + +class UniqueValidationError(ValidationError): + pass + + +class AppNotInstalledError(ValidationError): + pass + + +class IncorrectSitePath(NotFound): + pass + + +class ImplicitCommitError(ValidationError): + pass + + +class RetryBackgroundJobError(Exception): + pass + + +class DocumentLockedError(ValidationError): + pass + + +class CircularLinkingError(ValidationError): + pass + + +class SecurityException(Exception): + pass + + +class InvalidColumnName(ValidationError): + pass + + +class IncompatibleApp(ValidationError): + pass + + +class InvalidDates(ValidationError): + pass + + +class DataTooLongException(ValidationError): + pass + + +class FileAlreadyAttachedException(Exception): + pass + + +class DocumentAlreadyRestored(ValidationError): + pass + + +class AttachmentLimitReached(ValidationError): + pass + + +class QueryTimeoutError(Exception): + pass + + +class QueryDeadlockError(Exception): + pass + + +class TooManyWritesError(Exception): + pass + + # OAuth exceptions -class InvalidAuthorizationHeader(CSRFTokenError): pass -class InvalidAuthorizationPrefix(CSRFTokenError): pass -class InvalidAuthorizationToken(CSRFTokenError): pass -class InvalidDatabaseFile(ValidationError): pass -class ExecutableNotFound(FileNotFoundError): pass +class InvalidAuthorizationHeader(CSRFTokenError): + pass + + +class InvalidAuthorizationPrefix(CSRFTokenError): + pass + + +class InvalidAuthorizationToken(CSRFTokenError): + pass + + +class InvalidDatabaseFile(ValidationError): + pass + + +class ExecutableNotFound(FileNotFoundError): + pass + class InvalidRemoteException(Exception): pass diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index 7a1587aae0..04087463bc 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -1,6 +1,6 @@ -''' +""" FrappeClient is a library that helps you connect with other frappe systems -''' +""" import base64 import json @@ -13,20 +13,33 @@ from frappe.utils.data import cstr class AuthError(Exception): pass + class SiteExpiredError(Exception): pass + class SiteUnreachableError(Exception): pass + class FrappeException(Exception): pass + class FrappeClient(object): - def __init__(self, url, username=None, password=None, verify=True, api_key=None, api_secret=None, frappe_authorization_source=None): + def __init__( + self, + url, + username=None, + password=None, + verify=True, + api_key=None, + api_secret=None, + frappe_authorization_source=None, + ): self.headers = { - 'Accept': 'application/json', - 'content-type': 'application/x-www-form-urlencoded', + "Accept": "application/json", + "content-type": "application/x-www-form-urlencoded", } self.verify = verify self.session = requests.session() @@ -48,21 +61,22 @@ class FrappeClient(object): self.logout() def _login(self, username, password): - '''Login/start a sesion. Called internally on init''' - r = self.session.post(self.url, params={ - 'cmd': 'login', - 'usr': username, - 'pwd': password - }, verify=self.verify, headers=self.headers) - - if r.status_code==200 and r.json().get('message') in ("Logged In", "No App"): + """Login/start a sesion. Called internally on init""" + r = self.session.post( + self.url, + params={"cmd": "login", "usr": username, "pwd": password}, + verify=self.verify, + headers=self.headers, + ) + + if r.status_code == 200 and r.json().get("message") in ("Logged In", "No App"): return r.json() elif r.status_code == 502: raise SiteUnreachableError else: try: error = json.loads(r.text) - if error.get('exc_type') == "SiteExpiredError": + if error.get("exc_type") == "SiteExpiredError": raise SiteExpiredError except json.decoder.JSONDecodeError: error = r.text @@ -71,21 +85,28 @@ class FrappeClient(object): def setup_key_authentication_headers(self): if self.api_key and self.api_secret: - token = base64.b64encode(('{}:{}'.format(self.api_key, self.api_secret)).encode('utf-8')).decode('utf-8') + token = base64.b64encode( + ("{}:{}".format(self.api_key, self.api_secret)).encode("utf-8") + ).decode("utf-8") auth_header = { - 'Authorization': 'Basic {}'.format(token), + "Authorization": "Basic {}".format(token), } self.headers.update(auth_header) if self.frappe_authorization_source: - auth_source = {'Frappe-Authorization-Source': self.frappe_authorization_source} + auth_source = {"Frappe-Authorization-Source": self.frappe_authorization_source} self.headers.update(auth_source) def logout(self): - '''Logout session''' - self.session.get(self.url, params={ - 'cmd': 'logout', - }, verify=self.verify, headers=self.headers) + """Logout session""" + self.session.get( + self.url, + params={ + "cmd": "logout", + }, + verify=self.verify, + headers=self.headers, + ) def get_list(self, doctype, fields='["name"]', filters=None, limit_start=0, limit_page_length=0): """Returns list of records of a particular type""" @@ -99,144 +120,147 @@ class FrappeClient(object): if limit_page_length: params["limit_start"] = limit_start params["limit_page_length"] = limit_page_length - res = self.session.get(self.url + "/api/resource/" + doctype, params=params, verify=self.verify, headers=self.headers) + res = self.session.get( + self.url + "/api/resource/" + doctype, params=params, verify=self.verify, headers=self.headers + ) return self.post_process(res) def insert(self, doc): - '''Insert a document to the remote server - - :param doc: A dict or Document object to be inserted remotely''' - res = self.session.post(self.url + "/api/resource/" + doc.get("doctype"), - data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers) + """Insert a document to the remote server + + :param doc: A dict or Document object to be inserted remotely""" + res = self.session.post( + self.url + "/api/resource/" + doc.get("doctype"), + data={"data": frappe.as_json(doc)}, + verify=self.verify, + headers=self.headers, + ) return frappe._dict(self.post_process(res)) def insert_many(self, docs): - '''Insert multiple documents to the remote server + """Insert multiple documents to the remote server - :param docs: List of dict or Document objects to be inserted in one request''' - return self.post_request({ - "cmd": "frappe.client.insert_many", - "docs": frappe.as_json(docs) - }) + :param docs: List of dict or Document objects to be inserted in one request""" + return self.post_request({"cmd": "frappe.client.insert_many", "docs": frappe.as_json(docs)}) def update(self, doc): - '''Update a remote document + """Update a remote document - :param doc: dict or Document object to be updated remotely. `name` is mandatory for this''' + :param doc: dict or Document object to be updated remotely. `name` is mandatory for this""" url = self.url + "/api/resource/" + doc.get("doctype") + "/" + cstr(doc.get("name")) - res = self.session.put(url, data={"data":frappe.as_json(doc)}, verify=self.verify, headers=self.headers) + res = self.session.put( + url, data={"data": frappe.as_json(doc)}, verify=self.verify, headers=self.headers + ) return frappe._dict(self.post_process(res)) def bulk_update(self, docs): - '''Bulk update documents remotely + """Bulk update documents remotely - :param docs: List of dict or Document objects to be updated remotely (by `name`)''' - return self.post_request({ - "cmd": "frappe.client.bulk_update", - "docs": frappe.as_json(docs) - }) + :param docs: List of dict or Document objects to be updated remotely (by `name`)""" + return self.post_request({"cmd": "frappe.client.bulk_update", "docs": frappe.as_json(docs)}) def delete(self, doctype, name): - '''Delete remote document by name + """Delete remote document by name :param doctype: `doctype` to be deleted - :param name: `name` of document to be deleted''' - return self.post_request({ - "cmd": "frappe.client.delete", - "doctype": doctype, - "name": name - }) + :param name: `name` of document to be deleted""" + return self.post_request({"cmd": "frappe.client.delete", "doctype": doctype, "name": name}) def submit(self, doc): - '''Submit remote document + """Submit remote document - :param doc: dict or Document object to be submitted remotely''' - return self.post_request({ - "cmd": "frappe.client.submit", - "doc": frappe.as_json(doc) - }) + :param doc: dict or Document object to be submitted remotely""" + return self.post_request({"cmd": "frappe.client.submit", "doc": frappe.as_json(doc)}) def get_value(self, doctype, fieldname=None, filters=None): - '''Returns a value form a document + """Returns a value form a document :param doctype: DocType to be queried :param fieldname: Field to be returned (default `name`) - :param filters: dict or string for identifying the record''' - return self.get_request({ - "cmd": "frappe.client.get_value", - "doctype": doctype, - "fieldname": fieldname or "name", - "filters": frappe.as_json(filters) - }) + :param filters: dict or string for identifying the record""" + return self.get_request( + { + "cmd": "frappe.client.get_value", + "doctype": doctype, + "fieldname": fieldname or "name", + "filters": frappe.as_json(filters), + } + ) def set_value(self, doctype, docname, fieldname, value): - '''Set a value in a remote document + """Set a value in a remote document :param doctype: DocType of the document to be updated :param docname: name of the document to be updated :param fieldname: fieldname of the document to be updated - :param value: value to be updated''' - return self.post_request({ - "cmd": "frappe.client.set_value", - "doctype": doctype, - "name": docname, - "fieldname": fieldname, - "value": value - }) + :param value: value to be updated""" + return self.post_request( + { + "cmd": "frappe.client.set_value", + "doctype": doctype, + "name": docname, + "fieldname": fieldname, + "value": value, + } + ) def cancel(self, doctype, name): - '''Cancel a remote document + """Cancel a remote document :param doctype: DocType of the document to be cancelled - :param name: name of the document to be cancelled''' - return self.post_request({ - "cmd": "frappe.client.cancel", - "doctype": doctype, - "name": name - }) + :param name: name of the document to be cancelled""" + return self.post_request({"cmd": "frappe.client.cancel", "doctype": doctype, "name": name}) def get_doc(self, doctype, name="", filters=None, fields=None): - '''Returns a single remote document + """Returns a single remote document :param doctype: DocType of the document to be returned :param name: (optional) `name` of the document to be returned :param filters: (optional) Filter by this dict if name is not set - :param fields: (optional) Fields to be returned, will return everythign if not set''' + :param fields: (optional) Fields to be returned, will return everythign if not set""" params = {} if filters: params["filters"] = json.dumps(filters) if fields: params["fields"] = json.dumps(fields) - res = self.session.get(self.url + "/api/resource/" + doctype + "/" + cstr(name), - params=params, verify=self.verify, headers=self.headers) + res = self.session.get( + self.url + "/api/resource/" + doctype + "/" + cstr(name), + params=params, + verify=self.verify, + headers=self.headers, + ) return self.post_process(res) def rename_doc(self, doctype, old_name, new_name): - '''Rename remote document + """Rename remote document :param doctype: DocType of the document to be renamed :param old_name: Current `name` of the document to be renamed - :param new_name: New `name` to be set''' + :param new_name: New `name` to be set""" params = { "cmd": "frappe.client.rename_doc", "doctype": doctype, "old_name": old_name, - "new_name": new_name + "new_name": new_name, } return self.post_request(params) - def migrate_doctype(self, doctype, filters=None, update=None, verbose=1, exclude=None, preprocess=None): + def migrate_doctype( + self, doctype, filters=None, update=None, verbose=1, exclude=None, preprocess=None + ): """Migrate records from another doctype""" meta = frappe.get_meta(doctype) tables = {} for df in meta.get_table_fields(): - if verbose: print("getting " + df.options) + if verbose: + print("getting " + df.options) tables[df.fieldname] = self.get_list(df.options, limit_page_length=999999) # get links - if verbose: print("getting " + doctype) + if verbose: + print("getting " + doctype) docs = self.get_list(doctype, limit_page_length=999999, filters=filters) # build - attach children to parents @@ -250,7 +274,8 @@ class FrappeClient(object): if child.parent in docs_map: docs_map[child.parent].setdefault(fieldname, []).append(child) - if verbose: print("inserting " + doctype) + if verbose: + print("inserting " + doctype) for doc in docs: if exclude and doc["name"] in exclude: continue @@ -262,8 +287,9 @@ class FrappeClient(object): doc["owner"] = "Administrator" if doctype != "User" and not frappe.db.exists("User", doc.get("owner")): - frappe.get_doc({"doctype": "User", "email": doc.get("owner"), - "first_name": doc.get("owner").split("@")[0] }).insert() + frappe.get_doc( + {"doctype": "User", "email": doc.get("owner"), "first_name": doc.get("owner").split("@")[0]} + ).insert() if update: doc.update(update) @@ -274,12 +300,20 @@ class FrappeClient(object): if not meta.istable: if doctype != "Communication": - self.migrate_doctype("Communication", {"reference_doctype": doctype, "reference_name": doc["name"]}, - update={"reference_name": new_doc.name}, verbose=0) + self.migrate_doctype( + "Communication", + {"reference_doctype": doctype, "reference_name": doc["name"]}, + update={"reference_name": new_doc.name}, + verbose=0, + ) if doctype != "File": - self.migrate_doctype("File", {"attached_to_doctype": doctype, - "attached_to_name": doc["name"]}, update={"attached_to_name": new_doc.name}, verbose=0) + self.migrate_doctype( + "File", + {"attached_to_doctype": doctype, "attached_to_name": doc["name"]}, + update={"attached_to_name": new_doc.name}, + verbose=0, + ) def migrate_single(self, doctype): doc = self.get_doc(doctype, doctype) @@ -292,24 +326,30 @@ class FrappeClient(object): def get_api(self, method, params=None): if params is None: params = {} - res = self.session.get(f"{self.url}/api/method/{method}", - params=params, verify=self.verify, headers=self.headers) + res = self.session.get( + f"{self.url}/api/method/{method}", params=params, verify=self.verify, headers=self.headers + ) return self.post_process(res) def post_api(self, method, params=None): if params is None: params = {} - res = self.session.post(f"{self.url}/api/method/{method}", - params=params, verify=self.verify, headers=self.headers) + res = self.session.post( + f"{self.url}/api/method/{method}", params=params, verify=self.verify, headers=self.headers + ) return self.post_process(res) def get_request(self, params): - res = self.session.get(self.url, params=self.preprocess(params), verify=self.verify, headers=self.headers) + res = self.session.get( + self.url, params=self.preprocess(params), verify=self.verify, headers=self.headers + ) res = self.post_process(res) return res def post_request(self, data): - res = self.session.post(self.url, data=self.preprocess(data), verify=self.verify, headers=self.headers) + res = self.session.post( + self.url, data=self.preprocess(data), verify=self.verify, headers=self.headers + ) res = self.post_process(res) return res @@ -331,48 +371,57 @@ class FrappeClient(object): if rjson and ("exc" in rjson) and rjson["exc"]: try: exc = json.loads(rjson["exc"])[0] - exc = 'FrappeClient Request Failed\n\n' + exc + exc = "FrappeClient Request Failed\n\n" + exc except Exception: exc = rjson["exc"] raise FrappeException(exc) - if 'message' in rjson: - return rjson['message'] - elif 'data' in rjson: - return rjson['data'] + if "message" in rjson: + return rjson["message"] + elif "data" in rjson: + return rjson["data"] else: return None + class FrappeOAuth2Client(FrappeClient): def __init__(self, url, access_token, verify=True): self.access_token = access_token self.headers = { "Authorization": "Bearer " + access_token, - "content-type": "application/x-www-form-urlencoded" + "content-type": "application/x-www-form-urlencoded", } self.verify = verify self.session = OAuth2Session(self.headers) self.url = url def get_request(self, params): - res = requests.get(self.url, params=self.preprocess(params), headers=self.headers, verify=self.verify) + res = requests.get( + self.url, params=self.preprocess(params), headers=self.headers, verify=self.verify + ) res = self.post_process(res) return res def post_request(self, data): - res = requests.post(self.url, data=self.preprocess(data), headers=self.headers, verify=self.verify) + res = requests.post( + self.url, data=self.preprocess(data), headers=self.headers, verify=self.verify + ) res = self.post_process(res) return res -class OAuth2Session(): + +class OAuth2Session: def __init__(self, headers): self.headers = headers + def get(self, url, params, verify): res = requests.get(url, params=params, headers=self.headers, verify=verify) return res + def post(self, url, data, verify): res = requests.post(url, data=data, headers=self.headers, verify=verify) return res + def put(self, url, data, verify): res = requests.put(url, data=data, headers=self.headers, verify=verify) return res diff --git a/frappe/geo/country_info.py b/frappe/geo/country_info.py index 86f1d9bc2f..1de8467a6a 100644 --- a/frappe/geo/country_info.py +++ b/frappe/geo/country_info.py @@ -1,43 +1,48 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + # all country info -import os, json, frappe +import os + +import frappe from frappe.utils.momentjs import get_all_timezones + def get_country_info(country=None): data = get_all() data = frappe._dict(data.get(country, {})) - if 'date_format' not in data: + if "date_format" not in data: data.date_format = "dd-mm-yyyy" - if 'time_format' not in data: + if "time_format" not in data: data.time_format = "HH:mm:ss" return data + def get_all(): with open(os.path.join(os.path.dirname(__file__), "country_info.json"), "r") as local_info: all_data = json.loads(local_info.read()) return all_data + @frappe.whitelist() def get_country_timezone_info(): - return { - "country_info": get_all(), - "all_timezones": get_all_timezones() - } + return {"country_info": get_all(), "all_timezones": get_all_timezones()} + def get_translated_dict(): - from babel.dates import get_timezone, get_timezone_name, Locale + from babel.dates import Locale, get_timezone, get_timezone_name translated_dict = {} locale = Locale.parse(frappe.local.lang, sep="-") # timezones for tz in get_all_timezones(): - timezone_name = get_timezone_name(get_timezone(tz), locale=locale, width='short') + timezone_name = get_timezone_name(get_timezone(tz), locale=locale, width="short") if timezone_name: - translated_dict[tz] = timezone_name + ' - ' + tz + translated_dict[tz] = timezone_name + " - " + tz # country names && currencies for country, info in get_all().items(): @@ -52,6 +57,7 @@ def get_translated_dict(): return translated_dict + def update(): with open(os.path.join(os.path.dirname(__file__), "currency_info.json"), "r") as nformats: nformats = json.loads(nformats.read()) @@ -60,8 +66,9 @@ def update(): for country in all_data: data = all_data[country] - data["number_format"] = nformats.get(data.get("currency", "default"), - nformats.get("default"))["display"] + data["number_format"] = nformats.get(data.get("currency", "default"), nformats.get("default"))[ + "display" + ] with open(os.path.join(os.path.dirname(__file__), "country_info.json"), "w") as local_info: local_info.write(json.dumps(all_data, indent=1)) diff --git a/frappe/geo/doctype/country/country.py b/frappe/geo/doctype/country/country.py index a648744058..b3ba1b7127 100644 --- a/frappe/geo/doctype/country/country.py +++ b/frappe/geo/doctype/country/country.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class Country(Document): - pass \ No newline at end of file + pass diff --git a/frappe/geo/doctype/country/test_country.py b/frappe/geo/doctype/country/test_country.py index b4d15f81b3..ecc2fb6863 100644 --- a/frappe/geo/doctype/country/test_country.py +++ b/frappe/geo/doctype/country/test_country.py @@ -2,4 +2,5 @@ # License: MIT. See LICENSE import frappe -test_records = frappe.get_test_records('Country') \ No newline at end of file + +test_records = frappe.get_test_records("Country") diff --git a/frappe/geo/doctype/currency/currency.py b/frappe/geo/doctype/currency/currency.py index fbe37e73bd..dd5df57bab 100644 --- a/frappe/geo/doctype/currency/currency.py +++ b/frappe/geo/doctype/currency/currency.py @@ -2,10 +2,10 @@ # License: MIT. See LICENSE import frappe -from frappe import throw, _ - +from frappe import _, throw from frappe.model.document import Document + class Currency(Document): def validate(self): if not frappe.flags.in_install_app: diff --git a/frappe/geo/doctype/currency/test_currency.py b/frappe/geo/doctype/currency/test_currency.py index 71b963cc86..f93a452462 100644 --- a/frappe/geo/doctype/currency/test_currency.py +++ b/frappe/geo/doctype/currency/test_currency.py @@ -4,4 +4,5 @@ # pre loaded import frappe -test_records = frappe.get_test_records('Currency') \ No newline at end of file + +test_records = frappe.get_test_records("Currency") diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py index 9b44a2f3d8..577c5de2ff 100644 --- a/frappe/geo/utils.py +++ b/frappe/geo/utils.py @@ -2,92 +2,101 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe - from pymysql import InternalError +import frappe + @frappe.whitelist() def get_coords(doctype, filters, type): - '''Get a geojson dict representing a doctype.''' + """Get a geojson dict representing a doctype.""" filters_sql = get_coords_conditions(doctype, filters)[4:] coords = None - if type == 'location_field': + if type == "location_field": coords = return_location(doctype, filters_sql) - elif type == 'coordinates': + elif type == "coordinates": coords = return_coordinates(doctype, filters_sql) out = convert_to_geojson(type, coords) return out + def convert_to_geojson(type, coords): - '''Converts GPS coordinates to geoJSON string.''' + """Converts GPS coordinates to geoJSON string.""" geojson = {"type": "FeatureCollection", "features": None} - if type == 'location_field': - geojson['features'] = merge_location_features_in_one(coords) - elif type == 'coordinates': - geojson['features'] = create_gps_markers(coords) + if type == "location_field": + geojson["features"] = merge_location_features_in_one(coords) + elif type == "coordinates": + geojson["features"] = create_gps_markers(coords) return geojson def merge_location_features_in_one(coords): - '''Merging all features from location field.''' + """Merging all features from location field.""" geojson_dict = [] for element in coords: - geojson_loc = frappe.parse_json(element['location']) + geojson_loc = frappe.parse_json(element["location"]) if not geojson_loc: continue - for coord in geojson_loc['features']: - coord['properties']['name'] = element['name'] + for coord in geojson_loc["features"]: + coord["properties"]["name"] = element["name"] geojson_dict.append(coord.copy()) return geojson_dict def create_gps_markers(coords): - '''Build Marker based on latitude and longitude.''' + """Build Marker based on latitude and longitude.""" geojson_dict = [] for i in coords: node = {"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": None}} - node['properties']['name'] = i.name - node['geometry']['coordinates'] = [i.latitude, i.longitude] + node["properties"]["name"] = i.name + node["geometry"]["coordinates"] = [i.latitude, i.longitude] geojson_dict.append(node.copy()) return geojson_dict def return_location(doctype, filters_sql): - '''Get name and location fields for Doctype.''' + """Get name and location fields for Doctype.""" if filters_sql: try: - coords = frappe.db.sql('''SELECT name, location FROM `tab{}` WHERE {}'''.format(doctype, filters_sql), as_dict=True) + coords = frappe.db.sql( + """SELECT name, location FROM `tab{}` WHERE {}""".format(doctype, filters_sql), as_dict=True + ) except InternalError: - frappe.msgprint(frappe._('This Doctype does not contain location fields'), raise_exception=True) + frappe.msgprint(frappe._("This Doctype does not contain location fields"), raise_exception=True) return else: - coords = frappe.get_all(doctype, fields=['name', 'location']) + coords = frappe.get_all(doctype, fields=["name", "location"]) return coords def return_coordinates(doctype, filters_sql): - '''Get name, latitude and longitude fields for Doctype.''' + """Get name, latitude and longitude fields for Doctype.""" if filters_sql: try: - coords = frappe.db.sql('''SELECT name, latitude, longitude FROM `tab{}` WHERE {}'''.format(doctype, filters_sql), as_dict=True) + coords = frappe.db.sql( + """SELECT name, latitude, longitude FROM `tab{}` WHERE {}""".format(doctype, filters_sql), + as_dict=True, + ) except InternalError: - frappe.msgprint(frappe._('This Doctype does not contain latitude and longitude fields'), raise_exception=True) + frappe.msgprint( + frappe._("This Doctype does not contain latitude and longitude fields"), raise_exception=True + ) return else: - coords = frappe.get_all(doctype, fields=['name', 'latitude', 'longitude']) + coords = frappe.get_all(doctype, fields=["name", "latitude", "longitude"]) return coords def get_coords_conditions(doctype, filters=None): - '''Returns SQL conditions with user permissions and filters for event queries.''' + """Returns SQL conditions with user permissions and filters for event queries.""" from frappe.desk.reportview import get_filters_cond + if not frappe.has_permission(doctype): frappe.throw(frappe._("Not Permitted"), frappe.PermissionError) diff --git a/frappe/handler.py b/frappe/handler.py index ebc72da937..7b010eb716 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -1,24 +1,31 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from mimetypes import guess_type + from werkzeug.wrappers import Response import frappe -import frappe.utils import frappe.sessions -from frappe.utils import cint +import frappe.utils from frappe import _, is_whitelisted -from frappe.utils.response import build_response +from frappe.core.doctype.server_script.server_script_utils import get_server_script_map +from frappe.utils import cint from frappe.utils.csvutils import build_csv_response from frappe.utils.image import optimize_image -from mimetypes import guess_type -from frappe.core.doctype.server_script.server_script_utils import get_server_script_map - +from frappe.utils.response import build_response -ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/msword', - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - 'application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'application/vnd.oasis.opendocument.text', 'application/vnd.oasis.opendocument.spreadsheet') +ALLOWED_MIMETYPES = ( + "image/png", + "image/jpeg", + "application/pdf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.oasis.opendocument.text", + "application/vnd.oasis.opendocument.spreadsheet", +) def handle(): @@ -27,7 +34,7 @@ def handle(): cmd = frappe.local.form_dict.cmd data = None - if cmd != 'login': + if cmd != "login": data = execute_cmd(cmd) # data can be an empty string or list which are valid responses @@ -37,10 +44,11 @@ def handle(): return data # add the response to `message` label - frappe.response['message'] = data + frappe.response["message"] = data return build_response("json") + def execute_cmd(cmd, from_async=False): """execute a request as python module""" for hook in frappe.get_hooks("override_whitelisted_methods", {}).get(cmd, []): @@ -49,14 +57,14 @@ def execute_cmd(cmd, from_async=False): break # via server script - server_script = get_server_script_map().get('_api', {}).get(cmd) + server_script = get_server_script_map().get("_api", {}).get(cmd) if server_script: return run_server_script(server_script) try: method = get_attr(cmd) except Exception as e: - frappe.throw(_('Failed to get method for command {0} with {1}').format(cmd, e)) + frappe.throw(_("Failed to get method for command {0} with {1}").format(cmd, e)) if from_async: method = method.queue @@ -69,7 +77,7 @@ def execute_cmd(cmd, from_async=False): def run_server_script(server_script): - response = frappe.get_doc('Server Script', server_script).execute_method() + response = frappe.get_doc("Server Script", server_script).execute_method() # some server scripts return output using flags (empty dict by default), # while others directly modify frappe.response @@ -77,6 +85,7 @@ def run_server_script(server_script): if response != {}: return response + def is_valid_http_method(method): if frappe.flags.in_safe_exec: return @@ -86,65 +95,74 @@ def is_valid_http_method(method): if http_method not in frappe.allowed_http_methods_for_whitelisted_func[method]: throw_permission_error() + def throw_permission_error(): frappe.throw(_("Not permitted"), frappe.PermissionError) + @frappe.whitelist(allow_guest=True) def version(): return frappe.__version__ + @frappe.whitelist(allow_guest=True) def logout(): frappe.local.login_manager.logout() frappe.db.commit() + @frappe.whitelist(allow_guest=True) def web_logout(): frappe.local.login_manager.logout() frappe.db.commit() - frappe.respond_as_web_page(_("Logged Out"), _("You have been successfully logged out"), - indicator_color='green') + frappe.respond_as_web_page( + _("Logged Out"), _("You have been successfully logged out"), indicator_color="green" + ) + @frappe.whitelist() def uploadfile(): ret = None try: - if frappe.form_dict.get('from_form'): + if frappe.form_dict.get("from_form"): try: - ret = frappe.get_doc({ - "doctype": "File", - "attached_to_name": frappe.form_dict.docname, - "attached_to_doctype": frappe.form_dict.doctype, - "attached_to_field": frappe.form_dict.docfield, - "file_url": frappe.form_dict.file_url, - "file_name": frappe.form_dict.filename, - "is_private": frappe.utils.cint(frappe.form_dict.is_private), - "content": frappe.form_dict.filedata, - "decode": True - }) + ret = frappe.get_doc( + { + "doctype": "File", + "attached_to_name": frappe.form_dict.docname, + "attached_to_doctype": frappe.form_dict.doctype, + "attached_to_field": frappe.form_dict.docfield, + "file_url": frappe.form_dict.file_url, + "file_name": frappe.form_dict.filename, + "is_private": frappe.utils.cint(frappe.form_dict.is_private), + "content": frappe.form_dict.filedata, + "decode": True, + } + ) ret.save() except frappe.DuplicateEntryError: # ignore pass ret = None frappe.db.rollback() else: - if frappe.form_dict.get('method'): + if frappe.form_dict.get("method"): method = frappe.get_attr(frappe.form_dict.method) is_whitelisted(method) ret = method() except Exception: frappe.errprint(frappe.utils.get_traceback()) - frappe.response['http_status_code'] = 500 + frappe.response["http_status_code"] = 500 ret = None return ret + @frappe.whitelist(allow_guest=True) def upload_file(): user = None - if frappe.session.user == 'Guest': - if frappe.get_system_settings('allow_guests_to_upload_files'): + if frappe.session.user == "Guest": + if frappe.get_system_settings("allow_guests_to_upload_files"): ignore_permissions = True else: return @@ -158,23 +176,20 @@ def upload_file(): docname = frappe.form_dict.docname fieldname = frappe.form_dict.fieldname file_url = frappe.form_dict.file_url - folder = frappe.form_dict.folder or 'Home' + folder = frappe.form_dict.folder or "Home" method = frappe.form_dict.method filename = frappe.form_dict.file_name optimize = frappe.form_dict.optimize content = None - if 'file' in files: - file = files['file'] + if "file" in files: + file = files["file"] content = file.stream.read() filename = file.filename content_type = guess_type(filename)[0] if optimize and content_type.startswith("image/"): - args = { - "content": content, - "content_type": content_type - } + args = {"content": content, "content_type": content_type} if frappe.form_dict.max_width: args["max_width"] = int(frappe.form_dict.max_width) if frappe.form_dict.max_height: @@ -194,30 +209,33 @@ def upload_file(): is_whitelisted(method) return method() else: - ret = frappe.get_doc({ - "doctype": "File", - "attached_to_doctype": doctype, - "attached_to_name": docname, - "attached_to_field": fieldname, - "folder": folder, - "file_name": filename, - "file_url": file_url, - "is_private": cint(is_private), - "content": content - }) + ret = frappe.get_doc( + { + "doctype": "File", + "attached_to_doctype": doctype, + "attached_to_name": docname, + "attached_to_field": fieldname, + "folder": folder, + "file_name": filename, + "file_url": file_url, + "is_private": cint(is_private), + "content": content, + } + ) ret.save(ignore_permissions=ignore_permissions) return ret def get_attr(cmd): """get method object from cmd""" - if '.' in cmd: + if "." in cmd: method = frappe.get_attr(cmd) else: method = globals()[cmd] frappe.log("method:" + cmd) return method + @frappe.whitelist(allow_guest=True) def ping(): return "pong" @@ -230,9 +248,9 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): if not args and arg: args = arg - if dt: # not called from a doctype (from a page) + if dt: # not called from a doctype (from a page) if not dn: - dn = dt # single + dn = dt # single doc = frappe.get_doc(dt, dn) else: @@ -250,13 +268,13 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): pass method_obj = getattr(doc, method) - fn = getattr(method_obj, '__func__', method_obj) + fn = getattr(method_obj, "__func__", method_obj) is_whitelisted(fn) is_valid_http_method(fn) fnargs = getfullargspec(method_obj).args - if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"): + if not fnargs or (len(fnargs) == 1 and fnargs[0] == "self"): response = doc.run_method(method) elif "args" in fnargs or not isinstance(args, dict): @@ -270,11 +288,12 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): return # build output as csv - if cint(frappe.form_dict.get('as_csv')): - build_csv_response(response, _(doc.doctype).replace(' ', '')) + if cint(frappe.form_dict.get("as_csv")): + build_csv_response(response, _(doc.doctype).replace(" ", "")) return - frappe.response['message'] = response + frappe.response["message"] = response + # for backwards compatibility runserverobj = run_doc_method diff --git a/frappe/hooks.py b/frappe/hooks.py index b545c6a719..d3de3877ba 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -1,7 +1,5 @@ - from . import __version__ as app_version - app_name = "frappe" app_title = "Frappe Framework" app_publisher = "Frappe Technologies" @@ -10,9 +8,9 @@ app_icon = "octicon octicon-circuit-board" app_color = "orange" source_link = "https://github.com/frappe/frappe" app_license = "MIT" -app_logo_url = '/assets/frappe/images/frappe-framework-logo.svg' +app_logo_url = "/assets/frappe/images/frappe-framework-logo.svg" -develop_version = '14.x.x-develop' +develop_version = "14.x.x-develop" app_email = "developers@frappe.io" @@ -23,9 +21,7 @@ translator_url = "https://translate.erpnext.com" before_install = "frappe.utils.install.before_install" after_install = "frappe.utils.install.after_install" -page_js = { - "setup-wizard": "public/js/frappe/setup_wizard.js" -} +page_js = {"setup-wizard": "public/js/frappe/setup_wizard.js"} # website app_include_js = [ @@ -43,16 +39,14 @@ app_include_css = [ doctype_js = { "Web Page": "public/js/frappe/utils/web_template.js", - "Website Settings": "public/js/frappe/utils/web_template.js" + "Website Settings": "public/js/frappe/utils/web_template.js", } -web_include_js = [ - "website_script.js" -] +web_include_js = ["website_script.js"] web_include_css = [] -email_css = ['email.bundle.css'] +email_css = ["email.bundle.css"] website_route_rules = [ {"from_route": "/blog/", "to_route": "Blog Post"}, @@ -84,10 +78,12 @@ leaderboards = "frappe.desk.leaderboard.get_leaderboards" on_session_creation = [ "frappe.core.doctype.activity_log.feed.login_feed", - "frappe.core.doctype.user.user.notify_admin_access_to_system_manager" + "frappe.core.doctype.user.user.notify_admin_access_to_system_manager", ] -on_logout = "frappe.core.doctype.session_default_settings.session_default_settings.clear_session_defaults" +on_logout = ( + "frappe.core.doctype.session_default_settings.session_default_settings.clear_session_defaults" +) # permissions @@ -107,7 +103,7 @@ permission_query_conditions = { "Address": "frappe.contacts.address_and_contact.get_permission_query_conditions_for_address", "Communication": "frappe.core.doctype.communication.communication.get_permission_query_conditions_for_communication", "Workflow Action": "frappe.workflow.doctype.workflow_action.workflow_action.get_permission_query_conditions", - "Prepared Report": "frappe.core.doctype.prepared_report.prepared_report.get_permission_query_condition" + "Prepared Report": "frappe.core.doctype.prepared_report.prepared_report.get_permission_query_condition", } has_permission = { @@ -123,7 +119,7 @@ has_permission = { "Communication": "frappe.core.doctype.communication.communication.has_permission", "Workflow Action": "frappe.workflow.doctype.workflow_action.workflow_action.has_permission", "File": "frappe.core.doctype.file.file.has_permission", - "Prepared Report": "frappe.core.doctype.prepared_report.prepared_report.has_permission" + "Prepared Report": "frappe.core.doctype.prepared_report.prepared_report.has_permission", } has_website_permission = { @@ -137,12 +133,10 @@ jinja = { "frappe.utils.markdown", "frappe.website.utils.get_shade", "frappe.website.utils.abs_url", - ] + ], } -standard_queries = { - "User": "frappe.core.doctype.user.user.user_query" -} +standard_queries = {"User": "frappe.core.doctype.user.user.user_query"} doc_events = { "*": { @@ -157,26 +151,26 @@ doc_events = { "frappe.core.doctype.file.file.attach_files_to_document", "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", "frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date", - "frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type" + "frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type", ], "after_rename": "frappe.desk.notifications.clear_doctype_notifications", "on_cancel": [ "frappe.desk.notifications.clear_doctype_notifications", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", - "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" + "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", ], "on_trash": [ "frappe.desk.notifications.clear_doctype_notifications", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", - "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" + "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", ], "on_update_after_submit": [ "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions" ], "on_change": [ "frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points", - "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone" - ] + "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone", + ], }, "Event": { "after_insert": "frappe.integrations.doctype.google_calendar.google_calendar.insert_event_in_google_calendar", @@ -194,7 +188,7 @@ doc_events = { "Page": { "after_insert": "frappe.cache_manager.build_domain_restriced_page_cache", "after_save": "frappe.cache_manager.build_domain_restriced_page_cache", - } + }, } scheduler_events = { @@ -202,7 +196,7 @@ scheduler_events = { "0/15 * * * *": [ "frappe.oauth.delete_oauth2_data", "frappe.website.doctype.web_page.web_page.check_publish_status", - "frappe.twofactor.delete_all_barcodes_for_users" + "frappe.twofactor.delete_all_barcodes_for_users", ] }, "all": [ @@ -210,19 +204,19 @@ scheduler_events = { "frappe.email.doctype.email_account.email_account.pull", "frappe.email.doctype.email_account.email_account.notify_unreplied", "frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment", - 'frappe.utils.global_search.sync_global_search', + "frappe.utils.global_search.sync_global_search", "frappe.monitor.flush", ], "hourly": [ "frappe.model.utils.link_count.update_link_count", - 'frappe.model.utils.user_settings.sync_user_settings', + "frappe.model.utils.user_settings.sync_user_settings", "frappe.utils.error.collect_error_snapshots", "frappe.desk.page.backups.backups.delete_downloadable_backups", "frappe.deferred_insert.save_to_db", "frappe.desk.form.document_follow.send_hourly_updates", "frappe.integrations.doctype.google_calendar.google_calendar.sync", "frappe.email.doctype.newsletter.newsletter.send_scheduled_email", - "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.process_data_deletion_request" + "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.process_data_deletion_request", ], "daily": [ "frappe.email.queue.set_expiry_for_email_queue", @@ -241,13 +235,13 @@ scheduler_events = { "frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed", "frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails", "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports", - "frappe.core.doctype.log_settings.log_settings.run_log_clean_up" + "frappe.core.doctype.log_settings.log_settings.run_log_clean_up", ], "daily_long": [ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily", "frappe.utils.change_log.check_for_update", "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily", - "frappe.integrations.doctype.google_drive.google_drive.daily_backup" + "frappe.integrations.doctype.google_drive.google_drive.daily_backup", ], "weekly_long": [ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly", @@ -255,20 +249,20 @@ scheduler_events = { "frappe.desk.doctype.route_history.route_history.flush_old_route_records", "frappe.desk.form.document_follow.send_weekly_updates", "frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary", - "frappe.integrations.doctype.google_drive.google_drive.weekly_backup" + "frappe.integrations.doctype.google_drive.google_drive.weekly_backup", ], "monthly": [ "frappe.email.doctype.auto_email_report.auto_email_report.send_monthly", - "frappe.social.doctype.energy_point_log.energy_point_log.send_monthly_summary" + "frappe.social.doctype.energy_point_log.energy_point_log.send_monthly_summary", ], "monthly_long": [ "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_monthly" - ] + ], } get_translated_dict = { ("doctype", "System Settings"): "frappe.geo.country_info.get_translated_dict", - ("page", "setup-wizard"): "frappe.geo.country_info.get_translated_dict" + ("page", "setup-wizard"): "frappe.geo.country_info.get_translated_dict", } sounds = [ @@ -284,13 +278,13 @@ sounds = [ setup_wizard_exception = [ "frappe.desk.page.setup_wizard.setup_wizard.email_setup_wizard_exception", - "frappe.desk.page.setup_wizard.setup_wizard.log_setup_wizard_exception" + "frappe.desk.page.setup_wizard.setup_wizard.log_setup_wizard_exception", ] -before_migrate = ['frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute'] -after_migrate = ['frappe.website.doctype.website_theme.website_theme.after_migrate'] +before_migrate = ["frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute"] +after_migrate = ["frappe.website.doctype.website_theme.website_theme.after_migrate"] -otp_methods = ['OTP App','Email','SMS'] +otp_methods = ["OTP App", "Email", "SMS"] user_data_fields = [ {"doctype": "Access Log", "strict": True}, @@ -372,7 +366,7 @@ global_search_doctypes = { {"doctype": "Letter Head"}, {"doctype": "Workflow"}, {"doctype": "Web Page"}, - {"doctype": "Web Form"} + {"doctype": "Web Form"}, ] } diff --git a/frappe/installer.py b/frappe/installer.py index c7dacc4ac1..3ce2bdf293 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -5,7 +5,7 @@ import json import os import sys from collections import OrderedDict -from typing import List, Dict, Tuple +from typing import Dict, List, Tuple import frappe from frappe.defaults import _clear_cache @@ -47,6 +47,7 @@ def _new_site( if not db_name: import hashlib + db_name = "_" + hashlib.sha1(os.path.realpath(frappe.get_site_path()).encode()).hexdigest()[:16] try: @@ -86,27 +87,45 @@ def _new_site( scheduler.toggle_scheduler(enable_scheduler) frappe.db.commit() - scheduler_status = ( - "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled" - ) + scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled" print("*** Scheduler is", scheduler_status, "***") -def install_db(root_login=None, root_password=None, db_name=None, source_sql=None, - admin_password=None, verbose=True, force=0, site_config=None, reinstall=False, - db_password=None, db_type=None, db_host=None, db_port=None, no_mariadb_socket=False): +def install_db( + root_login=None, + root_password=None, + db_name=None, + source_sql=None, + admin_password=None, + verbose=True, + force=0, + site_config=None, + reinstall=False, + db_password=None, + db_type=None, + db_host=None, + db_port=None, + no_mariadb_socket=False, +): import frappe.database from frappe.database import setup_database if not db_type: - db_type = frappe.conf.db_type or 'mariadb' + db_type = frappe.conf.db_type or "mariadb" - if not root_login and db_type == 'mariadb': - root_login='root' - elif not root_login and db_type == 'postgres': - root_login='postgres' + if not root_login and db_type == "mariadb": + root_login = "root" + elif not root_login and db_type == "postgres": + root_login = "postgres" - make_conf(db_name, site_config=site_config, db_password=db_password, db_type=db_type, db_host=db_host, db_port=db_port) + make_conf( + db_name, + site_config=site_config, + db_password=db_password, + db_type=db_type, + db_host=db_host, + db_port=db_port, + ) frappe.flags.in_install_db = True frappe.flags.root_login = root_login @@ -125,7 +144,7 @@ def install_db(root_login=None, root_password=None, db_name=None, source_sql=Non def find_org(org_repo: str) -> Tuple[str, str]: - """ find the org a repo is in + """find the org a repo is in find_org() ref -> https://github.com/frappe/bench/blob/develop/bench/utils/__init__.py#L390 @@ -138,9 +157,10 @@ def find_org(org_repo: str) -> Tuple[str, str]: :return: organisation and repository :rtype: Tuple[str, str] """ - from frappe.exceptions import InvalidRemoteException import requests + from frappe.exceptions import InvalidRemoteException + for org in ["frappe", "erpnext"]: response = requests.head(f"https://api.github.com/repos/{org}/{org_repo}") if response.status_code == 400: @@ -152,7 +172,7 @@ def find_org(org_repo: str) -> Tuple[str, str]: def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]: - """ parse org, repo, tag from string + """parse org, repo, tag from string fetch_details_from_tag() ref -> https://github.com/frappe/bench/blob/develop/bench/utils/__init__.py#L403 @@ -252,7 +272,7 @@ def install_app(name, verbose=False, set_as_patched=True): add_to_installed_apps(name) - frappe.get_doc('Portal Settings', 'Portal Settings').sync_menu() + frappe.get_doc("Portal Settings", "Portal Settings").sync_menu() if set_as_patched: set_all_patches_as_completed(name) @@ -265,7 +285,7 @@ def install_app(name, verbose=False, set_as_patched=True): sync_customizations(name) for after_sync in app_hooks.after_sync or []: - frappe.get_attr(after_sync)() # + frappe.get_attr(after_sync)() # frappe.flags.in_install = False @@ -284,7 +304,9 @@ def remove_from_installed_apps(app_name): installed_apps = frappe.get_installed_apps() if app_name in installed_apps: installed_apps.remove(app_name) - frappe.db.set_value("DefaultValue", {"defkey": "installed_apps"}, "defvalue", json.dumps(installed_apps)) + frappe.db.set_value( + "DefaultValue", {"defkey": "installed_apps"}, "defvalue", json.dumps(installed_apps) + ) _clear_cache("__global") frappe.db.commit() if frappe.flags.in_install: @@ -332,7 +354,7 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False) if not dry_run: remove_from_installed_apps(app_name) - frappe.get_single('Installed Applications').update_versions() + frappe.get_single("Installed Applications").update_versions() frappe.db.commit() for after_uninstall in app_hooks.after_uninstall or []: @@ -343,11 +365,11 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False) def _delete_modules(modules: List[str], dry_run: bool) -> List[str]: - """ Delete modules belonging to the app and all related doctypes. + """Delete modules belonging to the app and all related doctypes. - Note: All record linked linked to Module Def are also deleted. + Note: All record linked linked to Module Def are also deleted. - Returns: list of deleted doctypes.""" + Returns: list of deleted doctypes.""" drop_doctypes = [] doctype_link_field_map = _get_module_linked_doctype_field_map() @@ -375,10 +397,8 @@ def _delete_modules(modules: List[str], dry_run: bool) -> List[str]: def _delete_linked_documents( - module_name: str, - doctype_linkfield_map: Dict[str, str], - dry_run: bool - ) -> None: + module_name: str, doctype_linkfield_map: Dict[str, str], dry_run: bool +) -> None: """Deleted all records linked with module def""" for doctype, fieldname in doctype_linkfield_map.items(): @@ -387,22 +407,25 @@ def _delete_linked_documents( if not dry_run: frappe.delete_doc(doctype, record, ignore_on_trash=True, force=True) + def _get_module_linked_doctype_field_map() -> Dict[str, str]: - """ Get all the doctypes which have module linked with them. + """Get all the doctypes which have module linked with them. - returns ordered dictionary with doctype->link field mapping.""" + returns ordered dictionary with doctype->link field mapping.""" # Hardcoded to change order of deletion ordered_doctypes = [ - ("Workspace", "module"), - ("Report", "module"), - ("Page", "module"), - ("Web Form", "module") + ("Workspace", "module"), + ("Report", "module"), + ("Page", "module"), + ("Web Form", "module"), ] doctype_to_field_map = OrderedDict(ordered_doctypes) linked_doctypes = frappe.get_all( - "DocField", filters={"fieldtype": "Link", "options": "Module Def"}, fields=["parent", "fieldname"] + "DocField", + filters={"fieldtype": "Link", "options": "Module Def"}, + fields=["parent", "fieldname"], ) existing_linked_doctypes = [d for d in linked_doctypes if frappe.db.exists("DocType", d.parent)] @@ -438,32 +461,35 @@ def set_all_patches_as_completed(app): patches = get_patches_from_app(app) for patch in patches: - frappe.get_doc({ - "doctype": "Patch Log", - "patch": patch - }).insert(ignore_permissions=True) + frappe.get_doc({"doctype": "Patch Log", "patch": patch}).insert(ignore_permissions=True) frappe.db.commit() def init_singles(): - singles = [single['name'] for single in frappe.get_all("DocType", filters={'issingle': True})] + singles = [single["name"] for single in frappe.get_all("DocType", filters={"issingle": True})] for single in singles: if not frappe.db.get_singles_dict(single): doc = frappe.new_doc(single) - doc.flags.ignore_mandatory=True - doc.flags.ignore_validate=True + doc.flags.ignore_mandatory = True + doc.flags.ignore_validate = True doc.save() -def make_conf(db_name=None, db_password=None, site_config=None, db_type=None, db_host=None, db_port=None): +def make_conf( + db_name=None, db_password=None, site_config=None, db_type=None, db_host=None, db_port=None +): site = frappe.local.site - make_site_config(db_name, db_password, site_config, db_type=db_type, db_host=db_host, db_port=db_port) + make_site_config( + db_name, db_password, site_config, db_type=db_type, db_host=db_host, db_port=db_port + ) sites_path = frappe.local.sites_path frappe.destroy() frappe.init(site, sites_path=sites_path) -def make_site_config(db_name=None, db_password=None, site_config=None, db_type=None, db_host=None, db_port=None): +def make_site_config( + db_name=None, db_password=None, site_config=None, db_type=None, db_host=None, db_port=None +): frappe.create_folder(os.path.join(frappe.local.site_path)) site_file = get_site_config_path() @@ -472,13 +498,13 @@ def make_site_config(db_name=None, db_password=None, site_config=None, db_type=N site_config = get_conf_params(db_name, db_password) if db_type: - site_config['db_type'] = db_type + site_config["db_type"] = db_type if db_host: - site_config['db_host'] = db_host + site_config["db_host"] = db_host if db_port: - site_config['db_port'] = db_port + site_config["db_port"] = db_port with open(site_file, "w") as f: f.write(json.dumps(site_config, indent=1, sort_keys=True)) @@ -493,12 +519,14 @@ def update_site_config(key, value, validate=True, site_config_path=None): site_config = json.loads(f.read()) # In case of non-int value - if value in ('0', '1'): + if value in ("0", "1"): value = int(value) # boolean - if value == 'false': value = False - if value == 'true': value = True + if value == "false": + value = False + if value == "true": + value = True # remove key if value is None if value == "None": @@ -526,6 +554,7 @@ def get_conf_params(db_name=None, db_password=None): if not db_password: from frappe.utils import random_string + db_password = random_string(16) return {"db_name": db_name, "db_password": db_password} @@ -556,7 +585,7 @@ def add_module_defs(app): def remove_missing_apps(): import importlib - apps = ('frappe_subscription', 'shopping_cart') + apps = ("frappe_subscription", "shopping_cart") installed_apps = json.loads(frappe.db.get_global("installed_apps") or "[]") for app in apps: if app in installed_apps: @@ -574,15 +603,16 @@ def extract_sql_from_archive(sql_file_path): root directory or the sites sub-directory. Args: - sql_file_path (str): Path of the SQL file + sql_file_path (str): Path of the SQL file Returns: - str: Path of the decompressed SQL file + str: Path of the decompressed SQL file """ from frappe.utils import get_bench_relative_path + sql_file_path = get_bench_relative_path(sql_file_path) # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file - if sql_file_path.endswith('sql.gz'): + if sql_file_path.endswith("sql.gz"): decompressed_file_name = extract_sql_gzip(sql_file_path) else: decompressed_file_name = sql_file_path @@ -597,9 +627,10 @@ def convert_archive_content(sql_file_path): if frappe.conf.db_type == "mariadb": # ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed # this step is added to ease restoring sites depending on older mariaDB servers - from frappe.utils import random_string from pathlib import Path + from frappe.utils import random_string + old_sql_file_path = Path(f"{sql_file_path}_{random_string(10)}") sql_file_path = Path(sql_file_path) @@ -619,7 +650,7 @@ def extract_sql_gzip(sql_gz_path): try: original_file = sql_gz_path decompressed_file = original_file.rstrip(".gz") - cmd = 'gzip --decompress --force < {0} > {1}'.format(original_file, decompressed_file) + cmd = "gzip --decompress --force < {0} > {1}".format(original_file, decompressed_file) subprocess.check_call(cmd, shell=True) except Exception: raise @@ -630,6 +661,7 @@ def extract_sql_gzip(sql_gz_path): def extract_files(site_name, file_path): import shutil import subprocess + from frappe.utils import get_bench_relative_path file_path = get_bench_relative_path(file_path) @@ -647,9 +679,9 @@ def extract_files(site_name, file_path): try: if file_path.endswith(".tar"): - subprocess.check_output(['tar', 'xvf', tar_path, '--strip', '2'], cwd=abs_site_path) + subprocess.check_output(["tar", "xvf", tar_path, "--strip", "2"], cwd=abs_site_path) elif file_path.endswith(".tgz"): - subprocess.check_output(['tar', 'zxvf', tar_path, '--strip', '2'], cwd=abs_site_path) + subprocess.check_output(["tar", "zxvf", tar_path, "--strip", "2"], cwd=abs_site_path) except: raise finally: @@ -667,6 +699,7 @@ def is_downgrade(sql_file_path, verbose=False): return False from semantic_version import Version + head = "INSERT INTO `tabInstalled Application` VALUES" with open(sql_file_path) as f: @@ -676,9 +709,13 @@ def is_downgrade(sql_file_path, verbose=False): line = line.strip().lstrip(head).rstrip(";").strip() app_rows = frappe.safe_eval(line) # check if iterable consists of tuples before trying to transform - apps_list = app_rows if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows) else (app_rows, ) + apps_list = ( + app_rows + if all(isinstance(app_row, (tuple, list, set)) for app_row in app_rows) + else (app_rows,) + ) # 'all_apps' (list) format: [('frappe', '12.x.x-develop ()', 'develop'), ('your_custom_app', '0.0.1', 'master')] - all_apps = [ x[-3:] for x in apps_list ] + all_apps = [x[-3:] for x in apps_list] for app in all_apps: app_name = app[0] @@ -713,13 +750,16 @@ def partial_restore(sql_file_path, verbose=False): if frappe.conf.db_type in (None, "mariadb"): from frappe.database.mariadb.setup_db import import_db_from_sql elif frappe.conf.db_type == "postgres": - from frappe.database.postgres.setup_db import import_db_from_sql import warnings + from click import style + + from frappe.database.postgres.setup_db import import_db_from_sql + warn = style( "Delete the tables you want to restore manually before attempting" " partial restore operation for PostreSQL databases", - fg="yellow" + fg="yellow", ) warnings.warn(warn) @@ -734,8 +774,8 @@ def validate_database_sql(path, _raise=True): """Check if file has contents and if DefaultValue table exists Args: - path (str): Path of the decompressed SQL file - _raise (bool, optional): Raise exception if invalid file. Defaults to True. + path (str): Path of the decompressed SQL file + _raise (bool, optional): Raise exception if invalid file. Defaults to True. """ empty_file = False missing_table = True @@ -750,7 +790,7 @@ def validate_database_sql(path, _raise=True): if not empty_file: with open(path, "r") as f: for line in f: - if 'tabDefaultValue' in line: + if "tabDefaultValue" in line: missing_table = False break @@ -759,6 +799,7 @@ def validate_database_sql(path, _raise=True): if error_message: import click + click.secho(error_message, fg="red") if _raise and (missing_table or empty_file): diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.py b/frappe/integrations/doctype/braintree_settings/braintree_settings.py index 59751185b9..ca7ab0cfdf 100644 --- a/frappe/integrations/doctype/braintree_settings/braintree_settings.py +++ b/frappe/integrations/doctype/braintree_settings/braintree_settings.py @@ -2,25 +2,154 @@ # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe -from frappe.model.document import Document +from urllib.parse import urlencode + import braintree + +import frappe from frappe import _ -from urllib.parse import urlencode -from frappe.utils import get_url, call_hook_method -from frappe.integrations.utils import create_request_log, create_payment_gateway +from frappe.integrations.utils import create_payment_gateway, create_request_log +from frappe.model.document import Document +from frappe.utils import call_hook_method, get_url + class BraintreeSettings(Document): supported_currencies = [ - "AED","AMD","AOA","ARS","AUD","AWG","AZN","BAM","BBD","BDT","BGN","BIF","BMD","BND","BOB", - "BRL","BSD","BWP","BYN","BZD","CAD","CHF","CLP","CNY","COP","CRC","CVE","CZK","DJF","DKK", - "DOP","DZD","EGP","ETB","EUR","FJD","FKP","GBP","GEL","GHS","GIP","GMD","GNF","GTQ","GYD", - "HKD","HNL","HRK","HTG","HUF","IDR","ILS","INR","ISK","JMD","JPY","KES","KGS","KHR","KMF", - "KRW","KYD","KZT","LAK","LBP","LKR","LRD","LSL","LTL","MAD","MDL","MKD","MNT","MOP","MUR", - "MVR","MWK","MXN","MYR","MZN","NAD","NGN","NIO","NOK","NPR","NZD","PAB","PEN","PGK","PHP", - "PKR","PLN","PYG","QAR","RON","RSD","RUB","RWF","SAR","SBD","SCR","SEK","SGD","SHP","SLL", - "SOS","SRD","STD","SVC","SYP","SZL","THB","TJS","TOP","TRY","TTD","TWD","TZS","UAH","UGX", - "USD","UYU","UZS","VEF","VND","VUV","WST","XAF","XCD","XOF","XPF","YER","ZAR","ZMK","ZWD" + "AED", + "AMD", + "AOA", + "ARS", + "AUD", + "AWG", + "AZN", + "BAM", + "BBD", + "BDT", + "BGN", + "BIF", + "BMD", + "BND", + "BOB", + "BRL", + "BSD", + "BWP", + "BYN", + "BZD", + "CAD", + "CHF", + "CLP", + "CNY", + "COP", + "CRC", + "CVE", + "CZK", + "DJF", + "DKK", + "DOP", + "DZD", + "EGP", + "ETB", + "EUR", + "FJD", + "FKP", + "GBP", + "GEL", + "GHS", + "GIP", + "GMD", + "GNF", + "GTQ", + "GYD", + "HKD", + "HNL", + "HRK", + "HTG", + "HUF", + "IDR", + "ILS", + "INR", + "ISK", + "JMD", + "JPY", + "KES", + "KGS", + "KHR", + "KMF", + "KRW", + "KYD", + "KZT", + "LAK", + "LBP", + "LKR", + "LRD", + "LSL", + "LTL", + "MAD", + "MDL", + "MKD", + "MNT", + "MOP", + "MUR", + "MVR", + "MWK", + "MXN", + "MYR", + "MZN", + "NAD", + "NGN", + "NIO", + "NOK", + "NPR", + "NZD", + "PAB", + "PEN", + "PGK", + "PHP", + "PKR", + "PLN", + "PYG", + "QAR", + "RON", + "RSD", + "RUB", + "RWF", + "SAR", + "SBD", + "SCR", + "SEK", + "SGD", + "SHP", + "SLL", + "SOS", + "SRD", + "STD", + "SVC", + "SYP", + "SZL", + "THB", + "TJS", + "TOP", + "TRY", + "TTD", + "TWD", + "TZS", + "UAH", + "UGX", + "USD", + "UYU", + "UZS", + "VEF", + "VND", + "VUV", + "WST", + "XAF", + "XCD", + "XOF", + "XPF", + "YER", + "ZAR", + "ZMK", + "ZWD", ] def validate(self): @@ -28,25 +157,31 @@ class BraintreeSettings(Document): self.configure_braintree() def on_update(self): - create_payment_gateway('Braintree-' + self.gateway_name, settings='Braintree Settings', controller=self.gateway_name) - call_hook_method('payment_gateway_enabled', gateway='Braintree-' + self.gateway_name) + create_payment_gateway( + "Braintree-" + self.gateway_name, settings="Braintree Settings", controller=self.gateway_name + ) + call_hook_method("payment_gateway_enabled", gateway="Braintree-" + self.gateway_name) def configure_braintree(self): if self.use_sandbox: - environment = 'sandbox' + environment = "sandbox" else: - environment = 'production' + environment = "production" braintree.Configuration.configure( environment=environment, merchant_id=self.merchant_id, public_key=self.public_key, - private_key=self.get_password(fieldname='private_key',raise_exception=False) + private_key=self.get_password(fieldname="private_key", raise_exception=False), ) def validate_transaction_currency(self, currency): if currency not in self.supported_currencies: - frappe.throw(_("Please select another payment method. Stripe does not support transactions in currency '{0}'").format(currency)) + frappe.throw( + _( + "Please select another payment method. Stripe does not support transactions in currency '{0}'" + ).format(currency) + ) def get_payment_url(self, **kwargs): return get_url("./integrations/braintree_checkout?{0}".format(urlencode(kwargs))) @@ -60,48 +195,62 @@ class BraintreeSettings(Document): except Exception: frappe.log_error(frappe.get_traceback()) - return{ - "redirect_to": frappe.redirect_to_message(_('Server Error'), _("There seems to be an issue with the server's braintree configuration. Don't worry, in case of failure, the amount will get refunded to your account.")), - "status": 401 + return { + "redirect_to": frappe.redirect_to_message( + _("Server Error"), + _( + "There seems to be an issue with the server's braintree configuration. Don't worry, in case of failure, the amount will get refunded to your account." + ), + ), + "status": 401, } def create_charge_on_braintree(self): self.configure_braintree() - redirect_to = self.data.get('redirect_to') or None - redirect_message = self.data.get('redirect_message') or None + redirect_to = self.data.get("redirect_to") or None + redirect_message = self.data.get("redirect_message") or None - result = braintree.Transaction.sale({ - "amount": self.data.amount, - "payment_method_nonce": self.data.payload_nonce, - "options": { - "submit_for_settlement": True + result = braintree.Transaction.sale( + { + "amount": self.data.amount, + "payment_method_nonce": self.data.payload_nonce, + "options": {"submit_for_settlement": True}, } - }) + ) if result.is_success: - self.integration_request.db_set('status', 'Completed', update_modified=False) + self.integration_request.db_set("status", "Completed", update_modified=False) self.flags.status_changed_to = "Completed" - self.integration_request.db_set('output', result.transaction.status, update_modified=False) + self.integration_request.db_set("output", result.transaction.status, update_modified=False) elif result.transaction: - self.integration_request.db_set('status', 'Failed', update_modified=False) - error_log = frappe.log_error("code: " + str(result.transaction.processor_response_code) + " | text: " + str(result.transaction.processor_response_text), "Braintree Payment Error") - self.integration_request.db_set('error', error_log.error, update_modified=False) + self.integration_request.db_set("status", "Failed", update_modified=False) + error_log = frappe.log_error( + "code: " + + str(result.transaction.processor_response_code) + + " | text: " + + str(result.transaction.processor_response_text), + "Braintree Payment Error", + ) + self.integration_request.db_set("error", error_log.error, update_modified=False) else: - self.integration_request.db_set('status', 'Failed', update_modified=False) + self.integration_request.db_set("status", "Failed", update_modified=False) for error in result.errors.deep_errors: - error_log = frappe.log_error("code: " + str(error.code) + " | message: " + str(error.message), "Braintree Payment Error") - self.integration_request.db_set('error', error_log.error, update_modified=False) + error_log = frappe.log_error( + "code: " + str(error.code) + " | message: " + str(error.message), "Braintree Payment Error" + ) + self.integration_request.db_set("error", error_log.error, update_modified=False) if self.flags.status_changed_to == "Completed": - status = 'Completed' + status = "Completed" if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: - custom_redirect_to = frappe.get_doc(self.data.reference_doctype, - self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) - braintree_success_page = frappe.get_hooks('braintree_success_page') + custom_redirect_to = frappe.get_doc( + self.data.reference_doctype, self.data.reference_docname + ).run_method("on_payment_authorized", self.flags.status_changed_to) + braintree_success_page = frappe.get_hooks("braintree_success_page") if braintree_success_page: custom_redirect_to = frappe.get_attr(braintree_success_page[-1])(self.data) except Exception: @@ -110,26 +259,27 @@ class BraintreeSettings(Document): if custom_redirect_to: redirect_to = custom_redirect_to - redirect_url = 'payment-success' + redirect_url = "payment-success" else: - status = 'Error' - redirect_url = 'payment-failed' + status = "Error" + redirect_url = "payment-failed" if redirect_to: - redirect_url += '?' + urlencode({'redirect_to': redirect_to}) + redirect_url += "?" + urlencode({"redirect_to": redirect_to}) if redirect_message: - redirect_url += '&' + urlencode({'redirect_message': redirect_message}) + redirect_url += "&" + urlencode({"redirect_message": redirect_message}) + + return {"redirect_to": redirect_url, "status": status} - return { - "redirect_to": redirect_url, - "status": status - } def get_gateway_controller(doc): payment_request = frappe.get_doc("Payment Request", doc) - gateway_controller = frappe.db.get_value("Payment Gateway", payment_request.payment_gateway, "gateway_controller") + gateway_controller = frappe.db.get_value( + "Payment Gateway", payment_request.payment_gateway, "gateway_controller" + ) return gateway_controller + def get_client_token(doc): gateway_controller = get_gateway_controller(doc) settings = frappe.get_doc("Braintree Settings", gateway_controller) diff --git a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py b/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py index 721158fb4a..475a62be79 100644 --- a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py +++ b/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestBraintreeSettings(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py index fcb5fe7ee9..e472193da8 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.py +++ b/frappe/integrations/doctype/connected_app/connected_app.py @@ -3,17 +3,18 @@ # License: MIT. See LICENSE import os -from urllib.parse import urljoin -from urllib.parse import urlencode +from urllib.parse import urlencode, urljoin + +from requests_oauthlib import OAuth2Session import frappe from frappe import _ from frappe.model.document import Document -from requests_oauthlib import OAuth2Session -if any((os.getenv('CI'), frappe.conf.developer_mode, frappe.conf.allow_tests)): +if any((os.getenv("CI"), frappe.conf.developer_mode, frappe.conf.allow_tests)): # Disable mandatory TLS in developer mode and tests - os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + class ConnectedApp(Document): """Connect to a remote oAuth Server. Retrieve and store user's access token @@ -22,7 +23,9 @@ class ConnectedApp(Document): def validate(self): base_url = frappe.utils.get_url() - callback_path = '/api/method/frappe.integrations.doctype.connected_app.connected_app.callback/' + self.name + callback_path = ( + "/api/method/frappe.integrations.doctype.connected_app.connected_app.callback/" + self.name + ) self.redirect_uri = urljoin(base_url, callback_path) def get_oauth2_session(self, user=None, init=False): @@ -36,10 +39,10 @@ class ConnectedApp(Document): token_cache = self.get_user_token(user) token = token_cache.get_json() token_updater = token_cache.update_data - auto_refresh_kwargs = {'client_id': self.client_id} - client_secret = self.get_password('client_secret') + auto_refresh_kwargs = {"client_id": self.client_id} + client_secret = self.get_password("client_secret") if client_secret: - auto_refresh_kwargs['client_secret'] = client_secret + auto_refresh_kwargs["client_secret"] = client_secret return OAuth2Session( client_id=self.client_id, @@ -48,7 +51,7 @@ class ConnectedApp(Document): auto_refresh_url=self.token_uri, auto_refresh_kwargs=auto_refresh_kwargs, redirect_uri=self.redirect_uri, - scope=self.get_scopes() + scope=self.get_scopes(), ) @frappe.whitelist() @@ -61,7 +64,7 @@ class ConnectedApp(Document): token_cache = self.get_token_cache(user) if not token_cache: - token_cache = frappe.new_doc('Token Cache') + token_cache = frappe.new_doc("Token Cache") token_cache.user = user token_cache.connected_app = self.name @@ -81,16 +84,16 @@ class ConnectedApp(Document): return token_cache redirect = self.initiate_web_application_flow(user, success_uri) - frappe.local.response['type'] = 'redirect' - frappe.local.response['location'] = redirect + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = redirect return redirect def get_token_cache(self, user): token_cache = None - token_cache_name = self.name + '-' + user + token_cache_name = self.name + "-" + user - if frappe.db.exists('Token Cache', token_cache_name): - token_cache = frappe.get_doc('Token Cache', token_cache_name) + if frappe.db.exists("Token Cache", token_cache_name): + token_cache = frappe.get_doc("Token Cache", token_cache_name) return token_cache @@ -109,33 +112,34 @@ def callback(code=None, state=None): transmit a code that can be used by the local server to obtain an access token. """ - if frappe.request.method != 'GET': - frappe.throw(_('Invalid request method: {}').format(frappe.request.method)) + if frappe.request.method != "GET": + frappe.throw(_("Invalid request method: {}").format(frappe.request.method)) - if frappe.session.user == 'Guest': - frappe.local.response['type'] = 'redirect' - frappe.local.response['location'] = '/login?' + urlencode({'redirect-to': frappe.request.url}) + if frappe.session.user == "Guest": + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = "/login?" + urlencode({"redirect-to": frappe.request.url}) return - path = frappe.request.path[1:].split('/') + path = frappe.request.path[1:].split("/") if len(path) != 4 or not path[3]: - frappe.throw(_('Invalid Parameters.')) + frappe.throw(_("Invalid Parameters.")) - connected_app = frappe.get_doc('Connected App', path[3]) - token_cache = frappe.get_doc('Token Cache', connected_app.name + '-' + frappe.session.user) + connected_app = frappe.get_doc("Connected App", path[3]) + token_cache = frappe.get_doc("Token Cache", connected_app.name + "-" + frappe.session.user) if state != token_cache.state: - frappe.throw(_('Invalid state.')) + frappe.throw(_("Invalid state.")) oauth_session = connected_app.get_oauth2_session(init=True) query_params = connected_app.get_query_params() - token = oauth_session.fetch_token(connected_app.token_uri, + token = oauth_session.fetch_token( + connected_app.token_uri, code=code, - client_secret=connected_app.get_password('client_secret'), + client_secret=connected_app.get_password("client_secret"), include_client_id=True, **query_params ) token_cache.update_data(token) - frappe.local.response['type'] = 'redirect' - frappe.local.response['location'] = token_cache.get('success_uri') or connected_app.get_url() + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = token_cache.get("success_uri") or connected_app.get_url() diff --git a/frappe/integrations/doctype/connected_app/test_connected_app.py b/frappe/integrations/doctype/connected_app/test_connected_app.py index eff7104ce0..1597ec75bf 100644 --- a/frappe/integrations/doctype/connected_app/test_connected_app.py +++ b/frappe/integrations/doctype/connected_app/test_connected_app.py @@ -2,48 +2,47 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE import unittest -import requests from urllib.parse import urljoin +import requests + import frappe -from frappe.integrations.doctype.social_login_key.test_social_login_key import create_or_update_social_login_key +from frappe.integrations.doctype.social_login_key.test_social_login_key import ( + create_or_update_social_login_key, +) def get_user(usr, pwd): - user = frappe.new_doc('User') + user = frappe.new_doc("User") user.email = usr user.enabled = 1 user.first_name = "_Test" user.new_password = pwd user.roles = [] - user.append('roles', { - 'doctype': 'Has Role', - 'parentfield': 'roles', - 'role': 'System Manager' - }) + user.append("roles", {"doctype": "Has Role", "parentfield": "roles", "role": "System Manager"}) user.insert() return user def get_connected_app(): - doctype = 'Connected App' + doctype = "Connected App" connected_app = frappe.new_doc(doctype) - connected_app.provider_name = 'frappe' + connected_app.provider_name = "frappe" connected_app.scopes = [] - connected_app.append('scopes', {'scope': 'all'}) + connected_app.append("scopes", {"scope": "all"}) connected_app.insert() return connected_app def get_oauth_client(): - oauth_client = frappe.new_doc('OAuth Client') - oauth_client.app_name = '_Test Connected App' - oauth_client.redirect_uris = 'to be replaced' - oauth_client.default_redirect_uri = 'to be replaced' - oauth_client.grant_type = 'Authorization Code' - oauth_client.response_type = 'Code' + oauth_client = frappe.new_doc("OAuth Client") + oauth_client.app_name = "_Test Connected App" + oauth_client.redirect_uris = "to be replaced" + oauth_client.default_redirect_uri = "to be replaced" + oauth_client.grant_type = "Authorization Code" + oauth_client.response_type = "Code" oauth_client.skip_authorization = 1 oauth_client.insert() @@ -51,7 +50,6 @@ def get_oauth_client(): class TestConnectedApp(unittest.TestCase): - def setUp(self): """Set up a Connected App that connects to our own oAuth provider. @@ -64,32 +62,31 @@ class TestConnectedApp(unittest.TestCase): just endpoints) are stored in "Social Login Key" so we get them from there. """ - self.user_name = 'test-connected-app@example.com' - self.user_password = 'Eastern_43A1W' + self.user_name = "test-connected-app@example.com" + self.user_password = "Eastern_43A1W" self.user = get_user(self.user_name, self.user_password) self.connected_app = get_connected_app() self.oauth_client = get_oauth_client() social_login_key = create_or_update_social_login_key() - self.base_url = social_login_key.get('base_url') + self.base_url = social_login_key.get("base_url") frappe.db.commit() self.connected_app.reload() self.oauth_client.reload() - redirect_uri = self.connected_app.get('redirect_uri') - self.oauth_client.update({ - 'redirect_uris': redirect_uri, - 'default_redirect_uri': redirect_uri - }) + redirect_uri = self.connected_app.get("redirect_uri") + self.oauth_client.update({"redirect_uris": redirect_uri, "default_redirect_uri": redirect_uri}) self.oauth_client.save() - self.connected_app.update({ - 'authorization_uri': urljoin(self.base_url, social_login_key.get('authorize_url')), - 'client_id': self.oauth_client.get('client_id'), - 'client_secret': self.oauth_client.get('client_secret'), - 'token_uri': urljoin(self.base_url, social_login_key.get('access_token_url')) - }) + self.connected_app.update( + { + "authorization_uri": urljoin(self.base_url, social_login_key.get("authorize_url")), + "client_id": self.oauth_client.get("client_id"), + "client_secret": self.oauth_client.get("client_secret"), + "token_uri": urljoin(self.base_url, social_login_key.get("access_token_url")), + } + ) self.connected_app.save() frappe.db.commit() @@ -98,11 +95,12 @@ class TestConnectedApp(unittest.TestCase): def test_web_application_flow(self): """Simulate a logged in user who opens the authorization URL.""" + def login(): - return session.get(urljoin(self.base_url, '/api/method/login'), params={ - 'usr': self.user_name, - 'pwd': self.user_password - }) + return session.get( + urljoin(self.base_url, "/api/method/login"), + params={"usr": self.user_name, "pwd": self.user_password}, + ) session = requests.Session() @@ -118,12 +116,12 @@ class TestConnectedApp(unittest.TestCase): self.assertEqual(callback_response.status_code, 200) self.token_cache = self.connected_app.get_token_cache(self.user_name) - token = self.token_cache.get_password('access_token') + token = self.token_cache.get_password("access_token") self.assertNotEqual(token, None) oauth2_session = self.connected_app.get_oauth2_session(self.user_name) - resp = oauth2_session.get(urljoin(self.base_url, '/api/method/frappe.auth.get_logged_user')) - self.assertEqual(resp.json().get('message'), self.user_name) + resp = oauth2_session.get(urljoin(self.base_url, "/api/method/frappe.auth.get_logged_user")) + self.assertEqual(resp.json().get("message"), self.user_name) def tearDown(self): def delete_if_exists(attribute): @@ -131,25 +129,21 @@ class TestConnectedApp(unittest.TestCase): if doc: doc.delete() - delete_if_exists('token_cache') - delete_if_exists('connected_app') + delete_if_exists("token_cache") + delete_if_exists("connected_app") - if getattr(self, 'oauth_client', None): - tokens = frappe.get_all('OAuth Bearer Token', filters={ - 'client': self.oauth_client.name - }) + if getattr(self, "oauth_client", None): + tokens = frappe.get_all("OAuth Bearer Token", filters={"client": self.oauth_client.name}) for token in tokens: - doc = frappe.get_doc('OAuth Bearer Token', token.name) + doc = frappe.get_doc("OAuth Bearer Token", token.name) doc.delete() - codes = frappe.get_all('OAuth Authorization Code', filters={ - 'client': self.oauth_client.name - }) + codes = frappe.get_all("OAuth Authorization Code", filters={"client": self.oauth_client.name}) for code in codes: - doc = frappe.get_doc('OAuth Authorization Code', code.name) + doc = frappe.get_doc("OAuth Authorization Code", code.name) doc.delete() - delete_if_exists('user') - delete_if_exists('oauth_client') + delete_if_exists("user") + delete_if_exists("oauth_client") frappe.db.commit() diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py index 9ccd1c0210..8a6a7a4bfb 100644 --- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py @@ -11,12 +11,22 @@ from rq.timeouts import JobTimeoutException import frappe from frappe import _ -from frappe.integrations.offsite_backup_utils import (get_chunk_site, - get_latest_backup_file, send_email, validate_file_size) +from frappe.integrations.offsite_backup_utils import ( + get_chunk_site, + get_latest_backup_file, + send_email, + validate_file_size, +) from frappe.integrations.utils import make_post_request from frappe.model.document import Document -from frappe.utils import (cint, encode, get_backups_path, get_files_path, - get_request_site_address, get_url) +from frappe.utils import ( + cint, + encode, + get_backups_path, + get_files_path, + get_request_site_address, + get_url, +) from frappe.utils.background_jobs import enqueue from frappe.utils.backups import new_backup @@ -30,24 +40,33 @@ class DropboxSettings(Document): def validate(self): if self.enabled and self.limit_no_of_backups and self.no_of_backups < 1: - frappe.throw(_('Number of DB backups cannot be less than 1')) + frappe.throw(_("Number of DB backups cannot be less than 1")) + @frappe.whitelist() def take_backup(): """Enqueue longjob for taking backup to dropbox""" - enqueue("frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox", queue='long', timeout=1500) + enqueue( + "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox", + queue="long", + timeout=1500, + ) frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour.")) + def take_backups_daily(): take_backups_if("Daily") + def take_backups_weekly(): take_backups_if("Weekly") + def take_backups_if(freq): if frappe.db.get_value("Dropbox Settings", None, "backup_frequency") == freq: take_backup_to_dropbox() + def take_backup_to_dropbox(retry_count=0, upload_db_backup=True): did_not_upload, error_log = [], [] try: @@ -55,7 +74,8 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True): validate_file_size() did_not_upload, error_log = backup_to_dropbox(upload_db_backup) - if did_not_upload: raise Exception + if did_not_upload: + raise Exception if cint(frappe.db.get_value("Dropbox Settings", None, "send_email_for_successful_backup")): send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to") @@ -63,19 +83,24 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True): if retry_count < 2: args = { "retry_count": retry_count + 1, - "upload_db_backup": False #considering till worker timeout db backup is uploaded + "upload_db_backup": False, # considering till worker timeout db backup is uploaded } - enqueue("frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox", - queue='long', timeout=1500, **args) + enqueue( + "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox", + queue="long", + timeout=1500, + **args + ) except Exception: if isinstance(error_log, str): error_message = error_log + "\n" + frappe.get_traceback() else: file_and_error = [" - ".join(f) for f in zip(did_not_upload, error_log)] - error_message = ("\n".join(file_and_error) + "\n" + frappe.get_traceback()) + error_message = "\n".join(file_and_error) + "\n" + frappe.get_traceback() send_email(False, "Dropbox", "Dropbox Settings", "send_notifications_to", error_message) + def backup_to_dropbox(upload_db_backup=True): if not frappe.db: frappe.connect() @@ -83,18 +108,20 @@ def backup_to_dropbox(upload_db_backup=True): # upload database dropbox_settings = get_dropbox_settings() - if not dropbox_settings['access_token']: + if not dropbox_settings["access_token"]: access_token = generate_oauth2_access_token_from_oauth1_token(dropbox_settings) - if not access_token.get('oauth2_token'): - return 'Failed backup upload', 'No Access Token exists! Please generate the access token for Dropbox.' + if not access_token.get("oauth2_token"): + return ( + "Failed backup upload", + "No Access Token exists! Please generate the access token for Dropbox.", + ) - dropbox_settings['access_token'] = access_token['oauth2_token'] - set_dropbox_access_token(access_token['oauth2_token']) + dropbox_settings["access_token"] = access_token["oauth2_token"] + set_dropbox_access_token(access_token["oauth2_token"]) dropbox_client = dropbox.Dropbox( - oauth2_access_token=dropbox_settings['access_token'], - timeout=None + oauth2_access_token=dropbox_settings["access_token"], timeout=None ) if upload_db_backup: @@ -109,20 +136,25 @@ def backup_to_dropbox(upload_db_backup=True): upload_file_to_dropbox(site_config, "/database", dropbox_client) # delete older databases - if dropbox_settings['no_of_backups']: - delete_older_backups(dropbox_client, "/database", dropbox_settings['no_of_backups']) + if dropbox_settings["no_of_backups"]: + delete_older_backups(dropbox_client, "/database", dropbox_settings["no_of_backups"]) # upload files to files folder did_not_upload = [] error_log = [] - if dropbox_settings['file_backup']: + if dropbox_settings["file_backup"]: upload_from_folder(get_files_path(), 0, "/files", dropbox_client, did_not_upload, error_log) - upload_from_folder(get_files_path(is_private=1), 1, "/private/files", dropbox_client, did_not_upload, error_log) + upload_from_folder( + get_files_path(is_private=1), 1, "/private/files", dropbox_client, did_not_upload, error_log + ) return did_not_upload, list(set(error_log)) -def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not_upload, error_log): + +def upload_from_folder( + path, is_private, dropbox_folder, dropbox_client, did_not_upload, error_log +): if not os.path.exists(path): return @@ -133,11 +165,14 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not path = str(path) - for f in frappe.get_all("File", filters={"is_folder": 0, "is_private": is_private, - "uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']): + for f in frappe.get_all( + "File", + filters={"is_folder": 0, "is_private": is_private, "uploaded_to_dropbox": 0}, + fields=["file_url", "name", "file_name"], + ): if not f.file_url: continue - filename = f.file_url.rsplit('/', 1)[-1] + filename = f.file_url.rsplit("/", 1)[-1] filepath = os.path.join(path, filename) @@ -147,8 +182,9 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not found = False for file_metadata in response.entries: try: - if (os.path.basename(filepath) == file_metadata.name - and os.stat(encode(filepath)).st_size == int(file_metadata.size)): + if os.path.basename(filepath) == file_metadata.name and os.stat( + encode(filepath) + ).st_size == int(file_metadata.size): found = True update_file_dropbox_status(f.name) break @@ -163,6 +199,7 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not did_not_upload.append(filepath) error_log.append(frappe.get_traceback()) + def upload_file_to_dropbox(filename, folder, dropbox_client): """upload files with chunk of 15 mb to reduce session append calls""" if not os.path.exists(filename): @@ -172,9 +209,9 @@ def upload_file_to_dropbox(filename, folder, dropbox_client): file_size = os.path.getsize(encode(filename)) chunk_size = get_chunk_site(file_size) - mode = (dropbox.files.WriteMode.overwrite) + mode = dropbox.files.WriteMode.overwrite - f = open(encode(filename), 'rb') + f = open(encode(filename), "rb") path = "{0}/{1}".format(folder, os.path.basename(filename)) try: @@ -182,14 +219,18 @@ def upload_file_to_dropbox(filename, folder, dropbox_client): dropbox_client.files_upload(f.read(), path, mode) else: upload_session_start_result = dropbox_client.files_upload_session_start(f.read(chunk_size)) - cursor = dropbox.files.UploadSessionCursor(session_id=upload_session_start_result.session_id, offset=f.tell()) + cursor = dropbox.files.UploadSessionCursor( + session_id=upload_session_start_result.session_id, offset=f.tell() + ) commit = dropbox.files.CommitInfo(path=path, mode=mode) while f.tell() < file_size: - if ((file_size - f.tell()) <= chunk_size): + if (file_size - f.tell()) <= chunk_size: dropbox_client.files_upload_session_finish(f.read(chunk_size), cursor, commit) else: - dropbox_client.files_upload_session_append(f.read(chunk_size), cursor.session_id,cursor.offset) + dropbox_client.files_upload_session_append( + f.read(chunk_size), cursor.session_id, cursor.offset + ) cursor.offset = f.tell() except dropbox.exceptions.ApiError as e: if isinstance(e.error, dropbox.files.UploadError): @@ -199,6 +240,7 @@ def upload_file_to_dropbox(filename, folder, dropbox_client): else: raise + def create_folder_if_not_exists(folder, dropbox_client): try: dropbox_client.files_get_metadata(folder) @@ -209,13 +251,16 @@ def create_folder_if_not_exists(folder, dropbox_client): else: raise + def update_file_dropbox_status(file_name): - frappe.db.set_value("File", file_name, 'uploaded_to_dropbox', 1, update_modified=False) + frappe.db.set_value("File", file_name, "uploaded_to_dropbox", 1, update_modified=False) + def is_fresh_upload(): - file_name = frappe.db.get_value("File", {'uploaded_to_dropbox': 1}, 'name') + file_name = frappe.db.get_value("File", {"uploaded_to_dropbox": 1}, "name") return not file_name + def get_uploaded_files_meta(dropbox_folder, dropbox_client): try: return dropbox_client.files_list_folder(dropbox_folder) @@ -226,54 +271,64 @@ def get_uploaded_files_meta(dropbox_folder, dropbox_client): else: raise + def get_dropbox_settings(redirect_uri=False): if not frappe.conf.dropbox_broker_site: - frappe.conf.dropbox_broker_site = 'https://dropbox.erpnext.com' + frappe.conf.dropbox_broker_site = "https://dropbox.erpnext.com" settings = frappe.get_doc("Dropbox Settings") app_details = { "app_key": settings.app_access_key or frappe.conf.dropbox_access_key, "app_secret": settings.get_password(fieldname="app_secret_key", raise_exception=False) - if settings.app_secret_key else frappe.conf.dropbox_secret_key, - 'access_token': settings.get_password('dropbox_access_token', raise_exception=False) - if settings.dropbox_access_token else '', - 'access_key': settings.get_password('dropbox_access_key', raise_exception=False), - 'access_secret': settings.get_password('dropbox_access_secret', raise_exception=False), - 'file_backup':settings.file_backup, - 'no_of_backups': settings.no_of_backups if settings.limit_no_of_backups else None + if settings.app_secret_key + else frappe.conf.dropbox_secret_key, + "access_token": settings.get_password("dropbox_access_token", raise_exception=False) + if settings.dropbox_access_token + else "", + "access_key": settings.get_password("dropbox_access_key", raise_exception=False), + "access_secret": settings.get_password("dropbox_access_secret", raise_exception=False), + "file_backup": settings.file_backup, + "no_of_backups": settings.no_of_backups if settings.limit_no_of_backups else None, } if redirect_uri: - app_details.update({ - 'redirect_uri': get_request_site_address(True) \ - + '/api/method/frappe.integrations.doctype.dropbox_settings.dropbox_settings.dropbox_auth_finish' \ - if settings.app_secret_key else frappe.conf.dropbox_broker_site\ - + '/api/method/dropbox_erpnext_broker.www.setup_dropbox.generate_dropbox_access_token', - }) - - if not app_details['app_key'] or not app_details['app_secret']: + app_details.update( + { + "redirect_uri": get_request_site_address(True) + + "/api/method/frappe.integrations.doctype.dropbox_settings.dropbox_settings.dropbox_auth_finish" + if settings.app_secret_key + else frappe.conf.dropbox_broker_site + + "/api/method/dropbox_erpnext_broker.www.setup_dropbox.generate_dropbox_access_token", + } + ) + + if not app_details["app_key"] or not app_details["app_secret"]: raise Exception(_("Please set Dropbox access keys in your site config")) return app_details + def delete_older_backups(dropbox_client, folder_path, to_keep): res = dropbox_client.files_list_folder(path=folder_path) files = [] for f in res.entries: - if isinstance(f, dropbox.files.FileMetadata) and 'sql' in f.name: + if isinstance(f, dropbox.files.FileMetadata) and "sql" in f.name: files.append(f) if len(files) <= to_keep: return - files.sort(key=lambda item:item.client_modified, reverse=True) + files.sort(key=lambda item: item.client_modified, reverse=True) for f in files[to_keep:]: dropbox_client.files_delete(os.path.join(folder_path, f.name)) + @frappe.whitelist() def get_redirect_url(): if not frappe.conf.dropbox_broker_site: - frappe.conf.dropbox_broker_site = 'https://dropbox.erpnext.com' - url = "{0}/api/method/dropbox_erpnext_broker.www.setup_dropbox.get_authotize_url".format(frappe.conf.dropbox_broker_site) + frappe.conf.dropbox_broker_site = "https://dropbox.erpnext.com" + url = "{0}/api/method/dropbox_erpnext_broker.www.setup_dropbox.get_authotize_url".format( + frappe.conf.dropbox_broker_site + ) try: response = make_post_request(url, data={"site": get_url()}) @@ -283,9 +338,12 @@ def get_redirect_url(): except Exception: frappe.log_error() frappe.throw( - _("Something went wrong while generating dropbox access token. Please check error log for more details.") + _( + "Something went wrong while generating dropbox access token. Please check error log for more details." + ) ) + @frappe.whitelist() def get_dropbox_authorize_url(): app_details = get_dropbox_settings(redirect_uri=True) @@ -294,52 +352,52 @@ def get_dropbox_authorize_url(): redirect_uri=app_details["redirect_uri"], session={}, csrf_token_session_key="dropbox-auth-csrf-token", - consumer_secret=app_details["app_secret"] + consumer_secret=app_details["app_secret"], ) auth_url = dropbox_oauth_flow.start() - return { - "auth_url": auth_url, - "args": parse_qs(urlparse(auth_url).query) - } + return {"auth_url": auth_url, "args": parse_qs(urlparse(auth_url).query)} + @frappe.whitelist() def dropbox_auth_finish(return_access_token=False): app_details = get_dropbox_settings(redirect_uri=True) callback = frappe.form_dict - close = '

' + _('Please close this window') + '

' + close = '

' + _("Please close this window") + "

" dropbox_oauth_flow = dropbox.DropboxOAuth2Flow( consumer_key=app_details["app_key"], redirect_uri=app_details["redirect_uri"], - session={ - 'dropbox-auth-csrf-token': callback.state - }, + session={"dropbox-auth-csrf-token": callback.state}, csrf_token_session_key="dropbox-auth-csrf-token", - consumer_secret=app_details["app_secret"] + consumer_secret=app_details["app_secret"], ) if callback.state or callback.code: - token = dropbox_oauth_flow.finish({'state': callback.state, 'code': callback.code}) + token = dropbox_oauth_flow.finish({"state": callback.state, "code": callback.code}) if return_access_token and token.access_token: return token.access_token, callback.state set_dropbox_access_token(token.access_token) else: - frappe.respond_as_web_page(_("Dropbox Setup"), + frappe.respond_as_web_page( + _("Dropbox Setup"), _("Illegal Access Token. Please try again") + close, - indicator_color='red', - http_status_code=frappe.AuthenticationError.http_status_code) + indicator_color="red", + http_status_code=frappe.AuthenticationError.http_status_code, + ) + + frappe.respond_as_web_page( + _("Dropbox Setup"), _("Dropbox access is approved!") + close, indicator_color="green" + ) - frappe.respond_as_web_page(_("Dropbox Setup"), - _("Dropbox access is approved!") + close, - indicator_color='green') def set_dropbox_access_token(access_token): - frappe.db.set_value("Dropbox Settings", None, 'dropbox_access_token', access_token) + frappe.db.set_value("Dropbox Settings", None, "dropbox_access_token", access_token) frappe.db.commit() + def generate_oauth2_access_token_from_oauth1_token(dropbox_settings=None): if not dropbox_settings.get("access_key") or not dropbox_settings.get("access_secret"): return {} @@ -349,7 +407,7 @@ def generate_oauth2_access_token_from_oauth1_token(dropbox_settings=None): auth = (dropbox_settings["app_key"], dropbox_settings["app_secret"]) data = { "oauth1_token": dropbox_settings["access_key"], - "oauth1_token_secret": dropbox_settings["access_secret"] + "oauth1_token_secret": dropbox_settings["access_secret"], } return make_post_request(url, auth=auth, headers=headers, data=json.dumps(data)) diff --git a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py index 458f876444..e73cf03268 100644 --- a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDropboxSettings(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index 0d4c5bbe5c..71f0e83f80 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -16,8 +16,15 @@ import frappe from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url from frappe.model.document import Document -from frappe.utils import (add_days, add_to_date, get_datetime, - get_request_site_address, get_time_zone, get_weekdays, now_datetime) +from frappe.utils import ( + add_days, + add_to_date, + get_datetime, + get_request_site_address, + get_time_zone, + get_weekdays, + now_datetime, +) from frappe.utils.password import set_encrypted_password SCOPES = "https://www.googleapis.com/auth/calendar" @@ -26,7 +33,7 @@ google_calendar_frequencies = { "RRULE:FREQ=DAILY": "Daily", "RRULE:FREQ=WEEKLY": "Weekly", "RRULE:FREQ=MONTHLY": "Monthly", - "RRULE:FREQ=YEARLY": "Yearly" + "RRULE:FREQ=YEARLY": "Yearly", } google_calendar_days = { @@ -36,14 +43,14 @@ google_calendar_days = { "TH": "thursday", "FR": "friday", "SA": "saturday", - "SU": "sunday" + "SU": "sunday", } framework_frequencies = { "Daily": "RRULE:FREQ=DAILY;", "Weekly": "RRULE:FREQ=WEEKLY;", "Monthly": "RRULE:FREQ=MONTHLY;", - "Yearly": "RRULE:FREQ=YEARLY;" + "Yearly": "RRULE:FREQ=YEARLY;", } framework_days = { @@ -53,11 +60,11 @@ framework_days = { "thursday": "TH", "friday": "FR", "saturday": "SA", - "sunday": "SU" + "sunday": "SU", } -class GoogleCalendar(Document): +class GoogleCalendar(Document): def validate(self): google_settings = frappe.get_single("Google Settings") if not google_settings.enable: @@ -80,27 +87,35 @@ class GoogleCalendar(Document): "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), "refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False), "grant_type": "refresh_token", - "scope": SCOPES + "scope": SCOPES, } try: r = requests.post(get_auth_url(), data=data).json() except requests.exceptions.HTTPError: button_label = frappe.bold(_("Allow Google Calendar Access")) - frappe.throw(_("Something went wrong during the token generation. Click on {0} to generate a new one.").format(button_label)) + frappe.throw( + _( + "Something went wrong during the token generation. Click on {0} to generate a new one." + ).format(button_label) + ) return r.get("access_token") + @frappe.whitelist() def authorize_access(g_calendar, reauthorize=None): """ - If no Authorization code get it from Google and then request for Refresh Token. - Google Calendar Name is set to flags to set_value after Authorization Code is obtained. + If no Authorization code get it from Google and then request for Refresh Token. + Google Calendar Name is set to flags to set_value after Authorization Code is obtained. """ google_settings = frappe.get_doc("Google Settings") google_calendar = frappe.get_doc("Google Calendar", g_calendar) - redirect_uri = get_request_site_address(True) + "?cmd=frappe.integrations.doctype.google_calendar.google_calendar.google_callback" + redirect_uri = ( + get_request_site_address(True) + + "?cmd=frappe.integrations.doctype.google_calendar.google_calendar.google_callback" + ) if not google_calendar.authorization_code or reauthorize: frappe.cache().hset("google_calendar", "google_calendar", google_calendar.name) @@ -110,32 +125,42 @@ def authorize_access(g_calendar, reauthorize=None): data = { "code": google_calendar.get_password(fieldname="authorization_code", raise_exception=False), "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), + "client_secret": google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), "redirect_uri": redirect_uri, - "grant_type": "authorization_code" + "grant_type": "authorization_code", } r = requests.post(get_auth_url(), data=data).json() if "refresh_token" in r: - frappe.db.set_value("Google Calendar", google_calendar.name, "refresh_token", r.get("refresh_token")) + frappe.db.set_value( + "Google Calendar", google_calendar.name, "refresh_token", r.get("refresh_token") + ) frappe.db.commit() frappe.local.response["type"] = "redirect" - frappe.local.response["location"] = "/app/Form/{0}/{1}".format(quote("Google Calendar"), quote(google_calendar.name)) + frappe.local.response["location"] = "/app/Form/{0}/{1}".format( + quote("Google Calendar"), quote(google_calendar.name) + ) frappe.msgprint(_("Google Calendar has been configured.")) except Exception as e: frappe.throw(e) + def get_authentication_url(client_id=None, redirect_uri=None): return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format(client_id, SCOPES, redirect_uri) + "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( + client_id, SCOPES, redirect_uri + ) } + @frappe.whitelist() def google_callback(code=None): """ - Authorization code is sent to callback as per the API configuration + Authorization code is sent to callback as per the API configuration """ google_calendar = frappe.cache().hget("google_calendar", "google_calendar") frappe.db.set_value("Google Calendar", google_calendar, "authorization_code", code) @@ -143,6 +168,7 @@ def google_callback(code=None): authorize_access(google_calendar) + @frappe.whitelist() def sync(g_calendar=None): filters = {"enable": 1} @@ -155,9 +181,10 @@ def sync(g_calendar=None): for g in google_calendars: return sync_events_from_google_calendar(g.name) + def get_google_calendar_object(g_calendar): """ - Returns an object of Google Calendar along with Google Calendar doc. + Returns an object of Google Calendar along with Google Calendar doc. """ google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Google Calendar", g_calendar) @@ -168,15 +195,12 @@ def get_google_calendar_object(g_calendar): "token_uri": get_auth_url(), "client_id": google_settings.client_id, "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/calendar/v3" + "scopes": "https://www.googleapis.com/auth/calendar/v3", } credentials = google.oauth2.credentials.Credentials(**credentials_dict) google_calendar = build( - serviceName="calendar", - version="v3", - credentials=credentials, - static_discovery=False + serviceName="calendar", version="v3", credentials=credentials, static_discovery=False ) check_google_calendar(account, google_calendar) @@ -184,10 +208,11 @@ def get_google_calendar_object(g_calendar): account.load_from_db() return google_calendar, account + def check_google_calendar(account, google_calendar): """ - Checks if Google Calendar is present with the specified name. - If not, creates one. + Checks if Google Calendar is present with the specified name. + If not, creates one. """ account.load_from_db() try: @@ -197,20 +222,27 @@ def check_google_calendar(account, google_calendar): # If no Calendar ID create a new Calendar calendar = { "summary": account.calendar_name, - "timeZone": frappe.db.get_single_value("System Settings", "time_zone") + "timeZone": frappe.db.get_single_value("System Settings", "time_zone"), } created_calendar = google_calendar.calendars().insert(body=calendar).execute() - frappe.db.set_value("Google Calendar", account.name, "google_calendar_id", created_calendar.get("id")) + frappe.db.set_value( + "Google Calendar", account.name, "google_calendar_id", created_calendar.get("id") + ) frappe.db.commit() except HttpError as err: - frappe.throw(_("Google Calendar - Could not create Calendar for {0}, error code {1}.").format(account.name, err.resp.status)) + frappe.throw( + _("Google Calendar - Could not create Calendar for {0}, error code {1}.").format( + account.name, err.resp.status + ) + ) + def sync_events_from_google_calendar(g_calendar, method=None): """ - Syncs Events from Google Calendar in Framework Calendar. - Google Calendar returns nextSyncToken when all the events in Google Calendar are fetched. - nextSyncToken is returned at the very last page - https://developers.google.com/calendar/v3/sync + Syncs Events from Google Calendar in Framework Calendar. + Google Calendar returns nextSyncToken when all the events in Google Calendar are fetched. + nextSyncToken is returned at the very last page + https://developers.google.com/calendar/v3/sync """ google_calendar, account = get_google_calendar_object(g_calendar) @@ -223,16 +255,28 @@ def sync_events_from_google_calendar(g_calendar, method=None): while True: try: # API Response listed at EOF - events = google_calendar.events().list(calendarId=account.google_calendar_id, maxResults=2000, - pageToken=events.get("nextPageToken"), singleEvents=False, showDeleted=True, syncToken=sync_token).execute() + events = ( + google_calendar.events() + .list( + calendarId=account.google_calendar_id, + maxResults=2000, + pageToken=events.get("nextPageToken"), + singleEvents=False, + showDeleted=True, + syncToken=sync_token, + ) + .execute() + ) except HttpError as err: - msg = _("Google Calendar - Could not fetch event from Google Calendar, error code {0}.").format(err.resp.status) + msg = _("Google Calendar - Could not fetch event from Google Calendar, error code {0}.").format( + err.resp.status + ) if err.resp.status == 410: set_encrypted_password("Google Calendar", account.name, "", "next_sync_token") frappe.db.commit() - msg += ' ' + _('Sync token was invalid and has been resetted, Retry syncing.') - frappe.msgprint(msg, title='Invalid Sync Token', indicator='blue') + msg += " " + _("Sync token was invalid and has been resetted, Retry syncing.") + frappe.msgprint(msg, title="Invalid Sync Token", indicator="blue") else: frappe.throw(msg) @@ -246,7 +290,9 @@ def sync_events_from_google_calendar(g_calendar, method=None): break for idx, event in enumerate(results): - frappe.publish_realtime("import_google_calendar", dict(progress=idx+1, total=len(results)), user=frappe.session.user) + frappe.publish_realtime( + "import_google_calendar", dict(progress=idx + 1, total=len(results)), user=frappe.session.user + ) # If Google Calendar Event if confirmed, then create an Event if event.get("status") == "confirmed": @@ -263,14 +309,31 @@ def sync_events_from_google_calendar(g_calendar, method=None): update_event_in_calendar(account, event, recurrence) elif event.get("status") == "cancelled": # If any synced Google Calendar Event is cancelled, then close the Event - frappe.db.set_value("Event", {"google_calendar_id": account.google_calendar_id, "google_calendar_event_id": event.get("id")}, "status", "Closed") - frappe.get_doc({ - "doctype": "Comment", - "comment_type": "Info", - "reference_doctype": "Event", - "reference_name": frappe.db.get_value("Event", {"google_calendar_id": account.google_calendar_id, "google_calendar_event_id": event.get("id")}, "name"), - "content": " - Event deleted from Google Calendar.", - }).insert(ignore_permissions=True) + frappe.db.set_value( + "Event", + { + "google_calendar_id": account.google_calendar_id, + "google_calendar_event_id": event.get("id"), + }, + "status", + "Closed", + ) + frappe.get_doc( + { + "doctype": "Comment", + "comment_type": "Info", + "reference_doctype": "Event", + "reference_name": frappe.db.get_value( + "Event", + { + "google_calendar_id": account.google_calendar_id, + "google_calendar_event_id": event.get("id"), + }, + "name", + ), + "content": " - Event deleted from Google Calendar.", + } + ).insert(ignore_permissions=True) else: pass @@ -281,9 +344,10 @@ def sync_events_from_google_calendar(g_calendar, method=None): else: return _("{0} Google Calendar Events synced.").format(len(results)) + def insert_event_to_calendar(account, event, recurrence=None): """ - Inserts event in Frappe Calendar during Sync + Inserts event in Frappe Calendar during Sync """ calendar_event = { "doctype": "Event", @@ -293,27 +357,40 @@ def insert_event_to_calendar(account, event, recurrence=None): "google_calendar": account.name, "google_calendar_id": account.google_calendar_id, "google_calendar_event_id": event.get("id"), - "pulled_from_google_calendar": 1 + "pulled_from_google_calendar": 1, } - calendar_event.update(google_calendar_to_repeat_on(recurrence=recurrence, start=event.get("start"), end=event.get("end"))) + calendar_event.update( + google_calendar_to_repeat_on( + recurrence=recurrence, start=event.get("start"), end=event.get("end") + ) + ) frappe.get_doc(calendar_event).insert(ignore_permissions=True) + def update_event_in_calendar(account, event, recurrence=None): """ - Updates Event in Frappe Calendar if any existing Google Calendar Event is updated + Updates Event in Frappe Calendar if any existing Google Calendar Event is updated """ calendar_event = frappe.get_doc("Event", {"google_calendar_event_id": event.get("id")}) calendar_event.subject = event.get("summary") calendar_event.description = event.get("description") - calendar_event.update(google_calendar_to_repeat_on(recurrence=recurrence, start=event.get("start"), end=event.get("end"))) + calendar_event.update( + google_calendar_to_repeat_on( + recurrence=recurrence, start=event.get("start"), end=event.get("end") + ) + ) calendar_event.save(ignore_permissions=True) + def insert_event_in_google_calendar(doc, method=None): """ - Insert Events in Google Calendar if sync_with_google_calendar is checked. + Insert Events in Google Calendar if sync_with_google_calendar is checked. """ - if not frappe.db.exists("Google Calendar", {"name": doc.google_calendar}) or doc.pulled_from_google_calendar \ - or not doc.sync_with_google_calendar: + if ( + not frappe.db.exists("Google Calendar", {"name": doc.google_calendar}) + or doc.pulled_from_google_calendar + or not doc.sync_with_google_calendar + ): return google_calendar, account = get_google_calendar_object(doc.google_calendar) @@ -321,31 +398,41 @@ def insert_event_in_google_calendar(doc, method=None): if not account.push_to_google_calendar: return - event = { - "summary": doc.subject, - "description": doc.description, - "google_calendar_event": 1 - } - event.update(format_date_according_to_google_calendar(doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on))) + event = {"summary": doc.subject, "description": doc.description, "google_calendar_event": 1} + event.update( + format_date_according_to_google_calendar( + doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on) + ) + ) if doc.repeat_on: event.update({"recurrence": repeat_on_to_google_calendar_recurrence_rule(doc)}) try: event = google_calendar.events().insert(calendarId=doc.google_calendar_id, body=event).execute() - frappe.db.set_value("Event", doc.name, "google_calendar_event_id", event.get("id"), update_modified=False) + frappe.db.set_value( + "Event", doc.name, "google_calendar_event_id", event.get("id"), update_modified=False + ) frappe.msgprint(_("Event Synced with Google Calendar.")) except HttpError as err: - frappe.throw(_("Google Calendar - Could not insert event in Google Calendar {0}, error code {1}.").format(account.name, err.resp.status)) + frappe.throw( + _("Google Calendar - Could not insert event in Google Calendar {0}, error code {1}.").format( + account.name, err.resp.status + ) + ) + def update_event_in_google_calendar(doc, method=None): """ - Updates Events in Google Calendar if any existing event is modified in Frappe Calendar + Updates Events in Google Calendar if any existing event is modified in Frappe Calendar """ # Workaround to avoid triggering updation when Event is being inserted since # creation and modified are same when inserting doc - if not frappe.db.exists("Google Calendar", {"name": doc.google_calendar}) or doc.modified == doc.creation \ - or not doc.sync_with_google_calendar: + if ( + not frappe.db.exists("Google Calendar", {"name": doc.google_calendar}) + or doc.modified == doc.creation + or not doc.sync_with_google_calendar + ): return if doc.sync_with_google_calendar and not doc.google_calendar_event_id: @@ -359,21 +446,38 @@ def update_event_in_google_calendar(doc, method=None): return try: - event = google_calendar.events().get(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id).execute() + event = ( + google_calendar.events() + .get(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id) + .execute() + ) event["summary"] = doc.subject event["description"] = doc.description event["recurrence"] = repeat_on_to_google_calendar_recurrence_rule(doc) - event["status"] = "cancelled" if doc.event_type == "Cancelled" or doc.status == "Closed" else event.get("status") - event.update(format_date_according_to_google_calendar(doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on))) - - google_calendar.events().update(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id, body=event).execute() + event["status"] = ( + "cancelled" if doc.event_type == "Cancelled" or doc.status == "Closed" else event.get("status") + ) + event.update( + format_date_according_to_google_calendar( + doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on) + ) + ) + + google_calendar.events().update( + calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id, body=event + ).execute() frappe.msgprint(_("Event Synced with Google Calendar.")) except HttpError as err: - frappe.throw(_("Google Calendar - Could not update Event {0} in Google Calendar, error code {1}.").format(doc.name, err.resp.status)) + frappe.throw( + _("Google Calendar - Could not update Event {0} in Google Calendar, error code {1}.").format( + doc.name, err.resp.status + ) + ) + def delete_event_from_google_calendar(doc, method=None): """ - Delete Events from Google Calendar if Frappe Event is deleted. + Delete Events from Google Calendar if Frappe Event is deleted. """ if not frappe.db.exists("Google Calendar", {"name": doc.google_calendar}): @@ -385,24 +489,39 @@ def delete_event_from_google_calendar(doc, method=None): return try: - event = google_calendar.events().get(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id).execute() + event = ( + google_calendar.events() + .get(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id) + .execute() + ) event["recurrence"] = None event["status"] = "cancelled" - google_calendar.events().update(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id, body=event).execute() + google_calendar.events().update( + calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id, body=event + ).execute() except HttpError as err: - frappe.msgprint(_("Google Calendar - Could not delete Event {0} from Google Calendar, error code {1}.").format(doc.name, err.resp.status)) + frappe.msgprint( + _("Google Calendar - Could not delete Event {0} from Google Calendar, error code {1}.").format( + doc.name, err.resp.status + ) + ) + def google_calendar_to_repeat_on(start, end, recurrence=None): """ - recurrence is in the form ['RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH'] - has the frequency and then the days on which the event recurs + recurrence is in the form ['RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH'] + has the frequency and then the days on which the event recurs - Both have been mapped in a dict for easier mapping. + Both have been mapped in a dict for easier mapping. """ repeat_on = { - "starts_on": get_datetime(start.get("date")) if start.get("date") else parser.parse(start.get("dateTime")).utcnow(), - "ends_on": get_datetime(end.get("date")) if end.get("date") else parser.parse(end.get("dateTime")).utcnow(), + "starts_on": get_datetime(start.get("date")) + if start.get("date") + else parser.parse(start.get("dateTime")).utcnow(), + "ends_on": get_datetime(end.get("date")) + if end.get("date") + else parser.parse(end.get("dateTime")).utcnow(), "all_day": 1 if start.get("date") else 0, "repeat_this_event": 1 if recurrence else 0, "repeat_on": None, @@ -441,7 +560,7 @@ def google_calendar_to_repeat_on(start, end, recurrence=None): repeat_day_week_number = num break - for day in ["MO","TU","WE","TH","FR","SA","SU"]: + for day in ["MO", "TU", "WE", "TH", "FR", "SA", "SU"]: if day in byday: repeat_day_name = google_calendar_days.get(day) break @@ -458,6 +577,7 @@ def google_calendar_to_repeat_on(start, end, recurrence=None): return repeat_on + def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None): if not ends_on: ends_on = starts_on + timedelta(minutes=10) @@ -466,11 +586,11 @@ def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None): "start": { "dateTime": starts_on.isoformat(), "timeZone": get_time_zone(), - }, + }, "end": { "dateTime": ends_on.isoformat(), "timeZone": get_time_zone(), - } + }, } if all_day: @@ -483,9 +603,10 @@ def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None): return date_format + def parse_google_calendar_recurrence_rule(repeat_day_week_number, repeat_day_name): """ - Returns (repeat_on) exact date for combination eg 4TH viz. 4th thursday of a month + Returns (repeat_on) exact date for combination eg 4TH viz. 4th thursday of a month """ if repeat_day_week_number < 0: # Consider a month with 5 weeks and event is to be repeated in last week of every month, google caledar considers @@ -511,9 +632,10 @@ def parse_google_calendar_recurrence_rule(repeat_day_week_number, repeat_day_nam return current_date + def repeat_on_to_google_calendar_recurrence_rule(doc): """ - Returns event (repeat_on) in Google Calendar format ie RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH + Returns event (repeat_on) in Google Calendar format ie RRULE:FREQ=WEEKLY;BYDAY=MO,TU,TH """ recurrence = framework_frequencies.get(doc.repeat_on) weekdays = get_weekdays() @@ -528,18 +650,21 @@ def repeat_on_to_google_calendar_recurrence_rule(doc): return [recurrence] + def get_week_number(dt): """ - Returns the week number of the month for the specified date. - https://stackoverflow.com/questions/3806473/python-week-number-of-the-month/16804556 + Returns the week number of the month for the specified date. + https://stackoverflow.com/questions/3806473/python-week-number-of-the-month/16804556 """ from math import ceil + first_day = dt.replace(day=1) dom = dt.day adjusted_dom = dom + first_day.weekday() - return int(ceil(adjusted_dom/7.0)) + return int(ceil(adjusted_dom / 7.0)) + def get_recurrence_parameters(recurrence): recurrence = recurrence.split(";") @@ -557,6 +682,7 @@ def get_recurrence_parameters(recurrence): return frequency, until, byday + """API Response { 'kind': 'calendar#events', diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index a63b0b6d80..c26366f71a 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -16,8 +16,8 @@ from frappe.utils import get_request_site_address SCOPES = "https://www.googleapis.com/auth/contacts" -class GoogleContacts(Document): +class GoogleContacts(Document): def validate(self): if not frappe.db.get_single_value("Google Settings", "enable"): frappe.throw(_("Enable Google API in Google Settings.")) @@ -29,7 +29,7 @@ class GoogleContacts(Document): frappe.throw(_("Google Contacts Integration is disabled.")) if not self.refresh_token: - button_label = frappe.bold(_('Allow Google Contacts Access')) + button_label = frappe.bold(_("Allow Google Contacts Access")) raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label)) data = { @@ -37,28 +37,36 @@ class GoogleContacts(Document): "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), "refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False), "grant_type": "refresh_token", - "scope": SCOPES + "scope": SCOPES, } try: r = requests.post(get_auth_url(), data=data).json() except requests.exceptions.HTTPError: - button_label = frappe.bold(_('Allow Google Contacts Access')) - frappe.throw(_("Something went wrong during the token generation. Click on {0} to generate a new one.").format(button_label)) + button_label = frappe.bold(_("Allow Google Contacts Access")) + frappe.throw( + _( + "Something went wrong during the token generation. Click on {0} to generate a new one." + ).format(button_label) + ) return r.get("access_token") + @frappe.whitelist() def authorize_access(g_contact, reauthorize=None): """ - If no Authorization code get it from Google and then request for Refresh Token. - Google Contact Name is set to flags to set_value after Authorization Code is obtained. + If no Authorization code get it from Google and then request for Refresh Token. + Google Contact Name is set to flags to set_value after Authorization Code is obtained. """ google_settings = frappe.get_doc("Google Settings") google_contact = frappe.get_doc("Google Contacts", g_contact) - redirect_uri = get_request_site_address(True) + "?cmd=frappe.integrations.doctype.google_contacts.google_contacts.google_callback" + redirect_uri = ( + get_request_site_address(True) + + "?cmd=frappe.integrations.doctype.google_contacts.google_contacts.google_callback" + ) if not google_contact.authorization_code or reauthorize: frappe.cache().hset("google_contacts", "google_contact", google_contact.name) @@ -68,14 +76,18 @@ def authorize_access(g_contact, reauthorize=None): data = { "code": google_contact.authorization_code, "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), + "client_secret": google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), "redirect_uri": redirect_uri, - "grant_type": "authorization_code" + "grant_type": "authorization_code", } r = requests.post(get_auth_url(), data=data).json() if "refresh_token" in r: - frappe.db.set_value("Google Contacts", google_contact.name, "refresh_token", r.get("refresh_token")) + frappe.db.set_value( + "Google Contacts", google_contact.name, "refresh_token", r.get("refresh_token") + ) frappe.db.commit() frappe.local.response["type"] = "redirect" @@ -85,15 +97,19 @@ def authorize_access(g_contact, reauthorize=None): except Exception as e: frappe.throw(e) + def get_authentication_url(client_id=None, redirect_uri=None): return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format(client_id, SCOPES, redirect_uri) + "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( + client_id, SCOPES, redirect_uri + ) } + @frappe.whitelist() def google_callback(code=None): """ - Authorization code is sent to callback as per the API configuration + Authorization code is sent to callback as per the API configuration """ google_contact = frappe.cache().hget("google_contacts", "google_contact") frappe.db.set_value("Google Contacts", google_contact, "authorization_code", code) @@ -101,9 +117,10 @@ def google_callback(code=None): authorize_access(google_contact) + def get_google_contacts_object(g_contact): """ - Returns an object of Google Calendar along with Google Calendar doc. + Returns an object of Google Calendar along with Google Calendar doc. """ google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Google Contacts", g_contact) @@ -114,19 +131,17 @@ def get_google_contacts_object(g_contact): "token_uri": get_auth_url(), "client_id": google_settings.client_id, "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/contacts" + "scopes": "https://www.googleapis.com/auth/contacts", } credentials = google.oauth2.credentials.Credentials(**credentials_dict) google_contacts = build( - serviceName="people", - version="v1", - credentials=credentials, - static_discovery=False + serviceName="people", version="v1", credentials=credentials, static_discovery=False ) return google_contacts, account + @frappe.whitelist() def sync(g_contact=None): filters = {"enable": 1} @@ -139,10 +154,11 @@ def sync(g_contact=None): for g in google_contacts: return sync_contacts_from_google_contacts(g.name) + def sync_contacts_from_google_contacts(g_contact): """ - Syncs Contacts from Google Contacts. - https://developers.google.com/people/api/rest/v1/people.connections/list + Syncs Contacts from Google Contacts. + https://developers.google.com/people/api/rest/v1/people.connections/list """ google_contacts, account = get_google_contacts_object(g_contact) @@ -157,58 +173,90 @@ def sync_contacts_from_google_contacts(g_contact): while True: try: - contacts = google_contacts.people().connections().list(resourceName='people/me', pageToken=contacts.get("nextPageToken"), - syncToken=sync_token, pageSize=2000, requestSyncToken=True, personFields="names,emailAddresses,organizations,phoneNumbers").execute() + contacts = ( + google_contacts.people() + .connections() + .list( + resourceName="people/me", + pageToken=contacts.get("nextPageToken"), + syncToken=sync_token, + pageSize=2000, + requestSyncToken=True, + personFields="names,emailAddresses,organizations,phoneNumbers", + ) + .execute() + ) except HttpError as err: - frappe.throw(_("Google Contacts - Could not sync contacts from Google Contacts {0}, error code {1}.").format(account.name, err.resp.status)) + frappe.throw( + _( + "Google Contacts - Could not sync contacts from Google Contacts {0}, error code {1}." + ).format(account.name, err.resp.status) + ) for contact in contacts.get("connections", []): results.append(contact) if not contacts.get("nextPageToken"): if contacts.get("nextSyncToken"): - frappe.db.set_value("Google Contacts", account.name, "next_sync_token", contacts.get("nextSyncToken")) + frappe.db.set_value( + "Google Contacts", account.name, "next_sync_token", contacts.get("nextSyncToken") + ) frappe.db.commit() break frappe.db.set_value("Google Contacts", account.name, "last_sync_on", frappe.utils.now_datetime()) for idx, connection in enumerate(results): - frappe.publish_realtime('import_google_contacts', dict(progress=idx+1, total=len(results)), user=frappe.session.user) + frappe.publish_realtime( + "import_google_contacts", dict(progress=idx + 1, total=len(results)), user=frappe.session.user + ) for name in connection.get("names"): if name.get("metadata").get("primary"): contacts_updated += 1 - contact = frappe.get_doc({ - "doctype": "Contact", - "first_name": name.get("givenName") or "", - "middle_name": name.get("middleName") or "", - "last_name": name.get("familyName") or "", - "designation": get_indexed_value(connection.get("organizations"), 0, "title"), - "pulled_from_google_contacts": 1, - "google_contacts": account.name, - "company_name": get_indexed_value(connection.get("organizations"), 0, "name") - }) + contact = frappe.get_doc( + { + "doctype": "Contact", + "first_name": name.get("givenName") or "", + "middle_name": name.get("middleName") or "", + "last_name": name.get("familyName") or "", + "designation": get_indexed_value(connection.get("organizations"), 0, "title"), + "pulled_from_google_contacts": 1, + "google_contacts": account.name, + "company_name": get_indexed_value(connection.get("organizations"), 0, "name"), + } + ) for email in connection.get("emailAddresses", []): - contact.add_email(email_id=email.get("value"), is_primary=1 if email.get("metadata").get("primary") else 0) + contact.add_email( + email_id=email.get("value"), is_primary=1 if email.get("metadata").get("primary") else 0 + ) for phone in connection.get("phoneNumbers", []): - contact.add_phone(phone=phone.get("value"), is_primary_phone=1 if phone.get("metadata").get("primary") else 0) + contact.add_phone( + phone=phone.get("value"), is_primary_phone=1 if phone.get("metadata").get("primary") else 0 + ) contact.insert(ignore_permissions=True) - return _("{0} Google Contacts synced.").format(contacts_updated) if contacts_updated > 0 \ + return ( + _("{0} Google Contacts synced.").format(contacts_updated) + if contacts_updated > 0 else _("No new Google Contacts synced.") + ) + def insert_contacts_to_google_contacts(doc, method=None): """ - Syncs Contacts from Google Contacts. - https://developers.google.com/people/api/rest/v1/people/createContact + Syncs Contacts from Google Contacts. + https://developers.google.com/people/api/rest/v1/people/createContact """ - if not frappe.db.exists("Google Contacts", {"name": doc.google_contacts}) or doc.pulled_from_google_contacts \ - or not doc.sync_with_google_contacts: + if ( + not frappe.db.exists("Google Contacts", {"name": doc.google_contacts}) + or doc.pulled_from_google_contacts + or not doc.sync_with_google_contacts + ): return google_contacts, account = get_google_contacts_object(doc.google_contacts) @@ -216,31 +264,40 @@ def insert_contacts_to_google_contacts(doc, method=None): if not account.push_to_google_contacts: return - names = { - "givenName": doc.first_name, - "middleName": doc.middle_name, - "familyName": doc.last_name - } + names = {"givenName": doc.first_name, "middleName": doc.middle_name, "familyName": doc.last_name} phoneNumbers = [{"value": phone_no.phone} for phone_no in doc.phone_nos] emailAddresses = [{"value": email_id.email_id} for email_id in doc.email_ids] try: - contact = google_contacts.people().createContact(body={"names": [names],"phoneNumbers": phoneNumbers, - "emailAddresses": emailAddresses}).execute() + contact = ( + google_contacts.people() + .createContact( + body={"names": [names], "phoneNumbers": phoneNumbers, "emailAddresses": emailAddresses} + ) + .execute() + ) frappe.db.set_value("Contact", doc.name, "google_contacts_id", contact.get("resourceName")) except HttpError as err: - frappe.msgprint(_("Google Calendar - Could not insert contact in Google Contacts {0}, error code {1}.").format(account.name, err.resp.status)) + frappe.msgprint( + _("Google Calendar - Could not insert contact in Google Contacts {0}, error code {1}.").format( + account.name, err.resp.status + ) + ) + def update_contacts_to_google_contacts(doc, method=None): """ - Syncs Contacts from Google Contacts. - https://developers.google.com/people/api/rest/v1/people/updateContact + Syncs Contacts from Google Contacts. + https://developers.google.com/people/api/rest/v1/people/updateContact """ # Workaround to avoid triggering updation when Event is being inserted since # creation and modified are same when inserting doc - if not frappe.db.exists("Google Contacts", {"name": doc.google_contacts}) or doc.modified == doc.creation \ - or not doc.sync_with_google_contacts: + if ( + not frappe.db.exists("Google Contacts", {"name": doc.google_contacts}) + or doc.modified == doc.creation + or not doc.sync_with_google_contacts + ): return if doc.sync_with_google_contacts and not doc.google_contacts_id: @@ -253,29 +310,43 @@ def update_contacts_to_google_contacts(doc, method=None): if not account.push_to_google_contacts: return - names = { - "givenName": doc.first_name, - "middleName": doc.middle_name, - "familyName": doc.last_name - } + names = {"givenName": doc.first_name, "middleName": doc.middle_name, "familyName": doc.last_name} phoneNumbers = [{"value": phone_no.phone} for phone_no in doc.phone_nos] emailAddresses = [{"value": email_id.email_id} for email_id in doc.email_ids] try: - contact = google_contacts.people().get(resourceName=doc.google_contacts_id, \ - personFields="names,emailAddresses,organizations,phoneNumbers").execute() + contact = ( + google_contacts.people() + .get( + resourceName=doc.google_contacts_id, + personFields="names,emailAddresses,organizations,phoneNumbers", + ) + .execute() + ) contact["names"] = [names] contact["phoneNumbers"] = phoneNumbers contact["emailAddresses"] = emailAddresses - google_contacts.people().updateContact(resourceName=doc.google_contacts_id,body={"names":[names], - "phoneNumbers":phoneNumbers,"emailAddresses":emailAddresses,"etag":contact.get("etag")}, - updatePersonFields="names,emailAddresses,organizations,phoneNumbers").execute() + google_contacts.people().updateContact( + resourceName=doc.google_contacts_id, + body={ + "names": [names], + "phoneNumbers": phoneNumbers, + "emailAddresses": emailAddresses, + "etag": contact.get("etag"), + }, + updatePersonFields="names,emailAddresses,organizations,phoneNumbers", + ).execute() frappe.msgprint(_("Contact Synced with Google Contacts.")) except HttpError as err: - frappe.msgprint(_("Google Contacts - Could not update contact in Google Contacts {0}, error code {1}.").format(account.name, err.resp.status)) + frappe.msgprint( + _("Google Contacts - Could not update contact in Google Contacts {0}, error code {1}.").format( + account.name, err.resp.status + ) + ) + def get_indexed_value(d, index, key): if not d: diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index beac7898a9..bbb1e8485e 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -14,11 +14,13 @@ from googleapiclient.errors import HttpError import frappe from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url -from frappe.integrations.offsite_backup_utils import (get_latest_backup_file, - send_email, validate_file_size) +from frappe.integrations.offsite_backup_utils import ( + get_latest_backup_file, + send_email, + validate_file_size, +) from frappe.model.document import Document -from frappe.utils import (get_backups_path, get_bench_path, - get_request_site_address) +from frappe.utils import get_backups_path, get_bench_path, get_request_site_address from frappe.utils.background_jobs import enqueue from frappe.utils.backups import new_backup @@ -26,11 +28,10 @@ SCOPES = "https://www.googleapis.com/auth/drive" class GoogleDrive(Document): - def validate(self): doc_before_save = self.get_doc_before_save() if doc_before_save and doc_before_save.backup_folder_name != self.backup_folder_name: - self.backup_folder_id = '' + self.backup_folder_id = "" def get_access_token(self): google_settings = frappe.get_doc("Google Settings") @@ -47,28 +48,36 @@ class GoogleDrive(Document): "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), "refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False), "grant_type": "refresh_token", - "scope": SCOPES + "scope": SCOPES, } try: r = requests.post(get_auth_url(), data=data).json() except requests.exceptions.HTTPError: button_label = frappe.bold(_("Allow Google Drive Access")) - frappe.throw(_("Something went wrong during the token generation. Click on {0} to generate a new one.").format(button_label)) + frappe.throw( + _( + "Something went wrong during the token generation. Click on {0} to generate a new one." + ).format(button_label) + ) return r.get("access_token") + @frappe.whitelist() def authorize_access(reauthorize=None): """ - If no Authorization code get it from Google and then request for Refresh Token. - Google Contact Name is set to flags to set_value after Authorization Code is obtained. + If no Authorization code get it from Google and then request for Refresh Token. + Google Contact Name is set to flags to set_value after Authorization Code is obtained. """ google_settings = frappe.get_doc("Google Settings") google_drive = frappe.get_doc("Google Drive") - redirect_uri = get_request_site_address(True) + "?cmd=frappe.integrations.doctype.google_drive.google_drive.google_callback" + redirect_uri = ( + get_request_site_address(True) + + "?cmd=frappe.integrations.doctype.google_drive.google_drive.google_callback" + ) if not google_drive.authorization_code or reauthorize: if reauthorize: @@ -79,9 +88,11 @@ def authorize_access(reauthorize=None): data = { "code": google_drive.authorization_code, "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), + "client_secret": google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), "redirect_uri": redirect_uri, - "grant_type": "authorization_code" + "grant_type": "authorization_code", } r = requests.post(get_auth_url(), data=data).json() @@ -96,24 +107,29 @@ def authorize_access(reauthorize=None): except Exception as e: frappe.throw(e) + def get_authentication_url(client_id, redirect_uri): return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format(client_id, SCOPES, redirect_uri) + "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( + client_id, SCOPES, redirect_uri + ) } + @frappe.whitelist() def google_callback(code=None): """ - Authorization code is sent to callback as per the API configuration + Authorization code is sent to callback as per the API configuration """ frappe.db.set_value("Google Drive", None, "authorization_code", code) frappe.db.commit() authorize_access() + def get_google_drive_object(): """ - Returns an object of Google Drive. + Returns an object of Google Drive. """ google_settings = frappe.get_doc("Google Settings") account = frappe.get_doc("Google Drive") @@ -124,25 +140,24 @@ def get_google_drive_object(): "token_uri": get_auth_url(), "client_id": google_settings.client_id, "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/drive/v3" + "scopes": "https://www.googleapis.com/auth/drive/v3", } credentials = google.oauth2.credentials.Credentials(**credentials_dict) google_drive = build( - serviceName="drive", - version="v3", - credentials=credentials, - static_discovery=False + serviceName="drive", version="v3", credentials=credentials, static_discovery=False ) return google_drive, account + def check_for_folder_in_google_drive(): """Checks if folder exists in Google Drive else create it.""" + def _create_folder_in_google_drive(google_drive, account): file_metadata = { "name": account.backup_folder_name, - "mimeType": "application/vnd.google-apps.folder" + "mimeType": "application/vnd.google-apps.folder", } try: @@ -150,7 +165,9 @@ def check_for_folder_in_google_drive(): frappe.db.set_value("Google Drive", None, "backup_folder_id", folder.get("id")) frappe.db.commit() except HttpError as e: - frappe.throw(_("Google Drive - Could not create folder in Google Drive - Error Code {0}").format(e)) + frappe.throw( + _("Google Drive - Could not create folder in Google Drive - Error Code {0}").format(e) + ) google_drive, account = get_google_drive_object() @@ -160,9 +177,13 @@ def check_for_folder_in_google_drive(): backup_folder_exists = False try: - google_drive_folders = google_drive.files().list(q="mimeType='application/vnd.google-apps.folder'").execute() + google_drive_folders = ( + google_drive.files().list(q="mimeType='application/vnd.google-apps.folder'").execute() + ) except HttpError as e: - frappe.throw(_("Google Drive - Could not find folder in Google Drive - Error Code {0}").format(e)) + frappe.throw( + _("Google Drive - Could not find folder in Google Drive - Error Code {0}").format(e) + ) for f in google_drive_folders.get("files"): if f.get("name") == account.backup_folder_name: @@ -174,15 +195,21 @@ def check_for_folder_in_google_drive(): if not backup_folder_exists: _create_folder_in_google_drive(google_drive, account) + @frappe.whitelist() def take_backup(): """Enqueue longjob for taking backup to Google Drive""" - enqueue("frappe.integrations.doctype.google_drive.google_drive.upload_system_backup_to_google_drive", queue='long', timeout=1500) + enqueue( + "frappe.integrations.doctype.google_drive.google_drive.upload_system_backup_to_google_drive", + queue="long", + timeout=1500, + ) frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour.")) + def upload_system_backup_to_google_drive(): """ - Upload system backup to Google Drive + Upload system backup to Google Drive """ # Get Google Drive Object google_drive, account = get_google_drive_object() @@ -210,13 +237,12 @@ def upload_system_backup_to_google_drive(): if not fileurl: continue - file_metadata = { - "name": fileurl, - "parents": [account.backup_folder_id] - } + file_metadata = {"name": fileurl, "parents": [account.backup_folder_id]} try: - media = MediaFileUpload(get_absolute_path(filename=fileurl), mimetype="application/gzip", resumable=True) + media = MediaFileUpload( + get_absolute_path(filename=fileurl), mimetype="application/gzip", resumable=True + ) except IOError as e: frappe.throw(_("Google Drive - Could not locate - {0}").format(e)) @@ -231,19 +257,27 @@ def upload_system_backup_to_google_drive(): send_email(True, "Google Drive", "Google Drive", "email") return _("Google Drive Backup Successful.") + def daily_backup(): - drive_settings = frappe.db.get_singles_dict('Google Drive') + drive_settings = frappe.db.get_singles_dict("Google Drive") if drive_settings.enable and drive_settings.frequency == "Daily": upload_system_backup_to_google_drive() + def weekly_backup(): - drive_settings = frappe.db.get_singles_dict('Google Drive') + drive_settings = frappe.db.get_singles_dict("Google Drive") if drive_settings.enable and drive_settings.frequency == "Weekly": upload_system_backup_to_google_drive() + def get_absolute_path(filename): file_path = os.path.join(get_backups_path()[2:], os.path.basename(filename)) return "{0}/sites/{1}".format(get_bench_path(), file_path) + def set_progress(progress, message): - frappe.publish_realtime("upload_to_google_drive", dict(progress=progress, total=3, message=message), user=frappe.session.user) + frappe.publish_realtime( + "upload_to_google_drive", + dict(progress=progress, total=3, message=message), + user=frappe.session.user, + ) diff --git a/frappe/integrations/doctype/google_drive/test_google_drive.py b/frappe/integrations/doctype/google_drive/test_google_drive.py index fbd9dce7f4..17f5b152ca 100644 --- a/frappe/integrations/doctype/google_drive/test_google_drive.py +++ b/frappe/integrations/doctype/google_drive/test_google_drive.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestGoogleDrive(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/google_settings/google_settings.py b/frappe/integrations/doctype/google_settings/google_settings.py index 94df43e69c..0d5f9cb00d 100644 --- a/frappe/integrations/doctype/google_settings/google_settings.py +++ b/frappe/integrations/doctype/google_settings/google_settings.py @@ -5,9 +5,11 @@ import frappe from frappe.model.document import Document + class GoogleSettings(Document): pass + def get_auth_url(): return "https://www.googleapis.com/oauth2/v4/token" @@ -23,5 +25,5 @@ def get_file_picker_settings(): "enabled": True, "appId": google_settings.app_id, "developerKey": google_settings.api_key, - "clientId": google_settings.client_id + "clientId": google_settings.client_id, } diff --git a/frappe/integrations/doctype/google_settings/test_google_settings.py b/frappe/integrations/doctype/google_settings/test_google_settings.py index cddf9f3697..53d59b1be0 100644 --- a/frappe/integrations/doctype/google_settings/test_google_settings.py +++ b/frappe/integrations/doctype/google_settings/test_google_settings.py @@ -3,13 +3,14 @@ # License: MIT. See LICENSE from __future__ import unicode_literals -import frappe import unittest +import frappe + from .google_settings import get_file_picker_settings -class TestGoogleSettings(unittest.TestCase): +class TestGoogleSettings(unittest.TestCase): def setUp(self): settings = frappe.get_single("Google Settings") settings.client_id = "test_client_id" diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py index ae0e024f58..4c99613161 100644 --- a/frappe/integrations/doctype/integration_request/integration_request.py +++ b/frappe/integrations/doctype/integration_request/integration_request.py @@ -2,10 +2,12 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe -from frappe.model.document import Document import json + +import frappe from frappe.integrations.utils import json_handler +from frappe.model.document import Document + class IntegrationRequest(Document): def autoname(self): @@ -33,4 +35,4 @@ class IntegrationRequest(Document): if isinstance(response, str): response = json.loads(response) self.db_set("status", "Failed") - self.db_set("error", json.dumps(response, default=json_handler)) \ No newline at end of file + self.db_set("error", json.dumps(response, default=json_handler)) diff --git a/frappe/integrations/doctype/integration_request/test_integration_request.py b/frappe/integrations/doctype/integration_request/test_integration_request.py index e26ccabc96..d14af481e8 100644 --- a/frappe/integrations/doctype/integration_request/test_integration_request.py +++ b/frappe/integrations/doctype/integration_request/test_integration_request.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Integration Request') + class TestIntegrationRequest(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py index b9838b996f..f1b242e4bb 100644 --- a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py +++ b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class LDAPGroupMapping(Document): pass diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index cfd6e1e133..a14124234f 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -5,7 +5,8 @@ import frappe from frappe import _, safe_encode from frappe.model.document import Document -from frappe.twofactor import (should_run_2fa, authenticate_for_2factor,confirm_otp_token) +from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, should_run_2fa + class LDAPSettings(Document): def validate(self): @@ -14,54 +15,74 @@ class LDAPSettings(Document): if not self.flags.ignore_mandatory: - if self.ldap_search_string.count('(') == self.ldap_search_string.count(')') and \ - self.ldap_search_string.startswith('(') and \ - self.ldap_search_string.endswith(')') and \ - self.ldap_search_string and \ - "{0}" in self.ldap_search_string: + if ( + self.ldap_search_string.count("(") == self.ldap_search_string.count(")") + and self.ldap_search_string.startswith("(") + and self.ldap_search_string.endswith(")") + and self.ldap_search_string + and "{0}" in self.ldap_search_string + ): - conn = self.connect_to_ldap(base_dn=self.base_dn, password=self.get_password(raise_exception=False)) + conn = self.connect_to_ldap( + base_dn=self.base_dn, password=self.get_password(raise_exception=False) + ) try: - if conn.result['type'] == 'bindResponse' and self.base_dn: + if conn.result["type"] == "bindResponse" and self.base_dn: import ldap3 conn.search( search_base=self.ldap_search_path_user, search_filter="(objectClass=*)", - attributes=self.get_ldap_attributes()) + attributes=self.get_ldap_attributes(), + ) conn.search( - search_base=self.ldap_search_path_group, - search_filter="(objectClass=*)", - attributes=['cn']) + search_base=self.ldap_search_path_group, search_filter="(objectClass=*)", attributes=["cn"] + ) except ldap3.core.exceptions.LDAPAttributeError as ex: - frappe.throw(_("LDAP settings incorrect. validation response was: {0}").format(ex), - title=_("Misconfigured")) + frappe.throw( + _("LDAP settings incorrect. validation response was: {0}").format(ex), + title=_("Misconfigured"), + ) except ldap3.core.exceptions.LDAPNoSuchObjectResult: - frappe.throw(_("Ensure the user and group search paths are correct."), - title=_("Misconfigured")) + frappe.throw( + _("Ensure the user and group search paths are correct."), title=_("Misconfigured") + ) - if self.ldap_directory_server.lower() == 'custom': + if self.ldap_directory_server.lower() == "custom": if not self.ldap_group_member_attribute or not self.ldap_group_objectclass: - frappe.throw(_("Custom LDAP Directoy Selected, please ensure 'LDAP Group Member attribute' and 'Group Object Class' are entered"), - title=_("Misconfigured")) + frappe.throw( + _( + "Custom LDAP Directoy Selected, please ensure 'LDAP Group Member attribute' and 'Group Object Class' are entered" + ), + title=_("Misconfigured"), + ) if self.ldap_custom_group_search and "{0}" not in self.ldap_custom_group_search: - frappe.throw(_("Custom Group Search if filled needs to contain the user placeholder {0}, eg uid={0},ou=users,dc=example,dc=com"), - title=_("Misconfigured")) + frappe.throw( + _( + "Custom Group Search if filled needs to contain the user placeholder {0}, eg uid={0},ou=users,dc=example,dc=com" + ), + title=_("Misconfigured"), + ) else: - frappe.throw(_("LDAP Search String must be enclosed in '()' and needs to contian the user placeholder {0}, eg sAMAccountName={0}")) + frappe.throw( + _( + "LDAP Search String must be enclosed in '()' and needs to contian the user placeholder {0}, eg sAMAccountName={0}" + ) + ) def connect_to_ldap(self, base_dn, password, read_only=True): try: - import ldap3 import ssl - if self.require_trusted_certificate == 'Yes': + import ldap3 + + if self.require_trusted_certificate == "Yes": tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLS_CLIENT) else: tls_configuration = ldap3.Tls(validate=ssl.CERT_NONE, version=ssl.PROTOCOL_TLS_CLIENT) @@ -82,7 +103,8 @@ class LDAPSettings(Document): password=password, auto_bind=bind_type, read_only=read_only, - raise_exceptions=True) + raise_exceptions=True, + ) return conn @@ -97,9 +119,7 @@ class LDAPSettings(Document): @staticmethod def get_ldap_client_settings(): # return the settings to be used on the client side. - result = { - "enabled": False - } + result = {"enabled": False} ldap = frappe.get_doc("LDAP Settings") if ldap.enabled: result["enabled"] = True @@ -109,7 +129,7 @@ class LDAPSettings(Document): @classmethod def update_user_fields(cls, user, user_data): - updatable_data = {key: value for key, value in user_data.items() if key != 'email'} + updatable_data = {key: value for key, value in user_data.items() if key != "email"} for key, value in updatable_data.items(): setattr(user, key, value) @@ -125,7 +145,9 @@ class LDAPSettings(Document): lower_groups = [g.lower() for g in additional_groups or []] all_mapped_roles = {r.erpnext_role for r in self.ldap_groups} - matched_roles = {r.erpnext_role for r in self.ldap_groups if r.ldap_group.lower() in lower_groups} + matched_roles = { + r.erpnext_role for r in self.ldap_groups if r.ldap_group.lower() in lower_groups + } unmatched_roles = all_mapped_roles.difference(matched_roles) needed_roles.update(matched_roles) roles_to_remove = current_roles.intersection(unmatched_roles) @@ -138,20 +160,22 @@ class LDAPSettings(Document): def create_or_update_user(self, user_data, groups=None): user = None - if frappe.db.exists("User", user_data['email']): - user = frappe.get_doc("User", user_data['email']) + if frappe.db.exists("User", user_data["email"]): + user = frappe.get_doc("User", user_data["email"]) LDAPSettings.update_user_fields(user=user, user_data=user_data) else: doc = user_data - doc.update({ - "doctype": "User", - "send_welcome_email": 0, - "language": "", - "user_type": "System User", - # "roles": [{ - # "role": self.default_role - # }] - }) + doc.update( + { + "doctype": "User", + "send_welcome_email": 0, + "language": "", + "user_type": "System User", + # "roles": [{ + # "role": self.default_role + # }] + } + ) user = frappe.get_doc(doc) user.insert(ignore_permissions=True) # always add default role. @@ -180,36 +204,39 @@ class LDAPSettings(Document): return ldap_attributes - def fetch_ldap_groups(self, user, conn): import ldap3 if type(user) is not ldap3.abstract.entry.Entry: - raise TypeError("Invalid type, attribute {0} must be of type '{1}'".format('user', 'ldap3.abstract.entry.Entry')) + raise TypeError( + "Invalid type, attribute {0} must be of type '{1}'".format( + "user", "ldap3.abstract.entry.Entry" + ) + ) if type(conn) is not ldap3.core.connection.Connection: - raise TypeError("Invalid type, attribute {0} must be of type '{1}'".format('conn', 'ldap3.Connection')) + raise TypeError( + "Invalid type, attribute {0} must be of type '{1}'".format("conn", "ldap3.Connection") + ) fetch_ldap_groups = None ldap_object_class = None ldap_group_members_attribute = None + if self.ldap_directory_server.lower() == "active directory": - if self.ldap_directory_server.lower() == 'active directory': - - ldap_object_class = 'Group' - ldap_group_members_attribute = 'member' + ldap_object_class = "Group" + ldap_group_members_attribute = "member" user_search_str = user.entry_dn + elif self.ldap_directory_server.lower() == "openldap": - elif self.ldap_directory_server.lower() == 'openldap': - - ldap_object_class = 'posixgroup' - ldap_group_members_attribute = 'memberuid' + ldap_object_class = "posixgroup" + ldap_group_members_attribute = "memberuid" user_search_str = getattr(user, self.ldap_username_field).value - elif self.ldap_directory_server.lower() == 'custom': + elif self.ldap_directory_server.lower() == "custom": ldap_object_class = self.ldap_group_objectclass ldap_group_members_attribute = self.ldap_group_member_attribute @@ -227,20 +254,20 @@ class LDAPSettings(Document): if ldap_object_class is not None: conn.search( search_base=self.ldap_search_path_group, - search_filter="(&(objectClass={0})({1}={2}))".format(ldap_object_class,ldap_group_members_attribute, user_search_str), - attributes=['cn']) # Build search query + search_filter="(&(objectClass={0})({1}={2}))".format( + ldap_object_class, ldap_group_members_attribute, user_search_str + ), + attributes=["cn"], + ) # Build search query if len(conn.entries) >= 1: fetch_ldap_groups = [] for group in conn.entries: - fetch_ldap_groups.append(group['cn'].value) + fetch_ldap_groups.append(group["cn"].value) return fetch_ldap_groups - - - def authenticate(self, username, password): if not self.enabled: @@ -257,7 +284,8 @@ class LDAPSettings(Document): conn.search( search_base=self.ldap_search_path_user, search_filter="{0}".format(user_filter), - attributes=ldap_attributes) + attributes=ldap_attributes, + ) if len(conn.entries) == 1 and conn.entries[0]: user = conn.entries[0] @@ -269,7 +297,7 @@ class LDAPSettings(Document): return self.create_or_update_user(self.convert_ldap_entry_to_dict(user), groups=groups) - raise ldap3.core.exceptions.LDAPInvalidCredentialsResult # even though nothing foundor failed authentication raise invalid credentials + raise ldap3.core.exceptions.LDAPInvalidCredentialsResult # even though nothing foundor failed authentication raise invalid credentials except ldap3.core.exceptions.LDAPInvalidFilterError: frappe.throw(_("Please use a valid LDAP search filter"), title=_("Misconfigured")) @@ -277,28 +305,29 @@ class LDAPSettings(Document): except ldap3.core.exceptions.LDAPInvalidCredentialsResult: frappe.throw(_("Invalid username or password")) - def reset_password(self, user, password, logout_sessions=False): from ldap3 import HASHED_SALTED_SHA, MODIFY_REPLACE from ldap3.utils.hashed import hashed search_filter = "({0}={1})".format(self.ldap_email_field, user) - conn = self.connect_to_ldap(self.base_dn, self.get_password(raise_exception=False), - read_only=False) + conn = self.connect_to_ldap( + self.base_dn, self.get_password(raise_exception=False), read_only=False + ) if conn.search( search_base=self.ldap_search_path_user, search_filter=search_filter, - attributes=self.get_ldap_attributes() + attributes=self.get_ldap_attributes(), ): if conn.entries and conn.entries[0]: entry_dn = conn.entries[0].entry_dn hashed_password = hashed(HASHED_SALTED_SHA, safe_encode(password)) - changes = {'userPassword': [(MODIFY_REPLACE, [hashed_password])]} + changes = {"userPassword": [(MODIFY_REPLACE, [hashed_password])]} if conn.modify(entry_dn, changes=changes): if logout_sessions: from frappe.sessions import clear_sessions + clear_sessions(user=user, force=True) frappe.msgprint(_("Password changed successfully.")) else: @@ -314,24 +343,24 @@ class LDAPSettings(Document): email = user_entry[self.ldap_email_field] data = { - 'username': user_entry[self.ldap_username_field].value, - 'email': str(email.value[0] if isinstance(email.value, list) else email.value), - 'first_name': user_entry[self.ldap_first_name_field].value + "username": user_entry[self.ldap_username_field].value, + "email": str(email.value[0] if isinstance(email.value, list) else email.value), + "first_name": user_entry[self.ldap_first_name_field].value, } # optional fields if self.ldap_middle_name_field: - data['middle_name'] = user_entry[self.ldap_middle_name_field].value + data["middle_name"] = user_entry[self.ldap_middle_name_field].value if self.ldap_last_name_field: - data['last_name'] = user_entry[self.ldap_last_name_field].value + data["last_name"] = user_entry[self.ldap_last_name_field].value if self.ldap_phone_field: - data['phone'] = user_entry[self.ldap_phone_field].value + data["phone"] = user_entry[self.ldap_phone_field].value if self.ldap_mobile_field: - data['mobile_no'] = user_entry[self.ldap_mobile_field].value + data["mobile_no"] = user_entry[self.ldap_mobile_field].value return data diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py index 41997fb4c7..0651932843 100644 --- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py @@ -1,20 +1,21 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -import unittest import functools -import ldap3 -import ssl import os - +import ssl +import unittest from unittest import mock + +import ldap3 +from ldap3 import MOCK_SYNC, OFFLINE_AD_2012_R2, OFFLINE_SLAPD_2_4, Connection, Server + +import frappe from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings -from ldap3 import Server, Connection, MOCK_SYNC, OFFLINE_SLAPD_2_4, OFFLINE_AD_2012_R2 -class LDAP_TestCase(): - TEST_LDAP_SERVER = None # must match the 'LDAP Settings' field option +class LDAP_TestCase: + TEST_LDAP_SERVER = None # must match the 'LDAP Settings' field option TEST_LDAP_SEARCH_STRING = None LDAP_USERNAME_FIELD = None DOCUMENT_GROUP_MAPPINGS = [] @@ -23,11 +24,12 @@ class LDAP_TestCase(): TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING = None def mock_ldap_connection(f): - @functools.wraps(f) def wrapped(self, *args, **kwargs): - with mock.patch('frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap') as mock_connection: + with mock.patch( + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap" + ) as mock_connection: mock_connection.return_value = self.connection self.test_class = LDAPSettings(self.doc) @@ -38,7 +40,6 @@ class LDAP_TestCase(): rv = f(self, *args, **kwargs) - # Clean-up self.test_class = None @@ -47,65 +48,66 @@ class LDAP_TestCase(): return wrapped def clean_test_users(): - try: # clean up test user 1 - frappe.get_doc("User", 'posix.user1@unit.testing').delete() + try: # clean up test user 1 + frappe.get_doc("User", "posix.user1@unit.testing").delete() except Exception: pass - try: # clean up test user 2 - frappe.get_doc("User", 'posix.user2@unit.testing').delete() + try: # clean up test user 2 + frappe.get_doc("User", "posix.user2@unit.testing").delete() except Exception: pass - @classmethod - def setUpClass(self, ldapServer='OpenLDAP'): + def setUpClass(self, ldapServer="OpenLDAP"): self.clean_test_users() # Save user data for restoration in tearDownClass() - self.user_ldap_settings = frappe.get_doc('LDAP Settings') + self.user_ldap_settings = frappe.get_doc("LDAP Settings") # Create test user1 self.user1doc = { - 'username': 'posix.user', - 'email': 'posix.user1@unit.testing', - 'first_name': 'posix' + "username": "posix.user", + "email": "posix.user1@unit.testing", + "first_name": "posix", } - self.user1doc.update({ - "doctype": "User", - "send_welcome_email": 0, - "language": "", - "user_type": "System User", - }) + self.user1doc.update( + { + "doctype": "User", + "send_welcome_email": 0, + "language": "", + "user_type": "System User", + } + ) user = frappe.get_doc(self.user1doc) user.insert(ignore_permissions=True) # Create test user1 self.user2doc = { - 'username': 'posix.user2', - 'email': 'posix.user2@unit.testing', - 'first_name': 'posix' + "username": "posix.user2", + "email": "posix.user2@unit.testing", + "first_name": "posix", } - self.user2doc.update({ - "doctype": "User", - "send_welcome_email": 0, - "language": "", - "user_type": "System User", - }) + self.user2doc.update( + { + "doctype": "User", + "send_welcome_email": 0, + "language": "", + "user_type": "System User", + } + ) user = frappe.get_doc(self.user2doc) user.insert(ignore_permissions=True) - # Setup Mock OpenLDAP Directory - self.ldap_dc_path = 'dc=unit,dc=testing' - self.ldap_user_path = 'ou=users,' + self.ldap_dc_path - self.ldap_group_path = 'ou=groups,' + self.ldap_dc_path - self.base_dn = 'cn=base_dn_user,' + self.ldap_dc_path - self.base_password = 'my_password' - self.ldap_server = 'ldap://my_fake_server:389' - + self.ldap_dc_path = "dc=unit,dc=testing" + self.ldap_user_path = "ou=users," + self.ldap_dc_path + self.ldap_group_path = "ou=groups," + self.ldap_dc_path + self.base_dn = "cn=base_dn_user," + self.ldap_dc_path + self.base_password = "my_password" + self.ldap_server = "ldap://my_fake_server:389" self.doc = { "doctype": "LDAP Settings", @@ -117,25 +119,26 @@ class LDAP_TestCase(): "ldap_search_path_user": self.ldap_user_path, "ldap_search_string": self.TEST_LDAP_SEARCH_STRING, "ldap_search_path_group": self.ldap_group_path, - "ldap_user_creation_and_mapping_section": '', - "ldap_email_field": 'mail', + "ldap_user_creation_and_mapping_section": "", + "ldap_email_field": "mail", "ldap_username_field": self.LDAP_USERNAME_FIELD, - "ldap_first_name_field": 'givenname', - "ldap_middle_name_field": '', - "ldap_last_name_field": 'sn', - "ldap_phone_field": 'telephonenumber', - "ldap_mobile_field": 'mobile', - "ldap_security": '', - "ssl_tls_mode": '', - "require_trusted_certificate": 'No', - "local_private_key_file": '', - "local_server_certificate_file": '', - "local_ca_certs_file": '', - "ldap_group_objectclass": '', - "ldap_group_member_attribute": '', - "default_role": 'Newsletter Manager', + "ldap_first_name_field": "givenname", + "ldap_middle_name_field": "", + "ldap_last_name_field": "sn", + "ldap_phone_field": "telephonenumber", + "ldap_mobile_field": "mobile", + "ldap_security": "", + "ssl_tls_mode": "", + "require_trusted_certificate": "No", + "local_private_key_file": "", + "local_server_certificate_file": "", + "local_ca_certs_file": "", + "ldap_group_objectclass": "", + "ldap_group_member_attribute": "", + "default_role": "Newsletter Manager", "ldap_groups": self.DOCUMENT_GROUP_MAPPINGS, - "ldap_group_field": ''} + "ldap_group_field": "", + } self.server = Server(host=self.ldap_server, port=389, get_info=self.LDAP_SCHEMA) @@ -144,17 +147,19 @@ class LDAP_TestCase(): user=self.base_dn, password=self.base_password, read_only=True, - client_strategy=MOCK_SYNC) + client_strategy=MOCK_SYNC, + ) - self.connection.strategy.entries_from_json(os.path.abspath(os.path.dirname(__file__)) + '/' + self.LDAP_LDIF_JSON) + self.connection.strategy.entries_from_json( + os.path.abspath(os.path.dirname(__file__)) + "/" + self.LDAP_LDIF_JSON + ) self.connection.bind() - @classmethod def tearDownClass(self): try: - frappe.get_doc('LDAP Settings').delete() + frappe.get_doc("LDAP Settings").delete() except Exception: pass @@ -172,78 +177,77 @@ class LDAP_TestCase(): # Clear OpenLDAP connection self.connection = None - @mock_ldap_connection def test_mandatory_fields(self): mandatory_fields = [ - 'ldap_server_url', - 'ldap_directory_server', - 'base_dn', - 'password', - 'ldap_search_path_user', - 'ldap_search_path_group', - 'ldap_search_string', - 'ldap_email_field', - 'ldap_username_field', - 'ldap_first_name_field', - 'require_trusted_certificate', - 'default_role' - ] # fields that are required to have ldap functioning need to be mandatory + "ldap_server_url", + "ldap_directory_server", + "base_dn", + "password", + "ldap_search_path_user", + "ldap_search_path_group", + "ldap_search_string", + "ldap_email_field", + "ldap_username_field", + "ldap_first_name_field", + "require_trusted_certificate", + "default_role", + ] # fields that are required to have ldap functioning need to be mandatory for mandatory_field in mandatory_fields: localdoc = self.doc.copy() - localdoc[mandatory_field] = '' + localdoc[mandatory_field] = "" try: frappe.get_doc(localdoc).save() - self.fail('Document LDAP Settings field [{0}] is not mandatory'.format(mandatory_field)) + self.fail("Document LDAP Settings field [{0}] is not mandatory".format(mandatory_field)) except frappe.exceptions.MandatoryError: pass except frappe.exceptions.ValidationError: - if mandatory_field == 'ldap_search_string': + if mandatory_field == "ldap_search_string": # additional validation is done on this field, pass in this instance pass + for non_mandatory_field in self.doc: # Ensure remaining fields have not been made mandatory - for non_mandatory_field in self.doc: # Ensure remaining fields have not been made mandatory - - if non_mandatory_field == 'doctype' or non_mandatory_field in mandatory_fields: + if non_mandatory_field == "doctype" or non_mandatory_field in mandatory_fields: continue localdoc = self.doc.copy() - localdoc[non_mandatory_field] = '' + localdoc[non_mandatory_field] = "" try: frappe.get_doc(localdoc).save() except frappe.exceptions.MandatoryError: - self.fail('Document LDAP Settings field [{0}] should not be mandatory'.format(non_mandatory_field)) - + self.fail( + "Document LDAP Settings field [{0}] should not be mandatory".format(non_mandatory_field) + ) @mock_ldap_connection def test_validation_ldap_search_string(self): invalid_ldap_search_strings = [ - '', - 'uid={0}', - '(uid={0}', - 'uid={0})', - '(&(objectclass=posixgroup)(uid={0})', - '&(objectclass=posixgroup)(uid={0}))', - '(uid=no_placeholder)' - ] # ldap search string must be enclosed in '()' for ldap search to work for finding user and have the same number of opening and closing brackets. + "", + "uid={0}", + "(uid={0}", + "uid={0})", + "(&(objectclass=posixgroup)(uid={0})", + "&(objectclass=posixgroup)(uid={0}))", + "(uid=no_placeholder)", + ] # ldap search string must be enclosed in '()' for ldap search to work for finding user and have the same number of opening and closing brackets. for invalid_search_string in invalid_ldap_search_strings: localdoc = self.doc.copy() - localdoc['ldap_search_string'] = invalid_search_string + localdoc["ldap_search_string"] = invalid_search_string try: frappe.get_doc(localdoc).save() @@ -253,95 +257,113 @@ class LDAP_TestCase(): except frappe.exceptions.ValidationError: pass - def test_connect_to_ldap(self): # setup a clean doc with ldap disabled so no validation occurs (this is tested seperatly) local_doc = self.doc.copy() - local_doc['enabled'] = False + local_doc["enabled"] = False self.test_class = LDAPSettings(self.doc) - with mock.patch('ldap3.Server') as ldap3_server_method: + with mock.patch("ldap3.Server") as ldap3_server_method: - with mock.patch('ldap3.Connection') as ldap3_connection_method: + with mock.patch("ldap3.Connection") as ldap3_connection_method: ldap3_connection_method.return_value = self.connection - with mock.patch('ldap3.Tls') as ldap3_Tls_method: + with mock.patch("ldap3.Tls") as ldap3_Tls_method: - function_return = self.test_class.connect_to_ldap(base_dn=self.base_dn, password=self.base_password) + function_return = self.test_class.connect_to_ldap( + base_dn=self.base_dn, password=self.base_password + ) args, kwargs = ldap3_connection_method.call_args prevent_connection_parameters = { # prevent these parameters for security or lack of the und user from being able to configure - 'mode': { - 'IP_V4_ONLY': 'Locks the user to IPv4 without frappe providing a way to configure', - 'IP_V6_ONLY': 'Locks the user to IPv6 without frappe providing a way to configure' + "mode": { + "IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure", + "IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure", + }, + "auto_bind": { + "NONE": "ldap3.Connection must autobind with base_dn", + "NO_TLS": "ldap3.Connection must have TLS", + "TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind", }, - 'auto_bind': { - 'NONE': 'ldap3.Connection must autobind with base_dn', - 'NO_TLS': 'ldap3.Connection must have TLS', - 'TLS_AFTER_BIND': '[Security] ldap3.Connection TLS bind must occur before bind' - } } for connection_arg in kwargs: - if connection_arg in prevent_connection_parameters and \ - kwargs[connection_arg] in prevent_connection_parameters[connection_arg]: + if ( + connection_arg in prevent_connection_parameters + and kwargs[connection_arg] in prevent_connection_parameters[connection_arg] + ): - self.fail('ldap3.Connection was called with {0}, failed reason: [{1}]'.format( - kwargs[connection_arg], - prevent_connection_parameters[connection_arg][kwargs[connection_arg]])) + self.fail( + "ldap3.Connection was called with {0}, failed reason: [{1}]".format( + kwargs[connection_arg], + prevent_connection_parameters[connection_arg][kwargs[connection_arg]], + ) + ) - if local_doc['require_trusted_certificate'] == 'Yes': + if local_doc["require_trusted_certificate"] == "Yes": tls_validate = ssl.CERT_REQUIRED tls_version = ssl.PROTOCOL_TLS_CLIENT tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version) - self.assertTrue(kwargs['auto_bind'] == ldap3.AUTO_BIND_TLS_BEFORE_BIND, - 'Security: [ldap3.Connection] autobind TLS before bind with value ldap3.AUTO_BIND_TLS_BEFORE_BIND') + self.assertTrue( + kwargs["auto_bind"] == ldap3.AUTO_BIND_TLS_BEFORE_BIND, + "Security: [ldap3.Connection] autobind TLS before bind with value ldap3.AUTO_BIND_TLS_BEFORE_BIND", + ) else: tls_validate = ssl.CERT_NONE tls_version = ssl.PROTOCOL_TLS_CLIENT tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version) - self.assertTrue(kwargs['auto_bind'], - 'ldap3.Connection must autobind') - + self.assertTrue(kwargs["auto_bind"], "ldap3.Connection must autobind") ldap3_Tls_method.assert_called_with(validate=tls_validate, version=tls_version) - ldap3_server_method.assert_called_with(host=self.doc['ldap_server_url'], tls=tls_configuration) + ldap3_server_method.assert_called_with( + host=self.doc["ldap_server_url"], tls=tls_configuration + ) - self.assertTrue(kwargs['password'] == self.base_password, - 'ldap3.Connection password does not match provided password') + self.assertTrue( + kwargs["password"] == self.base_password, + "ldap3.Connection password does not match provided password", + ) - self.assertTrue(kwargs['raise_exceptions'], - 'ldap3.Connection must raise exceptions for error handling') + self.assertTrue( + kwargs["raise_exceptions"], "ldap3.Connection must raise exceptions for error handling" + ) - self.assertTrue(kwargs['user'] == self.base_dn, - 'ldap3.Connection user does not match provided user') + self.assertTrue( + kwargs["user"] == self.base_dn, "ldap3.Connection user does not match provided user" + ) - ldap3_connection_method.assert_called_with(server=ldap3_server_method.return_value, + ldap3_connection_method.assert_called_with( + server=ldap3_server_method.return_value, auto_bind=True, password=self.base_password, raise_exceptions=True, read_only=True, - user=self.base_dn) + user=self.base_dn, + ) - self.assertTrue(type(function_return) is ldap3.core.connection.Connection, - 'The return type must be of ldap3.Connection') + self.assertTrue( + type(function_return) is ldap3.core.connection.Connection, + "The return type must be of ldap3.Connection", + ) - function_return = self.test_class.connect_to_ldap(base_dn=self.base_dn, password=self.base_password, read_only=False) + function_return = self.test_class.connect_to_ldap( + base_dn=self.base_dn, password=self.base_password, read_only=False + ) args, kwargs = ldap3_connection_method.call_args - self.assertFalse(kwargs['read_only'], 'connect_to_ldap() read_only parameter supplied as False but does not match the ldap3.Connection() read_only named parameter') - - - + self.assertFalse( + kwargs["read_only"], + "connect_to_ldap() read_only parameter supplied as False but does not match the ldap3.Connection() read_only named parameter", + ) @mock_ldap_connection def test_get_ldap_client_settings(self): @@ -350,133 +372,173 @@ class LDAP_TestCase(): self.assertIsInstance(result, dict) - self.assertTrue(result['enabled'] == self.doc['enabled']) # settings should match doc + self.assertTrue(result["enabled"] == self.doc["enabled"]) # settings should match doc localdoc = self.doc.copy() - localdoc['enabled'] = False + localdoc["enabled"] = False frappe.get_doc(localdoc).save() result = self.test_class.get_ldap_client_settings() - self.assertFalse(result['enabled']) # must match the edited doc - + self.assertFalse(result["enabled"]) # must match the edited doc @mock_ldap_connection def test_update_user_fields(self): test_user_data = { - 'username': 'posix.user', - 'email': 'posix.user1@unit.testing', - 'first_name': 'posix', - 'middle_name': 'another', - 'last_name': 'user', - 'phone': '08 1234 5678', - 'mobile_no': '0421 123 456' + "username": "posix.user", + "email": "posix.user1@unit.testing", + "first_name": "posix", + "middle_name": "another", + "last_name": "user", + "phone": "08 1234 5678", + "mobile_no": "0421 123 456", } - test_user = frappe.get_doc("User", test_user_data['email']) + test_user = frappe.get_doc("User", test_user_data["email"]) self.test_class.update_user_fields(test_user, test_user_data) - updated_user = frappe.get_doc("User", test_user_data['email']) - - self.assertTrue(updated_user.middle_name == test_user_data['middle_name']) - self.assertTrue(updated_user.last_name == test_user_data['last_name']) - self.assertTrue(updated_user.phone == test_user_data['phone']) - self.assertTrue(updated_user.mobile_no == test_user_data['mobile_no']) + updated_user = frappe.get_doc("User", test_user_data["email"]) + self.assertTrue(updated_user.middle_name == test_user_data["middle_name"]) + self.assertTrue(updated_user.last_name == test_user_data["last_name"]) + self.assertTrue(updated_user.phone == test_user_data["phone"]) + self.assertTrue(updated_user.mobile_no == test_user_data["mobile_no"]) @mock_ldap_connection def test_sync_roles(self): - if self.TEST_LDAP_SERVER.lower() == 'openldap': + if self.TEST_LDAP_SERVER.lower() == "openldap": test_user_data = { - 'posix.user1': ['Users', 'Administrators', 'default_role', 'frappe_default_all','frappe_default_guest'], - 'posix.user2': ['Users', 'Group3', 'default_role', 'frappe_default_all', 'frappe_default_guest'] + "posix.user1": [ + "Users", + "Administrators", + "default_role", + "frappe_default_all", + "frappe_default_guest", + ], + "posix.user2": [ + "Users", + "Group3", + "default_role", + "frappe_default_all", + "frappe_default_guest", + ], } - elif self.TEST_LDAP_SERVER.lower() == 'active directory': + elif self.TEST_LDAP_SERVER.lower() == "active directory": test_user_data = { - 'posix.user1': ['Domain Users', 'Domain Administrators', 'default_role', 'frappe_default_all','frappe_default_guest'], - 'posix.user2': ['Domain Users', 'Enterprise Administrators', 'default_role', 'frappe_default_all', 'frappe_default_guest'] + "posix.user1": [ + "Domain Users", + "Domain Administrators", + "default_role", + "frappe_default_all", + "frappe_default_guest", + ], + "posix.user2": [ + "Domain Users", + "Enterprise Administrators", + "default_role", + "frappe_default_all", + "frappe_default_guest", + ], } - role_to_group_map = { - self.doc['ldap_groups'][0]['erpnext_role']: self.doc['ldap_groups'][0]['ldap_group'], - self.doc['ldap_groups'][1]['erpnext_role']: self.doc['ldap_groups'][1]['ldap_group'], - self.doc['ldap_groups'][2]['erpnext_role']: self.doc['ldap_groups'][2]['ldap_group'], - 'Newsletter Manager': 'default_role', - 'All': 'frappe_default_all', - 'Guest': 'frappe_default_guest', - + self.doc["ldap_groups"][0]["erpnext_role"]: self.doc["ldap_groups"][0]["ldap_group"], + self.doc["ldap_groups"][1]["erpnext_role"]: self.doc["ldap_groups"][1]["ldap_group"], + self.doc["ldap_groups"][2]["erpnext_role"]: self.doc["ldap_groups"][2]["ldap_group"], + "Newsletter Manager": "default_role", + "All": "frappe_default_all", + "Guest": "frappe_default_guest", } # re-create user1 to ensure clean - frappe.get_doc("User", 'posix.user1@unit.testing').delete() + frappe.get_doc("User", "posix.user1@unit.testing").delete() user = frappe.get_doc(self.user1doc) user.insert(ignore_permissions=True) for test_user in test_user_data: - test_user_doc = frappe.get_doc("User", test_user + '@unit.testing') - test_user_roles = frappe.get_roles(test_user + '@unit.testing') + test_user_doc = frappe.get_doc("User", test_user + "@unit.testing") + test_user_roles = frappe.get_roles(test_user + "@unit.testing") - self.assertTrue(len(test_user_roles) == 2, - 'User should only be a part of the All and Guest roles') # check default frappe roles + self.assertTrue( + len(test_user_roles) == 2, "User should only be a part of the All and Guest roles" + ) # check default frappe roles - self.test_class.sync_roles(test_user_doc, test_user_data[test_user]) # update user roles + self.test_class.sync_roles(test_user_doc, test_user_data[test_user]) # update user roles - frappe.get_doc("User", test_user + '@unit.testing') - updated_user_roles = frappe.get_roles(test_user + '@unit.testing') + frappe.get_doc("User", test_user + "@unit.testing") + updated_user_roles = frappe.get_roles(test_user + "@unit.testing") - self.assertTrue(len(updated_user_roles) == len(test_user_data[test_user]), - 'syncing of the user roles failed. {0} != {1} for user {2}'.format(len(updated_user_roles), len(test_user_data[test_user]), test_user)) + self.assertTrue( + len(updated_user_roles) == len(test_user_data[test_user]), + "syncing of the user roles failed. {0} != {1} for user {2}".format( + len(updated_user_roles), len(test_user_data[test_user]), test_user + ), + ) - for user_role in updated_user_roles: # match each users role mapped to ldap groups + for user_role in updated_user_roles: # match each users role mapped to ldap groups - self.assertTrue(role_to_group_map[user_role] in test_user_data[test_user], - 'during sync_roles(), the user was given role {0} which should not have occured'.format(user_role)) + self.assertTrue( + role_to_group_map[user_role] in test_user_data[test_user], + "during sync_roles(), the user was given role {0} which should not have occured".format( + user_role + ), + ) @mock_ldap_connection def test_create_or_update_user(self): test_user_data = { - 'posix.user1': ['Users', 'Administrators', 'default_role', 'frappe_default_all','frappe_default_guest'], + "posix.user1": [ + "Users", + "Administrators", + "default_role", + "frappe_default_all", + "frappe_default_guest", + ], } - test_user = 'posix.user1' + test_user = "posix.user1" - frappe.get_doc("User", test_user + '@unit.testing').delete() # remove user 1 + frappe.get_doc("User", test_user + "@unit.testing").delete() # remove user 1 - with self.assertRaises(frappe.exceptions.DoesNotExistError): # ensure user deleted so function can be tested - frappe.get_doc("User", test_user + '@unit.testing') + with self.assertRaises( + frappe.exceptions.DoesNotExistError + ): # ensure user deleted so function can be tested + frappe.get_doc("User", test_user + "@unit.testing") - - with mock.patch('frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.update_user_fields') \ - as update_user_fields_method: + with mock.patch( + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.update_user_fields" + ) as update_user_fields_method: update_user_fields_method.return_value = None - - with mock.patch('frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.sync_roles') as sync_roles_method: + with mock.patch( + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.sync_roles" + ) as sync_roles_method: sync_roles_method.return_value = None # New user self.test_class.create_or_update_user(self.user1doc, test_user_data[test_user]) - self.assertTrue(sync_roles_method.called, 'User roles need to be updated for a new user') - self.assertFalse(update_user_fields_method.called, - 'User roles are not required to be updated for a new user, this will occur during logon') - + self.assertTrue(sync_roles_method.called, "User roles need to be updated for a new user") + self.assertFalse( + update_user_fields_method.called, + "User roles are not required to be updated for a new user, this will occur during logon", + ) # Existing user self.test_class.create_or_update_user(self.user1doc, test_user_data[test_user]) - self.assertTrue(sync_roles_method.called, 'User roles need to be updated for an existing user') - self.assertTrue(update_user_fields_method.called, 'User fields need to be updated for an existing user') - + self.assertTrue(sync_roles_method.called, "User roles need to be updated for an existing user") + self.assertTrue( + update_user_fields_method.called, "User fields need to be updated for an existing user" + ) @mock_ldap_connection def test_get_ldap_attributes(self): @@ -485,22 +547,15 @@ class LDAP_TestCase(): self.assertTrue(type(method_return) is list) - - @mock_ldap_connection def test_fetch_ldap_groups(self): - if self.TEST_LDAP_SERVER.lower() == 'openldap': + if self.TEST_LDAP_SERVER.lower() == "openldap": + test_users = {"posix.user": ["Users", "Administrators"], "posix.user2": ["Users", "Group3"]} + elif self.TEST_LDAP_SERVER.lower() == "active directory": test_users = { - 'posix.user': ['Users', 'Administrators'], - 'posix.user2': ['Users', 'Group3'] - - } - elif self.TEST_LDAP_SERVER.lower() == 'active directory': - test_users = { - 'posix.user': ['Domain Users', 'Domain Administrators'], - 'posix.user2': ['Domain Users', 'Enterprise Administrators'] - + "posix.user": ["Domain Users", "Domain Administrators"], + "posix.user2": ["Domain Users", "Enterprise Administrators"], } for test_user in test_users: @@ -508,7 +563,8 @@ class LDAP_TestCase(): self.connection.search( search_base=self.ldap_user_path, search_filter=self.TEST_LDAP_SEARCH_STRING.format(test_user), - attributes=self.test_class.get_ldap_attributes()) + attributes=self.test_class.get_ldap_attributes(), + ) method_return = self.test_class.fetch_ldap_groups(self.connection.entries[0], self.connection) @@ -519,30 +575,31 @@ class LDAP_TestCase(): self.assertTrue(returned_group in test_users[test_user]) - - @mock_ldap_connection def test_authenticate(self): - with mock.patch('frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.fetch_ldap_groups') as \ - fetch_ldap_groups_function: + with mock.patch( + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.fetch_ldap_groups" + ) as fetch_ldap_groups_function: fetch_ldap_groups_function.return_value = None - self.assertTrue(self.test_class.authenticate('posix.user', 'posix_user_password')) + self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password")) - self.assertTrue(fetch_ldap_groups_function.called, - 'As part of authentication function fetch_ldap_groups_function needs to be called') + self.assertTrue( + fetch_ldap_groups_function.called, + "As part of authentication function fetch_ldap_groups_function needs to be called", + ) invalid_users = [ - {'prefix_posix.user': 'posix_user_password'}, - {'posix.user_postfix': 'posix_user_password'}, - {'posix.user': 'posix_user_password_postfix'}, - {'posix.user': 'prefix_posix_user_password'}, - {'posix.user': ''}, - {'': 'posix_user_password'}, - {'': ''} - ] # All invalid users should return 'invalid username or password' + {"prefix_posix.user": "posix_user_password"}, + {"posix.user_postfix": "posix_user_password"}, + {"posix.user": "posix_user_password_postfix"}, + {"posix.user": "prefix_posix_user_password"}, + {"posix.user": ""}, + {"": "posix_user_password"}, + {"": ""}, + ] # All invalid users should return 'invalid username or password' for username, password in enumerate(invalid_users): @@ -550,9 +607,12 @@ class LDAP_TestCase(): self.test_class.authenticate(username, password) - self.assertTrue(str(display_massage.exception).lower() == 'invalid username or password', - 'invalid credentials passed authentication [user: {0}, password: {1}]'.format(username, password)) - + self.assertTrue( + str(display_massage.exception).lower() == "invalid username or password", + "invalid credentials passed authentication [user: {0}, password: {1}]".format( + username, password + ), + ) @mock_ldap_connection def test_complex_ldap_search_filter(self): @@ -563,17 +623,18 @@ class LDAP_TestCase(): self.test_class.ldap_search_string = search_filter - if 'ACCESS:test3' in search_filter: # posix.user does not have str in ldap.description auth should fail + if ( + "ACCESS:test3" in search_filter + ): # posix.user does not have str in ldap.description auth should fail with self.assertRaises(frappe.exceptions.ValidationError) as display_massage: - self.test_class.authenticate('posix.user', 'posix_user_password') + self.test_class.authenticate("posix.user", "posix_user_password") - self.assertTrue(str(display_massage.exception).lower() == 'invalid username or password') + self.assertTrue(str(display_massage.exception).lower() == "invalid username or password") else: - self.assertTrue(self.test_class.authenticate('posix.user', 'posix_user_password')) - + self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password")) def test_reset_password(self): @@ -582,103 +643,95 @@ class LDAP_TestCase(): # Create a clean doc localdoc = self.doc.copy() - localdoc['enabled'] = False + localdoc["enabled"] = False frappe.get_doc(localdoc).save() - with mock.patch('frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap') as connect_to_ldap: + with mock.patch( + "frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap" + ) as connect_to_ldap: connect_to_ldap.return_value = self.connection - with self.assertRaises(frappe.exceptions.ValidationError) as validation: # Fail if username string used - self.test_class.reset_password('posix.user', 'posix_user_password') + with self.assertRaises( + frappe.exceptions.ValidationError + ) as validation: # Fail if username string used + self.test_class.reset_password("posix.user", "posix_user_password") - self.assertTrue(str(validation.exception) == 'No LDAP User found for email: posix.user') + self.assertTrue(str(validation.exception) == "No LDAP User found for email: posix.user") try: - self.test_class.reset_password('posix.user1@unit.testing', 'posix_user_password') # Change Password + self.test_class.reset_password( + "posix.user1@unit.testing", "posix_user_password" + ) # Change Password - except Exception: # An exception from the tested class is ok, as long as the connection to LDAP was made writeable + except Exception: # An exception from the tested class is ok, as long as the connection to LDAP was made writeable pass connect_to_ldap.assert_called_with(self.base_dn, self.base_password, read_only=False) - @mock_ldap_connection def test_convert_ldap_entry_to_dict(self): self.connection.search( - search_base=self.ldap_user_path, - search_filter=self.TEST_LDAP_SEARCH_STRING.format("posix.user"), - attributes=self.test_class.get_ldap_attributes()) + search_base=self.ldap_user_path, + search_filter=self.TEST_LDAP_SEARCH_STRING.format("posix.user"), + attributes=self.test_class.get_ldap_attributes(), + ) test_ldap_entry = self.connection.entries[0] method_return = self.test_class.convert_ldap_entry_to_dict(test_ldap_entry) - self.assertTrue(type(method_return) is dict) # must be dict - self.assertTrue(len(method_return) == 6) # there are 6 fields in mock_ldap for use - + self.assertTrue(type(method_return) is dict) # must be dict + self.assertTrue(len(method_return) == 6) # there are 6 fields in mock_ldap for use class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase): - TEST_LDAP_SERVER = 'OpenLDAP' - TEST_LDAP_SEARCH_STRING = '(uid={0})' + TEST_LDAP_SERVER = "OpenLDAP" + TEST_LDAP_SEARCH_STRING = "(uid={0})" DOCUMENT_GROUP_MAPPINGS = [ - { - "doctype": "LDAP Group Mapping", - "ldap_group": "Administrators", - "erpnext_role": "System Manager" - }, - { - "doctype": "LDAP Group Mapping", - "ldap_group": "Users", - "erpnext_role": "Blogger" - }, - { - "doctype": "LDAP Group Mapping", - "ldap_group": "Group3", - "erpnext_role": "Accounts User" - } + { + "doctype": "LDAP Group Mapping", + "ldap_group": "Administrators", + "erpnext_role": "System Manager", + }, + {"doctype": "LDAP Group Mapping", "ldap_group": "Users", "erpnext_role": "Blogger"}, + {"doctype": "LDAP Group Mapping", "ldap_group": "Group3", "erpnext_role": "Accounts User"}, ] - LDAP_USERNAME_FIELD = 'uid' + LDAP_USERNAME_FIELD = "uid" LDAP_SCHEMA = OFFLINE_SLAPD_2_4 - LDAP_LDIF_JSON = 'test_data_ldif_openldap.json' + LDAP_LDIF_JSON = "test_data_ldif_openldap.json" TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING = [ - '(uid={0})', - '(&(objectclass=posixaccount)(uid={0}))', - '(&(description=*ACCESS:test1*)(uid={0}))', # OpenLDAP has no member of group, use description to filter posix.user has equivilent of AD 'memberOf' - '(&(objectclass=posixaccount)(description=*ACCESS:test3*)(uid={0}))' # OpenLDAP has no member of group, use description to filter posix.user doesn't have. equivilent of AD 'memberOf' + "(uid={0})", + "(&(objectclass=posixaccount)(uid={0}))", + "(&(description=*ACCESS:test1*)(uid={0}))", # OpenLDAP has no member of group, use description to filter posix.user has equivilent of AD 'memberOf' + "(&(objectclass=posixaccount)(description=*ACCESS:test3*)(uid={0}))", # OpenLDAP has no member of group, use description to filter posix.user doesn't have. equivilent of AD 'memberOf' ] class Test_ActiveDirectory(LDAP_TestCase, unittest.TestCase): - TEST_LDAP_SERVER = 'Active Directory' - TEST_LDAP_SEARCH_STRING = '(samaccountname={0})' + TEST_LDAP_SERVER = "Active Directory" + TEST_LDAP_SEARCH_STRING = "(samaccountname={0})" DOCUMENT_GROUP_MAPPINGS = [ - { - "doctype": "LDAP Group Mapping", - "ldap_group": "Domain Administrators", - "erpnext_role": "System Manager" - }, - { - "doctype": "LDAP Group Mapping", - "ldap_group": "Domain Users", - "erpnext_role": "Blogger" - }, - { - "doctype": "LDAP Group Mapping", - "ldap_group": "Enterprise Administrators", - "erpnext_role": "Accounts User" - } + { + "doctype": "LDAP Group Mapping", + "ldap_group": "Domain Administrators", + "erpnext_role": "System Manager", + }, + {"doctype": "LDAP Group Mapping", "ldap_group": "Domain Users", "erpnext_role": "Blogger"}, + { + "doctype": "LDAP Group Mapping", + "ldap_group": "Enterprise Administrators", + "erpnext_role": "Accounts User", + }, ] - LDAP_USERNAME_FIELD = 'samaccountname' + LDAP_USERNAME_FIELD = "samaccountname" LDAP_SCHEMA = OFFLINE_AD_2012_R2 - LDAP_LDIF_JSON = 'test_data_ldif_activedirectory.json' + LDAP_LDIF_JSON = "test_data_ldif_activedirectory.json" TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING = [ - '(samaccountname={0})', - '(&(objectclass=user)(samaccountname={0}))', - '(&(description=*ACCESS:test1*)(samaccountname={0}))', # OpenLDAP has no member of group, use description to filter posix.user has equivilent of AD 'memberOf' - '(&(objectclass=user)(description=*ACCESS:test3*)(samaccountname={0}))' # OpenLDAP has no member of group, use description to filter posix.user doesn't have. equivilent of AD 'memberOf' + "(samaccountname={0})", + "(&(objectclass=user)(samaccountname={0}))", + "(&(description=*ACCESS:test1*)(samaccountname={0}))", # OpenLDAP has no member of group, use description to filter posix.user has equivilent of AD 'memberOf' + "(&(objectclass=user)(description=*ACCESS:test3*)(samaccountname={0}))", # OpenLDAP has no member of group, use description to filter posix.user doesn't have. equivilent of AD 'memberOf' ] - diff --git a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py index 5a3f380e84..4ef6f65dc7 100644 --- a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py +++ b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class OAuthAuthorizationCode(Document): pass diff --git a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py index bc6d29cbdb..72cb789ebb 100644 --- a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py +++ b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('OAuth Authorization Code') + class TestOAuthAuthorizationCode(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py index ff6f96cc4d..515d3d2ba3 100644 --- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py +++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py @@ -5,8 +5,10 @@ import frappe from frappe.model.document import Document + class OAuthBearerToken(Document): def validate(self): if not self.expiration_time: - self.expiration_time = frappe.utils.datetime.datetime.strptime(self.creation, "%Y-%m-%d %H:%M:%S.%f") + frappe.utils.datetime.timedelta(seconds=self.expires_in) - + self.expiration_time = frappe.utils.datetime.datetime.strptime( + self.creation, "%Y-%m-%d %H:%M:%S.%f" + ) + frappe.utils.datetime.timedelta(seconds=self.expires_in) diff --git a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py index 965feb4f78..9dea8f482a 100644 --- a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py +++ b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('OAuth Bearer Token') + class TestOAuthBearerToken(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.py b/frappe/integrations/doctype/oauth_client/oauth_client.py index 42fba07ecb..09f6e3aced 100644 --- a/frappe/integrations/doctype/oauth_client/oauth_client.py +++ b/frappe/integrations/doctype/oauth_client/oauth_client.py @@ -6,13 +6,23 @@ import frappe from frappe import _ from frappe.model.document import Document + class OAuthClient(Document): def validate(self): self.client_id = self.name if not self.client_secret: self.client_secret = frappe.generate_hash(length=10) self.validate_grant_and_response() + def validate_grant_and_response(self): - if self.grant_type == "Authorization Code" and self.response_type != "Code" or \ - self.grant_type == "Implicit" and self.response_type != "Token": - frappe.throw(_("Combination of Grant Type ({0}) and Response Type ({1}) not allowed").format(self.grant_type, self.response_type)) + if ( + self.grant_type == "Authorization Code" + and self.response_type != "Code" + or self.grant_type == "Implicit" + and self.response_type != "Token" + ): + frappe.throw( + _( + "Combination of Grant Type ({0}) and Response Type ({1}) not allowed" + ).format(self.grant_type, self.response_type) + ) diff --git a/frappe/integrations/doctype/oauth_client/test_oauth_client.py b/frappe/integrations/doctype/oauth_client/test_oauth_client.py index fa03fa06e7..dd1b25239a 100644 --- a/frappe/integrations/doctype/oauth_client/test_oauth_client.py +++ b/frappe/integrations/doctype/oauth_client/test_oauth_client.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('OAuth Client') + class TestOAuthClient(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py index ec1636659f..2aefd591a1 100644 --- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py +++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py @@ -3,16 +3,20 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document + class OAuthProviderSettings(Document): pass + def get_oauth_settings(): """Returns oauth settings""" - out = frappe._dict({ - "skip_authorization" : frappe.db.get_value("OAuth Provider Settings", None, "skip_authorization") - }) + out = frappe._dict( + { + "skip_authorization": frappe.db.get_value("OAuth Provider Settings", None, "skip_authorization") + } + ) - return out \ No newline at end of file + return out diff --git a/frappe/integrations/doctype/oauth_scope/oauth_scope.py b/frappe/integrations/doctype/oauth_scope/oauth_scope.py index cf5fa1f341..a30d087cc0 100644 --- a/frappe/integrations/doctype/oauth_scope/oauth_scope.py +++ b/frappe/integrations/doctype/oauth_scope/oauth_scope.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class OAuthScope(Document): pass diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.py b/frappe/integrations/doctype/paypal_settings/paypal_settings.py index 30ac905792..ab7512f403 100644 --- a/frappe/integrations/doctype/paypal_settings/paypal_settings.py +++ b/frappe/integrations/doctype/paypal_settings/paypal_settings.py @@ -63,21 +63,48 @@ More Details: """ -import frappe import json +from urllib.parse import urlencode + import pytz + +import frappe from frappe import _ -from urllib.parse import urlencode +from frappe.integrations.utils import create_payment_gateway, create_request_log, make_post_request from frappe.model.document import Document -from frappe.integrations.utils import create_request_log, make_post_request, create_payment_gateway -from frappe.utils import get_url, call_hook_method, cint, get_datetime +from frappe.utils import call_hook_method, cint, get_datetime, get_url +api_path = "/api/method/frappe.integrations.doctype.paypal_settings.paypal_settings" -api_path = '/api/method/frappe.integrations.doctype.paypal_settings.paypal_settings' class PayPalSettings(Document): - supported_currencies = ["AUD", "BRL", "CAD", "CZK", "DKK", "EUR", "HKD", "HUF", "ILS", "JPY", "MYR", "MXN", - "TWD", "NZD", "NOK", "PHP", "PLN", "GBP", "RUB", "SGD", "SEK", "CHF", "THB", "TRY", "USD"] + supported_currencies = [ + "AUD", + "BRL", + "CAD", + "CZK", + "DKK", + "EUR", + "HKD", + "HUF", + "ILS", + "JPY", + "MYR", + "MXN", + "TWD", + "NZD", + "NOK", + "PHP", + "PLN", + "GBP", + "RUB", + "SGD", + "SEK", + "CHF", + "THB", + "TRY", + "USD", + ] def __setup__(self): setattr(self, "use_sandbox", 0) @@ -88,7 +115,7 @@ class PayPalSettings(Document): def validate(self): create_payment_gateway("PayPal") - call_hook_method('payment_gateway_enabled', gateway="PayPal") + call_hook_method("payment_gateway_enabled", gateway="PayPal") if not self.flags.ignore_mandatory: self.validate_paypal_credentails() @@ -97,7 +124,11 @@ class PayPalSettings(Document): def validate_transaction_currency(self, currency): if currency not in self.supported_currencies: - frappe.throw(_("Please select another payment method. PayPal does not support transactions in currency '{0}'").format(currency)) + frappe.throw( + _( + "Please select another payment method. PayPal does not support transactions in currency '{0}'" + ).format(currency) + ) def get_paypal_params_and_url(self): params = { @@ -105,17 +136,23 @@ class PayPalSettings(Document): "PWD": self.get_password(fieldname="api_password", raise_exception=False), "SIGNATURE": self.signature, "VERSION": "98", - "METHOD": "GetPalDetails" + "METHOD": "GetPalDetails", } if hasattr(self, "use_sandbox") and self.use_sandbox: - params.update({ - "USER": frappe.conf.sandbox_api_username, - "PWD": frappe.conf.sandbox_api_password, - "SIGNATURE": frappe.conf.sandbox_signature - }) - - api_url = "https://api-3t.sandbox.paypal.com/nvp" if (self.paypal_sandbox or self.use_sandbox) else "https://api-3t.paypal.com/nvp" + params.update( + { + "USER": frappe.conf.sandbox_api_username, + "PWD": frappe.conf.sandbox_api_password, + "SIGNATURE": frappe.conf.sandbox_signature, + } + ) + + api_url = ( + "https://api-3t.sandbox.paypal.com/nvp" + if (self.paypal_sandbox or self.use_sandbox) + else "https://api-3t.paypal.com/nvp" + ) return params, api_url @@ -142,27 +179,30 @@ class PayPalSettings(Document): else: return_url = "https://www.paypal.com/cgi-bin/webscr?cmd=_express-checkout&token={0}" - kwargs.update({ - "token": response.get("TOKEN")[0], - "correlation_id": response.get("CORRELATIONID")[0] - }) - self.integration_request = create_request_log(kwargs, "Remote", "PayPal", response.get("TOKEN")[0]) + kwargs.update( + {"token": response.get("TOKEN")[0], "correlation_id": response.get("CORRELATIONID")[0]} + ) + self.integration_request = create_request_log( + kwargs, "Remote", "PayPal", response.get("TOKEN")[0] + ) return return_url.format(kwargs["token"]) def execute_set_express_checkout(self, **kwargs): params, url = self.get_paypal_params_and_url() - params.update({ - "METHOD": "SetExpressCheckout", - "returnUrl": get_url("{0}.get_express_checkout_details".format(api_path)), - "cancelUrl": get_url("/payment-cancel"), - "PAYMENTREQUEST_0_PAYMENTACTION": "SALE", - "PAYMENTREQUEST_0_AMT": kwargs['amount'], - "PAYMENTREQUEST_0_CURRENCYCODE": kwargs['currency'].upper() - }) - - if kwargs.get('subscription_details'): + params.update( + { + "METHOD": "SetExpressCheckout", + "returnUrl": get_url("{0}.get_express_checkout_details".format(api_path)), + "cancelUrl": get_url("/payment-cancel"), + "PAYMENTREQUEST_0_PAYMENTACTION": "SALE", + "PAYMENTREQUEST_0_AMT": kwargs["amount"], + "PAYMENTREQUEST_0_CURRENCYCODE": kwargs["currency"].upper(), + } + ) + + if kwargs.get("subscription_details"): self.configure_recurring_payments(params, kwargs) params = urlencode(params) @@ -175,14 +215,20 @@ class PayPalSettings(Document): def configure_recurring_payments(self, params, kwargs): # removing the params as we have to setup rucurring payments - for param in ('PAYMENTREQUEST_0_PAYMENTACTION', 'PAYMENTREQUEST_0_AMT', - 'PAYMENTREQUEST_0_CURRENCYCODE'): + for param in ( + "PAYMENTREQUEST_0_PAYMENTACTION", + "PAYMENTREQUEST_0_AMT", + "PAYMENTREQUEST_0_CURRENCYCODE", + ): del params[param] - params.update({ - "L_BILLINGTYPE0": "RecurringPayments", #The type of billing agreement - "L_BILLINGAGREEMENTDESCRIPTION0": kwargs['description'] - }) + params.update( + { + "L_BILLINGTYPE0": "RecurringPayments", # The type of billing agreement + "L_BILLINGAGREEMENTDESCRIPTION0": kwargs["description"], + } + ) + def get_paypal_and_transaction_details(token): doc = frappe.get_doc("PayPal Settings") @@ -194,23 +240,25 @@ def get_paypal_and_transaction_details(token): return data, params, url + def setup_redirect(data, redirect_url, custom_redirect_to=None, redirect=True): - redirect_to = data.get('redirect_to') or None - redirect_message = data.get('redirect_message') or None + redirect_to = data.get("redirect_to") or None + redirect_message = data.get("redirect_message") or None if custom_redirect_to: redirect_to = custom_redirect_to if redirect_to: - redirect_url += '&' + urlencode({'redirect_to': redirect_to}) + redirect_url += "&" + urlencode({"redirect_to": redirect_to}) if redirect_message: - redirect_url += '&' + urlencode({'redirect_message': redirect_message}) + redirect_url += "&" + urlencode({"redirect_message": redirect_message}) # this is done so that functions called via hooks can update flags.redirect_to if redirect: frappe.local.response["type"] = "redirect" frappe.local.response["location"] = get_url(redirect_url) + @frappe.whitelist(allow_guest=True, xss_safe=True) def get_express_checkout_details(token): try: @@ -218,26 +266,29 @@ def get_express_checkout_details(token): doc.setup_sandbox_env(token) params, url = doc.get_paypal_params_and_url() - params.update({ - "METHOD": "GetExpressCheckoutDetails", - "TOKEN": token - }) + params.update({"METHOD": "GetExpressCheckoutDetails", "TOKEN": token}) response = make_post_request(url, data=params) if response.get("ACK")[0] != "Success": - frappe.respond_as_web_page(_("Something went wrong"), - _("Looks like something went wrong during the transaction. Since we haven't confirmed the payment, Paypal will automatically refund you this amount. If it doesn't, please send us an email and mention the Correlation ID: {0}.").format(response.get("CORRELATIONID", [None])[0]), - indicator_color='red', - http_status_code=frappe.ValidationError.http_status_code) + frappe.respond_as_web_page( + _("Something went wrong"), + _( + "Looks like something went wrong during the transaction. Since we haven't confirmed the payment, Paypal will automatically refund you this amount. If it doesn't, please send us an email and mention the Correlation ID: {0}." + ).format(response.get("CORRELATIONID", [None])[0]), + indicator_color="red", + http_status_code=frappe.ValidationError.http_status_code, + ) return doc = frappe.get_doc("Integration Request", token) - update_integration_request_status(token, { - "payerid": response.get("PAYERID")[0], - "payer_email": response.get("EMAIL")[0] - }, "Authorized", doc=doc) + update_integration_request_status( + token, + {"payerid": response.get("PAYERID")[0], "payer_email": response.get("EMAIL")[0]}, + "Authorized", + doc=doc, + ) frappe.local.response["type"] = "redirect" frappe.local.response["location"] = get_redirect_uri(doc, token, response.get("PAYERID")[0]) @@ -245,35 +296,45 @@ def get_express_checkout_details(token): except Exception: frappe.log_error(frappe.get_traceback()) + @frappe.whitelist(allow_guest=True, xss_safe=True) def confirm_payment(token): try: custom_redirect_to = None data, params, url = get_paypal_and_transaction_details(token) - params.update({ - "METHOD": "DoExpressCheckoutPayment", - "PAYERID": data.get("payerid"), - "TOKEN": token, - "PAYMENTREQUEST_0_PAYMENTACTION": "SALE", - "PAYMENTREQUEST_0_AMT": data.get("amount"), - "PAYMENTREQUEST_0_CURRENCYCODE": data.get("currency").upper() - }) + params.update( + { + "METHOD": "DoExpressCheckoutPayment", + "PAYERID": data.get("payerid"), + "TOKEN": token, + "PAYMENTREQUEST_0_PAYMENTACTION": "SALE", + "PAYMENTREQUEST_0_AMT": data.get("amount"), + "PAYMENTREQUEST_0_CURRENCYCODE": data.get("currency").upper(), + } + ) response = make_post_request(url, data=params) if response.get("ACK")[0] == "Success": - update_integration_request_status(token, { - "transaction_id": response.get("PAYMENTINFO_0_TRANSACTIONID")[0], - "correlation_id": response.get("CORRELATIONID")[0] - }, "Completed") + update_integration_request_status( + token, + { + "transaction_id": response.get("PAYMENTINFO_0_TRANSACTIONID")[0], + "correlation_id": response.get("CORRELATIONID")[0], + }, + "Completed", + ) if data.get("reference_doctype") and data.get("reference_docname"): - custom_redirect_to = frappe.get_doc(data.get("reference_doctype"), - data.get("reference_docname")).run_method("on_payment_authorized", "Completed") + custom_redirect_to = frappe.get_doc( + data.get("reference_doctype"), data.get("reference_docname") + ).run_method("on_payment_authorized", "Completed") frappe.db.commit() - redirect_url = '/integrations/payment-success?doctype={0}&docname={1}'.format(data.get("reference_doctype"), data.get("reference_docname")) + redirect_url = "/integrations/payment-success?doctype={0}&docname={1}".format( + data.get("reference_doctype"), data.get("reference_docname") + ) else: redirect_url = "/integrations/payment-failed" @@ -282,6 +343,7 @@ def confirm_payment(token): except Exception: frappe.log_error(frappe.get_traceback()) + @frappe.whitelist(allow_guest=True, xss_safe=True) def create_recurring_profile(token, payerid): try: @@ -292,49 +354,60 @@ def create_recurring_profile(token, payerid): addons = data.get("addons") subscription_details = data.get("subscription_details") - if data.get('subscription_id'): + if data.get("subscription_id"): if addons: updating = True - manage_recurring_payment_profile_status(data['subscription_id'], 'Cancel', params, url) - - params.update({ - "METHOD": "CreateRecurringPaymentsProfile", - "PAYERID": payerid, - "TOKEN": token, - "DESC": data.get("description"), - "BILLINGPERIOD": subscription_details.get("billing_period"), - "BILLINGFREQUENCY": subscription_details.get("billing_frequency"), - "AMT": data.get("amount") if data.get("subscription_amount") == data.get("amount") else data.get("subscription_amount"), - "CURRENCYCODE": data.get("currency").upper(), - "INITAMT": data.get("upfront_amount") - }) - - status_changed_to = 'Completed' if data.get("starting_immediately") or updating else 'Verified' + manage_recurring_payment_profile_status(data["subscription_id"], "Cancel", params, url) + + params.update( + { + "METHOD": "CreateRecurringPaymentsProfile", + "PAYERID": payerid, + "TOKEN": token, + "DESC": data.get("description"), + "BILLINGPERIOD": subscription_details.get("billing_period"), + "BILLINGFREQUENCY": subscription_details.get("billing_frequency"), + "AMT": data.get("amount") + if data.get("subscription_amount") == data.get("amount") + else data.get("subscription_amount"), + "CURRENCYCODE": data.get("currency").upper(), + "INITAMT": data.get("upfront_amount"), + } + ) + + status_changed_to = "Completed" if data.get("starting_immediately") or updating else "Verified" starts_at = get_datetime(subscription_details.get("start_date")) or frappe.utils.now_datetime() - starts_at = starts_at.replace(tzinfo=pytz.timezone(frappe.utils.get_time_zone())).astimezone(pytz.utc) + starts_at = starts_at.replace(tzinfo=pytz.timezone(frappe.utils.get_time_zone())).astimezone( + pytz.utc + ) - #"PROFILESTARTDATE": datetime.utcfromtimestamp(get_timestamp(starts_at)).isoformat() - params.update({ - "PROFILESTARTDATE": starts_at.isoformat() - }) + # "PROFILESTARTDATE": datetime.utcfromtimestamp(get_timestamp(starts_at)).isoformat() + params.update({"PROFILESTARTDATE": starts_at.isoformat()}) response = make_post_request(url, data=params) if response.get("ACK")[0] == "Success": - update_integration_request_status(token, { - "profile_id": response.get("PROFILEID")[0], - }, "Completed") + update_integration_request_status( + token, + { + "profile_id": response.get("PROFILEID")[0], + }, + "Completed", + ) if data.get("reference_doctype") and data.get("reference_docname"): - data['subscription_id'] = response.get("PROFILEID")[0] + data["subscription_id"] = response.get("PROFILEID")[0] frappe.flags.data = data - custom_redirect_to = frappe.get_doc(data.get("reference_doctype"), - data.get("reference_docname")).run_method("on_payment_authorized", status_changed_to) + custom_redirect_to = frappe.get_doc( + data.get("reference_doctype"), data.get("reference_docname") + ).run_method("on_payment_authorized", status_changed_to) frappe.db.commit() - redirect_url = '/integrations/payment-success?doctype={0}&docname={1}'.format(data.get("reference_doctype"), data.get("reference_docname")) + redirect_url = "/integrations/payment-success?doctype={0}&docname={1}".format( + data.get("reference_doctype"), data.get("reference_docname") + ) else: redirect_url = "/integrations/payment-failed" @@ -343,26 +416,29 @@ def create_recurring_profile(token, payerid): except Exception: frappe.log_error(frappe.get_traceback()) + def update_integration_request_status(token, data, status, error=False, doc=None): if not doc: doc = frappe.get_doc("Integration Request", token) doc.update_status(data, status) + def get_redirect_uri(doc, token, payerid): data = json.loads(doc.data) if data.get("subscription_details") or data.get("subscription_id"): - return get_url("{0}.create_recurring_profile?token={1}&payerid={2}".format(api_path, token, payerid)) + return get_url( + "{0}.create_recurring_profile?token={1}&payerid={2}".format(api_path, token, payerid) + ) else: return get_url("{0}.confirm_payment?token={1}".format(api_path, token)) + def manage_recurring_payment_profile_status(profile_id, action, args, url): - args.update({ - "METHOD": "ManageRecurringPaymentsProfileStatus", - "PROFILEID": profile_id, - "ACTION": action - }) + args.update( + {"METHOD": "ManageRecurringPaymentsProfileStatus", "PROFILEID": profile_id, "ACTION": action} + ) response = make_post_request(url, data=args) @@ -370,9 +446,10 @@ def manage_recurring_payment_profile_status(profile_id, action, args, url): # thus could not cancel the subscription. # thus raise an exception only if the error code is not equal to 11556 - if response.get("ACK")[0] != "Success" and response.get("L_ERRORCODE0", [])[0] != '11556': + if response.get("ACK")[0] != "Success" and response.get("L_ERRORCODE0", [])[0] != "11556": frappe.throw(_("Failed while amending subscription")) + @frappe.whitelist(allow_guest=True) def ipn_handler(): try: @@ -380,26 +457,32 @@ def ipn_handler(): validate_ipn_request(data) - data.update({ - "payment_gateway": "PayPal" - }) + data.update({"payment_gateway": "PayPal"}) - doc = frappe.get_doc({ - "data": json.dumps(frappe.local.form_dict), - "doctype": "Integration Request", - "integration_type": "Subscription Notification", - "status": "Queued" - }).insert(ignore_permissions=True) + doc = frappe.get_doc( + { + "data": json.dumps(frappe.local.form_dict), + "doctype": "Integration Request", + "integration_type": "Subscription Notification", + "status": "Queued", + } + ).insert(ignore_permissions=True) frappe.db.commit() - frappe.enqueue(method='frappe.integrations.doctype.paypal_settings.paypal_settings.handle_subscription_notification', - queue='long', timeout=600, is_async=True, **{"doctype": "Integration Request", "docname": doc.name}) + frappe.enqueue( + method="frappe.integrations.doctype.paypal_settings.paypal_settings.handle_subscription_notification", + queue="long", + timeout=600, + is_async=True, + **{"doctype": "Integration Request", "docname": doc.name} + ) except frappe.InvalidStatusError: pass except Exception as e: frappe.log(frappe.log_error(title=e)) + def validate_ipn_request(data): def _throw(): frappe.throw(_("In Valid Request"), exc=frappe.InvalidStatusError) @@ -410,16 +493,16 @@ def validate_ipn_request(data): doc = frappe.get_doc("PayPal Settings") params, url = doc.get_paypal_params_and_url() - params.update({ - "METHOD": "GetRecurringPaymentsProfileDetails", - "PROFILEID": data.get("recurring_payment_id") - }) + params.update( + {"METHOD": "GetRecurringPaymentsProfileDetails", "PROFILEID": data.get("recurring_payment_id")} + ) params = urlencode(params) res = make_post_request(url=url, data=params.encode("utf-8")) - if res['ACK'][0] != 'Success': + if res["ACK"][0] != "Success": _throw() + def handle_subscription_notification(doctype, docname): call_hook_method("handle_subscription_notification", doctype=doctype, docname=docname) diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py index 5255360242..0888fd35b7 100644 --- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py +++ b/frappe/integrations/doctype/paytm_settings/paytm_settings.py @@ -3,112 +3,131 @@ # License: MIT. See LICENSE import json -import requests from urllib.parse import urlencode +import requests +from paytmchecksum import generateSignature, verifySignature + import frappe -from frappe.model.document import Document from frappe import _ -from frappe.utils import get_url, call_hook_method, cint, flt, cstr -from frappe.integrations.utils import create_request_log, create_payment_gateway -from frappe.utils import get_request_site_address -from paytmchecksum import generateSignature, verifySignature +from frappe.integrations.utils import create_payment_gateway, create_request_log +from frappe.model.document import Document +from frappe.utils import call_hook_method, cint, cstr, flt, get_request_site_address, get_url from frappe.utils.password import get_decrypted_password + class PaytmSettings(Document): supported_currencies = ["INR"] def validate(self): - create_payment_gateway('Paytm') - call_hook_method('payment_gateway_enabled', gateway='Paytm') + create_payment_gateway("Paytm") + call_hook_method("payment_gateway_enabled", gateway="Paytm") def validate_transaction_currency(self, currency): if currency not in self.supported_currencies: - frappe.throw(_("Please select another payment method. Paytm does not support transactions in currency '{0}'").format(currency)) + frappe.throw( + _( + "Please select another payment method. Paytm does not support transactions in currency '{0}'" + ).format(currency) + ) def get_payment_url(self, **kwargs): - '''Return payment url with several params''' + """Return payment url with several params""" # create unique order id by making it equal to the integration request integration_request = create_request_log(kwargs, "Host", "Paytm") kwargs.update(dict(order_id=integration_request.name)) return get_url("./integrations/paytm_checkout?{0}".format(urlencode(kwargs))) + def get_paytm_config(): - ''' Returns paytm config ''' + """Returns paytm config""" - paytm_config = frappe.db.get_singles_dict('Paytm Settings') - paytm_config.update(dict(merchant_key=get_decrypted_password('Paytm Settings', 'Paytm Settings', 'merchant_key'))) + paytm_config = frappe.db.get_singles_dict("Paytm Settings") + paytm_config.update( + dict(merchant_key=get_decrypted_password("Paytm Settings", "Paytm Settings", "merchant_key")) + ) if cint(paytm_config.staging): - paytm_config.update(dict( - website="WEBSTAGING", - url='https://securegw-stage.paytm.in/order/process', - transaction_status_url='https://securegw-stage.paytm.in/order/status', - industry_type_id='RETAIL' - )) + paytm_config.update( + dict( + website="WEBSTAGING", + url="https://securegw-stage.paytm.in/order/process", + transaction_status_url="https://securegw-stage.paytm.in/order/status", + industry_type_id="RETAIL", + ) + ) else: - paytm_config.update(dict( - url='https://securegw.paytm.in/order/process', - transaction_status_url='https://securegw.paytm.in/order/status', - )) + paytm_config.update( + dict( + url="https://securegw.paytm.in/order/process", + transaction_status_url="https://securegw.paytm.in/order/status", + ) + ) return paytm_config + def get_paytm_params(payment_details, order_id, paytm_config): # initialize a dictionary paytm_params = dict() - redirect_uri = get_request_site_address(True) + "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.verify_transaction" - + redirect_uri = ( + get_request_site_address(True) + + "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.verify_transaction" + ) - paytm_params.update({ - "MID" : paytm_config.merchant_id, - "WEBSITE" : paytm_config.website, - "INDUSTRY_TYPE_ID" : paytm_config.industry_type_id, - "CHANNEL_ID" : "WEB", - "ORDER_ID" : order_id, - "CUST_ID" : payment_details['payer_email'], - "EMAIL" : payment_details['payer_email'], - "TXN_AMOUNT" : cstr(flt(payment_details['amount'], 2)), - "CALLBACK_URL" : redirect_uri, - }) + paytm_params.update( + { + "MID": paytm_config.merchant_id, + "WEBSITE": paytm_config.website, + "INDUSTRY_TYPE_ID": paytm_config.industry_type_id, + "CHANNEL_ID": "WEB", + "ORDER_ID": order_id, + "CUST_ID": payment_details["payer_email"], + "EMAIL": payment_details["payer_email"], + "TXN_AMOUNT": cstr(flt(payment_details["amount"], 2)), + "CALLBACK_URL": redirect_uri, + } + ) checksum = generateSignature(paytm_params, paytm_config.merchant_key) - paytm_params.update({ - "CHECKSUMHASH" : checksum - }) + paytm_params.update({"CHECKSUMHASH": checksum}) return paytm_params + @frappe.whitelist(allow_guest=True) def verify_transaction(**paytm_params): - '''Verify checksum for received data in the callback and then verify the transaction''' + """Verify checksum for received data in the callback and then verify the transaction""" paytm_config = get_paytm_config() is_valid_checksum = False - paytm_params.pop('cmd', None) - paytm_checksum = paytm_params.pop('CHECKSUMHASH', None) + paytm_params.pop("cmd", None) + paytm_checksum = paytm_params.pop("CHECKSUMHASH", None) if paytm_params and paytm_config and paytm_checksum: # Verify checksum is_valid_checksum = verifySignature(paytm_params, paytm_config.merchant_key, paytm_checksum) - if is_valid_checksum and paytm_params.get('RESPCODE') == '01': - verify_transaction_status(paytm_config, paytm_params['ORDERID']) + if is_valid_checksum and paytm_params.get("RESPCODE") == "01": + verify_transaction_status(paytm_config, paytm_params["ORDERID"]) else: - frappe.respond_as_web_page("Payment Failed", + frappe.respond_as_web_page( + "Payment Failed", "Transaction failed to complete. In case of any deductions, deducted amount will get refunded to your account.", - http_status_code=401, indicator_color='red') - frappe.log_error("Order unsuccessful. Failed Response:"+cstr(paytm_params), 'Paytm Payment Failed') + http_status_code=401, + indicator_color="red", + ) + frappe.log_error( + "Order unsuccessful. Failed Response:" + cstr(paytm_params), "Paytm Payment Failed" + ) + def verify_transaction_status(paytm_config, order_id): - '''Verify transaction completion after checksum has been verified''' - paytm_params=dict( - MID=paytm_config.merchant_id, - ORDERID= order_id - ) + """Verify transaction completion after checksum has been verified""" + paytm_params = dict(MID=paytm_config.merchant_id, ORDERID=order_id) checksum = generateSignature(paytm_params, paytm_config.merchant_key) paytm_params["CHECKSUMHASH"] = checksum @@ -116,43 +135,48 @@ def verify_transaction_status(paytm_config, order_id): post_data = json.dumps(paytm_params) url = paytm_config.transaction_status_url - response = requests.post(url, data = post_data, headers = {"Content-type": "application/json"}).json() + response = requests.post(url, data=post_data, headers={"Content-type": "application/json"}).json() finalize_request(order_id, response) + def finalize_request(order_id, transaction_response): - request = frappe.get_doc('Integration Request', order_id) + request = frappe.get_doc("Integration Request", order_id) transaction_data = frappe._dict(json.loads(request.data)) - redirect_to = transaction_data.get('redirect_to') or None - redirect_message = transaction_data.get('redirect_message') or None + redirect_to = transaction_data.get("redirect_to") or None + redirect_message = transaction_data.get("redirect_message") or None - if transaction_response['STATUS'] == "TXN_SUCCESS": + if transaction_response["STATUS"] == "TXN_SUCCESS": if transaction_data.reference_doctype and transaction_data.reference_docname: custom_redirect_to = None try: - custom_redirect_to = frappe.get_doc(transaction_data.reference_doctype, - transaction_data.reference_docname).run_method("on_payment_authorized", 'Completed') - request.db_set('status', 'Completed') + custom_redirect_to = frappe.get_doc( + transaction_data.reference_doctype, transaction_data.reference_docname + ).run_method("on_payment_authorized", "Completed") + request.db_set("status", "Completed") except Exception: - request.db_set('status', 'Failed') + request.db_set("status", "Failed") frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to - redirect_url = '/integrations/payment-success' + redirect_url = "/integrations/payment-success" else: - request.db_set('status', 'Failed') - redirect_url = '/integrations/payment-failed' + request.db_set("status", "Failed") + redirect_url = "/integrations/payment-failed" if redirect_to: - redirect_url += '?' + urlencode({'redirect_to': redirect_to}) + redirect_url += "?" + urlencode({"redirect_to": redirect_to}) if redirect_message: - redirect_url += '&' + urlencode({'redirect_message': redirect_message}) + redirect_url += "&" + urlencode({"redirect_message": redirect_message}) + + frappe.local.response["type"] = "redirect" + frappe.local.response["location"] = redirect_url - frappe.local.response['type'] = 'redirect' - frappe.local.response['location'] = redirect_url def get_gateway_controller(doctype, docname): reference_doc = frappe.get_doc(doctype, docname) - gateway_controller = frappe.db.get_value("Payment Gateway", reference_doc.payment_gateway, "gateway_controller") - return gateway_controller \ No newline at end of file + gateway_controller = frappe.db.get_value( + "Payment Gateway", reference_doc.payment_gateway, "gateway_controller" + ) + return gateway_controller diff --git a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py index 425fc87a3f..d9e72e344c 100644 --- a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py +++ b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestPaytmSettings(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/query_parameters/query_parameters.py b/frappe/integrations/doctype/query_parameters/query_parameters.py index 68e97e9071..09f039a764 100644 --- a/frappe/integrations/doctype/query_parameters/query_parameters.py +++ b/frappe/integrations/doctype/query_parameters/query_parameters.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class QueryParameters(Document): pass diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py index 9bbab9db9b..c4ffb74325 100644 --- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py +++ b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py @@ -60,17 +60,24 @@ For razorpay payment status is Authorized """ -import frappe -from frappe import _ -import json -import hmac -import razorpay import hashlib +import hmac +import json from urllib.parse import urlencode + +import razorpay + +import frappe +from frappe import _ +from frappe.integrations.utils import ( + create_payment_gateway, + create_request_log, + make_get_request, + make_post_request, +) from frappe.model.document import Document -from frappe.utils import get_url, call_hook_method, cint, get_timestamp -from frappe.integrations.utils import (make_get_request, make_post_request, create_request_log, - create_payment_gateway) +from frappe.utils import call_hook_method, cint, get_timestamp, get_url + class RazorpaySettings(Document): supported_currencies = ["INR"] @@ -81,37 +88,45 @@ class RazorpaySettings(Document): self.client = razorpay.Client(auth=(self.api_key, secret)) def validate(self): - create_payment_gateway('Razorpay') - call_hook_method('payment_gateway_enabled', gateway='Razorpay') + create_payment_gateway("Razorpay") + call_hook_method("payment_gateway_enabled", gateway="Razorpay") if not self.flags.ignore_mandatory: self.validate_razorpay_credentails() def validate_razorpay_credentails(self): if self.api_key and self.api_secret: try: - make_get_request(url="https://api.razorpay.com/v1/payments", - auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False))) + make_get_request( + url="https://api.razorpay.com/v1/payments", + auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False)), + ) except Exception: frappe.throw(_("Seems API Key or API Secret is wrong !!!")) def validate_transaction_currency(self, currency): if currency not in self.supported_currencies: - frappe.throw(_("Please select another payment method. Razorpay does not support transactions in currency '{0}'").format(currency)) + frappe.throw( + _( + "Please select another payment method. Razorpay does not support transactions in currency '{0}'" + ).format(currency) + ) def setup_addon(self, settings, **kwargs): """ - Addon template: - { - "item": { - "name": row.upgrade_type, - "amount": row.amount, - "currency": currency, - "description": "add-on description" - }, - "quantity": 1 (The total amount is calculated as item.amount * quantity) - } + Addon template: + { + "item": { + "name": row.upgrade_type, + "amount": row.amount, + "currency": currency, + "description": "add-on description" + }, + "quantity": 1 (The total amount is calculated as item.amount * quantity) + } """ - url = "https://api.razorpay.com/v1/subscriptions/{0}/addons".format(kwargs.get('subscription_id')) + url = "https://api.razorpay.com/v1/subscriptions/{0}/addons".format( + kwargs.get("subscription_id") + ) try: if not frappe.conf.converted_rupee_to_paisa: @@ -122,52 +137,49 @@ class RazorpaySettings(Document): url, auth=(settings.api_key, settings.api_secret), data=json.dumps(addon), - headers={ - "content-type": "application/json" - } + headers={"content-type": "application/json"}, ) - if not resp.get('id'): - frappe.log_error(str(resp), 'Razorpay Failed while creating subscription') + if not resp.get("id"): + frappe.log_error(str(resp), "Razorpay Failed while creating subscription") except: frappe.log_error(frappe.get_traceback()) # failed pass def setup_subscription(self, settings, **kwargs): - start_date = get_timestamp(kwargs.get('subscription_details').get("start_date")) \ - if kwargs.get('subscription_details').get("start_date") else None + start_date = ( + get_timestamp(kwargs.get("subscription_details").get("start_date")) + if kwargs.get("subscription_details").get("start_date") + else None + ) subscription_details = { - "plan_id": kwargs.get('subscription_details').get("plan_id"), - "total_count": kwargs.get('subscription_details').get("billing_frequency"), - "customer_notify": kwargs.get('subscription_details').get("customer_notify") + "plan_id": kwargs.get("subscription_details").get("plan_id"), + "total_count": kwargs.get("subscription_details").get("billing_frequency"), + "customer_notify": kwargs.get("subscription_details").get("customer_notify"), } if start_date: - subscription_details['start_at'] = cint(start_date) + subscription_details["start_at"] = cint(start_date) - if kwargs.get('addons'): + if kwargs.get("addons"): convert_rupee_to_paisa(**kwargs) - subscription_details.update({ - "addons": kwargs.get('addons') - }) + subscription_details.update({"addons": kwargs.get("addons")}) try: resp = make_post_request( "https://api.razorpay.com/v1/subscriptions", auth=(settings.api_key, settings.api_secret), data=json.dumps(subscription_details), - headers={ - "content-type": "application/json" - } + headers={"content-type": "application/json"}, ) - if resp.get('status') == 'created': - kwargs['subscription_id'] = resp.get('id') - frappe.flags.status = 'created' + if resp.get("status") == "created": + kwargs["subscription_id"] = resp.get("id") + frappe.flags.status = "created" return kwargs else: - frappe.log_error(str(resp), 'Razorpay Failed while creating subscription') + frappe.log_error(str(resp), "Razorpay Failed while creating subscription") except: frappe.log_error(frappe.get_traceback()) @@ -178,8 +190,8 @@ class RazorpaySettings(Document): if not kwargs.get("subscription_id"): kwargs = self.setup_subscription(settings, **kwargs) - if frappe.flags.status !='created': - kwargs['subscription_id'] = None + if frappe.flags.status != "created": + kwargs["subscription_id"] = None return kwargs @@ -191,25 +203,27 @@ class RazorpaySettings(Document): # Creating Orders https://razorpay.com/docs/api/orders/ # convert rupees to paisa - kwargs['amount'] *= 100 + kwargs["amount"] *= 100 # Create integration log integration_request = create_request_log(kwargs, "Host", "Razorpay") # Setup payment options payment_options = { - "amount": kwargs.get('amount'), - "currency": kwargs.get('currency', 'INR'), - "receipt": kwargs.get('receipt'), - "payment_capture": kwargs.get('payment_capture') + "amount": kwargs.get("amount"), + "currency": kwargs.get("currency", "INR"), + "receipt": kwargs.get("receipt"), + "payment_capture": kwargs.get("payment_capture"), } if self.api_key and self.api_secret: try: - order = make_post_request("https://api.razorpay.com/v1/orders", + order = make_post_request( + "https://api.razorpay.com/v1/orders", auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False)), - data=payment_options) - order['integration_request'] = integration_request.name - return order # Order returned to be consumed by razorpay.js + data=payment_options, + ) + order["integration_request"] = integration_request.name + return order # Order returned to be consumed by razorpay.js except Exception: frappe.log(frappe.get_traceback()) frappe.throw(_("Could not create razorpay order")) @@ -219,14 +233,19 @@ class RazorpaySettings(Document): try: self.integration_request = frappe.get_doc("Integration Request", self.data.token) - self.integration_request.update_status(self.data, 'Queued') + self.integration_request.update_status(self.data, "Queued") return self.authorize_payment() except Exception: frappe.log_error(frappe.get_traceback()) - return{ - "redirect_to": frappe.redirect_to_message(_('Server Error'), _("Seems issue with server's razorpay config. Don't worry, in case of failure amount will get refunded to your account.")), - "status": 401 + return { + "redirect_to": frappe.redirect_to_message( + _("Server Error"), + _( + "Seems issue with server's razorpay config. Don't worry, in case of failure amount will get refunded to your account." + ), + ), + "status": 401, } def authorize_payment(self): @@ -239,29 +258,30 @@ class RazorpaySettings(Document): settings = self.get_settings(data) try: - resp = make_get_request("https://api.razorpay.com/v1/payments/{0}" - .format(self.data.razorpay_payment_id), auth=(settings.api_key, - settings.api_secret)) + resp = make_get_request( + "https://api.razorpay.com/v1/payments/{0}".format(self.data.razorpay_payment_id), + auth=(settings.api_key, settings.api_secret), + ) if resp.get("status") == "authorized": - self.integration_request.update_status(data, 'Authorized') + self.integration_request.update_status(data, "Authorized") self.flags.status_changed_to = "Authorized" if resp.get("status") == "captured": - self.integration_request.update_status(data, 'Completed') + self.integration_request.update_status(data, "Completed") self.flags.status_changed_to = "Completed" - elif data.get('subscription_id'): + elif data.get("subscription_id"): if resp.get("status") == "refunded": # if subscription start date is in future then # razorpay refunds the amount after authorizing the card details # thus changing status to Verified - self.integration_request.update_status(data, 'Completed') + self.integration_request.update_status(data, "Completed") self.flags.status_changed_to = "Verified" else: - frappe.log_error(str(resp), 'Razorpay Payment not authorized') + frappe.log_error(str(resp), "Razorpay Payment not authorized") except: frappe.log_error(frappe.get_traceback()) @@ -270,15 +290,16 @@ class RazorpaySettings(Document): status = frappe.flags.integration_request.status_code - redirect_to = data.get('redirect_to') or None - redirect_message = data.get('redirect_message') or None + redirect_to = data.get("redirect_to") or None + redirect_message = data.get("redirect_message") or None if self.flags.status_changed_to in ("Authorized", "Verified", "Completed"): if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: frappe.flags.data = data - custom_redirect_to = frappe.get_doc(self.data.reference_doctype, - self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) + custom_redirect_to = frappe.get_doc( + self.data.reference_doctype, self.data.reference_docname + ).run_method("on_payment_authorized", self.flags.status_changed_to) except Exception: frappe.log_error(frappe.get_traceback()) @@ -286,31 +307,34 @@ class RazorpaySettings(Document): if custom_redirect_to: redirect_to = custom_redirect_to - redirect_url = 'payment-success?doctype={0}&docname={1}'.format(self.data.reference_doctype, self.data.reference_docname) + redirect_url = "payment-success?doctype={0}&docname={1}".format( + self.data.reference_doctype, self.data.reference_docname + ) else: - redirect_url = 'payment-failed' + redirect_url = "payment-failed" if redirect_to: - redirect_url += '&' + urlencode({'redirect_to': redirect_to}) + redirect_url += "&" + urlencode({"redirect_to": redirect_to}) if redirect_message: - redirect_url += '&' + urlencode({'redirect_message': redirect_message}) + redirect_url += "&" + urlencode({"redirect_message": redirect_message}) - return { - "redirect_to": redirect_url, - "status": status - } + return {"redirect_to": redirect_url, "status": status} def get_settings(self, data): - settings = frappe._dict({ - "api_key": self.api_key, - "api_secret": self.get_password(fieldname="api_secret", raise_exception=False) - }) + settings = frappe._dict( + { + "api_key": self.api_key, + "api_secret": self.get_password(fieldname="api_secret", raise_exception=False), + } + ) - if cint(data.get('notes', {}).get('use_sandbox')) or data.get("use_sandbox"): - settings.update({ - "api_key": frappe.conf.sandbox_api_key, - "api_secret": frappe.conf.sandbox_api_secret, - }) + if cint(data.get("notes", {}).get("use_sandbox")) or data.get("use_sandbox"): + settings.update( + { + "api_key": frappe.conf.sandbox_api_key, + "api_secret": frappe.conf.sandbox_api_secret, + } + ) return settings @@ -318,15 +342,16 @@ class RazorpaySettings(Document): settings = self.get_settings({}) try: - resp = make_post_request("https://api.razorpay.com/v1/subscriptions/{0}/cancel" - .format(subscription_id), auth=(settings.api_key, - settings.api_secret)) + resp = make_post_request( + "https://api.razorpay.com/v1/subscriptions/{0}/cancel".format(subscription_id), + auth=(settings.api_key, settings.api_secret), + ) except Exception: frappe.log_error(frappe.get_traceback()) def verify_signature(self, body, signature, key): - key = bytes(key, 'utf-8') - body = bytes(body, 'utf-8') + key = bytes(key, "utf-8") + body = bytes(body, "utf-8") dig = hmac.new(key=key, msg=body, digestmod=hashlib.sha256) @@ -334,22 +359,26 @@ class RazorpaySettings(Document): result = hmac.compare_digest(generated_signature, signature) if not result: - frappe.throw(_('Razorpay Signature Verification Failed'), exc=frappe.PermissionError) + frappe.throw(_("Razorpay Signature Verification Failed"), exc=frappe.PermissionError) return result + def capture_payment(is_sandbox=False, sanbox_response=None): """ - Verifies the purchase as complete by the merchant. - After capture, the amount is transferred to the merchant within T+3 days - where T is the day on which payment is captured. + Verifies the purchase as complete by the merchant. + After capture, the amount is transferred to the merchant within T+3 days + where T is the day on which payment is captured. - Note: Attempting to capture a payment whose status is not authorized will produce an error. + Note: Attempting to capture a payment whose status is not authorized will produce an error. """ controller = frappe.get_doc("Razorpay Settings") - for doc in frappe.get_all("Integration Request", filters={"status": "Authorized", - "integration_request_service": "Razorpay"}, fields=["name", "data"]): + for doc in frappe.get_all( + "Integration Request", + filters={"status": "Authorized", "integration_request_service": "Razorpay"}, + fields=["name", "data"], + ): try: if is_sandbox: resp = sanbox_response @@ -357,12 +386,18 @@ def capture_payment(is_sandbox=False, sanbox_response=None): data = json.loads(doc.data) settings = controller.get_settings(data) - resp = make_get_request("https://api.razorpay.com/v1/payments/{0}".format(data.get("razorpay_payment_id")), - auth=(settings.api_key, settings.api_secret), data={"amount": data.get("amount")}) + resp = make_get_request( + "https://api.razorpay.com/v1/payments/{0}".format(data.get("razorpay_payment_id")), + auth=(settings.api_key, settings.api_secret), + data={"amount": data.get("amount")}, + ) - if resp.get('status') == "authorized": - resp = make_post_request("https://api.razorpay.com/v1/payments/{0}/capture".format(data.get("razorpay_payment_id")), - auth=(settings.api_key, settings.api_secret), data={"amount": data.get("amount")}) + if resp.get("status") == "authorized": + resp = make_post_request( + "https://api.razorpay.com/v1/payments/{0}/capture".format(data.get("razorpay_payment_id")), + auth=(settings.api_key, settings.api_secret), + data={"amount": data.get("amount")}, + ) if resp.get("status") == "captured": frappe.db.set_value("Integration Request", doc.name, "status", "Completed") @@ -372,7 +407,7 @@ def capture_payment(is_sandbox=False, sanbox_response=None): doc.status = "Failed" doc.error = frappe.get_traceback() doc.save() - frappe.log_error(doc.error, '{0} Failed'.format(doc.name)) + frappe.log_error(doc.error, "{0} Failed".format(doc.name)) @frappe.whitelist(allow_guest=True) @@ -380,6 +415,7 @@ def get_api_key(): controller = frappe.get_doc("Razorpay Settings") return controller.api_key + @frappe.whitelist(allow_guest=True) def get_order(doctype, docname): # Order returned to be consumed by razorpay.js @@ -391,6 +427,7 @@ def get_order(doctype, docname): frappe.log_error(frappe.get_traceback(), _("Controller method get_razorpay_order missing")) frappe.throw(_("Could not create Razorpay order. Please contact Administrator")) + @frappe.whitelist(allow_guest=True) def order_payment_success(integration_request, params): """Called by razorpay.js on order payment success, the params @@ -398,8 +435,8 @@ def order_payment_success(integration_request, params): that is updated in the data field of integration request Args: - integration_request (string): Name for integration request doc - params (string): Params to be updated for integration request. + integration_request (string): Name for integration request doc + params (string): Params to be updated for integration request. """ params = json.loads(params) integration = frappe.get_doc("Integration Request", integration_request) @@ -418,25 +455,28 @@ def order_payment_success(integration_request, params): # Authorize payment controller.authorize_payment() + @frappe.whitelist(allow_guest=True) def order_payment_failure(integration_request, params): """Called by razorpay.js on failure Args: - integration_request (TYPE): Description - params (TYPE): error data to be updated + integration_request (TYPE): Description + params (TYPE): error data to be updated """ - frappe.log_error(params, 'Razorpay Payment Failure') + frappe.log_error(params, "Razorpay Payment Failure") params = json.loads(params) integration = frappe.get_doc("Integration Request", integration_request) integration.update_status(params, integration.status) + def convert_rupee_to_paisa(**kwargs): - for addon in kwargs.get('addons'): - addon['item']['amount'] *= 100 + for addon in kwargs.get("addons"): + addon["item"]["amount"] *= 100 frappe.conf.converted_rupee_to_paisa = True + @frappe.whitelist(allow_guest=True) def razorpay_subscription_callback(): try: @@ -444,44 +484,53 @@ def razorpay_subscription_callback(): validate_payment_callback(data) - data.update({ - "payment_gateway": "Razorpay" - }) + data.update({"payment_gateway": "Razorpay"}) - doc = frappe.get_doc({ - "data": json.dumps(frappe.local.form_dict), - "doctype": "Integration Request", - "integration_type": "Subscription Notification", - "status": "Queued" - }).insert(ignore_permissions=True) + doc = frappe.get_doc( + { + "data": json.dumps(frappe.local.form_dict), + "doctype": "Integration Request", + "integration_type": "Subscription Notification", + "status": "Queued", + } + ).insert(ignore_permissions=True) frappe.db.commit() - frappe.enqueue(method='frappe.integrations.doctype.razorpay_settings.razorpay_settings.handle_subscription_notification', - queue='long', timeout=600, is_async=True, **{"doctype": "Integration Request", "docname": doc.name}) + frappe.enqueue( + method="frappe.integrations.doctype.razorpay_settings.razorpay_settings.handle_subscription_notification", + queue="long", + timeout=600, + is_async=True, + **{"doctype": "Integration Request", "docname": doc.name} + ) except frappe.InvalidStatusError: pass except Exception as e: frappe.log(frappe.log_error(title=e)) + def validate_payment_callback(data): def _throw(): frappe.throw(_("Invalid Subscription"), exc=frappe.InvalidStatusError) - subscription_id = data.get('payload').get("subscription").get("entity").get("id") + subscription_id = data.get("payload").get("subscription").get("entity").get("id") - if not(subscription_id): + if not (subscription_id): _throw() controller = frappe.get_doc("Razorpay Settings") settings = controller.get_settings(data) - resp = make_get_request("https://api.razorpay.com/v1/subscriptions/{0}".format(subscription_id), - auth=(settings.api_key, settings.api_secret)) + resp = make_get_request( + "https://api.razorpay.com/v1/subscriptions/{0}".format(subscription_id), + auth=(settings.api_key, settings.api_secret), + ) if resp.get("status") != "active": _throw() + def handle_subscription_notification(doctype, docname): call_hook_method("handle_subscription_notification", doctype=doctype, docname=docname) diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py index dc824e18b9..015d4f0467 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py @@ -3,31 +3,37 @@ # License: MIT. See LICENSE import os import os.path -import frappe + import boto3 +from botocore.exceptions import ClientError +from rq.timeouts import JobTimeoutException + +import frappe from frappe import _ -from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, generate_files_backup +from frappe.integrations.offsite_backup_utils import ( + generate_files_backup, + get_latest_backup_file, + send_email, + validate_file_size, +) from frappe.model.document import Document from frappe.utils import cint from frappe.utils.background_jobs import enqueue -from rq.timeouts import JobTimeoutException -from botocore.exceptions import ClientError class S3BackupSettings(Document): - def validate(self): if not self.enabled: return if not self.endpoint_url: - self.endpoint_url = 'https://s3.amazonaws.com' + self.endpoint_url = "https://s3.amazonaws.com" conn = boto3.client( - 's3', + "s3", aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.get_password('secret_access_key'), - endpoint_url=self.endpoint_url + aws_secret_access_key=self.get_password("secret_access_key"), + endpoint_url=self.endpoint_url, ) try: @@ -35,11 +41,11 @@ class S3BackupSettings(Document): # Requires ListBucket permission conn.head_bucket(Bucket=self.bucket) except ClientError as e: - error_code = e.response['Error']['Code'] + error_code = e.response["Error"]["Code"] bucket_name = frappe.bold(self.bucket) - if error_code == '403': + if error_code == "403": msg = _("Do not have permission to access bucket {0}.").format(bucket_name) - elif error_code == '404': + elif error_code == "404": msg = _("Bucket {0} not found.").format(bucket_name) else: msg = e.args[0] @@ -50,7 +56,11 @@ class S3BackupSettings(Document): @frappe.whitelist() def take_backup(): """Enqueue longjob for taking backup to s3""" - enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", queue='long', timeout=1500) + enqueue( + "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", + queue="long", + timeout=1500, + ) frappe.msgprint(_("Queued for backup. It may take a few minutes to an hour.")) @@ -80,11 +90,13 @@ def take_backups_s3(retry_count=0): send_email(True, "Amazon S3", "S3 Backup Settings", "notify_email") except JobTimeoutException: if retry_count < 2: - args = { - "retry_count": retry_count + 1 - } - enqueue("frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", - queue='long', timeout=1500, **args) + args = {"retry_count": retry_count + 1} + enqueue( + "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3", + queue="long", + timeout=1500, + **args + ) else: notify() except Exception: @@ -93,44 +105,55 @@ def take_backups_s3(retry_count=0): def notify(): error_message = frappe.get_traceback() - send_email(False, 'Amazon S3', "S3 Backup Settings", "notify_email", error_message) + send_email(False, "Amazon S3", "S3 Backup Settings", "notify_email", error_message) def backup_to_s3(): - from frappe.utils.backups import new_backup from frappe.utils import get_backups_path + from frappe.utils.backups import new_backup doc = frappe.get_single("S3 Backup Settings") bucket = doc.bucket backup_files = cint(doc.backup_files) conn = boto3.client( - 's3', - aws_access_key_id=doc.access_key_id, - aws_secret_access_key=doc.get_password('secret_access_key'), - endpoint_url=doc.endpoint_url or 'https://s3.amazonaws.com' - ) + "s3", + aws_access_key_id=doc.access_key_id, + aws_secret_access_key=doc.get_password("secret_access_key"), + endpoint_url=doc.endpoint_url or "https://s3.amazonaws.com", + ) if frappe.flags.create_new_backup: - backup = new_backup(ignore_files=False, backup_path_db=None, - backup_path_files=None, backup_path_private_files=None, force=True) + backup = new_backup( + ignore_files=False, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True, + ) db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db)) site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf)) if backup_files: files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files)) - private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files)) + private_files = os.path.join( + get_backups_path(), os.path.basename(backup.backup_path_private_files) + ) else: if backup_files: - db_filename, site_config, files_filename, private_files = get_latest_backup_file(with_files=backup_files) + db_filename, site_config, files_filename, private_files = get_latest_backup_file( + with_files=backup_files + ) if not files_filename or not private_files: generate_files_backup() - db_filename, site_config, files_filename, private_files = get_latest_backup_file(with_files=backup_files) + db_filename, site_config, files_filename, private_files = get_latest_backup_file( + with_files=backup_files + ) else: db_filename, site_config = get_latest_backup_file() - folder = os.path.basename(db_filename)[:15] + '/' + folder = os.path.basename(db_filename)[:15] + "/" # for adding datetime to folder name upload_file_to_s3(db_filename, folder, conn, bucket) @@ -148,7 +171,7 @@ def upload_file_to_s3(filename, folder, conn, bucket): destpath = os.path.join(folder, os.path.basename(filename)) try: print("Uploading file:", filename) - conn.upload_file(filename, bucket, destpath) # Requires PutObject permission + conn.upload_file(filename, bucket, destpath) # Requires PutObject permission except Exception as e: frappe.log_error() diff --git a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py index 2a586c30d4..c6dbee2c20 100755 --- a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestS3BackupSettings(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py index a74c0a36ca..cf6a23fb34 100644 --- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py +++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py @@ -2,20 +2,21 @@ # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + +import requests + import frappe +from frappe import _ from frappe.model.document import Document from frappe.utils import get_url_to_form -from frappe import _ -import requests -import json - error_messages = { 400: "400: Invalid Payload or User not found", 403: "403: Action Prohibited", 404: "404: Channel not found", 410: "410: The Channel is Archived", - 500: "500: Rollup Error, Slack seems to be down" + 500: "500: Rollup Error, Slack seems to be down", } @@ -49,7 +50,7 @@ def send_slack_message(webhook_url, message, reference_doctype, reference_name): if not r.ok: message = error_messages.get(r.status_code, r.status_code) - frappe.log_error(message, _('Slack Webhook Error')) - return 'error' + frappe.log_error(message, _("Slack Webhook Error")) + return "error" - return 'success' + return "success" diff --git a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py index a256735f81..4bd71033bb 100644 --- a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py +++ b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestSlackWebhookURL(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.py b/frappe/integrations/doctype/social_login_key/social_login_key.py index 195d6800be..8c19767107 100644 --- a/frappe/integrations/doctype/social_login_key/social_login_key.py +++ b/frappe/integrations/doctype/social_login_key/social_login_key.py @@ -2,19 +2,38 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe from frappe import _ from frappe.model.document import Document -class BaseUrlNotSetError(frappe.ValidationError): pass -class AuthorizeUrlNotSetError(frappe.ValidationError): pass -class AccessTokenUrlNotSetError(frappe.ValidationError): pass -class RedirectUrlNotSetError(frappe.ValidationError): pass -class ClientIDNotSetError(frappe.ValidationError): pass -class ClientSecretNotSetError(frappe.ValidationError): pass -class SocialLoginKey(Document): +class BaseUrlNotSetError(frappe.ValidationError): + pass + + +class AuthorizeUrlNotSetError(frappe.ValidationError): + pass + + +class AccessTokenUrlNotSetError(frappe.ValidationError): + pass + + +class RedirectUrlNotSetError(frappe.ValidationError): + pass + +class ClientIDNotSetError(frappe.ValidationError): + pass + + +class ClientSecretNotSetError(frappe.ValidationError): + pass + + +class SocialLoginKey(Document): def autoname(self): self.name = frappe.scrub(self.provider_name) @@ -29,9 +48,13 @@ class SocialLoginKey(Document): if not self.redirect_url: frappe.throw(_("Please enter Redirect URL"), exc=RedirectUrlNotSetError) if self.enable_social_login and not self.client_id: - frappe.throw(_("Please enter Client ID before social login is enabled"), exc=ClientIDNotSetError) + frappe.throw( + _("Please enter Client ID before social login is enabled"), exc=ClientIDNotSetError + ) if self.enable_social_login and not self.client_secret: - frappe.throw(_("Please enter Client Secret before social login is enabled"), exc=ClientSecretNotSetError) + frappe.throw( + _("Please enter Client Secret before social login is enabled"), exc=ClientSecretNotSetError + ) def set_icon(self): icon_map = { @@ -41,12 +64,12 @@ class SocialLoginKey(Document): "Office 365": "office_365.svg", "GitHub": "github.svg", "Salesforce": "salesforce.svg", - "fairlogin": "fair.svg" + "fairlogin": "fair.svg", } if self.provider_name in icon_map: icon_file = icon_map[self.provider_name] - self.icon = '/assets/frappe/icons/social/{0}'.format(icon_file) + self.icon = "/assets/frappe/icons/social/{0}".format(icon_file) @frappe.whitelist() def get_social_login_provider(self, provider, initialize=False): @@ -57,32 +80,27 @@ class SocialLoginKey(Document): "enable_social_login": 1, "base_url": "https://login.microsoftonline.com", "custom_base_url": 0, - "icon":"fa fa-windows", + "icon": "fa fa-windows", "authorize_url": "https://login.microsoftonline.com/common/oauth2/authorize", "access_token_url": "https://login.microsoftonline.com/common/oauth2/token", "redirect_url": "/api/method/frappe.integrations.oauth2_logins.login_via_office365", "api_endpoint": None, - "api_endpoint_args":None, - "auth_url_data": json.dumps({ - "response_type": "code", - "scope":"openid" - }) + "api_endpoint_args": None, + "auth_url_data": json.dumps({"response_type": "code", "scope": "openid"}), } providers["GitHub"] = { - "provider_name":"GitHub", + "provider_name": "GitHub", "enable_social_login": 1, - "base_url":"https://api.github.com/", - "custom_base_url":0, - "icon":"fa fa-github", - "authorize_url":"https://github.com/login/oauth/authorize", - "access_token_url":"https://github.com/login/oauth/access_token", - "redirect_url":"/api/method/frappe.www.login.login_via_github", - "api_endpoint":"user", - "api_endpoint_args":None, - "auth_url_data": json.dumps({ - "scope": "user:email" - }) + "base_url": "https://api.github.com/", + "custom_base_url": 0, + "icon": "fa fa-github", + "authorize_url": "https://github.com/login/oauth/authorize", + "access_token_url": "https://github.com/login/oauth/access_token", + "redirect_url": "/api/method/frappe.www.login.login_via_github", + "api_endpoint": "user", + "api_endpoint_args": None, + "auth_url_data": json.dumps({"scope": "user:email"}), } providers["Google"] = { @@ -90,16 +108,18 @@ class SocialLoginKey(Document): "enable_social_login": 1, "base_url": "https://www.googleapis.com", "custom_base_url": 0, - "icon":"fa fa-google", + "icon": "fa fa-google", "authorize_url": "https://accounts.google.com/o/oauth2/auth", "access_token_url": "https://accounts.google.com/o/oauth2/token", "redirect_url": "/api/method/frappe.www.login.login_via_google", "api_endpoint": "oauth2/v2/userinfo", - "api_endpoint_args":None, - "auth_url_data": json.dumps({ - "scope": "https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email", - "response_type": "code" - }) + "api_endpoint_args": None, + "auth_url_data": json.dumps( + { + "scope": "https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email", + "response_type": "code", + } + ), } providers["Facebook"] = { @@ -112,30 +132,25 @@ class SocialLoginKey(Document): "access_token_url": "https://graph.facebook.com/oauth/access_token", "redirect_url": "/api/method/frappe.www.login.login_via_facebook", "api_endpoint": "/v2.5/me", - "api_endpoint_args": json.dumps({ - "fields": "first_name,last_name,email,gender,location,verified,picture" - }), - "auth_url_data": json.dumps({ - "display": "page", - "response_type": "code", - "scope": "email,public_profile" - }) + "api_endpoint_args": json.dumps( + {"fields": "first_name,last_name,email,gender,location,verified,picture"} + ), + "auth_url_data": json.dumps( + {"display": "page", "response_type": "code", "scope": "email,public_profile"} + ), } providers["Frappe"] = { "provider_name": "Frappe", "enable_social_login": 1, "custom_base_url": 1, - "icon":"/assets/frappe/images/frappe-favicon.svg", + "icon": "/assets/frappe/images/frappe-favicon.svg", "redirect_url": "/api/method/frappe.www.login.login_via_frappe", "api_endpoint": "/api/method/frappe.integrations.oauth2.openid_profile", - "api_endpoint_args":None, + "api_endpoint_args": None, "authorize_url": "/api/method/frappe.integrations.oauth2.authorize", "access_token_url": "/api/method/frappe.integrations.oauth2.get_token", - "auth_url_data": json.dumps({ - "response_type": "code", - "scope": "openid" - }) + "auth_url_data": json.dumps({"response_type": "code", "scope": "openid"}), } providers["Salesforce"] = { @@ -143,16 +158,13 @@ class SocialLoginKey(Document): "enable_social_login": 1, "base_url": "https://login.salesforce.com", "custom_base_url": 0, - "icon":"fa fa-cloud", #https://github.com/FortAwesome/Font-Awesome/issues/1744 + "icon": "fa fa-cloud", # https://github.com/FortAwesome/Font-Awesome/issues/1744 "redirect_url": "/api/method/frappe.integrations.oauth2_logins.login_via_salesforce", "api_endpoint": "https://login.salesforce.com/services/oauth2/userinfo", - "api_endpoint_args":None, + "api_endpoint_args": None, "authorize_url": "https://login.salesforce.com/services/oauth2/authorize", "access_token_url": "https://login.salesforce.com/services/oauth2/token", - "auth_url_data": json.dumps({ - "response_type": "code", - "scope": "openid" - }) + "auth_url_data": json.dumps({"response_type": "code", "scope": "openid"}), } providers["fairlogin"] = { @@ -160,24 +172,20 @@ class SocialLoginKey(Document): "enable_social_login": 1, "base_url": "https://id.fairkom.net/auth/realms/fairlogin/", "custom_base_url": 0, - "icon":"fa fa-key", + "icon": "fa fa-key", "redirect_url": "/api/method/frappe.integrations.oauth2_logins.login_via_fairlogin", "api_endpoint": "https://id.fairkom.net/auth/realms/fairlogin/protocol/openid-connect/userinfo", - "api_endpoint_args":None, + "api_endpoint_args": None, "authorize_url": "https://id.fairkom.net/auth/realms/fairlogin/protocol/openid-connect/auth", "access_token_url": "https://id.fairkom.net/auth/realms/fairlogin/protocol/openid-connect/token", - "auth_url_data": json.dumps({ - "response_type": "code", - "scope": "openid" - }) + "auth_url_data": json.dumps({"response_type": "code", "scope": "openid"}), } - # Initialize the doc and return, used in patch # Or can be used for creating key from controller if initialize and provider: for k, v in providers[provider].items(): - setattr(self,k,v) + setattr(self, k, v) return return providers.get(provider) if provider else providers diff --git a/frappe/integrations/doctype/social_login_key/test_social_login_key.py b/frappe/integrations/doctype/social_login_key/test_social_login_key.py index 73e6a072cb..be4c1f7c49 100644 --- a/frappe/integrations/doctype/social_login_key/test_social_login_key.py +++ b/frappe/integrations/doctype/social_login_key/test_social_login_key.py @@ -1,22 +1,22 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -from frappe.integrations.doctype.social_login_key.social_login_key import BaseUrlNotSetError import unittest -from frappe.utils.oauth import login_via_oauth2 -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + from rauth import OAuth2Service -from frappe.auth import LoginManager, CookieManager + +import frappe +from frappe.auth import CookieManager, LoginManager +from frappe.integrations.doctype.social_login_key.social_login_key import BaseUrlNotSetError from frappe.utils import set_request +from frappe.utils.oauth import login_via_oauth2 class TestSocialLoginKey(unittest.TestCase): def test_adding_frappe_social_login_provider(self): provider_name = "Frappe" - social_login_key = make_social_login_key( - social_login_provider=provider_name - ) + social_login_key = make_social_login_key(social_login_provider=provider_name) social_login_key.get_social_login_provider(provider_name, initialize=True) self.assertRaises(BaseUrlNotSetError, social_login_key.insert) @@ -27,7 +27,7 @@ class TestSocialLoginKey(unittest.TestCase): mock_session.get.side_effect = github_response_for_private_email with patch.object(OAuth2Service, "get_auth_session", return_value=mock_session): - login_via_oauth2("github", "iwriu", {"token": "ewrwerwer"}) # Dummy code and state token + login_via_oauth2("github", "iwriu", {"token": "ewrwerwer"}) # Dummy code and state token def test_github_login_with_public_email(self): github_social_login_setup() @@ -36,17 +36,15 @@ class TestSocialLoginKey(unittest.TestCase): mock_session.get.side_effect = github_response_for_public_email with patch.object(OAuth2Service, "get_auth_session", return_value=mock_session): - login_via_oauth2("github", "iwriu", {"token": "ewrwerwer"}) # Dummy code and state token + login_via_oauth2("github", "iwriu", {"token": "ewrwerwer"}) # Dummy code and state token def test_normal_signup_and_github_login(self): github_social_login_setup() if not frappe.db.exists("User", "githublogin@example.com"): - user = frappe.get_doc({ - "doctype": "User", - "email": "githublogin@example.com", - "first_name": "GitHub Login" - }) + user = frappe.get_doc( + {"doctype": "User", "email": "githublogin@example.com", "first_name": "GitHub Login"} + ) user.save(ignore_permissions=True) mock_session = MagicMock() @@ -55,6 +53,7 @@ class TestSocialLoginKey(unittest.TestCase): with patch.object(OAuth2Service, "get_auth_session", return_value=mock_session): login_via_oauth2("github", "iwriu", {"token": "ewrwerwer"}) + def make_social_login_key(**kwargs): kwargs["doctype"] = "Social Login Key" if not "provider_name" in kwargs: @@ -62,6 +61,7 @@ def make_social_login_key(**kwargs): doc = frappe.get_doc(kwargs) return doc + def create_or_update_social_login_key(): # used in other tests (connected app, oauth20) try: @@ -76,14 +76,13 @@ def create_or_update_social_login_key(): return social_login_key + def create_github_social_login_key(): if frappe.db.exists("Social Login Key", "github"): return frappe.get_doc("Social Login Key", "github") else: provider_name = "GitHub" - social_login_key = make_social_login_key( - social_login_provider=provider_name - ) + social_login_key = make_social_login_key(social_login_provider=provider_name) social_login_key.get_social_login_provider(provider_name, initialize=True) # Dummy client_id and client_secret @@ -92,28 +91,47 @@ def create_github_social_login_key(): social_login_key.insert(ignore_permissions=True) return social_login_key + def github_response_for_private_email(url, *args, **kwargs): if url == "user": - return_value = {"login": "dummy_username", "id": "223342", "email": None, "first_name": "Github Private"} + return_value = { + "login": "dummy_username", + "id": "223342", + "email": None, + "first_name": "Github Private", + } else: return_value = [{"email": "github@example.com", "primary": True, "verified": True}] return MagicMock(status_code=200, json=MagicMock(return_value=return_value)) + def github_response_for_public_email(url, *args, **kwargs): if url == "user": - return_value = {"login": "dummy_username", "id": "223343", "email": "github_public@example.com", "first_name": "Github Public"} + return_value = { + "login": "dummy_username", + "id": "223343", + "email": "github_public@example.com", + "first_name": "Github Public", + } return MagicMock(status_code=200, json=MagicMock(return_value=return_value)) + def github_response_for_login(url, *args, **kwargs): if url == "user": - return_value = {"login": "dummy_username", "id": "223346", "email": None, "first_name": "Github Login"} + return_value = { + "login": "dummy_username", + "id": "223346", + "email": None, + "first_name": "Github Login", + } else: return_value = [{"email": "githublogin@example.com", "primary": True, "verified": True}] return MagicMock(status_code=200, json=MagicMock(return_value=return_value)) + def github_social_login_setup(): set_request(path="/random") frappe.local.cookie_manager = CookieManager() diff --git a/frappe/integrations/doctype/social_login_keys/social_login_keys.py b/frappe/integrations/doctype/social_login_keys/social_login_keys.py index da9e21cd8e..16d59e9690 100644 --- a/frappe/integrations/doctype/social_login_keys/social_login_keys.py +++ b/frappe/integrations/doctype/social_login_keys/social_login_keys.py @@ -1,5 +1,6 @@ # see license from frappe.model.document import Document + class SocialLoginKeys(Document): pass diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.py b/frappe/integrations/doctype/stripe_settings/stripe_settings.py index 81e40fa72f..4ebf902e84 100644 --- a/frappe/integrations/doctype/stripe_settings/stripe_settings.py +++ b/frappe/integrations/doctype/stripe_settings/stripe_settings.py @@ -2,41 +2,171 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE +from urllib.parse import urlencode + import frappe -from frappe.model.document import Document from frappe import _ -from urllib.parse import urlencode -from frappe.utils import get_url, call_hook_method, cint, flt -from frappe.integrations.utils import make_get_request, make_post_request, create_request_log, create_payment_gateway +from frappe.integrations.utils import ( + create_payment_gateway, + create_request_log, + make_get_request, + make_post_request, +) +from frappe.model.document import Document +from frappe.utils import call_hook_method, cint, flt, get_url + class StripeSettings(Document): supported_currencies = [ - "AED", "ALL", "ANG", "ARS", "AUD", "AWG", "BBD", "BDT", "BIF", "BMD", "BND", - "BOB", "BRL", "BSD", "BWP", "BZD", "CAD", "CHF", "CLP", "CNY", "COP", "CRC", "CVE", "CZK", "DJF", - "DKK", "DOP", "DZD", "EGP", "ETB", "EUR", "FJD", "FKP", "GBP", "GIP", "GMD", "GNF", "GTQ", "GYD", - "HKD", "HNL", "HRK", "HTG", "HUF", "IDR", "ILS", "INR", "ISK", "JMD", "JPY", "KES", "KHR", "KMF", - "KRW", "KYD", "KZT", "LAK", "LBP", "LKR", "LRD", "MAD", "MDL", "MNT", "MOP", "MRO", "MUR", "MVR", - "MWK", "MXN", "MYR", "NAD", "NGN", "NIO", "NOK", "NPR", "NZD", "PAB", "PEN", "PGK", "PHP", "PKR", - "PLN", "PYG", "QAR", "RUB", "SAR", "SBD", "SCR", "SEK", "SGD", "SHP", "SLL", "SOS", "STD", "SVC", - "SZL", "THB", "TOP", "TTD", "TWD", "TZS", "UAH", "UGX", "USD", "UYU", "UZS", "VND", "VUV", "WST", - "XAF", "XOF", "XPF", "YER", "ZAR" + "AED", + "ALL", + "ANG", + "ARS", + "AUD", + "AWG", + "BBD", + "BDT", + "BIF", + "BMD", + "BND", + "BOB", + "BRL", + "BSD", + "BWP", + "BZD", + "CAD", + "CHF", + "CLP", + "CNY", + "COP", + "CRC", + "CVE", + "CZK", + "DJF", + "DKK", + "DOP", + "DZD", + "EGP", + "ETB", + "EUR", + "FJD", + "FKP", + "GBP", + "GIP", + "GMD", + "GNF", + "GTQ", + "GYD", + "HKD", + "HNL", + "HRK", + "HTG", + "HUF", + "IDR", + "ILS", + "INR", + "ISK", + "JMD", + "JPY", + "KES", + "KHR", + "KMF", + "KRW", + "KYD", + "KZT", + "LAK", + "LBP", + "LKR", + "LRD", + "MAD", + "MDL", + "MNT", + "MOP", + "MRO", + "MUR", + "MVR", + "MWK", + "MXN", + "MYR", + "NAD", + "NGN", + "NIO", + "NOK", + "NPR", + "NZD", + "PAB", + "PEN", + "PGK", + "PHP", + "PKR", + "PLN", + "PYG", + "QAR", + "RUB", + "SAR", + "SBD", + "SCR", + "SEK", + "SGD", + "SHP", + "SLL", + "SOS", + "STD", + "SVC", + "SZL", + "THB", + "TOP", + "TTD", + "TWD", + "TZS", + "UAH", + "UGX", + "USD", + "UYU", + "UZS", + "VND", + "VUV", + "WST", + "XAF", + "XOF", + "XPF", + "YER", + "ZAR", ] currency_wise_minimum_charge_amount = { - 'JPY': 50, 'MXN': 10, 'DKK': 2.50, 'HKD': 4.00, 'NOK': 3.00, 'SEK': 3.00, - 'USD': 0.50, 'AUD': 0.50, 'BRL': 0.50, 'CAD': 0.50, 'CHF': 0.50, 'EUR': 0.50, - 'GBP': 0.30, 'NZD': 0.50, 'SGD': 0.50 + "JPY": 50, + "MXN": 10, + "DKK": 2.50, + "HKD": 4.00, + "NOK": 3.00, + "SEK": 3.00, + "USD": 0.50, + "AUD": 0.50, + "BRL": 0.50, + "CAD": 0.50, + "CHF": 0.50, + "EUR": 0.50, + "GBP": 0.30, + "NZD": 0.50, + "SGD": 0.50, } def on_update(self): - create_payment_gateway('Stripe-' + self.gateway_name, settings='Stripe Settings', controller=self.gateway_name) - call_hook_method('payment_gateway_enabled', gateway='Stripe-' + self.gateway_name) + create_payment_gateway( + "Stripe-" + self.gateway_name, settings="Stripe Settings", controller=self.gateway_name + ) + call_hook_method("payment_gateway_enabled", gateway="Stripe-" + self.gateway_name) if not self.flags.ignore_mandatory: self.validate_stripe_credentails() def validate_stripe_credentails(self): if self.publishable_key and self.secret_key: - header = {"Authorization": "Bearer {0}".format(self.get_password(fieldname="secret_key", raise_exception=False))} + header = { + "Authorization": "Bearer {0}".format( + self.get_password(fieldname="secret_key", raise_exception=False) + ) + } try: make_get_request(url="https://api.stripe.com/v1/charges", headers=header) except Exception: @@ -44,19 +174,27 @@ class StripeSettings(Document): def validate_transaction_currency(self, currency): if currency not in self.supported_currencies: - frappe.throw(_("Please select another payment method. Stripe does not support transactions in currency '{0}'").format(currency)) + frappe.throw( + _( + "Please select another payment method. Stripe does not support transactions in currency '{0}'" + ).format(currency) + ) def validate_minimum_transaction_amount(self, currency, amount): if currency in self.currency_wise_minimum_charge_amount: if flt(amount) < self.currency_wise_minimum_charge_amount.get(currency, 0.0): - frappe.throw(_("For currency {0}, the minimum transaction amount should be {1}").format(currency, - self.currency_wise_minimum_charge_amount.get(currency, 0.0))) + frappe.throw( + _("For currency {0}, the minimum transaction amount should be {1}").format( + currency, self.currency_wise_minimum_charge_amount.get(currency, 0.0) + ) + ) def get_payment_url(self, **kwargs): return get_url("./integrations/stripe_checkout?{0}".format(urlencode(kwargs))) def create_request(self, data): import stripe + self.data = frappe._dict(data) stripe.api_key = self.get_password(fieldname="secret_key", raise_exception=False) stripe.default_http_client = stripe.http_client.RequestsClient() @@ -67,65 +205,77 @@ class StripeSettings(Document): except Exception: frappe.log_error(frappe.get_traceback()) - return{ - "redirect_to": frappe.redirect_to_message(_('Server Error'), _("It seems that there is an issue with the server's stripe configuration. In case of failure, the amount will get refunded to your account.")), - "status": 401 + return { + "redirect_to": frappe.redirect_to_message( + _("Server Error"), + _( + "It seems that there is an issue with the server's stripe configuration. In case of failure, the amount will get refunded to your account." + ), + ), + "status": 401, } def create_charge_on_stripe(self): import stripe + try: - charge = stripe.Charge.create(amount=cint(flt(self.data.amount)*100), currency=self.data.currency, source=self.data.stripe_token_id, description=self.data.description, receipt_email=self.data.payer_email) + charge = stripe.Charge.create( + amount=cint(flt(self.data.amount) * 100), + currency=self.data.currency, + source=self.data.stripe_token_id, + description=self.data.description, + receipt_email=self.data.payer_email, + ) if charge.captured == True: - self.integration_request.db_set('status', 'Completed', update_modified=False) + self.integration_request.db_set("status", "Completed", update_modified=False) self.flags.status_changed_to = "Completed" else: - frappe.log_error(charge.failure_message, 'Stripe Payment not completed') + frappe.log_error(charge.failure_message, "Stripe Payment not completed") except Exception: frappe.log_error(frappe.get_traceback()) return self.finalize_request() - def finalize_request(self): - redirect_to = self.data.get('redirect_to') or None - redirect_message = self.data.get('redirect_message') or None + redirect_to = self.data.get("redirect_to") or None + redirect_message = self.data.get("redirect_message") or None status = self.integration_request.status if self.flags.status_changed_to == "Completed": if self.data.reference_doctype and self.data.reference_docname: custom_redirect_to = None try: - custom_redirect_to = frappe.get_doc(self.data.reference_doctype, - self.data.reference_docname).run_method("on_payment_authorized", self.flags.status_changed_to) + custom_redirect_to = frappe.get_doc( + self.data.reference_doctype, self.data.reference_docname + ).run_method("on_payment_authorized", self.flags.status_changed_to) except Exception: frappe.log_error(frappe.get_traceback()) if custom_redirect_to: redirect_to = custom_redirect_to - redirect_url = 'payment-success' + redirect_url = "payment-success" if self.redirect_url: redirect_url = self.redirect_url redirect_to = None else: - redirect_url = 'payment-failed' + redirect_url = "payment-failed" if redirect_to: - redirect_url += '?' + urlencode({'redirect_to': redirect_to}) + redirect_url += "?" + urlencode({"redirect_to": redirect_to}) if redirect_message: - redirect_url += '&' + urlencode({'redirect_message': redirect_message}) + redirect_url += "&" + urlencode({"redirect_message": redirect_message}) + + return {"redirect_to": redirect_url, "status": status} - return { - "redirect_to": redirect_url, - "status": status - } def get_gateway_controller(doctype, docname): reference_doc = frappe.get_doc(doctype, docname) - gateway_controller = frappe.db.get_value("Payment Gateway", reference_doc.payment_gateway, "gateway_controller") + gateway_controller = frappe.db.get_value( + "Payment Gateway", reference_doc.payment_gateway, "gateway_controller" + ) return gateway_controller diff --git a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py b/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py index e7113d3bd9..e13359fa6d 100644 --- a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py +++ b/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestStripeSettings(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/token_cache/test_token_cache.py b/frappe/integrations/doctype/token_cache/test_token_cache.py index 5fe648d225..6a3b16e72c 100644 --- a/frappe/integrations/doctype/token_cache/test_token_cache.py +++ b/frappe/integrations/doctype/token_cache/test_token_cache.py @@ -2,28 +2,31 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE import unittest + import frappe -test_dependencies = ['User', 'Connected App', 'Token Cache'] +test_dependencies = ["User", "Connected App", "Token Cache"] -class TestTokenCache(unittest.TestCase): +class TestTokenCache(unittest.TestCase): def setUp(self): - self.token_cache = frappe.get_last_doc('Token Cache') - self.token_cache.update({'connected_app': frappe.get_last_doc('Connected App').name}) + self.token_cache = frappe.get_last_doc("Token Cache") + self.token_cache.update({"connected_app": frappe.get_last_doc("Connected App").name}) self.token_cache.save(ignore_permissions=True) def test_get_auth_header(self): self.token_cache.get_auth_header() def test_update_data(self): - self.token_cache.update_data({ - 'access_token': 'new-access-token', - 'refresh_token': 'new-refresh-token', - 'token_type': 'bearer', - 'expires_in': 2000, - 'scope': 'new scope' - }) + self.token_cache.update_data( + { + "access_token": "new-access-token", + "refresh_token": "new-refresh-token", + "token_type": "bearer", + "expires_in": 2000, + "scope": "new scope", + } + ) def test_get_expires_in(self): self.token_cache.get_expires_in() diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index ea86100cc2..7d961fe1cc 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -6,14 +6,14 @@ from datetime import datetime, timedelta import frappe from frappe import _ -from frappe.utils import cstr, cint from frappe.model.document import Document +from frappe.utils import cint, cstr -class TokenCache(Document): +class TokenCache(Document): def get_auth_header(self): if self.access_token: - headers = {'Authorization': 'Bearer ' + self.get_password('access_token')} + headers = {"Authorization": "Bearer " + self.get_password("access_token")} return headers raise frappe.exceptions.DoesNotExistError @@ -25,25 +25,25 @@ class TokenCache(Document): Params: data - Dict with access_token, refresh_token, expires_in and scope. """ - token_type = cstr(data.get('token_type', '')).lower() - if token_type not in ['bearer', 'mac']: - frappe.throw(_('Received an invalid token type.')) + token_type = cstr(data.get("token_type", "")).lower() + if token_type not in ["bearer", "mac"]: + frappe.throw(_("Received an invalid token type.")) # 'Bearer' or 'MAC' - token_type = token_type.title() if token_type == 'bearer' else token_type.upper() + token_type = token_type.title() if token_type == "bearer" else token_type.upper() self.token_type = token_type - self.access_token = cstr(data.get('access_token', '')) - self.refresh_token = cstr(data.get('refresh_token', '')) - self.expires_in = cint(data.get('expires_in', 0)) + self.access_token = cstr(data.get("access_token", "")) + self.refresh_token = cstr(data.get("refresh_token", "")) + self.expires_in = cint(data.get("expires_in", 0)) - new_scopes = data.get('scope') + new_scopes = data.get("scope") if new_scopes: if isinstance(new_scopes, str): - new_scopes = new_scopes.split(' ') + new_scopes = new_scopes.split(" ") if isinstance(new_scopes, list): self.scopes = None for scope in new_scopes: - self.append('scopes', {'scope': scope}) + self.append("scopes", {"scope": scope}) self.state = None self.save(ignore_permissions=True) @@ -59,8 +59,8 @@ class TokenCache(Document): def get_json(self): return { - 'access_token': self.get_password('access_token', ''), - 'refresh_token': self.get_password('refresh_token', ''), - 'expires_in': self.get_expires_in(), - 'token_type': self.token_type + "access_token": self.get_password("access_token", ""), + "refresh_token": self.get_password("refresh_token", ""), + "expires_in": self.get_expires_in(), + "token_type": self.token_type, } diff --git a/frappe/integrations/doctype/webhook/__init__.py b/frappe/integrations/doctype/webhook/__init__.py index cb7a9963b7..915d2819ee 100644 --- a/frappe/integrations/doctype/webhook/__init__.py +++ b/frappe/integrations/doctype/webhook/__init__.py @@ -6,8 +6,13 @@ import frappe def run_webhooks(doc, method): - '''Run webhooks for this method''' - if frappe.flags.in_import or frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_migrate: + """Run webhooks for this method""" + if ( + frappe.flags.in_import + or frappe.flags.in_patch + or frappe.flags.in_install + or frappe.flags.in_migrate + ): return if frappe.flags.webhooks_executed is None: @@ -15,19 +20,20 @@ def run_webhooks(doc, method): if frappe.flags.webhooks is None: # load webhooks from cache - webhooks = frappe.cache().get_value('webhooks') + webhooks = frappe.cache().get_value("webhooks") if webhooks is None: # query webhooks - webhooks_list = frappe.get_all('Webhook', - fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"], - filters={"enabled": True} - ) + webhooks_list = frappe.get_all( + "Webhook", + fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"], + filters={"enabled": True}, + ) # make webhooks map for cache webhooks = {} for w in webhooks_list: webhooks.setdefault(w.webhook_doctype, []).append(w) - frappe.cache().set_value('webhooks', webhooks) + frappe.cache().set_value("webhooks", webhooks) frappe.flags.webhooks = webhooks @@ -40,8 +46,12 @@ def run_webhooks(doc, method): def _webhook_request(webhook): if webhook.name not in frappe.flags.webhooks_executed.get(doc.name, []): - frappe.enqueue("frappe.integrations.doctype.webhook.webhook.enqueue_webhook", - enqueue_after_commit=True, doc=doc, webhook=webhook) + frappe.enqueue( + "frappe.integrations.doctype.webhook.webhook.enqueue_webhook", + enqueue_after_commit=True, + doc=doc, + webhook=webhook, + ) # keep list of webhooks executed for this doc in this request # so that we don't run the same webhook for the same document multiple times @@ -52,8 +62,8 @@ def run_webhooks(doc, method): if not doc.flags.in_insert: # value change is not applicable in insert - event_list.append('on_change') - event_list.append('before_update_after_submit') + event_list.append("on_change") + event_list.append("before_update_after_submit") from frappe.integrations.doctype.webhook.webhook import get_context diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py index a1176aa38b..5386a75573 100644 --- a/frappe/integrations/doctype/webhook/test_webhook.py +++ b/frappe/integrations/doctype/webhook/test_webhook.py @@ -4,7 +4,11 @@ import unittest import frappe -from frappe.integrations.doctype.webhook.webhook import get_webhook_headers, get_webhook_data, enqueue_webhook +from frappe.integrations.doctype.webhook.webhook import ( + enqueue_webhook, + get_webhook_data, + get_webhook_headers, +) class TestWebhook(unittest.TestCase): @@ -25,15 +29,15 @@ class TestWebhook(unittest.TestCase): "webhook_docevent": "after_insert", "request_url": "https://httpbin.org/post", "condition": "doc.email", - "enabled": True + "enabled": True, }, { "webhook_doctype": "User", "webhook_docevent": "after_insert", "request_url": "https://httpbin.org/post", "condition": "doc.first_name", - "enabled": False - } + "enabled": False, + }, ] cls.sample_webhooks = [] @@ -53,7 +57,7 @@ class TestWebhook(unittest.TestCase): webhook_fields = { "webhook_doctype": "User", "webhook_docevent": "after_insert", - "request_url": "https://httpbin.org/post" + "request_url": "https://httpbin.org/post", } if frappe.db.exists("Webhook", webhook_fields): @@ -81,7 +85,7 @@ class TestWebhook(unittest.TestCase): def test_webhook_trigger_with_enabled_webhooks(self): """Test webhook trigger for enabled webhooks""" - frappe.cache().delete_value('webhooks') + frappe.cache().delete_value("webhooks") frappe.flags.webhooks = None # Insert the user to db @@ -89,14 +93,10 @@ class TestWebhook(unittest.TestCase): self.assertTrue("User" in frappe.flags.webhooks) # only 1 hook (enabled) must be queued - self.assertEqual( - len(frappe.flags.webhooks.get("User")), - 1 - ) + self.assertEqual(len(frappe.flags.webhooks.get("User")), 1) self.assertTrue(self.test_user.email in frappe.flags.webhooks_executed) self.assertEqual( - frappe.flags.webhooks_executed.get(self.test_user.email)[0], - self.sample_webhooks[0].name + frappe.flags.webhooks_executed.get(self.test_user.email)[0], self.sample_webhooks[0].name ) def test_validate_doc_events(self): @@ -115,18 +115,13 @@ class TestWebhook(unittest.TestCase): "Test validation for request headers" # test incomplete headers - self.webhook.set("webhook_headers", [{ - "key": "Content-Type" - }]) + self.webhook.set("webhook_headers", [{"key": "Content-Type"}]) self.webhook.save() headers = get_webhook_headers(doc=None, webhook=self.webhook) self.assertEqual(headers, {}) # test complete headers - self.webhook.set("webhook_headers", [{ - "key": "Content-Type", - "value": "application/json" - }]) + self.webhook.set("webhook_headers", [{"key": "Content-Type", "value": "application/json"}]) self.webhook.save() headers = get_webhook_headers(doc=None, webhook=self.webhook) self.assertEqual(headers, {"Content-Type": "application/json"}) @@ -135,10 +130,7 @@ class TestWebhook(unittest.TestCase): "Test validation of Form URL-Encoded request body" self.webhook.request_structure = "Form URL-Encoded" - self.webhook.set("webhook_data", [{ - "fieldname": "name", - "key": "name" - }]) + self.webhook.set("webhook_data", [{"fieldname": "name", "key": "name"}]) self.webhook.webhook_json = """{ "name": "{{ doc.name }}" }""" @@ -152,10 +144,7 @@ class TestWebhook(unittest.TestCase): "Test validation of JSON request body" self.webhook.request_structure = "JSON" - self.webhook.set("webhook_data", [{ - "fieldname": "name", - "key": "name" - }]) + self.webhook.set("webhook_data", [{"fieldname": "name", "key": "name"}]) self.webhook.webhook_json = """{ "name": "{{ doc.name }}" }""" @@ -166,16 +155,14 @@ class TestWebhook(unittest.TestCase): self.assertEqual(data, {"name": self.user.name}) def test_webhook_req_log_creation(self): - if not frappe.db.get_value('User', 'user2@integration.webhooks.test.com'): - user = frappe.get_doc({ - 'doctype': 'User', - 'email': 'user2@integration.webhooks.test.com', - 'first_name': 'user2' - }).insert() + if not frappe.db.get_value("User", "user2@integration.webhooks.test.com"): + user = frappe.get_doc( + {"doctype": "User", "email": "user2@integration.webhooks.test.com", "first_name": "user2"} + ).insert() else: - user = frappe.get_doc('User', 'user2@integration.webhooks.test.com') + user = frappe.get_doc("User", "user2@integration.webhooks.test.com") - webhook = frappe.get_doc('Webhook', {'webhook_doctype': 'User'}) + webhook = frappe.get_doc("Webhook", {"webhook_doctype": "User"}) enqueue_webhook(user, webhook) - self.assertTrue(frappe.db.get_all('Webhook Request Log', pluck='name')) \ No newline at end of file + self.assertTrue(frappe.db.get_all("Webhook Request Log", pluck="name")) diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index 8546a9d2f8..04a1c6c21e 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -8,9 +8,9 @@ import hashlib import hmac import json from time import sleep +from urllib.parse import urlparse import requests -from urllib.parse import urlparse import frappe from frappe import _ @@ -30,12 +30,16 @@ class Webhook(Document): self.validate_repeating_fields() def on_update(self): - frappe.cache().delete_value('webhooks') + frappe.cache().delete_value("webhooks") def validate_docevent(self): if self.webhook_doctype: is_submittable = frappe.get_value("DocType", self.webhook_doctype, "is_submittable") - if not is_submittable and self.webhook_docevent in ["on_submit", "on_cancel", "on_update_after_submit"]: + if not is_submittable and self.webhook_docevent in [ + "on_submit", + "on_cancel", + "on_update_after_submit", + ]: frappe.throw(_("DocType must be Submittable for the selected Doc Event")) def validate_condition(self): @@ -73,7 +77,8 @@ class Webhook(Document): def get_context(doc): - return {'doc': doc, 'utils': get_safe_globals().get('frappe').get('utils')} + return {"doc": doc, "utils": get_safe_globals().get("frappe").get("utils")} + def enqueue_webhook(doc, webhook): webhook = frappe.get_doc("Webhook", webhook.get("name")) @@ -82,8 +87,13 @@ def enqueue_webhook(doc, webhook): for i in range(3): try: - r = requests.request(method=webhook.request_method, url=webhook.request_url, - data=json.dumps(data, default=str), headers=headers, timeout=5) + r = requests.request( + method=webhook.request_method, + url=webhook.request_url, + data=json.dumps(data, default=str), + headers=headers, + timeout=5, + ) r.raise_for_status() frappe.logger().debug({"webhook_success": r.text}) log_request(webhook.request_url, headers, data, r) @@ -97,18 +107,22 @@ def enqueue_webhook(doc, webhook): else: raise e + def log_request(url, headers, data, res): - request_log = frappe.get_doc({ - "doctype": "Webhook Request Log", - "user": frappe.session.user if frappe.session.user else None, - "url": url, - "headers": json.dumps(headers, indent=4) if headers else None, - "data": json.dumps(data, indent=4) if isinstance(data, dict) else data, - "response": json.dumps(res.json(), indent=4) if res else None - }) + request_log = frappe.get_doc( + { + "doctype": "Webhook Request Log", + "user": frappe.session.user if frappe.session.user else None, + "url": url, + "headers": json.dumps(headers, indent=4) if headers else None, + "data": json.dumps(data, indent=4) if isinstance(data, dict) else data, + "response": json.dumps(res.json(), indent=4) if res else None, + } + ) request_log.save(ignore_permissions=True) + def get_webhook_headers(doc, webhook): headers = {} @@ -118,7 +132,7 @@ def get_webhook_headers(doc, webhook): hmac.new( webhook.get_password("webhook_secret").encode("utf8"), json.dumps(data).encode("utf8"), - hashlib.sha256 + hashlib.sha256, ).digest() ) headers[WEBHOOK_SECRET_HEADER] = signature diff --git a/frappe/integrations/doctype/webhook_data/webhook_data.py b/frappe/integrations/doctype/webhook_data/webhook_data.py index 6037ed5390..1bb0d901df 100644 --- a/frappe/integrations/doctype/webhook_data/webhook_data.py +++ b/frappe/integrations/doctype/webhook_data/webhook_data.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WebhookData(Document): pass diff --git a/frappe/integrations/doctype/webhook_header/webhook_header.py b/frappe/integrations/doctype/webhook_header/webhook_header.py index e1944c84bc..9478d227e4 100644 --- a/frappe/integrations/doctype/webhook_header/webhook_header.py +++ b/frappe/integrations/doctype/webhook_header/webhook_header.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WebhookHeader(Document): pass diff --git a/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py b/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py index 5de26a35ed..e3ad5a88a4 100644 --- a/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py +++ b/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestWebhookRequestLog(unittest.TestCase): pass diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py index 3f0558ce80..8fbc73f5e5 100644 --- a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py +++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class WebhookRequestLog(Document): pass diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py index f60344ee8f..0a01989d5e 100644 --- a/frappe/integrations/frappe_providers/frappecloud.py +++ b/frappe/integrations/frappe_providers/frappecloud.py @@ -12,15 +12,22 @@ def frappecloud_migrator(local_site): request = requests.get(request_url) if request.status_code / 100 != 2: - print("Request exitted with Status Code: {}\nPayload: {}".format(request.status_code, html2text(request.text))) - click.secho("Some errors occurred while recovering the migration script. Please contact us @ Frappe Cloud if this issue persists", fg="yellow") + print( + "Request exitted with Status Code: {}\nPayload: {}".format( + request.status_code, html2text(request.text) + ) + ) + click.secho( + "Some errors occurred while recovering the migration script. Please contact us @ Frappe Cloud if this issue persists", + fg="yellow", + ) return script_contents = request.json()["message"] - import tempfile import os import sys + import tempfile py = sys.executable script = tempfile.NamedTemporaryFile(mode="w") diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py index 2b227f503d..39f98e2f1b 100644 --- a/frappe/integrations/oauth2.py +++ b/frappe/integrations/oauth2.py @@ -1,9 +1,8 @@ import json from urllib.parse import quote, urlencode + from oauthlib.oauth2 import FatalClientError, OAuth2Error -from oauthlib.openid.connect.core.endpoints.pre_configured import ( - Server as WebApplicationServer, -) +from oauthlib.openid.connect.core.endpoints.pre_configured import Server as WebApplicationServer import frappe from frappe.integrations.doctype.oauth_provider_settings.oauth_provider_settings import ( @@ -50,10 +49,7 @@ def approve(*args, **kwargs): r = frappe.request try: - ( - scopes, - frappe.flags.oauth_credentials, - ) = get_oauth_server().validate_authorization_request( + (scopes, frappe.flags.oauth_credentials,) = get_oauth_server().validate_authorization_request( r.url, r.method, r.get_data(), r.headers ) @@ -90,10 +86,7 @@ def authorize(**kwargs): else: try: r = frappe.request - ( - scopes, - frappe.flags.oauth_credentials, - ) = get_oauth_server().validate_authorization_request( + (scopes, frappe.flags.oauth_credentials,) = get_oauth_server().validate_authorization_request( r.url, r.method, r.get_data(), r.headers ) @@ -102,22 +95,16 @@ def authorize(**kwargs): frappe.flags.oauth_credentials["client_id"], "skip_authorization", ) - unrevoked_tokens = frappe.get_all( - "OAuth Bearer Token", filters={"status": "Active"} - ) + unrevoked_tokens = frappe.get_all("OAuth Bearer Token", filters={"status": "Active"}) - if skip_auth or ( - get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens - ): + if skip_auth or (get_oauth_settings().skip_authorization == "Auto" and unrevoked_tokens): frappe.local.response["type"] = "redirect" frappe.local.response["location"] = success_url else: # Show Allow/Deny screen. response_html_params = frappe._dict( { - "client_id": frappe.db.get_value( - "OAuth Client", kwargs["client_id"], "app_name" - ), + "client_id": frappe.db.get_value("OAuth Client", kwargs["client_id"], "app_name"), "success_url": success_url, "failure_url": failure_url, "details": scopes, @@ -222,9 +209,7 @@ def introspect_token(token=None, token_type_hint=None): if token_type_hint == "access_token": bearer_token = frappe.get_doc("OAuth Bearer Token", {"access_token": token}) elif token_type_hint == "refresh_token": - bearer_token = frappe.get_doc( - "OAuth Bearer Token", {"refresh_token": token} - ) + bearer_token = frappe.get_doc("OAuth Bearer Token", {"refresh_token": token}) client = frappe.get_doc("OAuth Client", bearer_token.client) diff --git a/frappe/integrations/oauth2_logins.py b/frappe/integrations/oauth2_logins.py index b187d29b34..c8252b0f70 100644 --- a/frappe/integrations/oauth2_logins.py +++ b/frappe/integrations/oauth2_logins.py @@ -1,39 +1,48 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe import frappe.utils from frappe.utils.oauth import login_via_oauth2, login_via_oauth2_id_token -import json + @frappe.whitelist(allow_guest=True) def login_via_google(code, state): login_via_oauth2("google", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_github(code, state): login_via_oauth2("github", code, state) + @frappe.whitelist(allow_guest=True) def login_via_facebook(code, state): login_via_oauth2("facebook", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_frappe(code, state): login_via_oauth2("frappe", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_office365(code, state): login_via_oauth2_id_token("office_365", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_salesforce(code, state): login_via_oauth2("salesforce", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_fairlogin(code, state): login_via_oauth2("fairlogin", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def custom(code, state): """ @@ -48,6 +57,7 @@ def custom(code, state): if frappe.db.exists("Social Login Key", provider): login_via_oauth2(provider, code, state, decoder=decoder_compat) + def decoder_compat(b): # https://github.com/litl/rauth/issues/145#issuecomment-31199471 return json.loads(bytes(b).decode("utf-8")) diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 4242676d94..307f1525fe 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -2,10 +2,12 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import glob import os -from frappe.utils import split_emails, cint + +import frappe +from frappe.utils import cint, split_emails + def send_email(success, service_name, doctype, email_field, error_status=None): recipients = get_recipients(doctype, email_field) @@ -65,7 +67,7 @@ def get_latest_backup_file(with_files=False): return database, config -def get_file_size(file_path, unit='MB'): +def get_file_size(file_path, unit="MB"): file_size = os.path.getsize(file_path) memory_size_unit_mapper = {"KB": 1, "MB": 2, "GB": 3, "TB": 4} @@ -76,10 +78,11 @@ def get_file_size(file_path, unit='MB'): return file_size + def get_chunk_site(file_size): - ''' this function will return chunk size in megabytes based on file size ''' + """this function will return chunk size in megabytes based on file size""" - file_size_in_gb = cint(file_size/1024/1024) + file_size_in_gb = cint(file_size / 1024 / 1024) MB = 1024 * 1024 if file_size_in_gb > 5000: @@ -93,6 +96,7 @@ def get_chunk_site(file_size): else: return 15 * MB + def validate_file_size(): frappe.flags.create_new_backup = True latest_file, site_config = get_latest_backup_file() @@ -101,12 +105,18 @@ def validate_file_size(): if file_size > 1: frappe.flags.create_new_backup = False + def generate_files_backup(): from frappe.utils.backups import BackupGenerator - backup = BackupGenerator(frappe.conf.db_name, frappe.conf.db_name, - frappe.conf.db_password, db_host = frappe.db.host, - db_type=frappe.conf.db_type, db_port=frappe.conf.db_port) + backup = BackupGenerator( + frappe.conf.db_name, + frappe.conf.db_name, + frappe.conf.db_password, + db_host=frappe.db.host, + db_type=frappe.conf.db_type, + db_port=frappe.conf.db_port, + ) backup.set_backup_file_name() backup.zip_files() diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index bda45a765d..191cd1f23b 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -2,14 +2,17 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe -import json,datetime +import datetime +import json from urllib.parse import parse_qs -from frappe.utils import get_request_session + +import frappe from frappe import _ +from frappe.utils import get_request_session + def make_request(method, url, auth=None, headers=None, data=None): - auth = auth or '' + auth = auth or "" data = data or {} headers = headers or {} @@ -26,14 +29,18 @@ def make_request(method, url, auth=None, headers=None, data=None): frappe.log_error() raise exc + def make_get_request(url, **kwargs): - return make_request('GET', url, **kwargs) + return make_request("GET", url, **kwargs) + def make_post_request(url, **kwargs): - return make_request('POST', url, **kwargs) + return make_request("POST", url, **kwargs) + def make_put_request(url, **kwargs): - return make_request('PUT', url, **kwargs) + return make_request("PUT", url, **kwargs) + def create_request_log(data, integration_type, service_name, name=None, error=None): if isinstance(data, str): @@ -42,15 +49,17 @@ def create_request_log(data, integration_type, service_name, name=None, error=No if isinstance(error, str): error = json.loads(error) - integration_request = frappe.get_doc({ - "doctype": "Integration Request", - "integration_type": integration_type, - "integration_request_service": service_name, - "reference_doctype": data.get("reference_doctype"), - "reference_docname": data.get("reference_docname"), - "error": json.dumps(error, default=json_handler), - "data": json.dumps(data, default=json_handler) - }) + integration_request = frappe.get_doc( + { + "doctype": "Integration Request", + "integration_type": integration_type, + "integration_request_service": service_name, + "reference_doctype": data.get("reference_doctype"), + "reference_docname": data.get("reference_docname"), + "error": json.dumps(error, default=json_handler), + "data": json.dumps(data, default=json_handler), + } + ) if name: integration_request.flags._name = name @@ -60,8 +69,9 @@ def create_request_log(data, integration_type, service_name, name=None, error=No return integration_request + def get_payment_gateway_controller(payment_gateway): - '''Return payment gateway controller''' + """Return payment gateway controller""" gateway = frappe.get_doc("Payment Gateway", payment_gateway) if gateway.gateway_controller is None: try: @@ -78,28 +88,36 @@ def get_payment_gateway_controller(payment_gateway): @frappe.whitelist(allow_guest=True, xss_safe=True) def get_checkout_url(**kwargs): try: - if kwargs.get('payment_gateway'): - doc = frappe.get_doc("{0} Settings".format(kwargs.get('payment_gateway'))) + if kwargs.get("payment_gateway"): + doc = frappe.get_doc("{0} Settings".format(kwargs.get("payment_gateway"))) return doc.get_payment_url(**kwargs) else: raise Exception except Exception: - frappe.respond_as_web_page(_("Something went wrong"), - _("Looks like something is wrong with this site's payment gateway configuration. No payment has been made."), - indicator_color='red', - http_status_code=frappe.ValidationError.http_status_code) + frappe.respond_as_web_page( + _("Something went wrong"), + _( + "Looks like something is wrong with this site's payment gateway configuration. No payment has been made." + ), + indicator_color="red", + http_status_code=frappe.ValidationError.http_status_code, + ) + def create_payment_gateway(gateway, settings=None, controller=None): # NOTE: we don't translate Payment Gateway name because it is an internal doctype if not frappe.db.exists("Payment Gateway", gateway): - payment_gateway = frappe.get_doc({ - "doctype": "Payment Gateway", - "gateway": gateway, - "gateway_settings": settings, - "gateway_controller": controller - }) + payment_gateway = frappe.get_doc( + { + "doctype": "Payment Gateway", + "gateway": gateway, + "gateway_settings": settings, + "gateway_controller": controller, + } + ) payment_gateway.insert(ignore_permissions=True) + def json_handler(obj): if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime)): return str(obj) diff --git a/frappe/middlewares.py b/frappe/middlewares.py index 38cb4cea21..cd47b7210f 100644 --- a/frappe/middlewares.py +++ b/frappe/middlewares.py @@ -1,11 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import os + from werkzeug.exceptions import NotFound from werkzeug.middleware.shared_data import SharedDataMiddleware -from frappe.utils import get_site_name, cstr + +import frappe +from frappe.utils import cstr, get_site_name class StaticDataMiddleware(SharedDataMiddleware): @@ -15,8 +17,8 @@ class StaticDataMiddleware(SharedDataMiddleware): def get_directory_loader(self, directory): def loader(path): - site = get_site_name(frappe.app._site or self.environ.get('HTTP_HOST')) - path = os.path.join(directory, site, 'public', 'files', cstr(path)) + site = get_site_name(frappe.app._site or self.environ.get("HTTP_HOST")) + path = os.path.join(directory, site, "public", "files", cstr(path)) if os.path.isfile(path): return os.path.basename(path), self._opener(path) else: diff --git a/frappe/migrate.py b/frappe/migrate.py index eabd0ff3e0..bb83fa5b6d 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -66,8 +66,7 @@ class SiteMigration: self.skip_search_index = skip_search_index def setUp(self): - """Complete setup required for site migration - """ + """Complete setup required for site migration""" frappe.flags.touched_tables = set() self.touched_tables_file = frappe.get_site_path("touched_tables.json") add_column(doctype="DocType", column_name="migration_hash", fieldtype="Data") @@ -99,19 +98,21 @@ class SiteMigration: @atomic def pre_schema_updates(self): - """Executes `before_migrate` hooks - """ + """Executes `before_migrate` hooks""" for app in frappe.get_installed_apps(): for fn in frappe.get_hooks("before_migrate", app_name=app): frappe.get_attr(fn)() @atomic def run_schema_updates(self): - """Run patches as defined in patches.txt, sync schema changes as defined in the {doctype}.json files - """ - frappe.modules.patch_handler.run_all(skip_failing=self.skip_failing, patch_type=PatchType.pre_model_sync) + """Run patches as defined in patches.txt, sync schema changes as defined in the {doctype}.json files""" + frappe.modules.patch_handler.run_all( + skip_failing=self.skip_failing, patch_type=PatchType.pre_model_sync + ) frappe.model.sync.sync_all() - frappe.modules.patch_handler.run_all(skip_failing=self.skip_failing, patch_type=PatchType.post_model_sync) + frappe.modules.patch_handler.run_all( + skip_failing=self.skip_failing, patch_type=PatchType.post_model_sync + ) @atomic def post_schema_updates(self): diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index ab792d90e5..6c9bab2dff 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -5,180 +5,163 @@ import frappe data_fieldtypes = ( - 'Currency', - 'Int', - 'Long Int', - 'Float', - 'Percent', - 'Check', - 'Small Text', - 'Long Text', - 'Code', - 'Text Editor', - 'Markdown Editor', - 'HTML Editor', - 'Date', - 'Datetime', - 'Time', - 'Text', - 'Data', - 'Link', - 'Dynamic Link', - 'Password', - 'Select', - 'Rating', - 'Read Only', - 'Attach', - 'Attach Image', - 'Signature', - 'Color', - 'Barcode', - 'Geolocation', - 'Duration', - 'Icon', - 'Autocomplete', + "Currency", + "Int", + "Long Int", + "Float", + "Percent", + "Check", + "Small Text", + "Long Text", + "Code", + "Text Editor", + "Markdown Editor", + "HTML Editor", + "Date", + "Datetime", + "Time", + "Text", + "Data", + "Link", + "Dynamic Link", + "Password", + "Select", + "Rating", + "Read Only", + "Attach", + "Attach Image", + "Signature", + "Color", + "Barcode", + "Geolocation", + "Duration", + "Icon", + "Autocomplete", ) attachment_fieldtypes = ( - 'Attach', - 'Attach Image', + "Attach", + "Attach Image", ) no_value_fields = ( - 'Section Break', - 'Column Break', - 'Tab Break', - 'HTML', - 'Table', - 'Table MultiSelect', - 'Button', - 'Image', - 'Fold', - 'Heading' + "Section Break", + "Column Break", + "Tab Break", + "HTML", + "Table", + "Table MultiSelect", + "Button", + "Image", + "Fold", + "Heading", ) display_fieldtypes = ( - 'Section Break', - 'Column Break', - 'Tab Break', - 'HTML', - 'Button', - 'Image', - 'Fold', - 'Heading') - -numeric_fieldtypes = ( - 'Currency', - 'Int', - 'Long Int', - 'Float', - 'Percent', - 'Check' + "Section Break", + "Column Break", + "Tab Break", + "HTML", + "Button", + "Image", + "Fold", + "Heading", ) -data_field_options = ( - 'Email', - 'Name', - 'Phone', - 'URL', - 'Barcode' -) +numeric_fieldtypes = ("Currency", "Int", "Long Int", "Float", "Percent", "Check") + +data_field_options = ("Email", "Name", "Phone", "URL", "Barcode") default_fields = ( - 'doctype', - 'name', - 'owner', - 'creation', - 'modified', - 'modified_by', - 'docstatus', - 'idx' + "doctype", + "name", + "owner", + "creation", + "modified", + "modified_by", + "docstatus", + "idx", ) -child_table_fields = ( - 'parent', - 'parentfield', - 'parenttype' -) +child_table_fields = ("parent", "parentfield", "parenttype") -optional_fields = ( - "_user_tags", - "_comments", - "_assign", - "_liked_by", - "_seen" -) +optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen") -table_fields = ( - 'Table', - 'Table MultiSelect' -) +table_fields = ("Table", "Table MultiSelect") core_doctypes_list = ( - 'DocType', - 'DocField', - 'DocPerm', - 'DocType Action', - 'DocType Link', - 'User', - 'Role', - 'Has Role', - 'Page', - 'Module Def', - 'Print Format', - 'Report', - 'Customize Form', - 'Customize Form Field', - 'Property Setter', - 'Custom Field', - 'Client Script' + "DocType", + "DocField", + "DocPerm", + "DocType Action", + "DocType Link", + "User", + "Role", + "Has Role", + "Page", + "Module Def", + "Print Format", + "Report", + "Customize Form", + "Customize Form Field", + "Property Setter", + "Custom Field", + "Client Script", ) log_types = ( - 'Version', - 'Error Log', - 'Scheduled Job Log', - 'Event Sync Log', - 'Event Update Log', - 'Access Log', - 'View Log', - 'Activity Log', - 'Energy Point Log', - 'Notification Log', - 'Email Queue', - 'DocShare', - 'Document Follow', - 'Console Log' + "Version", + "Error Log", + "Scheduled Job Log", + "Event Sync Log", + "Event Update Log", + "Access Log", + "View Log", + "Activity Log", + "Energy Point Log", + "Notification Log", + "Email Queue", + "DocShare", + "Document Follow", + "Console Log", ) + def delete_fields(args_dict, delete=0): """ - Delete a field. - * Deletes record from `tabDocField` - * If not single doctype: Drops column from table - * If single, deletes record from `tabSingles` - args_dict = { dt: [field names] } + Delete a field. + * Deletes record from `tabDocField` + * If not single doctype: Drops column from table + * If single, deletes record from `tabSingles` + args_dict = { dt: [field names] } """ import frappe.utils + for dt in args_dict: fields = args_dict[dt] if not fields: continue - frappe.db.delete("DocField", { - "parent": dt, - "fieldname": ("in", fields), - }) + frappe.db.delete( + "DocField", + { + "parent": dt, + "fieldname": ("in", fields), + }, + ) # Delete the data/column only if delete is specified if not delete: continue if frappe.db.get_value("DocType", dt, "issingle"): - frappe.db.delete("Singles", { - "doctype": dt, - "field": ("in", fields), - }) + frappe.db.delete( + "Singles", + { + "doctype": dt, + "field": ("in", fields), + }, + ) else: existing_fields = frappe.db.describe(dt) existing_fields = existing_fields and [e[0] for e in existing_fields] or [] @@ -186,14 +169,15 @@ def delete_fields(args_dict, delete=0): if not fields_need_to_delete: continue - if frappe.db.db_type == 'mariadb': + if frappe.db.db_type == "mariadb": # mariadb implicitly commits before DDL, make it explicit frappe.db.commit() - query = "ALTER TABLE `tab%s` " % dt + \ - ", ".join("DROP COLUMN `%s`" % f for f in fields_need_to_delete) + query = "ALTER TABLE `tab%s` " % dt + ", ".join( + "DROP COLUMN `%s`" % f for f in fields_need_to_delete + ) frappe.db.sql(query) - if frappe.db.db_type == 'postgres': + if frappe.db.db_type == "postgres": # commit the results to db frappe.db.commit() diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 57591d01d5..59c631eb95 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -5,20 +5,16 @@ import datetime import frappe from frappe import _ from frappe.model import child_table_fields, default_fields, display_fieldtypes, table_fields +from frappe.model.docstatus import DocStatus from frappe.model.naming import set_new_name from frappe.model.utils.link_count import notify_link_count from frappe.modules import load_doctype_module from frappe.utils import cast_fieldtype, cint, cstr, flt, now, sanitize_html, strip_html from frappe.utils.html_utils import unescape_html -from frappe.model.docstatus import DocStatus -max_positive_value = { - 'smallint': 2 ** 15, - 'int': 2 ** 31, - 'bigint': 2 ** 63 -} +max_positive_value = {"smallint": 2**15, "int": 2**31, "bigint": 2**63} -DOCTYPES_FOR_DOCTYPE = ('DocType', 'DocField', 'DocPerm', 'DocType Action', 'DocType Link') +DOCTYPES_FOR_DOCTYPE = ("DocType", "DocField", "DocPerm", "DocType Action", "DocType Link") def get_controller(doctype): @@ -36,18 +32,18 @@ def get_controller(doctype): ) or ("Core", False) if custom: - is_tree = frappe.db.get_value( - "DocType", doctype, "is_tree", ignore=True, cache=True - ) + is_tree = frappe.db.get_value("DocType", doctype, "is_tree", ignore=True, cache=True) _class = NestedSet if is_tree else Document else: - class_overrides = frappe.get_hooks('override_doctype_class') + class_overrides = frappe.get_hooks("override_doctype_class") if class_overrides and class_overrides.get(doctype): import_path = class_overrides[doctype][-1] - module_path, classname = import_path.rsplit('.', 1) + module_path, classname = import_path.rsplit(".", 1) module = frappe.get_module(module_path) if not hasattr(module, classname): - raise ImportError('{0}: {1} does not exist in module {2}'.format(doctype, classname, module_path)) + raise ImportError( + "{0}: {1} does not exist in module {2}".format(doctype, classname, module_path) + ) else: module = load_doctype_module(doctype, module_name) classname = doctype.replace(" ", "").replace("-", "") @@ -71,6 +67,7 @@ def get_controller(doctype): return site_controllers[doctype] + class BaseDocument(object): ignore_in_setter = ("doctype", "_meta", "meta", "_table_fields", "_valid_columns") @@ -96,13 +93,13 @@ class BaseDocument(object): return self.__dict__ def update(self, d): - """ Update multiple fields of a doctype using a dictionary of key-value pairs. + """Update multiple fields of a doctype using a dictionary of key-value pairs. Example: - doc.update({ - "user": "admin", - "balance": 42000 - }) + doc.update({ + "user": "admin", + "balance": 42000 + }) """ # set name first, as it is used a reference in child document @@ -146,9 +143,7 @@ class BaseDocument(object): else: value = self.__dict__.get(key, default) - if value is None and key in ( - d.fieldname for d in self.meta.get_table_fields() - ): + if value is None and key in (d.fieldname for d in self.meta.get_table_fields()): value = [] self.set(key, value) @@ -175,17 +170,17 @@ class BaseDocument(object): del self.__dict__[key] def append(self, key, value=None): - """ Append an item to a child table. + """Append an item to a child table. Example: - doc.append("childtable", { - "child_table_field": "value", - "child_table_int_field": 0, - ... - }) + doc.append("childtable", { + "child_table_field": "value", + "child_table_int_field": 0, + ... + }) """ if value is None: - value={} + value = {} if isinstance(value, (dict, BaseDocument)): if not self.__dict__.get(key): self.__dict__[key] = [] @@ -201,13 +196,17 @@ class BaseDocument(object): # metaclasses may have arbitrary lists # which we can ignore - if (getattr(self, '_metaclass', None) - or self.__class__.__name__ in ('Meta', 'FormMeta', 'DocField')): + if getattr(self, "_metaclass", None) or self.__class__.__name__ in ( + "Meta", + "FormMeta", + "DocField", + ): return value raise ValueError( - 'Document for field "{0}" attached to child table of "{1}" must be a dict or BaseDocument, not {2} ({3})'.format(key, - self.name, str(type(value))[1:-1], value) + 'Document for field "{0}" attached to child table of "{1}" must be a dict or BaseDocument, not {2} ({3})'.format( + key, self.name, str(type(value))[1:-1], value + ) ) def extend(self, key, value): @@ -246,11 +245,13 @@ class BaseDocument(object): value.idx = len(self.get(key) or []) + 1 if not getattr(value, "name", None): - value.__dict__['__islocal'] = 1 + value.__dict__["__islocal"] = 1 return value - def get_valid_dict(self, sanitize=True, convert_dates_to_str=False, ignore_nulls=False, ignore_virtual=False): + def get_valid_dict( + self, sanitize=True, convert_dates_to_str=False, ignore_nulls=False, ignore_virtual=False + ): d = frappe._dict() for fieldname in self.meta.get_valid_columns(): d[fieldname] = self.get(fieldname) @@ -280,31 +281,28 @@ class BaseDocument(object): if _val and not callable(_val): d[fieldname] = _val elif df: - if df.fieldtype=="Check": + if df.fieldtype == "Check": d[fieldname] = 1 if cint(d[fieldname]) else 0 - elif df.fieldtype=="Int" and not isinstance(d[fieldname], int): + elif df.fieldtype == "Int" and not isinstance(d[fieldname], int): d[fieldname] = cint(d[fieldname]) elif df.fieldtype in ("Currency", "Float", "Percent") and not isinstance(d[fieldname], float): d[fieldname] = flt(d[fieldname]) - elif df.fieldtype in ("Datetime", "Date", "Time") and d[fieldname]=="": + elif df.fieldtype in ("Datetime", "Date", "Time") and d[fieldname] == "": d[fieldname] = None - elif df.get("unique") and cstr(d[fieldname]).strip()=="": + elif df.get("unique") and cstr(d[fieldname]).strip() == "": # unique empty field should be set to None d[fieldname] = None if isinstance(d[fieldname], list) and df.fieldtype not in table_fields: - frappe.throw(_('Value for {0} cannot be a list').format(_(df.label))) - - if convert_dates_to_str and isinstance(d[fieldname], ( - datetime.datetime, - datetime.date, - datetime.time, - datetime.timedelta - )): + frappe.throw(_("Value for {0} cannot be a list").format(_(df.label))) + + if convert_dates_to_str and isinstance( + d[fieldname], (datetime.datetime, datetime.date, datetime.time, datetime.timedelta) + ): d[fieldname] = str(d[fieldname]) if d[fieldname] is None and ignore_nulls: @@ -331,6 +329,7 @@ class BaseDocument(object): if self.doctype not in frappe.local.valid_columns: if self.doctype in DOCTYPES_FOR_DOCTYPE: from frappe.model.meta import get_table_columns + valid = get_table_columns(self.doctype) else: valid = self.meta.get_valid_columns() @@ -350,7 +349,13 @@ class BaseDocument(object): def docstatus(self, value): self.__dict__["docstatus"] = DocStatus(cint(value)) - def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False): + def as_dict( + self, + no_nulls=False, + no_default_fields=False, + convert_dates_to_str=False, + no_child_table_fields=False, + ): doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str) doc["doctype"] = self.doctype @@ -361,8 +366,9 @@ class BaseDocument(object): convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields, - no_child_table_fields=no_child_table_fields - ) for d in children + no_child_table_fields=no_child_table_fields, + ) + for d in children ] if no_nulls: @@ -380,7 +386,14 @@ class BaseDocument(object): if k in child_table_fields: del doc[k] - for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"): + for key in ( + "_user_tags", + "__islocal", + "__onload", + "_liked_by", + "__run_link_triggers", + "__unsaved", + ): if self.get(key): doc[key] = self.get(key) @@ -393,21 +406,23 @@ class BaseDocument(object): try: return self.meta.get_field(fieldname).options except AttributeError: - if self.doctype == 'DocType': - return dict(links='DocType Link', actions='DocType Action', states='DocType State').get(fieldname) + if self.doctype == "DocType": + return dict(links="DocType Link", actions="DocType Action", states="DocType State").get( + fieldname + ) raise def get_parentfield_of_doctype(self, doctype): - fieldname = [df.fieldname for df in self.meta.get_table_fields() if df.options==doctype] + fieldname = [df.fieldname for df in self.meta.get_table_fields() if df.options == doctype] return fieldname[0] if fieldname else None def db_insert(self, ignore_if_duplicate=False): """INSERT the document (with valid columns) in the database. - args: - ignore_if_duplicate: ignore primary key collision - at database level (postgres) - in python (mariadb) + args: + ignore_if_duplicate: ignore primary key collision + at database level (postgres) + in python (mariadb) """ if not self.name: # name will be set by document class in most cases @@ -432,16 +447,19 @@ class BaseDocument(object): columns = list(d) try: - frappe.db.sql("""INSERT INTO `tab{doctype}` ({columns}) + frappe.db.sql( + """INSERT INTO `tab{doctype}` ({columns}) VALUES ({values}) {conflict_handler}""".format( doctype=self.doctype, - columns=", ".join("`"+c+"`" for c in columns), + columns=", ".join("`" + c + "`" for c in columns), values=", ".join(["%s"] * len(columns)), - conflict_handler=conflict_handler - ), list(d.values())) + conflict_handler=conflict_handler, + ), + list(d.values()), + ) except Exception as e: if frappe.db.is_primary_key_violation(e): - if self.meta.autoname=="hash": + if self.meta.autoname == "hash": # hash collision? try again frappe.flags.retry_count = (frappe.flags.retry_count or 0) + 1 if frappe.flags.retry_count > 5 and not frappe.flags.in_test: @@ -451,9 +469,11 @@ class BaseDocument(object): return if not ignore_if_duplicate: - frappe.msgprint(_("{0} {1} already exists") - .format(self.doctype, frappe.bold(self.name)), - title=_("Duplicate Name"), indicator="red") + frappe.msgprint( + _("{0} {1} already exists").format(self.doctype, frappe.bold(self.name)), + title=_("Duplicate Name"), + indicator="red", + ) raise frappe.DuplicateEntryError(self.doctype, self.name, e) elif frappe.db.is_unique_key_violation(e): @@ -470,20 +490,24 @@ class BaseDocument(object): self.db_insert() return - d = self.get_valid_dict(convert_dates_to_str=True, ignore_nulls = self.doctype in DOCTYPES_FOR_DOCTYPE) + d = self.get_valid_dict( + convert_dates_to_str=True, ignore_nulls=self.doctype in DOCTYPES_FOR_DOCTYPE + ) # don't update name, as case might've been changed - name = cstr(d['name']) - del d['name'] + name = cstr(d["name"]) + del d["name"] columns = list(d) try: - frappe.db.sql("""UPDATE `tab{doctype}` + frappe.db.sql( + """UPDATE `tab{doctype}` SET {values} WHERE `name`=%s""".format( - doctype = self.doctype, - values = ", ".join("`"+c+"`=%s" for c in columns) - ), list(d.values()) + [name]) + doctype=self.doctype, values=", ".join("`" + c + "`=%s" for c in columns) + ), + list(d.values()) + [name], + ) except Exception as e: if frappe.db.is_unique_key_violation(e): self.show_unique_validation_message(e) @@ -499,7 +523,7 @@ class BaseDocument(object): doc.db_update() def show_unique_validation_message(self, e): - if frappe.db.db_type != 'postgres': + if frappe.db.db_type != "postgres": fieldname = str(e).split("'")[-2] label = None @@ -521,15 +545,16 @@ class BaseDocument(object): This function returns the `column_name` associated with the `key_name` passed Args: - key_name (str): The name of the database index. + key_name (str): The name of the database index. Raises: - IndexError: If the key is not found in the table. + IndexError: If the key is not found in the table. Returns: - str: The column name associated with the key. + str: The column name associated with the key. """ - return frappe.db.sql(f""" + return frappe.db.sql( + f""" SHOW INDEX FROM @@ -538,16 +563,19 @@ class BaseDocument(object): key_name=%s AND Non_unique=0 - """, key_name, as_dict=True)[0].get("Column_name") + """, + key_name, + as_dict=True, + )[0].get("Column_name") def get_label_from_fieldname(self, fieldname): """Returns the associated label for fieldname Args: - fieldname (str): The fieldname in the DocType to use to pull the label. + fieldname (str): The fieldname in the DocType to use to pull the label. Returns: - str: The label associated with the fieldname, if found, otherwise `None`. + str: The label associated with the fieldname, if found, otherwise `None`. """ df = self.meta.get_field(fieldname) if df: @@ -556,7 +584,7 @@ class BaseDocument(object): def update_modified(self): """Update modified timestamp""" self.set("modified", now()) - frappe.db.set_value(self.doctype, self.name, 'modified', self.modified, update_modified=False) + frappe.db.set_value(self.doctype, self.name, "modified", self.modified, update_modified=False) def _fix_numeric_types(self): for df in self.meta.get("fields"): @@ -575,20 +603,27 @@ class BaseDocument(object): def _get_missing_mandatory_fields(self): """Get mandatory fields that do not have any values""" + def get_msg(df): if df.fieldtype in table_fields: return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label)) # check if parentfield exists (only applicable for child table doctype) elif self.get("parentfield"): - return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)), - _("Row"), self.idx, _("Value missing for"), _(df.label)) + return "{}: {} {} #{}: {}: {}".format( + _("Error"), + frappe.bold(_(self.doctype)), + _("Row"), + self.idx, + _("Value missing for"), + _(df.label), + ) return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label)) missing = [] - for df in self.meta.get("fields", {"reqd": ('=', 1)}): + for df in self.meta.get("fields", {"reqd": ("=", 1)}): if self.get(df.fieldname) in (None, []) or not strip_html(cstr(self.get(df.fieldname))).strip(): missing.append((df.fieldname, get_msg(df))) @@ -602,6 +637,7 @@ class BaseDocument(object): def get_invalid_links(self, is_submittable=False): """Returns list of invalid links and also updates fetch values if not set""" + def get_msg(df, docname): # check if parentfield exists (only applicable for child table doctype) if self.get("parentfield"): @@ -612,12 +648,13 @@ class BaseDocument(object): invalid_links = [] cancelled_links = [] - for df in (self.meta.get_link_fields() - + self.meta.get("fields", {"fieldtype": ('=', "Dynamic Link")})): + for df in self.meta.get_link_fields() + self.meta.get( + "fields", {"fieldtype": ("=", "Dynamic Link")} + ): docname = self.get(df.fieldname) if docname: - if df.fieldtype=="Link": + if df.fieldtype == "Link": doctype = df.options if not doctype: frappe.throw(_("Options not set for link field {0}").format(df.fieldname)) @@ -633,28 +670,25 @@ class BaseDocument(object): # Readonly or Data or Text type fields fields_to_fetch = [ - _df for _df in self.meta.get_fields_to_fetch(df.fieldname) - if - not _df.get('fetch_if_empty') - or (_df.get('fetch_if_empty') and not self.get(_df.fieldname)) + _df + for _df in self.meta.get_fields_to_fetch(df.fieldname) + if not _df.get("fetch_if_empty") + or (_df.get("fetch_if_empty") and not self.get(_df.fieldname)) ] - if not frappe.get_meta(doctype).get('is_virtual'): + if not frappe.get_meta(doctype).get("is_virtual"): if not fields_to_fetch: # cache a single value type - values = frappe._dict(name=frappe.db.get_value(doctype, docname, - 'name', cache=True)) + values = frappe._dict(name=frappe.db.get_value(doctype, docname, "name", cache=True)) else: - values_to_fetch = ['name'] + [_df.fetch_from.split('.')[-1] - for _df in fields_to_fetch] + values_to_fetch = ["name"] + [_df.fetch_from.split(".")[-1] for _df in fields_to_fetch] # don't cache if fetching other values too - values = frappe.db.get_value(doctype, docname, - values_to_fetch, as_dict=True) + values = frappe.db.get_value(doctype, docname, values_to_fetch, as_dict=True) if frappe.get_meta(doctype).issingle: values.name = doctype - if frappe.get_meta(doctype).get('is_virtual'): + if frappe.get_meta(doctype).get("is_virtual"): values = frappe.get_doc(doctype, docname) if values: @@ -669,29 +703,35 @@ class BaseDocument(object): if not values.name: invalid_links.append((df.fieldname, docname, get_msg(df, docname))) - elif (df.fieldname != "amended_from" - and (is_submittable or self.meta.is_submittable) and frappe.get_meta(doctype).is_submittable - and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled()): + elif ( + df.fieldname != "amended_from" + and (is_submittable or self.meta.is_submittable) + and frappe.get_meta(doctype).is_submittable + and cint(frappe.db.get_value(doctype, docname, "docstatus")) == DocStatus.cancelled() + ): cancelled_links.append((df.fieldname, docname, get_msg(df, docname))) return invalid_links, cancelled_links def set_fetch_from_value(self, doctype, df, values): - fetch_from_fieldname = df.fetch_from.split('.')[-1] + fetch_from_fieldname = df.fetch_from.split(".")[-1] value = values[fetch_from_fieldname] - if df.fieldtype in ['Small Text', 'Text', 'Data']: + if df.fieldtype in ["Small Text", "Text", "Data"]: from frappe.model.meta import get_default_df - fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname) + + fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field( + fetch_from_fieldname + ) if not fetch_from_df: frappe.throw( _('Please check the value of "Fetch From" set for field {0}').format(frappe.bold(df.label)), - title = _('Wrong Fetch From value') + title=_("Wrong Fetch From value"), ) - fetch_from_ft = fetch_from_df.get('fieldtype') - if fetch_from_ft == 'Text Editor' and value: + fetch_from_ft = fetch_from_df.get("fieldtype") + if fetch_from_ft == "Text Editor" and value: value = unescape_html(strip_html(value)) setattr(self, df.fieldname, value) @@ -700,7 +740,7 @@ class BaseDocument(object): return for df in self.meta.get_select_fields(): - if df.fieldname=="naming_series" or not (self.get(df.fieldname) and df.options): + if df.fieldname == "naming_series" or not (self.get(df.fieldname) and df.options): continue options = (df.options or "").split("\n") @@ -719,8 +759,11 @@ class BaseDocument(object): label = _(self.meta.get_label(df.fieldname)) comma_options = '", "'.join(_(each) for each in options) - frappe.throw(_('{0} {1} cannot be "{2}". It should be one of "{3}"').format(prefix, label, - value, comma_options)) + frappe.throw( + _('{0} {1} cannot be "{2}". It should be one of "{3}"').format( + prefix, label, value, comma_options + ) + ) def _validate_data_fields(self): # data_field options defined in frappe.model.data_field_options @@ -754,7 +797,7 @@ class BaseDocument(object): if frappe.flags.in_import or self.is_new() or self.flags.ignore_validate_constants: return - constants = [d.fieldname for d in self.meta.get("fields", {"set_only_once": ('=',1)})] + constants = [d.fieldname for d in self.meta.get("fields", {"set_only_once": ("=", 1)})] if constants: values = frappe.db.get_value(self.doctype, self.name, constants, as_dict=True) @@ -762,15 +805,17 @@ class BaseDocument(object): df = self.meta.get_field(fieldname) # This conversion to string only when fieldtype is Date - if df.fieldtype == 'Date' or df.fieldtype == 'Datetime': + if df.fieldtype == "Date" or df.fieldtype == "Datetime": value = str(values.get(fieldname)) else: - value = values.get(fieldname) + value = values.get(fieldname) if self.get(fieldname) != value: - frappe.throw(_("Value cannot be changed for {0}").format(self.meta.get_label(fieldname)), - frappe.CannotChangeConstantError) + frappe.throw( + _("Value cannot be changed for {0}").format(self.meta.get_label(fieldname)), + frappe.CannotChangeConstantError, + ) def _validate_length(self): if frappe.flags.in_install: @@ -785,20 +830,20 @@ class BaseDocument(object): for fieldname, value in self.get_valid_dict(ignore_virtual=True).items(): df = self.meta.get_field(fieldname) - if not df or df.fieldtype == 'Check': + if not df or df.fieldtype == "Check": # skip standard fields and Check fields continue column_type = type_map[df.fieldtype][0] or None - if column_type == 'varchar': + if column_type == "varchar": default_column_max_length = type_map[df.fieldtype][1] or None max_length = cint(df.get("length")) or cint(default_column_max_length) if len(cstr(value)) > max_length: self.throw_length_exceeded_error(df, max_length, value) - elif column_type in ('int', 'bigint', 'smallint'): + elif column_type in ("int", "bigint", "smallint"): max_length = max_positive_value[column_type] if abs(cint(value)) > max_length: @@ -830,8 +875,13 @@ class BaseDocument(object): else: reference = "{0} {1}".format(_(self.doctype), self.name) - frappe.throw(_("{0}: '{1}' ({3}) will get truncated, as max characters allowed is {2}")\ - .format(reference, _(df.label), max_length, value), frappe.CharacterLengthExceededError, title=_('Value too big')) + frappe.throw( + _("{0}: '{1}' ({3}) will get truncated, as max characters allowed is {2}").format( + reference, _(df.label), max_length, value + ), + frappe.CharacterLengthExceededError, + title=_("Value too big"), + ) def _validate_update_after_submit(self): # get the full doc with children @@ -852,15 +902,22 @@ class BaseDocument(object): self_value = self.get_value(key) # Postgres stores values as `datetime.time`, MariaDB as `timedelta` if isinstance(self_value, datetime.timedelta) and isinstance(db_value, datetime.time): - db_value = datetime.timedelta(hours=db_value.hour, minutes=db_value.minute, seconds=db_value.second, microseconds=db_value.microsecond) + db_value = datetime.timedelta( + hours=db_value.hour, + minutes=db_value.minute, + seconds=db_value.second, + microseconds=db_value.microsecond, + ) if self_value != db_value: - frappe.throw(_("Not allowed to change {0} after submission").format(df.label), - frappe.UpdateAfterSubmitError) + frappe.throw( + _("Not allowed to change {0} after submission").format(df.label), + frappe.UpdateAfterSubmitError, + ) def _sanitize_content(self): """Sanitize HTML and Email in field values. Used to prevent XSS. - - Ignore if 'Ignore XSS Filter' is checked or fieldtype is 'Code' + - Ignore if 'Ignore XSS Filter' is checked or fieldtype is 'Code' """ from bs4 import BeautifulSoup @@ -873,7 +930,7 @@ class BaseDocument(object): value = frappe.as_unicode(value) - if (u"<" not in value and u">" not in value): + if "<" not in value and ">" not in value: # doesn't look like html so no need continue @@ -884,29 +941,31 @@ class BaseDocument(object): df = self.meta.get_field(fieldname) sanitized_value = value - if df and (df.get("ignore_xss_filter") - or (df.get("fieldtype") in ("Data", "Small Text", "Text") and df.get("options")=="Email") + if df and ( + df.get("ignore_xss_filter") + or (df.get("fieldtype") in ("Data", "Small Text", "Text") and df.get("options") == "Email") or df.get("fieldtype") in ("Attach", "Attach Image", "Barcode", "Code") - # cancelled and submit but not update after submit should be ignored or self.docstatus.is_cancelled() - or (self.docstatus.is_submitted() and not df.get("allow_on_submit"))): + or (self.docstatus.is_submitted() and not df.get("allow_on_submit")) + ): continue else: - sanitized_value = sanitize_html(value, linkify=df and df.fieldtype=='Text Editor') + sanitized_value = sanitize_html(value, linkify=df and df.fieldtype == "Text Editor") self.set(fieldname, sanitized_value) def _save_passwords(self): """Save password field values in __Auth table""" - from frappe.utils.password import set_encrypted_password, remove_encrypted_password + from frappe.utils.password import remove_encrypted_password, set_encrypted_password if self.flags.ignore_save_passwords is True: return - for df in self.meta.get('fields', {'fieldtype': ('=', 'Password')}): - if self.flags.ignore_save_passwords and df.fieldname in self.flags.ignore_save_passwords: continue + for df in self.meta.get("fields", {"fieldtype": ("=", "Password")}): + if self.flags.ignore_save_passwords and df.fieldname in self.flags.ignore_save_passwords: + continue new_password = self.get(df.fieldname) if not new_password: @@ -917,18 +976,20 @@ class BaseDocument(object): set_encrypted_password(self.doctype, self.name, new_password, df.fieldname) # set dummy password like '*****' - self.set(df.fieldname, '*'*len(new_password)) + self.set(df.fieldname, "*" * len(new_password)) - def get_password(self, fieldname='password', raise_exception=True): + def get_password(self, fieldname="password", raise_exception=True): from frappe.utils.password import get_decrypted_password if self.get(fieldname) and not self.is_dummy_password(self.get(fieldname)): return self.get(fieldname) - return get_decrypted_password(self.doctype, self.name, fieldname, raise_exception=raise_exception) + return get_decrypted_password( + self.doctype, self.name, fieldname, raise_exception=raise_exception + ) def is_dummy_password(self, pwd): - return ''.join(set(pwd))=='*' + return "".join(set(pwd)) == "*" def precision(self, fieldname, parentfield=None): """Returns float precision for a particular field (or get global default). @@ -959,13 +1020,15 @@ class BaseDocument(object): return self._precision[cache_key][fieldname] - - def get_formatted(self, fieldname, doc=None, currency=None, absolute_value=False, translated=False, format=None): + def get_formatted( + self, fieldname, doc=None, currency=None, absolute_value=False, translated=False, format=None + ): from frappe.utils.formatters import format_value df = self.meta.get_field(fieldname) if not df: from frappe.model.meta import get_default_df + df = get_default_df(fieldname) if ( @@ -974,7 +1037,7 @@ class BaseDocument(object): and (currency_field := df.get("options")) and (currency_value := self.get(currency_field)) ): - currency = frappe.db.get_value('Currency', currency_value, cache=True) + currency = frappe.db.get_value("Currency", currency_value, cache=True) val = self.get(fieldname) @@ -984,7 +1047,7 @@ class BaseDocument(object): if not doc: doc = getattr(self, "parent_doc", None) or self - if (absolute_value or doc.get('absolute_value')) and isinstance(val, (int, float)): + if (absolute_value or doc.get("absolute_value")) and isinstance(val, (int, float)): val = abs(self.get(fieldname)) return format_value(val, df=df, doc=doc, currency=currency, format=format) @@ -995,9 +1058,9 @@ class BaseDocument(object): Print Hide can be set via the Print Format Builder or in the controller as a list of hidden fields. Example - class MyDoc(Document): - def __setup__(self): - self.print_hide = ["field1", "field2"] + class MyDoc(Document): + def __setup__(self): + self.print_hide = ["field1", "field2"] :param fieldname: Fieldname to be checked if hidden. """ @@ -1007,8 +1070,8 @@ class BaseDocument(object): print_hide = 0 - if self.get(fieldname)==0 and not self.meta.istable: - print_hide = ( df and df.print_hide_if_no_value ) or ( meta_df and meta_df.print_hide_if_no_value ) + if self.get(fieldname) == 0 and not self.meta.istable: + print_hide = (df and df.print_hide_if_no_value) or (meta_df and meta_df.print_hide_if_no_value) if not print_hide: if df and df.print_hide is not None: @@ -1020,7 +1083,7 @@ class BaseDocument(object): def in_format_data(self, fieldname): """Returns True if shown via Print Format::`format_data` property. - Called from within standard print format.""" + Called from within standard print format.""" doc = getattr(self, "parent_doc", self) if hasattr(doc, "format_data_map"): @@ -1042,7 +1105,7 @@ class BaseDocument(object): ref_doc = frappe.new_doc(self.doctype) else: # get values from old doc - if self.get('parent_doc'): + if self.get("parent_doc"): parent_doc = self.parent_doc.get_latest() ref_doc = [d for d in parent_doc.get(self.parentfield) if d.name == self.name][0] else: @@ -1062,15 +1125,17 @@ class BaseDocument(object): def _extract_images_from_text_editor(self): from frappe.core.doctype.file.file import extract_images_from_doc + if self.doctype != "DocType": - for df in self.meta.get("fields", {"fieldtype": ('=', "Text Editor")}): + for df in self.meta.get("fields", {"fieldtype": ("=", "Text Editor")}): extract_images_from_doc(self, df.fieldname) + def _filter(data, filters, limit=None): """pass filters as: - {"key": "val", "key": ["!=", "val"], - "key": ["in", "val"], "key": ["not in", "val"], "key": "^val", - "key" : True (exists), "key": False (does not exist) }""" + {"key": "val", "key": ["!=", "val"], + "key": ["in", "val"], "key": ["not in", "val"], "key": "^val", + "key" : True (exists), "key": False (does not exist) }""" out, _filters = [], {} diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py index fff2156a10..8671008f82 100644 --- a/frappe/model/create_new.py +++ b/frappe/model/create_new.py @@ -6,14 +6,16 @@ Create a new document with defaults set """ import copy + import frappe import frappe.defaults -from frappe.model import data_fieldtypes -from frappe.utils import nowdate, nowtime, now_datetime, cstr from frappe.core.doctype.user_permission.user_permission import get_user_permissions +from frappe.model import data_fieldtypes from frappe.permissions import filter_allowed_docs_for_doctype +from frappe.utils import cstr, now_datetime, nowdate, nowtime -def get_new_doc(doctype, parent_doc = None, parentfield = None, as_dict=False): + +def get_new_doc(doctype, parent_doc=None, parentfield=None, as_dict=False): if doctype not in frappe.local.new_doc_templates: # cache a copy of new doc as it is called # frequently for inserts @@ -30,13 +32,11 @@ def get_new_doc(doctype, parent_doc = None, parentfield = None, as_dict=False): else: return frappe.get_doc(doc) + def make_new_doc(doctype): - doc = frappe.get_doc({ - "doctype": doctype, - "__islocal": 1, - "owner": frappe.session.user, - "docstatus": 0 - }) + doc = frappe.get_doc( + {"doctype": doctype, "__islocal": 1, "owner": frappe.session.user, "docstatus": 0} + ) set_user_and_static_default_values(doc) @@ -50,6 +50,7 @@ def make_new_doc(doctype): return doc + def set_user_and_static_default_values(doc): user_permissions = get_user_permissions() defaults = frappe.defaults.get_defaults() @@ -59,10 +60,13 @@ def set_user_and_static_default_values(doc): # user permissions for link options doctype_user_permissions = user_permissions.get(df.options, []) # Allowed records for the reference doctype (link field) along with default doc - allowed_records, default_doc = filter_allowed_docs_for_doctype(doctype_user_permissions, - df.parent, with_default_doc=True) + allowed_records, default_doc = filter_allowed_docs_for_doctype( + doctype_user_permissions, df.parent, with_default_doc=True + ) - user_default_value = get_user_default_value(df, defaults, doctype_user_permissions, allowed_records, default_doc) + user_default_value = get_user_default_value( + df, defaults, doctype_user_permissions, allowed_records, default_doc + ) if user_default_value is not None: # if fieldtype is link check if doc exists if not df.fieldtype == "Link" or frappe.db.exists(df.options, user_default_value): @@ -74,23 +78,26 @@ def set_user_and_static_default_values(doc): if static_default_value is not None: doc.set(df.fieldname, static_default_value) + def get_user_default_value(df, defaults, doctype_user_permissions, allowed_records, default_doc): # don't set defaults for "User" link field using User Permissions! if df.fieldtype == "Link" and df.options != "User": # If user permission has Is Default enabled or single-user permission has found against respective doctype. - if (not df.ignore_user_permissions and default_doc): + if not df.ignore_user_permissions and default_doc: return default_doc # 2 - Look in user defaults user_default = defaults.get(df.fieldname) - allowed_by_user_permission = validate_value_via_user_permissions(df, doctype_user_permissions, - allowed_records, user_default=user_default) + allowed_by_user_permission = validate_value_via_user_permissions( + df, doctype_user_permissions, allowed_records, user_default=user_default + ) # is this user default also allowed as per user permissions? if user_default and allowed_by_user_permission: return user_default + def get_static_default_value(df, doctype_user_permissions, allowed_records): # 3 - look in default of docfield if df.get("default"): @@ -102,16 +109,20 @@ def get_static_default_value(df, doctype_user_permissions, allowed_records): elif not cstr(df.default).startswith(":"): # a simple default value - is_allowed_default_value = validate_value_via_user_permissions(df, doctype_user_permissions, - allowed_records) + is_allowed_default_value = validate_value_via_user_permissions( + df, doctype_user_permissions, allowed_records + ) - if df.fieldtype!="Link" or df.options=="User" or is_allowed_default_value: + if df.fieldtype != "Link" or df.options == "User" or is_allowed_default_value: return df.default - elif (df.fieldtype == "Select" and df.options and df.options not in ("[Select]", "Loading...")): + elif df.fieldtype == "Select" and df.options and df.options not in ("[Select]", "Loading..."): return df.options.split("\n")[0] -def validate_value_via_user_permissions(df, doctype_user_permissions, allowed_records, user_default=None): + +def validate_value_via_user_permissions( + df, doctype_user_permissions, allowed_records, user_default=None +): is_valid = True # If User Permission exists and allowed records is empty, # that means there are User Perms, but none applicable to this new doctype. @@ -124,6 +135,7 @@ def validate_value_via_user_permissions(df, doctype_user_permissions, allowed_re return is_valid + def set_dynamic_default_values(doc, parent_doc, parentfield): # these values should not be cached user_permissions = get_user_permissions() @@ -148,21 +160,28 @@ def set_dynamic_default_values(doc, parent_doc, parentfield): if parentfield: doc["parentfield"] = parentfield + def user_permissions_exist(df, doctype_user_permissions): - return (df.fieldtype=="Link" + return ( + df.fieldtype == "Link" and not getattr(df, "ignore_user_permissions", False) - and doctype_user_permissions) + and doctype_user_permissions + ) + def get_default_based_on_another_field(df, user_permissions, parent_doc): # default value based on another document from frappe.permissions import get_allowed_docs_for_doctype - ref_doctype = df.default[1:] + ref_doctype = df.default[1:] ref_fieldname = ref_doctype.lower().replace(" ", "_") - reference_name = parent_doc.get(ref_fieldname) if parent_doc else frappe.db.get_default(ref_fieldname) + reference_name = ( + parent_doc.get(ref_fieldname) if parent_doc else frappe.db.get_default(ref_fieldname) + ) default_value = frappe.db.get_value(ref_doctype, reference_name, df.fieldname) - is_allowed_default_value = (not user_permissions_exist(df, user_permissions.get(df.options)) or - (default_value in get_allowed_docs_for_doctype(user_permissions[df.options], df.parent))) + is_allowed_default_value = not user_permissions_exist(df, user_permissions.get(df.options)) or ( + default_value in get_allowed_docs_for_doctype(user_permissions[df.options], df.parent) + ) # is this allowed as per user permissions if is_allowed_default_value: diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index b573e1d301..a7d9536ebc 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -2,19 +2,33 @@ # License: MIT. See LICENSE """build query for doclistview and return results""" +import copy +import json +import re +from datetime import datetime from typing import List + +import frappe import frappe.defaults -from frappe.query_builder.utils import Column +import frappe.permissions import frappe.share from frappe import _ -import frappe.permissions -from datetime import datetime -import frappe, json, copy, re +from frappe.core.doctype.server_script.server_script_utils import get_server_script_map from frappe.model import optional_fields -from frappe.model.utils.user_settings import get_user_settings, update_user_settings -from frappe.utils import flt, cint, get_time, make_filter_tuple, get_filter, add_to_date, cstr, get_timespan_date_range from frappe.model.meta import get_table_columns -from frappe.core.doctype.server_script.server_script_utils import get_server_script_map +from frappe.model.utils.user_settings import get_user_settings, update_user_settings +from frappe.query_builder.utils import Column +from frappe.utils import ( + add_to_date, + cint, + cstr, + flt, + get_filter, + get_time, + get_timespan_date_range, + make_filter_tuple, +) + class DatabaseQuery(object): def __init__(self, doctype, user=None): @@ -28,49 +42,74 @@ class DatabaseQuery(object): self.flags = frappe._dict() self.reference_doctype = None - def execute(self, fields=None, filters=None, or_filters=None, - docstatus=None, group_by=None, order_by="KEEP_DEFAULT_ORDERING", limit_start=False, - limit_page_length=None, as_list=False, with_childnames=False, debug=False, - ignore_permissions=False, user=None, with_comment_count=False, - join='left join', distinct=False, start=None, page_length=None, limit=None, - ignore_ifnull=False, save_user_settings=False, save_user_settings_fields=False, - update=None, add_total_row=None, user_settings=None, reference_doctype=None, - run=True, strict=True, pluck=None, ignore_ddl=False, parent_doctype=None) -> List: + def execute( + self, + fields=None, + filters=None, + or_filters=None, + docstatus=None, + group_by=None, + order_by="KEEP_DEFAULT_ORDERING", + limit_start=False, + limit_page_length=None, + as_list=False, + with_childnames=False, + debug=False, + ignore_permissions=False, + user=None, + with_comment_count=False, + join="left join", + distinct=False, + start=None, + page_length=None, + limit=None, + ignore_ifnull=False, + save_user_settings=False, + save_user_settings_fields=False, + update=None, + add_total_row=None, + user_settings=None, + reference_doctype=None, + run=True, + strict=True, + pluck=None, + ignore_ddl=False, + parent_doctype=None, + ) -> List: if ( not ignore_permissions and not frappe.has_permission(self.doctype, "select", user=user, parent_doctype=parent_doctype) and not frappe.has_permission(self.doctype, "read", user=user, parent_doctype=parent_doctype) ): - frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(self.doctype)) + frappe.flags.error_message = _("Insufficient Permission for {0}").format( + frappe.bold(self.doctype) + ) raise frappe.PermissionError(self.doctype) # filters and fields swappable # its hard to remember what comes first - if ( - isinstance(fields, dict) - or ( - fields - and isinstance(fields, list) - and isinstance(fields[0], list) - ) + if isinstance(fields, dict) or ( + fields and isinstance(fields, list) and isinstance(fields[0], list) ): # if fields is given as dict/list of list, its probably filters filters, fields = fields, filters - elif fields and isinstance(filters, list) \ - and len(filters) > 1 and isinstance(filters[0], str): + elif fields and isinstance(filters, list) and len(filters) > 1 and isinstance(filters[0], str): # if `filters` is a list of strings, its probably fields filters, fields = fields, filters if fields: self.fields = fields else: - self.fields = [f"`tab{self.doctype}`.`{pluck or 'name'}`"] + self.fields = [f"`tab{self.doctype}`.`{pluck or 'name'}`"] - if start: limit_start = start - if page_length: limit_page_length = page_length - if limit: limit_page_length = limit + if start: + limit_start = start + if page_length: + limit_page_length = page_length + if limit: + limit_page_length = limit self.filters = filters or [] self.or_filters = or_filters or [] @@ -103,7 +142,8 @@ class DatabaseQuery(object): self.columns = self.get_table_columns() # no table & ignore_ddl, return - if not self.columns: return [] + if not self.columns: + return [] result = self.build_and_run() @@ -127,23 +167,32 @@ class DatabaseQuery(object): args.conditions = "where " + args.conditions if self.distinct: - args.fields = 'distinct ' + args.fields - args.order_by = '' # TODO: recheck for alternative + args.fields = "distinct " + args.fields + args.order_by = "" # TODO: recheck for alternative # Postgres requires any field that appears in the select clause to also # appear in the order by and group by clause - if frappe.db.db_type == 'postgres' and args.order_by and args.group_by: + if frappe.db.db_type == "postgres" and args.order_by and args.group_by: args = self.prepare_select_args(args) - query = """select %(fields)s + query = ( + """select %(fields)s from %(tables)s %(conditions)s %(group_by)s %(order_by)s - %(limit)s""" % args - - return frappe.db.sql(query, as_dict=not self.as_list, debug=self.debug, - update=self.update, ignore_ddl=self.ignore_ddl, run=self.run) + %(limit)s""" + % args + ) + + return frappe.db.sql( + query, + as_dict=not self.as_list, + debug=self.debug, + update=self.update, + ignore_ddl=self.ignore_ddl, + run=self.run, + ) def prepare_args(self): self.parse_args() @@ -170,11 +219,10 @@ class DatabaseQuery(object): if self.grouped_or_conditions: self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})") - args.conditions = ' and '.join(self.conditions) + args.conditions = " and ".join(self.conditions) if self.or_conditions: - args.conditions += (' or ' if args.conditions else "") + \ - ' or '.join(self.or_conditions) + args.conditions += (" or " if args.conditions else "") + " or ".join(self.or_conditions) self.set_field_tables() @@ -184,11 +232,13 @@ class DatabaseQuery(object): # TODO: Add support for wrapping fields with sql functions and distinct keyword for field in self.fields: stripped_field = field.strip().lower() - skip_wrapping = any([ - stripped_field.startswith(("`", "*", '"', "'")), - "(" in stripped_field, - "distinct" in stripped_field, - ]) + skip_wrapping = any( + [ + stripped_field.startswith(("`", "*", '"', "'")), + "(" in stripped_field, + "distinct" in stripped_field, + ] + ) if skip_wrapping: fields.append(field) elif "as" in field.lower().split(" "): @@ -249,30 +299,46 @@ class DatabaseQuery(object): setattr(self, filter_name, filters) def sanitize_fields(self): - ''' - regex : ^.*[,();].* - purpose : The regex will look for malicious patterns like `,`, '(', ')', '@', ;' in each - field which may leads to sql injection. - example : - field = "`DocType`.`issingle`, version()" - As field contains `,` and mysql function `version()`, with the help of regex - the system will filter out this field. - ''' + """ + regex : ^.*[,();].* + purpose : The regex will look for malicious patterns like `,`, '(', ')', '@', ;' in each + field which may leads to sql injection. + example : + field = "`DocType`.`issingle`, version()" + As field contains `,` and mysql function `version()`, with the help of regex + the system will filter out this field. + """ sub_query_regex = re.compile("^.*[,();@].*") - blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'show'] - blacklisted_functions = ['concat', 'concat_ws', 'if', 'ifnull', 'nullif', 'coalesce', - 'connection_id', 'current_user', 'database', 'last_insert_id', 'session_user', - 'system_user', 'user', 'version', 'global'] + blacklisted_keywords = ["select", "create", "insert", "delete", "drop", "update", "case", "show"] + blacklisted_functions = [ + "concat", + "concat_ws", + "if", + "ifnull", + "nullif", + "coalesce", + "connection_id", + "current_user", + "database", + "last_insert_id", + "session_user", + "system_user", + "user", + "version", + "global", + ] def _raise_exception(): - frappe.throw(_('Use of sub-query or function is restricted'), frappe.DataError) + frappe.throw(_("Use of sub-query or function is restricted"), frappe.DataError) def _is_query(field): if re.compile(r"^(select|delete|update|drop|create)\s").match(field): _raise_exception() - elif re.compile(r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )").match(field): + elif re.compile(r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )").match( + field + ): _raise_exception() for field in self.fields: @@ -286,7 +352,7 @@ class DatabaseQuery(object): if any(f"{keyword}(" in field.lower() for keyword in blacklisted_functions): _raise_exception() - if '@' in field.lower(): + if "@" in field.lower(): # prevent access to global variables _raise_exception() @@ -300,10 +366,10 @@ class DatabaseQuery(object): if self.strict: if re.compile(r".*/\*.*").match(field): - frappe.throw(_('Illegal SQL Query')) + frappe.throw(_("Illegal SQL Query")) if re.compile(r".*\s(union).*\s").match(field.lower()): - frappe.throw(_('Illegal SQL Query')) + frappe.throw(_("Illegal SQL Query")) def extract_tables(self): """extract tables from fields""" @@ -331,16 +397,20 @@ class DatabaseQuery(object): if func_found or not ("tab" in field and "." in field): continue - table_name = field.split('.')[0] + table_name = field.split(".")[0] - if table_name.lower().startswith('group_concat('): + if table_name.lower().startswith("group_concat("): table_name = table_name[13:] - if not table_name[0]=='`': + if not table_name[0] == "`": table_name = f"`{table_name}`" if table_name not in self.tables: self.append_table(table_name) - def cast_name(self, column: str, sql_function: str = "",) -> str: + def cast_name( + self, + column: str, + sql_function: str = "", + ) -> str: if frappe.db.db_type == "postgres": if "name" in column.lower(): if "cast(" not in column.lower() or "::" not in column: @@ -349,50 +419,47 @@ class DatabaseQuery(object): elif sql_function == "locate(": return re.sub( - r'locate\(([^,]+),([^)]+)\)', - r'locate(\1, cast(\2 as varchar))', + r"locate\(([^,]+),([^)]+)\)", + r"locate(\1, cast(\2 as varchar))", column, - flags=re.IGNORECASE + flags=re.IGNORECASE, ) elif sql_function == "strpos(": return re.sub( - r'strpos\(([^,]+),([^)]+)\)', - r'strpos(cast(\1 as varchar), \2)', + r"strpos\(([^,]+),([^)]+)\)", + r"strpos(cast(\1 as varchar), \2)", column, - flags=re.IGNORECASE + flags=re.IGNORECASE, ) elif sql_function == "ifnull(": - return re.sub( - r"ifnull\(([^,]+)", - r"ifnull(cast(\1 as varchar)", - column, - flags=re.IGNORECASE - ) + return re.sub(r"ifnull\(([^,]+)", r"ifnull(cast(\1 as varchar)", column, flags=re.IGNORECASE) return column def append_table(self, table_name): self.tables.append(table_name) doctype = table_name[4:-1] - ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read' + ptype = "select" if frappe.only_has_select_perm(doctype) else "read" - if not self.flags.ignore_permissions and \ - not frappe.has_permission(doctype, ptype=ptype, parent_doctype=self.doctype): - frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(doctype)) + if not self.flags.ignore_permissions and not frappe.has_permission( + doctype, ptype=ptype, parent_doctype=self.doctype + ): + frappe.flags.error_message = _("Insufficient Permission for {0}").format(frappe.bold(doctype)) raise frappe.PermissionError(doctype) def set_field_tables(self): - '''If there are more than one table, the fieldname must not be ambiguous. - If the fieldname is not explicitly mentioned, set the default table''' + """If there are more than one table, the fieldname must not be ambiguous. + If the fieldname is not explicitly mentioned, set the default table""" + def _in_standard_sql_methods(field): - methods = ('count(', 'avg(', 'sum(', 'extract(', 'dayofyear(') + methods = ("count(", "avg(", "sum(", "extract(", "dayofyear(") return field.lower().startswith(methods) if len(self.tables) > 1: for idx, field in enumerate(self.fields): - if '.' not in field and not _in_standard_sql_methods(field): + if "." not in field and not _in_standard_sql_methods(field): self.fields[idx] = f"{self.tables[0]}.{field}" def get_table_columns(self): @@ -460,20 +527,21 @@ class DatabaseQuery(object): def prepare_filter_condition(self, f): """Returns a filter condition in the format: - ifnull(`tabDocType`.`fieldname`, fallback) operator "value" + ifnull(`tabDocType`.`fieldname`, fallback) operator "value" """ # TODO: refactor from frappe.boot import get_additional_filters_from_hooks + additional_filters_config = get_additional_filters_from_hooks() f = get_filter(self.doctype, f, additional_filters_config) - tname = ('`tab' + f.doctype + '`') + tname = "`tab" + f.doctype + "`" if tname not in self.tables: self.append_table(tname) - if 'ifnull(' in f.fieldname: + if "ifnull(" in f.fieldname: column_name = self.cast_name(f.fieldname, "ifnull(") else: column_name = self.cast_name(f"{tname}.`{f.fieldname}`") @@ -485,8 +553,13 @@ class DatabaseQuery(object): can_be_null = True # prepare in condition - if f.operator.lower() in ('ancestors of', 'descendants of', 'not ancestors of', 'not descendants of'): - values = f.value or '' + if f.operator.lower() in ( + "ancestors of", + "descendants of", + "not ancestors of", + "not descendants of", + ): + values = f.value or "" # TODO: handle list and tuple # if not isinstance(values, (list, tuple)): @@ -495,25 +568,23 @@ class DatabaseQuery(object): field = meta.get_field(f.fieldname) ref_doctype = field.options if field else f.doctype - lft, rgt = '', '' + lft, rgt = "", "" if f.value: lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"]) # Get descendants elements of a DocType with a tree structure - if f.operator.lower() in ('descendants of', 'not descendants of') : - result = frappe.get_all(ref_doctype, filters={ - 'lft': ['>', lft], - 'rgt': ['<', rgt] - }, order_by='`lft` ASC') - else : + if f.operator.lower() in ("descendants of", "not descendants of"): + result = frappe.get_all( + ref_doctype, filters={"lft": [">", lft], "rgt": ["<", rgt]}, order_by="`lft` ASC" + ) + else: # Get ancestor elements of a DocType with a tree structure - result = frappe.get_all(ref_doctype, filters={ - 'lft': ['<', lft], - 'rgt': ['>', rgt] - }, order_by='`lft` DESC') + result = frappe.get_all( + ref_doctype, filters={"lft": ["<", lft], "rgt": [">", rgt]}, order_by="`lft` DESC" + ) fallback = "''" - value = [frappe.db.escape((cstr(v.name) or '').strip(), percent=False) for v in result] + value = [frappe.db.escape((cstr(v.name) or "").strip(), percent=False) for v in result] if len(value): value = f"({', '.join(value)})" else: @@ -521,15 +592,17 @@ class DatabaseQuery(object): # changing operator to IN as the above code fetches all the parent / child values and convert into tuple # which can be directly used with IN operator to query. - f.operator = 'not in' if f.operator.lower() in ('not ancestors of', 'not descendants of') else 'in' + f.operator = ( + "not in" if f.operator.lower() in ("not ancestors of", "not descendants of") else "in" + ) - elif f.operator.lower() in ('in', 'not in'): - values = f.value or '' + elif f.operator.lower() in ("in", "not in"): + values = f.value or "" if isinstance(values, str): values = values.split(",") fallback = "''" - value = [frappe.db.escape((cstr(v) or '').strip(), percent=False) for v in values] + value = [frappe.db.escape((cstr(v) or "").strip(), percent=False) for v in values] if len(value): value = f"({', '.join(value)})" else: @@ -542,62 +615,67 @@ class DatabaseQuery(object): if df and df.fieldtype in ("Check", "Float", "Int", "Currency", "Percent"): can_be_null = False - if f.operator.lower() in ('previous', 'next', 'timespan'): + if f.operator.lower() in ("previous", "next", "timespan"): date_range = get_date_range(f.operator.lower(), f.value) f.operator = "Between" f.value = date_range fallback = "'0001-01-01 00:00:00'" - if f.operator in ('>', '<') and (f.fieldname in ('creation', 'modified')): + if f.operator in (">", "<") and (f.fieldname in ("creation", "modified")): value = cstr(f.value) fallback = "'0001-01-01 00:00:00'" - elif f.operator.lower() in ('between') and \ - (f.fieldname in ('creation', 'modified') or - (df and (df.fieldtype=="Date" or df.fieldtype=="Datetime"))): + elif f.operator.lower() in ("between") and ( + f.fieldname in ("creation", "modified") + or (df and (df.fieldtype == "Date" or df.fieldtype == "Datetime")) + ): value = get_between_date_filter(f.value, df) fallback = "'0001-01-01 00:00:00'" elif f.operator.lower() == "is": - if f.value == 'set': - f.operator = '!=' - elif f.value == 'not set': - f.operator = '=' + if f.value == "set": + f.operator = "!=" + elif f.value == "not set": + f.operator = "=" value = "" fallback = "''" can_be_null = True - if 'ifnull' not in column_name.lower(): - column_name = f'ifnull({column_name}, {fallback})' + if "ifnull" not in column_name.lower(): + column_name = f"ifnull({column_name}, {fallback})" - elif df and df.fieldtype=="Date": + elif df and df.fieldtype == "Date": value = frappe.db.format_date(f.value) fallback = "'0001-01-01'" - elif (df and df.fieldtype=="Datetime") or isinstance(f.value, datetime): + elif (df and df.fieldtype == "Datetime") or isinstance(f.value, datetime): value = frappe.db.format_datetime(f.value) fallback = "'0001-01-01 00:00:00'" - elif df and df.fieldtype=="Time": + elif df and df.fieldtype == "Time": value = get_time(f.value).strftime("%H:%M:%S.%f") fallback = "'00:00:00'" - elif f.operator.lower() in ("like", "not like") or (isinstance(f.value, str) and - (not df or df.fieldtype not in ["Float", "Int", "Currency", "Percent", "Check"])): - value = "" if f.value is None else f.value - fallback = "''" + elif f.operator.lower() in ("like", "not like") or ( + isinstance(f.value, str) + and (not df or df.fieldtype not in ["Float", "Int", "Currency", "Percent", "Check"]) + ): + value = "" if f.value is None else f.value + fallback = "''" - if f.operator.lower() in ("like", "not like") and isinstance(value, str): - # because "like" uses backslash (\) for escaping - value = value.replace("\\", "\\\\").replace("%", "%%") + if f.operator.lower() in ("like", "not like") and isinstance(value, str): + # because "like" uses backslash (\) for escaping + value = value.replace("\\", "\\\\").replace("%", "%%") - elif f.operator == '=' and df and df.fieldtype in ['Link', 'Data']: # TODO: Refactor if possible + elif ( + f.operator == "=" and df and df.fieldtype in ["Link", "Data"] + ): # TODO: Refactor if possible value = f.value or "''" fallback = "''" - elif f.fieldname == 'name': + elif f.fieldname == "name": value = f.value or "''" fallback = "''" @@ -606,25 +684,25 @@ class DatabaseQuery(object): fallback = 0 if isinstance(f.value, Column): - can_be_null = False # added to avoid the ifnull/coalesce addition - quote = '"' if frappe.conf.db_type == 'postgres' else "`" + can_be_null = False # added to avoid the ifnull/coalesce addition + quote = '"' if frappe.conf.db_type == "postgres" else "`" value = f"{tname}.{quote}{f.value.name}{quote}" # escape value - elif isinstance(value, str) and f.operator.lower() != 'between': + elif isinstance(value, str) and f.operator.lower() != "between": value = f"{frappe.db.escape(value, percent=False)}" if ( self.ignore_ifnull or not can_be_null - or (f.value and f.operator.lower() in ('=', 'like')) - or 'ifnull(' in column_name.lower() + or (f.value and f.operator.lower() in ("=", "like")) + or "ifnull(" in column_name.lower() ): - if f.operator.lower() == 'like' and frappe.conf.get('db_type') == 'postgres': - f.operator = 'ilike' - condition = f'{column_name} {f.operator} {value}' + if f.operator.lower() == "like" and frappe.conf.get("db_type") == "postgres": + f.operator = "ilike" + condition = f"{column_name} {f.operator} {value}" else: - condition = f'ifnull({column_name}, {fallback}) {f.operator} {value}' + condition = f"ifnull({column_name}, {fallback}) {f.operator} {value}" return condition @@ -636,17 +714,18 @@ class DatabaseQuery(object): if not self.user: self.user = frappe.session.user - if not self.tables: self.extract_tables() + if not self.tables: + self.extract_tables() meta = frappe.get_meta(self.doctype) role_permissions = frappe.permissions.get_role_permissions(meta, user=self.user) self.shared = frappe.share.get_shared(self.doctype, self.user) if ( - not meta.istable and - not (role_permissions.get("select") or role_permissions.get("read")) and - not self.flags.ignore_permissions and - not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype) + not meta.istable + and not (role_permissions.get("select") or role_permissions.get("read")) + and not self.flags.ignore_permissions + and not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype) ): only_if_shared = True if not self.shared: @@ -675,11 +754,11 @@ class DatabaseQuery(object): doctype_conditions = self.get_permission_query_conditions() if doctype_conditions: - conditions += (' and ' + doctype_conditions) if conditions else doctype_conditions + conditions += (" and " + doctype_conditions) if conditions else doctype_conditions # share is an OR condition, if there is a role permission if not only_if_shared and self.shared and conditions: - conditions = f"({conditions}) or ({self.get_share_condition()})" + conditions = f"({conditions}) or ({self.get_share_condition()})" return conditions @@ -695,17 +774,20 @@ class DatabaseQuery(object): doctype_link_fields = meta.get_link_fields() # append current doctype with fieldname as 'name' as first link field - doctype_link_fields.append(dict( - options=self.doctype, - fieldname='name', - )) + doctype_link_fields.append( + dict( + options=self.doctype, + fieldname="name", + ) + ) match_filters = {} match_conditions = [] for df in doctype_link_fields: - if df.get('ignore_user_permissions'): continue + if df.get("ignore_user_permissions"): + continue - user_permission_values = user_permissions.get(df.get('options'), {}) + user_permission_values = user_permissions.get(df.get("options"), {}) if user_permission_values: docs = [] @@ -716,26 +798,26 @@ class DatabaseQuery(object): condition = empty_value_condition + " or " for permission in user_permission_values: - if not permission.get('applicable_for'): - docs.append(permission.get('doc')) + if not permission.get("applicable_for"): + docs.append(permission.get("doc")) # append docs based on user permission applicable on reference doctype # this is useful when getting list of docs from a link field # in this case parent doctype of the link # will be the reference doctype - elif df.get('fieldname') == 'name' and self.reference_doctype: - if permission.get('applicable_for') == self.reference_doctype: - docs.append(permission.get('doc')) + elif df.get("fieldname") == "name" and self.reference_doctype: + if permission.get("applicable_for") == self.reference_doctype: + docs.append(permission.get("doc")) - elif permission.get('applicable_for') == self.doctype: - docs.append(permission.get('doc')) + elif permission.get("applicable_for") == self.doctype: + docs.append(permission.get("doc")) if docs: values = ", ".join(frappe.db.escape(doc, percent=False) for doc in docs) condition += f"`tab{self.doctype}`.`{df.get('fieldname')}` in ({values})" match_conditions.append(f"({condition})") - match_filters[df.get('options')] = docs + match_filters[df.get("options")] = docs if match_conditions: self.match_conditions.append(" and ".join(match_conditions)) @@ -770,31 +852,36 @@ class DatabaseQuery(object): args.order_by = "" # don't add order by from meta if a mysql group function is used without group by clause - group_function_without_group_by = (len(self.fields)==1 and - ( self.fields[0].lower().startswith("count(") + group_function_without_group_by = ( + len(self.fields) == 1 + and ( + self.fields[0].lower().startswith("count(") or self.fields[0].lower().startswith("min(") or self.fields[0].lower().startswith("max(") - ) and not self.group_by) + ) + and not self.group_by + ) if not group_function_without_group_by: sort_field = sort_order = None - if meta.sort_field and ',' in meta.sort_field: + if meta.sort_field and "," in meta.sort_field: # multiple sort given in doctype definition # Example: # `idx desc, modified desc` # will covert to # `tabItem`.`idx` desc, `tabItem`.`modified` desc - args.order_by = ', '.join( - f"`tab{self.doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" for f in meta.sort_field.split(',') + args.order_by = ", ".join( + f"`tab{self.doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" + for f in meta.sort_field.split(",") ) else: - sort_field = meta.sort_field or 'modified' - sort_order = (meta.sort_field and meta.sort_order) or 'desc' + sort_field = meta.sort_field or "modified" + sort_order = (meta.sort_field and meta.sort_order) or "desc" if self.order_by: args.order_by = f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" # draft docs always on top - if hasattr(meta, 'is_submittable') and meta.is_submittable: + if hasattr(meta, "is_submittable") and meta.is_submittable: if self.order_by: args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}" @@ -804,25 +891,25 @@ class DatabaseQuery(object): return _lower = parameters.lower() - if 'select' in _lower and 'from' in _lower: - frappe.throw(_('Cannot use sub-query in order by')) + if "select" in _lower and "from" in _lower: + frappe.throw(_("Cannot use sub-query in order by")) if re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*").match(_lower): - frappe.throw(_('Illegal SQL Query')) + frappe.throw(_("Illegal SQL Query")) for field in parameters.split(","): if "." in field and field.strip().startswith("`tab"): - tbl = field.strip().split('.')[0] + tbl = field.strip().split(".")[0] if tbl not in self.tables: - if tbl.startswith('`'): + if tbl.startswith("`"): tbl = tbl[4:-1] frappe.throw(_("Please select atleast 1 column from {0} to sort/group").format(tbl)) def add_limit(self): if self.limit_page_length: - return 'limit %s offset %s' % (self.limit_page_length, self.limit_start) + return "limit %s offset %s" % (self.limit_page_length, self.limit_start) else: - return '' + return "" def add_comment_count(self, result): for r in result: @@ -837,20 +924,21 @@ class DatabaseQuery(object): # update user settings if new search user_settings = json.loads(get_user_settings(self.doctype)) - if hasattr(self, 'user_settings'): + if hasattr(self, "user_settings"): user_settings.update(self.user_settings) if self.save_user_settings_fields: - user_settings['fields'] = self.user_settings_fields + user_settings["fields"] = self.user_settings_fields update_user_settings(self.doctype, user_settings) + def check_parent_permission(parent, child_doctype): if parent: # User may pass fake parent and get the information from the child table if child_doctype and not ( - frappe.db.exists('DocField', {'parent': parent, 'options': child_doctype}) - or frappe.db.exists('Custom Field', {'dt': parent, 'options': child_doctype}) + frappe.db.exists("DocField", {"parent": parent, "options": child_doctype}) + or frappe.db.exists("Custom Field", {"dt": parent, "options": child_doctype}) ): raise frappe.PermissionError @@ -860,21 +948,25 @@ def check_parent_permission(parent, child_doctype): # Either parent not passed or the user doesn't have permission on parent doctype of child table! raise frappe.PermissionError + def get_order_by(doctype, meta): order_by = "" sort_field = sort_order = None - if meta.sort_field and ',' in meta.sort_field: + if meta.sort_field and "," in meta.sort_field: # multiple sort given in doctype definition # Example: # `idx desc, modified desc` # will covert to # `tabItem`.`idx` desc, `tabItem`.`modified` desc - order_by = ', '.join(f"`tab{doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" for f in meta.sort_field.split(',')) + order_by = ", ".join( + f"`tab{doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" + for f in meta.sort_field.split(",") + ) else: - sort_field = meta.sort_field or 'modified' - sort_order = (meta.sort_field and meta.sort_order) or 'desc' + sort_field = meta.sort_field or "modified" + sort_order = (meta.sort_field and meta.sort_order) or "desc" order_by = f"`tab{doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" # draft docs always on top @@ -883,19 +975,21 @@ def get_order_by(doctype, meta): return order_by + def is_parent_only_filter(doctype, filters): - #check if filters contains only parent doctype + # check if filters contains only parent doctype only_parent_doctype = True if isinstance(filters, list): for flt in filters: if doctype not in flt: only_parent_doctype = False - if 'Between' in flt: + if "Between" in flt: flt[3] = get_between_date_filter(flt[3]) return only_parent_doctype + def has_any_user_permission_for_doctype(doctype, user, applicable_for): user_permissions = frappe.permissions.get_user_permissions(user=user) doctype_user_permissions = user_permissions.get(doctype, []) @@ -906,35 +1000,38 @@ def has_any_user_permission_for_doctype(doctype, user, applicable_for): return False + def get_between_date_filter(value, df=None): - ''' - return the formattted date as per the given example - [u'2017-11-01', u'2017-11-03'] => '2017-11-01 00:00:00.000000' AND '2017-11-04 00:00:00.000000' - ''' + """ + return the formattted date as per the given example + [u'2017-11-01', u'2017-11-03'] => '2017-11-01 00:00:00.000000' AND '2017-11-04 00:00:00.000000' + """ from_date = frappe.utils.nowdate() to_date = frappe.utils.nowdate() if value and isinstance(value, (list, tuple)): - if len(value) >= 1: from_date = value[0] - if len(value) >= 2: to_date = value[1] + if len(value) >= 1: + from_date = value[0] + if len(value) >= 2: + to_date = value[1] - if not df or (df and df.fieldtype == 'Datetime'): + if not df or (df and df.fieldtype == "Datetime"): to_date = add_to_date(to_date, days=1) - if df and df.fieldtype == 'Datetime': + if df and df.fieldtype == "Datetime": data = "'%s' AND '%s'" % ( frappe.db.format_datetime(from_date), - frappe.db.format_datetime(to_date)) + frappe.db.format_datetime(to_date), + ) else: - data = "'%s' AND '%s'" % ( - frappe.db.format_date(from_date), - frappe.db.format_date(to_date)) + data = "'%s' AND '%s'" % (frappe.db.format_date(from_date), frappe.db.format_date(to_date)) return data + def get_additional_filter_field(additional_filters_config, f, value): additional_filter = additional_filters_config[f.operator.lower()] - f = frappe._dict(frappe.get_attr(additional_filter['get_field'])()) + f = frappe._dict(frappe.get_attr(additional_filter["get_field"])()) if f.query_value: for option in f.options: option = frappe._dict(option) @@ -942,23 +1039,25 @@ def get_additional_filter_field(additional_filters_config, f, value): f.value = option.query_value return f + def get_date_range(operator, value): timespan_map = { - '1 week': 'week', - '1 month': 'month', - '3 months': 'quarter', - '6 months': '6 months', - '1 year': 'year', + "1 week": "week", + "1 month": "month", + "3 months": "quarter", + "6 months": "6 months", + "1 year": "year", } period_map = { - 'previous': 'last', - 'next': 'next', + "previous": "last", + "next": "next", } - timespan = period_map[operator] + ' ' + timespan_map[value] if operator != 'timespan' else value + timespan = period_map[operator] + " " + timespan_map[value] if operator != "timespan" else value return get_timespan_date_range(timespan) + def requires_owner_constraint(role_permissions): """Returns True if "select" or "read" isn't available without being creator.""" diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index f055cd79d0..733e8ca367 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -7,30 +7,53 @@ import shutil import frappe import frappe.defaults import frappe.model.meta -from frappe import _ -from frappe import get_module_path +from frappe import _, get_module_path +from frappe.desk.doctype.tag.tag import delete_tags_for_document from frappe.model.dynamic_links import get_dynamic_link_map -from frappe.utils.file_manager import remove_all -from frappe.utils.password import delete_all_passwords_for from frappe.model.naming import revert_series_if_last +from frappe.utils.file_manager import remove_all from frappe.utils.global_search import delete_for_document -from frappe.desk.doctype.tag.tag import delete_tags_for_document - - -doctypes_to_skip = ("Communication", "ToDo", "DocShare", "Email Unsubscribe", "Activity Log", "File", - "Version", "Document Follow", "Comment" , "View Log", "Tag Link", "Notification Log", "Email Queue") +from frappe.utils.password import delete_all_passwords_for -def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reload=False, ignore_permissions=False, - flags=None, ignore_on_trash=False, ignore_missing=True, delete_permanently=False): +doctypes_to_skip = ( + "Communication", + "ToDo", + "DocShare", + "Email Unsubscribe", + "Activity Log", + "File", + "Version", + "Document Follow", + "Comment", + "View Log", + "Tag Link", + "Notification Log", + "Email Queue", +) + + +def delete_doc( + doctype=None, + name=None, + force=0, + ignore_doctypes=None, + for_reload=False, + ignore_permissions=False, + flags=None, + ignore_on_trash=False, + ignore_missing=True, + delete_permanently=False, +): """ - Deletes a doc(dt, dn) and validates if it is not submitted and not linked in a live record + Deletes a doc(dt, dn) and validates if it is not submitted and not linked in a live record """ - if not ignore_doctypes: ignore_doctypes = [] + if not ignore_doctypes: + ignore_doctypes = [] # get from form if not doctype: - doctype = frappe.form_dict.get('dt') - name = frappe.form_dict.get('dn') + doctype = frappe.form_dict.get("dt") + name = frappe.form_dict.get("dn") names = name if isinstance(name, str) or isinstance(name, int): @@ -49,7 +72,7 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa delete_all_passwords_for(doctype, name) doc = None - if doctype=="DocType": + if doctype == "DocType": if for_reload: try: @@ -74,11 +97,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa delete_from_table(doctype, name, ignore_doctypes, None) - if frappe.conf.developer_mode and not doc.custom and not ( - for_reload - or frappe.flags.in_migrate - or frappe.flags.in_install - or frappe.flags.in_uninstall + if ( + frappe.conf.developer_mode + and not doc.custom + and not ( + for_reload or frappe.flags.in_migrate or frappe.flags.in_install or frappe.flags.in_uninstall + ) ): try: delete_controllers(name, doc.module) @@ -96,7 +120,7 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa if not ignore_on_trash: doc.run_method("on_trash") doc.flags.in_delete = True - doc.run_method('on_change') + doc.run_method("on_change") # check if links exist if not force: @@ -113,9 +137,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa if not for_reload: # Enqueued at the end, because it gets committed # All the linked docs should be checked beforehand - frappe.enqueue('frappe.model.delete_doc.delete_dynamic_links', - doctype=doc.doctype, name=doc.name, - now=frappe.flags.in_test) + frappe.enqueue( + "frappe.model.delete_doc.delete_dynamic_links", + doctype=doc.doctype, + name=doc.name, + now=frappe.flags.in_test, + ) # clear cache for Document doc.clear_cache() @@ -141,28 +168,32 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa # delete user_permissions frappe.defaults.clear_default(parenttype="User Permission", key=doctype, value=name) + def add_to_deleted_document(doc): - '''Add this document to Deleted Document table. Called after delete''' - if doc.doctype != 'Deleted Document' and frappe.flags.in_install != 'frappe': - frappe.get_doc(dict( - doctype='Deleted Document', - deleted_doctype=doc.doctype, - deleted_name=doc.name, - data=doc.as_json(), - owner=frappe.session.user - )).db_insert() + """Add this document to Deleted Document table. Called after delete""" + if doc.doctype != "Deleted Document" and frappe.flags.in_install != "frappe": + frappe.get_doc( + dict( + doctype="Deleted Document", + deleted_doctype=doc.doctype, + deleted_name=doc.name, + data=doc.as_json(), + owner=frappe.session.user, + ) + ).db_insert() + def update_naming_series(doc): if doc.meta.autoname: - if doc.meta.autoname.startswith("naming_series:") \ - and getattr(doc, "naming_series", None): + if doc.meta.autoname.startswith("naming_series:") and getattr(doc, "naming_series", None): revert_series_if_last(doc.naming_series, doc.name, doc) elif doc.meta.autoname.split(":")[0] not in ("Prompt", "field", "hash", "autoincrement"): revert_series_if_last(doc.meta.autoname, doc.name, doc) + def delete_from_table(doctype, name, ignore_doctypes, doc): - if doctype!="DocType" and doctype==name: + if doctype != "DocType" and doctype == name: frappe.db.delete("Singles", {"doctype": name}) else: frappe.db.delete(doctype, {"name": name}) @@ -171,21 +202,23 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): tables = [d.options for d in doc.meta.get_table_fields()] else: + def get_table_fields(field_doctype): - if field_doctype == 'Custom Field': + if field_doctype == "Custom Field": return [] - return [r[0] for r in frappe.get_all(field_doctype, - fields='options', - filters={ - 'fieldtype': ['in', frappe.model.table_fields], - 'parent': doctype - }, - as_list=1 - )] + return [ + r[0] + for r in frappe.get_all( + field_doctype, + fields="options", + filters={"fieldtype": ["in", frappe.model.table_fields], "parent": doctype}, + as_list=1, + ) + ] tables = get_table_fields("DocField") - if not frappe.flags.in_install=="frappe": + if not frappe.flags.in_install == "frappe": tables += get_table_fields("Custom Field") # delete from child tables @@ -193,51 +226,67 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): if t not in ignore_doctypes: frappe.db.delete(t, {"parenttype": doctype, "parent": name}) + def update_flags(doc, flags=None, ignore_permissions=False): if ignore_permissions: - if not flags: flags = {} + if not flags: + flags = {} flags["ignore_permissions"] = ignore_permissions if flags: doc.flags.update(flags) + def check_permission_and_not_submitted(doc): # permission - if (not doc.flags.ignore_permissions - and frappe.session.user!="Administrator" - and ( - not doc.has_permission("delete") - or (doc.doctype=="DocType" and not doc.custom))): - frappe.msgprint(_("User not allowed to delete {0}: {1}") - .format(doc.doctype, doc.name), raise_exception=frappe.PermissionError) + if ( + not doc.flags.ignore_permissions + and frappe.session.user != "Administrator" + and (not doc.has_permission("delete") or (doc.doctype == "DocType" and not doc.custom)) + ): + frappe.msgprint( + _("User not allowed to delete {0}: {1}").format(doc.doctype, doc.name), + raise_exception=frappe.PermissionError, + ) # check if submitted if doc.docstatus.is_submitted(): - frappe.msgprint(_("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format(_(doc.doctype), doc.name, "", ""), - raise_exception=True) + frappe.msgprint( + _("{0} {1}: Submitted Record cannot be deleted. You must {2} Cancel {3} it first.").format( + _(doc.doctype), + doc.name, + "", + "", + ), + raise_exception=True, + ) + def check_if_doc_is_linked(doc, method="Delete"): """ - Raises excption if the given doc(dt, dn) is linked in another record. + Raises excption if the given doc(dt, dn) is linked in another record. """ from frappe.model.rename_doc import get_link_fields + link_fields = get_link_fields(doc.doctype) - ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + ignore_linked_doctypes = doc.get("ignore_linked_doctypes") or [] for lf in link_fields: - link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle'] + link_dt, link_field, issingle = lf["parent"], lf["fieldname"], lf["issingle"] if not issingle: fields = ["name", "docstatus"] if frappe.get_meta(link_dt).istable: fields.extend(["parent", "parenttype"]) - for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True): + for item in frappe.db.get_values(link_dt, {link_field: doc.name}, fields, as_dict=True): # available only in child table cases item_parent = getattr(item, "parent", None) linked_doctype = item.parenttype if item_parent else link_dt - if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'): + if linked_doctype in doctypes_to_skip or ( + linked_doctype in ignore_linked_doctypes and method == "Cancel" + ): # don't check for communication and todo! continue @@ -257,13 +306,14 @@ def check_if_doc_is_linked(doc, method="Delete"): if frappe.db.get_value(link_dt, None, link_field) == doc.name: raise_link_exists_exception(doc, link_dt, link_dt) + def check_if_doc_is_dynamically_linked(doc, method="Delete"): - '''Raise `frappe.LinkExistsError` if the document is dynamically linked''' + """Raise `frappe.LinkExistsError` if the document is dynamically linked""" for df in get_dynamic_link_map().get(doc.doctype, []): - ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or [] + ignore_linked_doctypes = doc.get("ignore_linked_doctypes") or [] - if df.parent in doctypes_to_skip or (df.parent in ignore_linked_doctypes and method == 'Cancel'): + if df.parent in doctypes_to_skip or (df.parent in ignore_linked_doctypes and method == "Cancel"): # don't check for communication and todo! continue @@ -271,11 +321,14 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"): if meta.issingle: # dynamic link in single doc refdoc = frappe.db.get_singles_dict(df.parent) - if (refdoc.get(df.options)==doc.doctype - and refdoc.get(df.fieldname)==doc.name - and ((method=="Delete" and refdoc.docstatus < 2) - or (method=="Cancel" and refdoc.docstatus==1)) - ): + if ( + refdoc.get(df.options) == doc.doctype + and refdoc.get(df.fieldname) == doc.name + and ( + (method == "Delete" and refdoc.docstatus < 2) + or (method == "Cancel" and refdoc.docstatus == 1) + ) + ): # raise exception only if # linked to an non-cancelled doc when deleting # or linked to a submitted doc when cancelling @@ -283,10 +336,18 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"): else: # dynamic link in table df["table"] = ", `parent`, `parenttype`, `idx`" if meta.istable else "" - for refdoc in frappe.db.sql("""select `name`, `docstatus` {table} from `tab{parent}` where - {options}=%s and {fieldname}=%s""".format(**df), (doc.doctype, doc.name), as_dict=True): + for refdoc in frappe.db.sql( + """select `name`, `docstatus` {table} from `tab{parent}` where + {options}=%s and {fieldname}=%s""".format( + **df + ), + (doc.doctype, doc.name), + as_dict=True, + ): - if ((method=="Delete" and refdoc.docstatus < 2) or (method=="Cancel" and refdoc.docstatus==1)): + if (method == "Delete" and refdoc.docstatus < 2) or ( + method == "Cancel" and refdoc.docstatus == 1 + ): # raise exception only if # linked to an non-cancelled doc when deleting # or linked to a submitted doc when cancelling @@ -297,56 +358,78 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"): raise_link_exists_exception(doc, reference_doctype, reference_docname, at_position) -def raise_link_exists_exception(doc, reference_doctype, reference_docname, row=''): + +def raise_link_exists_exception(doc, reference_doctype, reference_docname, row=""): doc_link = '{1}'.format(doc.doctype, doc.name) - reference_link = '{1}'.format(reference_doctype, reference_docname) + reference_link = '{1}'.format( + reference_doctype, reference_docname + ) - #hack to display Single doctype only once in message + # hack to display Single doctype only once in message if reference_doctype == reference_docname: - reference_doctype = '' + reference_doctype = "" + + frappe.throw( + _("Cannot delete or cancel because {0} {1} is linked with {2} {3} {4}").format( + doc.doctype, doc_link, reference_doctype, reference_link, row + ), + frappe.LinkExistsError, + ) - frappe.throw(_('Cannot delete or cancel because {0} {1} is linked with {2} {3} {4}') - .format(doc.doctype, doc_link, reference_doctype, reference_link, row), frappe.LinkExistsError) def delete_dynamic_links(doctype, name): - delete_references('ToDo', doctype, name, 'reference_type') - delete_references('Email Unsubscribe', doctype, name) - delete_references('DocShare', doctype, name, 'share_doctype', 'share_name') - delete_references('Version', doctype, name, 'ref_doctype', 'docname') - delete_references('Comment', doctype, name) - delete_references('View Log', doctype, name) - delete_references('Document Follow', doctype, name, 'ref_doctype', 'ref_docname') - delete_references('Notification Log', doctype, name, 'document_type', 'document_name') + delete_references("ToDo", doctype, name, "reference_type") + delete_references("Email Unsubscribe", doctype, name) + delete_references("DocShare", doctype, name, "share_doctype", "share_name") + delete_references("Version", doctype, name, "ref_doctype", "docname") + delete_references("Comment", doctype, name) + delete_references("View Log", doctype, name) + delete_references("Document Follow", doctype, name, "ref_doctype", "ref_docname") + delete_references("Notification Log", doctype, name, "document_type", "document_name") # unlink communications clear_timeline_references(doctype, name) - clear_references('Communication', doctype, name) - - clear_references('Activity Log', doctype, name) - clear_references('Activity Log', doctype, name, 'timeline_doctype', 'timeline_name') - -def delete_references(doctype, reference_doctype, reference_name, - reference_doctype_field = 'reference_doctype', reference_name_field = 'reference_name'): - frappe.db.delete(doctype, { - reference_doctype_field: reference_doctype, - reference_name_field: reference_name - }) - -def clear_references(doctype, reference_doctype, reference_name, - reference_doctype_field = 'reference_doctype', reference_name_field = 'reference_name'): - frappe.db.sql('''update + clear_references("Communication", doctype, name) + + clear_references("Activity Log", doctype, name) + clear_references("Activity Log", doctype, name, "timeline_doctype", "timeline_name") + + +def delete_references( + doctype, + reference_doctype, + reference_name, + reference_doctype_field="reference_doctype", + reference_name_field="reference_name", +): + frappe.db.delete( + doctype, {reference_doctype_field: reference_doctype, reference_name_field: reference_name} + ) + + +def clear_references( + doctype, + reference_doctype, + reference_name, + reference_doctype_field="reference_doctype", + reference_name_field="reference_name", +): + frappe.db.sql( + """update `tab{0}` set {1}=NULL, {2}=NULL where - {1}=%s and {2}=%s'''.format(doctype, reference_doctype_field, reference_name_field), # nosec - (reference_doctype, reference_name)) + {1}=%s and {2}=%s""".format( + doctype, reference_doctype_field, reference_name_field + ), # nosec + (reference_doctype, reference_name), + ) + def clear_timeline_references(link_doctype, link_name): - frappe.db.delete("Communication Link", { - "link_doctype": link_doctype, - "link_name": link_name - }) + frappe.db.delete("Communication Link", {"link_doctype": link_doctype, "link_name": link_name}) + def insert_feed(doc): if ( @@ -359,13 +442,15 @@ def insert_feed(doc): from frappe.utils import get_fullname - frappe.get_doc({ - "doctype": "Comment", - "comment_type": "Deleted", - "reference_doctype": doc.doctype, - "subject": "{0} {1}".format(_(doc.doctype), doc.name), - "full_name": get_fullname(doc.owner), - }).insert(ignore_permissions=True) + frappe.get_doc( + { + "doctype": "Comment", + "comment_type": "Deleted", + "reference_doctype": doc.doctype, + "subject": "{0} {1}".format(_(doc.doctype), doc.name), + "full_name": get_fullname(doc.owner), + } + ).insert(ignore_permissions=True) def delete_controllers(doctype, module): @@ -373,6 +458,6 @@ def delete_controllers(doctype, module): Delete controller code in the doctype folder """ module_path = get_module_path(module) - dir_path = os.path.join(module_path, 'doctype', frappe.scrub(doctype)) + dir_path = os.path.join(module_path, "doctype", frappe.scrub(doctype)) shutil.rmtree(dir_path) diff --git a/frappe/model/docfield.py b/frappe/model/docfield.py index c173561b1e..195385a2e1 100644 --- a/frappe/model/docfield.py +++ b/frappe/model/docfield.py @@ -5,49 +5,59 @@ import frappe + def rename(doctype, fieldname, newname): """rename docfield""" - df = frappe.db.sql("""select * from tabDocField where parent=%s and fieldname=%s""", - (doctype, fieldname), as_dict=1) + df = frappe.db.sql( + """select * from tabDocField where parent=%s and fieldname=%s""", (doctype, fieldname), as_dict=1 + ) if not df: return df = df[0] - if frappe.db.get_value('DocType', doctype, 'issingle'): + if frappe.db.get_value("DocType", doctype, "issingle"): update_single(df, newname) else: update_table(df, newname) update_parent_field(df, newname) + def update_single(f, new): """update in tabSingles""" frappe.db.begin() - frappe.db.sql("""update tabSingles set field=%s where doctype=%s and field=%s""", - (new, f['parent'], f['fieldname'])) + frappe.db.sql( + """update tabSingles set field=%s where doctype=%s and field=%s""", + (new, f["parent"], f["fieldname"]), + ) frappe.db.commit() + def update_table(f, new): """update table""" query = get_change_column_query(f, new) if query: frappe.db.sql(query) + def update_parent_field(f, new): """update 'parentfield' in tables""" - if f['fieldtype'] in frappe.model.table_fields: + if f["fieldtype"] in frappe.model.table_fields: frappe.db.begin() - frappe.db.sql("""update `tab%s` set parentfield=%s where parentfield=%s""" \ - % (f['options'], '%s', '%s'), (new, f['fieldname'])) + frappe.db.sql( + """update `tab%s` set parentfield=%s where parentfield=%s""" % (f["options"], "%s", "%s"), + (new, f["fieldname"]), + ) frappe.db.commit() + def get_change_column_query(f, new): """generate change fieldname query""" - desc = frappe.db.sql("desc `tab%s`" % f['parent']) + desc = frappe.db.sql("desc `tab%s`" % f["parent"]) for d in desc: - if d[0]== f['fieldname']: - return 'alter table `tab%s` change `%s` `%s` %s' % \ - (f['parent'], f['fieldname'], new, d[1]) + if d[0] == f["fieldname"]: + return "alter table `tab%s` change `%s` `%s` %s" % (f["parent"], f["fieldname"], new, d[1]) + def supports_translation(fieldtype): - return fieldtype in ["Data", "Select", "Text", "Small Text", "Text Editor"] \ No newline at end of file + return fieldtype in ["Data", "Select", "Text", "Small Text", "Text Editor"] diff --git a/frappe/model/document.py b/frappe/model/document.py index 15e9c28d83..ef2aa9a6dc 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -3,27 +3,27 @@ import hashlib import json import time + from werkzeug.exceptions import NotFound import frappe -from frappe import _, msgprint, is_whitelisted -from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff +from frappe import _, is_whitelisted, msgprint +from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event +from frappe.desk.form.document_follow import follow_document +from frappe.integrations.doctype.webhook import run_webhooks +from frappe.model import optional_fields, table_fields from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, validate_name from frappe.model.docstatus import DocStatus -from frappe.model import optional_fields, table_fields -from frappe.model.workflow import validate_workflow -from frappe.model.workflow import set_workflow_state_on_action -from frappe.utils.global_search import update_global_search -from frappe.integrations.doctype.webhook import run_webhooks -from frappe.desk.form.document_follow import follow_document -from frappe.core.doctype.server_script.server_script_utils import run_server_script_for_doc_event +from frappe.model.naming import set_new_name, validate_name +from frappe.model.workflow import set_workflow_state_on_action, validate_workflow +from frappe.utils import cstr, date_diff, file_lock, flt, get_datetime_str, now from frappe.utils.data import get_absolute_url - +from frappe.utils.global_search import update_global_search # once_only validation # methods + def get_doc(*args, **kwargs): """returns a frappe.model.Document object. @@ -33,23 +33,23 @@ def get_doc(*args, **kwargs): There are multiple ways to call `get_doc` - # will fetch the latest user object (with child table) from the database - user = get_doc("User", "test@example.com") + # will fetch the latest user object (with child table) from the database + user = get_doc("User", "test@example.com") - # create a new object - user = get_doc({ - "doctype":"User" - "email_id": "test@example.com", - "roles: [ - {"role": "System Manager"} - ] - }) + # create a new object + user = get_doc({ + "doctype":"User" + "email_id": "test@example.com", + "roles: [ + {"role": "System Manager"} + ] + }) - # create new object with keyword arguments - user = get_doc(doctype='User', email_id='test@example.com') + # create new object with keyword arguments + user = get_doc(doctype='User', email_id='test@example.com') - # select a document for update - user = get_doc("User", "test@example.com", for_update=True) + # select a document for update + user = get_doc("User", "test@example.com", for_update=True) """ if args: if isinstance(args[0], BaseDocument): @@ -63,11 +63,11 @@ def get_doc(*args, **kwargs): kwargs = args[0] else: - raise ValueError('First non keyword argument must be a string or dict') + raise ValueError("First non keyword argument must be a string or dict") if len(args) < 2 and kwargs: - if 'doctype' in kwargs: - doctype = kwargs['doctype'] + if "doctype" in kwargs: + doctype = kwargs["doctype"] else: raise ValueError('"doctype" is a required key') @@ -77,8 +77,10 @@ def get_doc(*args, **kwargs): raise ImportError(doctype) + class Document(BaseDocument): """All controllers inherit from `Document`.""" + def __init__(self, *args, **kwargs): """Constructor. @@ -117,7 +119,7 @@ class Document(BaseDocument): else: # incorrect arguments. let's not proceed. - raise ValueError('Illegal arguments') + raise ValueError("Illegal arguments") @staticmethod def whitelist(fn): @@ -129,9 +131,7 @@ class Document(BaseDocument): """Load document and children from database and create properties from fields""" if not getattr(self, "_metaclass", False) and self.meta.issingle: - single_doc = frappe.db.get_singles_dict( - self.doctype, for_update=self.flags.for_update - ) + single_doc = frappe.db.get_singles_dict(self.doctype, for_update=self.flags.for_update) if not single_doc: single_doc = frappe.new_doc(self.doctype, as_dict=True) single_doc["name"] = self.doctype @@ -142,22 +142,31 @@ class Document(BaseDocument): self._fix_numeric_types() else: - d = frappe.db.get_value(self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update) + d = frappe.db.get_value( + self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update + ) if not d: - frappe.throw(_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError) + frappe.throw( + _("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError + ) super(Document, self).__init__(d) - if self.name=="DocType" and self.doctype=="DocType": + if self.name == "DocType" and self.doctype == "DocType": from frappe.model.meta import DOCTYPE_TABLE_FIELDS + table_fields = DOCTYPE_TABLE_FIELDS else: table_fields = self.meta.get_table_fields() for df in table_fields: - children = frappe.db.get_values(df.options, + children = frappe.db.get_values( + df.options, {"parent": self.name, "parenttype": self.doctype, "parentfield": df.fieldname}, - "*", as_dict=True, order_by="idx asc") + "*", + as_dict=True, + order_by="idx asc", + ) if children: self.set(df.fieldname, children) else: @@ -174,7 +183,7 @@ class Document(BaseDocument): self.latest = frappe.get_doc(self.doctype, self.name) return self.latest - def check_permission(self, permtype='read', permlevel=None): + def check_permission(self, permtype="read", permlevel=None): """Raise `frappe.PermissionError` if not permitted""" if not self.has_permission(permtype): self.raise_no_permission_to(permlevel or permtype) @@ -192,11 +201,18 @@ class Document(BaseDocument): def raise_no_permission_to(self, perm_type): """Raise `frappe.PermissionError`.""" - frappe.flags.error_message = _('Insufficient Permission for {0}').format(self.doctype) + frappe.flags.error_message = _("Insufficient Permission for {0}").format(self.doctype) raise frappe.PermissionError - def insert(self, ignore_permissions=None, ignore_links=None, ignore_if_duplicate=False, - ignore_mandatory=None, set_name=None, set_child_names=True): + def insert( + self, + ignore_permissions=None, + ignore_links=None, + ignore_if_duplicate=False, + ignore_mandatory=None, + set_name=None, + set_child_names=True, + ): """Insert the document in the database (as a new document). This will check for user permissions and execute `before_insert`, `validate`, `on_update`, `after_insert` methods if they are written. @@ -267,7 +283,9 @@ class Document(BaseDocument): if hasattr(self, "__unsaved"): delattr(self, "__unsaved") - if not (frappe.flags.in_migrate or frappe.local.flags.in_install or frappe.flags.in_setup_wizard): + if not ( + frappe.flags.in_migrate or frappe.local.flags.in_install or frappe.flags.in_setup_wizard + ): if frappe.get_cached_value("User", frappe.session.user, "follow_created_documents"): follow_document(self.doctype, self.name, frappe.session.user) return self @@ -337,20 +355,22 @@ class Document(BaseDocument): """Copy attachments from `amended_from`""" from frappe.desk.form.load import get_attachments - #loop through attachments + # loop through attachments for attach_item in get_attachments(self.doctype, self.amended_from): - #save attachments to new doc - _file = frappe.get_doc({ - "doctype": "File", - "file_url": attach_item.file_url, - "file_name": attach_item.file_name, - "attached_to_name": self.name, - "attached_to_doctype": self.doctype, - "folder": "Home/Attachments"}) + # save attachments to new doc + _file = frappe.get_doc( + { + "doctype": "File", + "file_url": attach_item.file_url, + "file_name": attach_item.file_name, + "attached_to_name": self.name, + "attached_to_doctype": self.doctype, + "folder": "Home/Attachments", + } + ) _file.save() - def update_children(self): """update child tables""" for df in self.meta.get_table_fields(): @@ -373,29 +393,31 @@ class Document(BaseDocument): if rows: # select rows that do not match the ones in the document - deleted_rows = frappe.db.sql("""select name from `tab{0}` where parent=%s + deleted_rows = frappe.db.sql( + """select name from `tab{0}` where parent=%s and parenttype=%s and parentfield=%s - and name not in ({1})""".format(df.options, ','.join(['%s'] * len(rows))), - [self.name, self.doctype, fieldname] + rows) + and name not in ({1})""".format( + df.options, ",".join(["%s"] * len(rows)) + ), + [self.name, self.doctype, fieldname] + rows, + ) if len(deleted_rows) > 0: # delete rows that do not match the ones in the document frappe.db.delete(df.options, {"name": ("in", tuple(row[0] for row in deleted_rows))}) else: # no rows found, delete all rows - frappe.db.delete(df.options, { - "parent": self.name, - "parenttype": self.doctype, - "parentfield": fieldname - }) + frappe.db.delete( + df.options, {"parent": self.name, "parenttype": self.doctype, "parentfield": fieldname} + ) def get_doc_before_save(self): - return getattr(self, '_doc_before_save', None) + return getattr(self, "_doc_before_save", None) def has_value_changed(self, fieldname): - '''Returns true if value is changed before and after saving''' + """Returns true if value is changed before and after saving""" previous = self.get_doc_before_save() - return previous.get(fieldname)!=self.get(fieldname) if previous else True + return previous.get(fieldname) != self.get(fieldname) if previous else True def set_new_name(self, force=False, set_name=None, set_child_names=True): """Calls `frappe.naming.set_new_name` for parent and child docs.""" @@ -427,6 +449,7 @@ class Document(BaseDocument): def set_title_field(self): """Set title field based on template""" + def get_values(): values = self.as_dict() # format values @@ -435,7 +458,7 @@ class Document(BaseDocument): values[key] = "" return values - if self.meta.get("title_field")=="title": + if self.meta.get("title_field") == "title": df = self.meta.get_field(self.meta.title_field) if df.options: @@ -446,13 +469,14 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" - frappe.db.delete("Singles", { - "doctype": self.doctype - }) + frappe.db.delete("Singles", {"doctype": self.doctype}) for field, value in d.items(): if field != "doctype": - frappe.db.sql("""insert into `tabSingles` (doctype, field, value) - values (%s, %s, %s)""", (self.doctype, field, value)) + frappe.db.sql( + """insert into `tabSingles` (doctype, field, value) + values (%s, %s, %s)""", + (self.doctype, field, value), + ) if self.doctype in frappe.db.value_cache: del frappe.db.value_cache[self.doctype] @@ -464,7 +488,9 @@ class Document(BaseDocument): # We'd probably want the creation and owner to be set via API # or Data import at some point, that'd have to be handled here - if self.is_new() and not (frappe.flags.in_install or frappe.flags.in_patch or frappe.flags.in_migrate): + if self.is_new() and not ( + frappe.flags.in_install or frappe.flags.in_patch or frappe.flags.in_migrate + ): self.creation = self.modified self.owner = self.modified_by @@ -518,13 +544,21 @@ class Document(BaseDocument): def _validate_non_negative(self): def get_msg(df): if self.get("parentfield"): - return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)), - _("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label))) + return "{} {} #{}: {} {}".format( + frappe.bold(_(self.doctype)), + _("Row"), + self.idx, + _("Value cannot be negative for"), + frappe.bold(_(df.label)), + ) else: - return _("Value cannot be negative for {0}: {1}").format(_(df.parent), frappe.bold(_(df.label))) + return _("Value cannot be negative for {0}: {1}").format( + _(df.parent), frappe.bold(_(df.label)) + ) - for df in self.meta.get('fields', {'non_negative': ('=', 1), - 'fieldtype': ('in', ['Int', 'Float', 'Currency'])}): + for df in self.meta.get( + "fields", {"non_negative": ("=", 1), "fieldtype": ("in", ["Int", "Float", "Currency"])} + ): if flt(self.get(df.fieldname)) < 0: msg = get_msg(df) @@ -532,11 +566,12 @@ class Document(BaseDocument): def validate_workflow(self): """Validate if the workflow transition is valid""" - if frappe.flags.in_install == 'frappe': return + if frappe.flags.in_install == "frappe": + return workflow = self.meta.get_workflow() if workflow: validate_workflow(self) - if not self._action == 'save': + if not self._action == "save": set_workflow_state_on_action(self, workflow, self._action) def validate_set_only_once(self): @@ -552,7 +587,7 @@ class Document(BaseDocument): if field.fieldtype in table_fields: fail = not self.is_child_table_same(field.fieldname) - elif field.fieldtype in ('Date', 'Datetime', 'Time'): + elif field.fieldtype in ("Date", "Datetime", "Time"): fail = str(value) != str(original_value) else: fail = value != original_value @@ -562,7 +597,7 @@ class Document(BaseDocument): _("Value cannot be changed for {0}").format( frappe.bold(self.meta.get_label(field.fieldname)) ), - exc=frappe.CannotChangeConstantError + exc=frappe.CannotChangeConstantError, ) return False @@ -578,11 +613,11 @@ class Document(BaseDocument): else: # check all child entries for i, d in enumerate(original_value): - new_child = value[i].as_dict(convert_dates_to_str = True) - original_child = d.as_dict(convert_dates_to_str = True) + new_child = value[i].as_dict(convert_dates_to_str=True) + original_child = d.as_dict(convert_dates_to_str=True) # all fields must be same other than modified and modified_by - for key in ('modified', 'modified_by', 'creation'): + for key in ("modified", "modified_by", "creation"): del new_child[key] del original_child[key] @@ -612,7 +647,7 @@ class Document(BaseDocument): if not has_higher_permlevel: return - has_access_to = self.get_permlevel_access('read') + has_access_to = self.get_permlevel_access("read") for df in self.meta.fields: if df.permlevel and not df.permlevel in has_access_to: @@ -639,7 +674,8 @@ class Document(BaseDocument): self.reset_values_if_no_permlevel_access(has_access_to, high_permlevel_fields) # If new record then don't reset the values for child table - if self.is_new(): return + if self.is_new(): + return # check for child tables for df in self.meta.get_table_fields(): @@ -648,7 +684,7 @@ class Document(BaseDocument): for d in self.get(df.fieldname): d.reset_values_if_no_permlevel_access(has_access_to, high_permlevel_fields) - def get_permlevel_access(self, permission_type='write'): + def get_permlevel_access(self, permission_type="write"): if not hasattr(self, "_has_access_to"): self._has_access_to = {} @@ -661,7 +697,7 @@ class Document(BaseDocument): return self._has_access_to[permission_type] - def has_permlevel_access_to(self, fieldname, df=None, permission_type='read'): + def has_permlevel_access_to(self, fieldname, df=None, permission_type="read"): if not df: df = self.meta.get_field(fieldname) @@ -701,16 +737,25 @@ class Document(BaseDocument): `self.check_docstatus_transition`.""" conflict = False self._action = "save" - if not self.get('__islocal') and not self.meta.get('is_virtual'): + if not self.get("__islocal") and not self.meta.get("is_virtual"): if self.meta.issingle: - modified = frappe.db.sql("""select value from tabSingles - where doctype=%s and field='modified' for update""", self.doctype) + modified = frappe.db.sql( + """select value from tabSingles + where doctype=%s and field='modified' for update""", + self.doctype, + ) modified = modified and modified[0][0] if modified and modified != cstr(self._original_modified): conflict = True else: - tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` - where name = %s for update""".format(self.doctype), self.name, as_dict=True) + tmp = frappe.db.sql( + """select modified, docstatus from `tab{0}` + where name = %s for update""".format( + self.doctype + ), + self.name, + as_dict=True, + ) if not tmp: frappe.throw(_("Record does not exist")) @@ -725,10 +770,12 @@ class Document(BaseDocument): self.check_docstatus_transition(tmp.docstatus) if conflict: - frappe.msgprint(_("Error: Document has been modified after you have opened it") \ - + (" (%s, %s). " % (modified, self.modified)) \ - + _("Please refresh to get the latest document."), - raise_exception=frappe.TimestampMismatchError) + frappe.msgprint( + _("Error: Document has been modified after you have opened it") + + (" (%s, %s). " % (modified, self.modified)) + + _("Please refresh to get the latest document."), + raise_exception=frappe.TimestampMismatchError, + ) else: self.check_docstatus_transition(0) @@ -752,7 +799,9 @@ class Document(BaseDocument): self._action = "submit" self.check_permission("submit") elif self.docstatus.is_cancelled(): - raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)")) + raise frappe.DocstatusTransitionError( + _("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)") + ) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) @@ -764,7 +813,9 @@ class Document(BaseDocument): self._action = "cancel" self.check_permission("cancel") elif self.docstatus.is_draft(): - raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)")) + raise frappe.DocstatusTransitionError( + _("Cannot change docstatus from 1 (Submitted) to 0 (Draft)") + ) else: raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) @@ -814,10 +865,11 @@ class Document(BaseDocument): if frappe.flags.print_messages: print(self.as_json().encode("utf-8")) - raise frappe.MandatoryError('[{doctype}, {name}]: {fields}'.format( - fields=", ".join((each[0] for each in missing)), - doctype=self.doctype, - name=self.name)) + raise frappe.MandatoryError( + "[{doctype}, {name}]: {fields}".format( + fields=", ".join((each[0] for each in missing)), doctype=self.doctype, name=self.name + ) + ) def _validate_links(self): if self.flags.ignore_links or self._action == "cancel": @@ -832,13 +884,11 @@ class Document(BaseDocument): if invalid_links: msg = ", ".join((each[2] for each in invalid_links)) - frappe.throw(_("Could not find {0}").format(msg), - frappe.LinkValidationError) + frappe.throw(_("Could not find {0}").format(msg), frappe.LinkValidationError) if cancelled_links: msg = ", ".join((each[2] for each in cancelled_links)) - frappe.throw(_("Cannot link cancelled document: {0}").format(msg), - frappe.CancelledLinkError) + frappe.throw(_("Cannot link cancelled document: {0}").format(msg), frappe.CancelledLinkError) def get_all_children(self, parenttype=None): """Returns all children documents from **Table** type fields in a list.""" @@ -880,7 +930,11 @@ class Document(BaseDocument): def run_notifications(self, method): """Run notifications for this method""" - if (frappe.flags.in_import and frappe.flags.mute_emails) or frappe.flags.in_patch or frappe.flags.in_install: + if ( + (frappe.flags.in_import and frappe.flags.mute_emails) + or frappe.flags.in_patch + or frappe.flags.in_install + ): return if self.flags.notifications_executed is None: @@ -889,11 +943,14 @@ class Document(BaseDocument): from frappe.email.doctype.notification.notification import evaluate_alert if self.flags.notifications is None: - alerts = frappe.cache().hget('notifications', self.doctype) + alerts = frappe.cache().hget("notifications", self.doctype) if alerts is None: - alerts = frappe.get_all('Notification', fields=['name', 'event', 'method'], - filters={'enabled': 1, 'document_type': self.doctype}) - frappe.cache().hset('notifications', self.doctype, alerts) + alerts = frappe.get_all( + "Notification", + fields=["name", "event", "method"], + filters={"enabled": 1, "document_type": self.doctype}, + ) + frappe.cache().hset("notifications", self.doctype, alerts) self.flags.notifications = alerts if not self.flags.notifications: @@ -908,18 +965,18 @@ class Document(BaseDocument): "on_update": "Save", "after_insert": "New", "on_submit": "Submit", - "on_cancel": "Cancel" + "on_cancel": "Cancel", } if not self.flags.in_insert: # value change is not applicable in insert - event_map['on_change'] = 'Value Change' + event_map["on_change"] = "Value Change" for alert in self.flags.notifications: event = event_map.get(method, None) if event and alert.event == event: _evaluate_alert(alert) - elif alert.event=='Method' and method == alert.method: + elif alert.event == "Method" and method == alert.method: _evaluate_alert(alert) @whitelist.__func__ @@ -930,8 +987,7 @@ class Document(BaseDocument): @whitelist.__func__ def _cancel(self): - """Cancel the document. Sets `docstatus` = 2, then saves. - """ + """Cancel the document. Sets `docstatus` = 2, then saves.""" self.docstatus = DocStatus.cancelled() return self.save() @@ -947,7 +1003,9 @@ class Document(BaseDocument): def delete(self, ignore_permissions=False): """Delete document.""" - frappe.delete_doc(self.doctype, self.name, ignore_permissions = ignore_permissions, flags=self.flags) + frappe.delete_doc( + self.doctype, self.name, ignore_permissions=ignore_permissions, flags=self.flags + ) def run_before_save_methods(self): """Run standard methods before `INSERT` or `UPDATE`. Standard Methods are: @@ -969,15 +1027,15 @@ class Document(BaseDocument): if self.flags.ignore_validate: return - if self._action=="save": + if self._action == "save": self.run_method("validate") self.run_method("before_save") - elif self._action=="submit": + elif self._action == "submit": self.run_method("validate") self.run_method("before_submit") - elif self._action=="cancel": + elif self._action == "cancel": self.run_method("before_cancel") - elif self._action=="update_after_submit": + elif self._action == "update_after_submit": self.run_method("before_update_after_submit") self.set_title_field() @@ -1000,18 +1058,17 @@ class Document(BaseDocument): - `on_cancel` for **Cancel** - `update_after_submit` for **Update after Submit**""" - if self._action=="save": + if self._action == "save": self.run_method("on_update") - elif self._action=="submit": + elif self._action == "submit": self.run_method("on_update") self.run_method("on_submit") - elif self._action=="cancel": + elif self._action == "cancel": self.run_method("on_cancel") self.check_no_back_links_exist() - elif self._action=="update_after_submit": + elif self._action == "update_after_submit": self.run_method("on_update_after_submit") - self.clear_cache() self.notify_update() @@ -1019,7 +1076,7 @@ class Document(BaseDocument): self.save_version() - self.run_method('on_change') + self.run_method("on_change") if (self.doctype, self.name) in frappe.flags.currently_saving: frappe.flags.currently_saving.remove((self.doctype, self.name)) @@ -1031,23 +1088,30 @@ class Document(BaseDocument): def reset_seen(self): """Clear _seen property and set current user as seen""" - if getattr(self.meta, 'track_seen', False): - frappe.db.set_value(self.doctype, self.name, "_seen", json.dumps([frappe.session.user]), update_modified=False) + if getattr(self.meta, "track_seen", False): + frappe.db.set_value( + self.doctype, self.name, "_seen", json.dumps([frappe.session.user]), update_modified=False + ) def notify_update(self): """Publish realtime that the current document is modified""" - if frappe.flags.in_patch: return - - frappe.publish_realtime("doc_update", {"modified": self.modified, "doctype": self.doctype, "name": self.name}, - doctype=self.doctype, docname=self.name, after_commit=True) + if frappe.flags.in_patch: + return - if not self.meta.get("read_only") and not self.meta.get("issingle") and \ - not self.meta.get("istable"): - data = { - "doctype": self.doctype, - "name": self.name, - "user": frappe.session.user - } + frappe.publish_realtime( + "doc_update", + {"modified": self.modified, "doctype": self.doctype, "name": self.name}, + doctype=self.doctype, + docname=self.name, + after_commit=True, + ) + + if ( + not self.meta.get("read_only") + and not self.meta.get("issingle") + and not self.meta.get("istable") + ): + data = {"doctype": self.doctype, "name": self.name, "user": frappe.session.user} frappe.publish_realtime("list_update", data, after_commit=True) def db_set(self, fieldname, value=None, update_modified=True, notify=False, commit=False): @@ -1078,12 +1142,19 @@ class Document(BaseDocument): self.load_doc_before_save() # to trigger notification on value change - self.run_method('before_change') + self.run_method("before_change") - frappe.db.set_value(self.doctype, self.name, fieldname, value, - self.modified, self.modified_by, update_modified=update_modified) + frappe.db.set_value( + self.doctype, + self.name, + fieldname, + value, + self.modified, + self.modified_by, + update_modified=update_modified, + ) - self.run_method('on_change') + self.run_method("on_change") if notify: self.notify_update() @@ -1098,7 +1169,8 @@ class Document(BaseDocument): def check_no_back_links_exist(self): """Check if document links to any active document before Cancel.""" - from frappe.model.delete_doc import check_if_doc_is_linked, check_if_doc_is_dynamically_linked + from frappe.model.delete_doc import check_if_doc_is_dynamically_linked, check_if_doc_is_linked + if not self.flags.ignore_links: check_if_doc_is_linked(self, method="Cancel") check_if_doc_is_dynamically_linked(self, method="Cancel") @@ -1107,14 +1179,16 @@ class Document(BaseDocument): """Save version info""" # don't track version under following conditions - if (not getattr(self.meta, 'track_changes', False) - or self.doctype == 'Version' + if ( + not getattr(self.meta, "track_changes", False) + or self.doctype == "Version" or self.flags.ignore_version or frappe.flags.in_install - or (not self._doc_before_save and frappe.flags.in_patch)): + or (not self._doc_before_save and frappe.flags.in_patch) + ): return - version = frappe.new_doc('Version') + version = frappe.new_doc("Version") if not self._doc_before_save: version.for_insert(self) version.insert(ignore_permissions=True) @@ -1132,6 +1206,7 @@ class Document(BaseDocument): Note: If each hooked method returns a value (dict), then all returns are collated in one dict and returned. Ideally, don't return values in hookable methods, set properties in the document.""" + def add_to_return_value(self, new_return_value): if new_return_value is None: self._return_value = self.get("_return_value") @@ -1158,8 +1233,9 @@ class Document(BaseDocument): hooks = [] method = f.__name__ doc_events = frappe.get_doc_hooks() - for handler in doc_events.get(self.doctype, {}).get(method, []) \ - + doc_events.get("*", {}).get(method, []): + for handler in doc_events.get(self.doctype, {}).get(method, []) + doc_events.get("*", {}).get( + method, [] + ): hooks.append(frappe.get_attr(handler)) composed = compose(f, *hooks) @@ -1172,11 +1248,11 @@ class Document(BaseDocument): if not method: raise NotFound("Method {0} not found".format(method_name)) - is_whitelisted(getattr(method, '__func__', method)) + is_whitelisted(getattr(method, "__func__", method)) def validate_value(self, fieldname, condition, val2, doc=None, raise_exception=None): """Check that value of fieldname should be 'condition' val2 - else throw Exception.""" + else throw Exception.""" error_condition_map = { "in": _("one of"), "not in": _("none of"), @@ -1195,7 +1271,9 @@ class Document(BaseDocument): label = doc.meta.get_label(fieldname) condition_str = error_condition_map.get(condition, condition) if doc.get("parentfield"): - msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2) + msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format( + doc.idx, label, condition_str, val2 + ) else: msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2) @@ -1206,7 +1284,9 @@ class Document(BaseDocument): """Raise exception if Table field is empty.""" if not (isinstance(self.get(parentfield), list) and len(self.get(parentfield)) > 0): label = self.meta.get_label(parentfield) - frappe.throw(_("Table {0} cannot be empty").format(label), raise_exception or frappe.EmptyTableError) + frappe.throw( + _("Table {0} cannot be empty").format(label), raise_exception or frappe.EmptyTableError + ) def round_floats_in(self, doc, fieldnames=None): """Round floats for all `Currency`, `Float`, `Percent` fields for the given doc. @@ -1214,8 +1294,10 @@ class Document(BaseDocument): :param doc: Document whose numeric properties are to be rounded. :param fieldnames: [Optional] List of fields to be rounded.""" if not fieldnames: - fieldnames = (df.fieldname for df in - doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]})) + fieldnames = ( + df.fieldname + for df in doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]}) + ) for fieldname in fieldnames: doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield")))) @@ -1224,22 +1306,32 @@ class Document(BaseDocument): """Returns Desk URL for this document.""" return get_absolute_url(self.doctype, self.name) - def add_comment(self, comment_type='Comment', text=None, comment_email=None, link_doctype=None, link_name=None, comment_by=None): + def add_comment( + self, + comment_type="Comment", + text=None, + comment_email=None, + link_doctype=None, + link_name=None, + comment_by=None, + ): """Add a comment to this document. :param comment_type: e.g. `Comment`. See Communication for more info.""" - out = frappe.get_doc({ - "doctype":"Comment", - 'comment_type': comment_type, - "comment_email": comment_email or frappe.session.user, - "comment_by": comment_by, - "reference_doctype": self.doctype, - "reference_name": self.name, - "content": text or comment_type, - "link_doctype": link_doctype, - "link_name": link_name - }).insert(ignore_permissions=True) + out = frappe.get_doc( + { + "doctype": "Comment", + "comment_type": comment_type, + "comment_email": comment_email or frappe.session.user, + "comment_by": comment_by, + "reference_doctype": self.doctype, + "reference_name": self.name, + "content": text or comment_type, + "link_doctype": link_doctype, + "link_name": link_name, + } + ).insert(ignore_permissions=True) return out def add_seen(self, user=None): @@ -1248,12 +1340,12 @@ class Document(BaseDocument): user = frappe.session.user if self.meta.track_seen: - _seen = self.get('_seen') or [] + _seen = self.get("_seen") or [] _seen = frappe.parse_json(_seen) if user not in _seen: _seen.append(user) - frappe.db.set_value(self.doctype, self.name, '_seen', json.dumps(_seen), update_modified=False) + frappe.db.set_value(self.doctype, self.name, "_seen", json.dumps(_seen), update_modified=False) frappe.local.flags.commit = True def add_viewed(self, user=None): @@ -1261,13 +1353,15 @@ class Document(BaseDocument): if not user: user = frappe.session.user - if hasattr(self.meta, 'track_views') and self.meta.track_views: - frappe.get_doc({ - "doctype": "View Log", - "viewed_by": frappe.session.user, - "reference_doctype": self.doctype, - "reference_name": self.name, - }).insert(ignore_permissions=True) + if hasattr(self.meta, "track_views") and self.meta.track_views: + frappe.get_doc( + { + "doctype": "View Log", + "viewed_by": frappe.session.user, + "reference_doctype": self.doctype, + "reference_name": self.name, + } + ).insert(ignore_permissions=True) frappe.local.flags.commit = True def get_signature(self): @@ -1290,7 +1384,7 @@ class Document(BaseDocument): if not key: return self.get("__onload", frappe._dict()) - return self.get('__onload')[key] + return self.get("__onload")[key] def queue_action(self, action, **kwargs): """Run an action in background. If the action has an inner function, @@ -1300,16 +1394,23 @@ class Document(BaseDocument): # See: Stock Reconciliation from frappe.utils.background_jobs import enqueue - if hasattr(self, '_' + action): - action = '_' + action + if hasattr(self, "_" + action): + action = "_" + action if file_lock.lock_exists(self.get_signature()): - frappe.throw(_('This document is currently queued for execution. Please try again'), - title=_('Document Queued')) + frappe.throw( + _("This document is currently queued for execution. Please try again"), + title=_("Document Queued"), + ) self.lock() - enqueue('frappe.model.document.execute_action', doctype=self.doctype, name=self.name, - action=action, **kwargs) + enqueue( + "frappe.model.document.execute_action", + doctype=self.doctype, + name=self.name, + action=action, + **kwargs, + ) def lock(self, timeout=None): """Creates a lock file for the given document. If timeout is set, @@ -1339,19 +1440,25 @@ class Document(BaseDocument): Generic validation to verify date sequence """ if date_diff(self.get(to_date_field), self.get(from_date_field)) < 0: - frappe.throw(_('{0} must be after {1}').format( - frappe.bold(self.meta.get_label(to_date_field)), - frappe.bold(self.meta.get_label(from_date_field)), - ), frappe.exceptions.InvalidDates) + frappe.throw( + _("{0} must be after {1}").format( + frappe.bold(self.meta.get_label(to_date_field)), + frappe.bold(self.meta.get_label(from_date_field)), + ), + frappe.exceptions.InvalidDates, + ) def get_assigned_users(self): - assigned_users = frappe.get_all('ToDo', - fields=['allocated_to'], + assigned_users = frappe.get_all( + "ToDo", + fields=["allocated_to"], filters={ - 'reference_type': self.doctype, - 'reference_name': self.name, - 'status': ('!=', 'Cancelled'), - }, pluck='allocated_to') + "reference_type": self.doctype, + "reference_name": self.name, + "status": ("!=", "Cancelled"), + }, + pluck="allocated_to", + ) users = set(assigned_users) return users @@ -1359,11 +1466,13 @@ class Document(BaseDocument): def add_tag(self, tag): """Add a Tag to this document""" from frappe.desk.doctype.tag.tag import DocTags + DocTags(self.doctype).add(self.name, tag) def get_tags(self): """Return a list of Tags attached to this document""" from frappe.desk.doctype.tag.tag import DocTags + return DocTags(self.doctype).get_tags(self.name).split(",")[1:] def __repr__(self): @@ -1393,12 +1502,9 @@ def execute_action(doctype, name, action, **kwargs): # add a comment (?) if frappe.local.message_log: - msg = json.loads(frappe.local.message_log[-1]).get('message') + msg = json.loads(frappe.local.message_log[-1]).get("message") else: - msg = '
' + frappe.get_traceback() + '
' + msg = "
" + frappe.get_traceback() + "
" - doc.add_comment('Comment', _('Action Failed') + '

' + msg) + doc.add_comment("Comment", _("Action Failed") + "

" + msg) doc.notify_update() - - - diff --git a/frappe/model/dynamic_links.py b/frappe/model/dynamic_links.py index 03f616ef60..523d587389 100644 --- a/frappe/model/dynamic_links.py +++ b/frappe/model/dynamic_links.py @@ -7,7 +7,7 @@ import frappe # the validation message shows the user-facing doctype first. # For example Journal Entry should be validated before GL Entry (which is an internal doctype) -dynamic_link_queries = [ +dynamic_link_queries = [ """select `tabDocField`.parent, `tabDocType`.read_only, `tabDocType`.in_create, `tabDocField`.fieldname, `tabDocField`.options @@ -15,7 +15,6 @@ dynamic_link_queries = [ where `tabDocField`.fieldtype='Dynamic Link' and `tabDocType`.`name`=`tabDocField`.parent order by `tabDocType`.read_only, `tabDocType`.in_create""", - """select `tabCustom Field`.dt as parent, `tabDocType`.read_only, `tabDocType`.in_create, `tabCustom Field`.fieldname, `tabCustom Field`.options @@ -25,14 +24,15 @@ dynamic_link_queries = [ order by `tabDocType`.read_only, `tabDocType`.in_create""", ] + def get_dynamic_link_map(for_delete=False): - '''Build a map of all dynamically linked tables. For example, - if Note is dynamically linked to ToDo, the function will return - `{"Note": ["ToDo"], "Sales Invoice": ["Journal Entry Detail"]}` + """Build a map of all dynamically linked tables. For example, + if Note is dynamically linked to ToDo, the function will return + `{"Note": ["ToDo"], "Sales Invoice": ["Journal Entry Detail"]}` Note: Will not map single doctypes - ''' - if getattr(frappe.local, 'dynamic_link_map', None) is None or frappe.flags.in_test: + """ + if getattr(frappe.local, "dynamic_link_map", None) is None or frappe.flags.in_test: # Build from scratch dynamic_link_map = {} for df in get_dynamic_links(): @@ -45,15 +45,16 @@ def get_dynamic_link_map(for_delete=False): links = frappe.db.sql_list("""select distinct {options} from `tab{parent}`""".format(**df)) for doctype in links: dynamic_link_map.setdefault(doctype, []).append(df) - except frappe.db.TableMissingError: # noqa: E722 + except frappe.db.TableMissingError: # noqa: E722 pass frappe.local.dynamic_link_map = dynamic_link_map return frappe.local.dynamic_link_map + def get_dynamic_links(): - '''Return list of dynamic link fields as DocField. - Uses cache if possible''' + """Return list of dynamic link fields as DocField. + Uses cache if possible""" df = [] for query in dynamic_link_queries: df += frappe.db.sql(query, as_dict=True) diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py index f40a43bb73..6a6522ad07 100644 --- a/frappe/model/mapper.py +++ b/frappe/model/mapper.py @@ -4,16 +4,16 @@ import json import frappe from frappe import _ -from frappe.model import default_fields, table_fields, child_table_fields +from frappe.model import child_table_fields, default_fields, table_fields from frappe.utils import cstr @frappe.whitelist() def make_mapped_doc(method, source_name, selected_children=None, args=None): - '''Returns the mapped document calling the given mapper method. + """Returns the mapped document calling the given mapper method. Sets selected_children as flags for the `get_mapped_doc` method. - Called from `open_mapped_doc` from create_new.js''' + Called from `open_mapped_doc` from create_new.js""" for hook in frappe.get_hooks("override_whitelisted_methods", {}).get(method, []): # override using the first hook @@ -35,13 +35,14 @@ def make_mapped_doc(method, source_name, selected_children=None, args=None): return method(source_name) + @frappe.whitelist() def map_docs(method, source_names, target_doc, args=None): - ''' Returns the mapped document calling the given mapper method + '''Returns the mapped document calling the given mapper method with each of the given source docs on the target doc :param args: Args as string to pass to the mapper method - E.g. args: "{ 'supplier': 'XYZ' }" ''' + E.g. args: "{ 'supplier': 'XYZ' }"''' method = frappe.get_attr(method) if method not in frappe.whitelisted: @@ -52,8 +53,16 @@ def map_docs(method, source_names, target_doc, args=None): target_doc = method(*_args) return target_doc -def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, - postprocess=None, ignore_permissions=False, ignore_child_tables=False): + +def get_mapped_doc( + from_doctype, + from_docname, + table_maps, + target_doc=None, + postprocess=None, + ignore_permissions=False, + ignore_child_tables=False, +): apply_strict_user_permissions = frappe.get_system_settings("apply_strict_user_permissions") @@ -63,8 +72,11 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, elif isinstance(target_doc, str): target_doc = frappe.get_doc(json.loads(target_doc)) - if (not apply_strict_user_permissions - and not ignore_permissions and not target_doc.has_permission("create")): + if ( + not apply_strict_user_permissions + and not ignore_permissions + and not target_doc.has_permission("create") + ): target_doc.raise_no_permission_to("create") source_doc = frappe.get_doc(from_doctype, from_docname) @@ -88,10 +100,13 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, target_df = target_doc.meta.get_field(df.fieldname) if target_df: target_child_doctype = target_df.options - if target_df and target_child_doctype==source_child_doctype and not df.no_copy and not target_df.no_copy: - table_map = { - "doctype": target_child_doctype - } + if ( + target_df + and target_child_doctype == source_child_doctype + and not df.no_copy + and not target_df.no_copy + ): + table_map = {"doctype": target_child_doctype} if table_map: for source_d in source_doc.get(df.fieldname): @@ -101,9 +116,11 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, # if children are selected (checked from UI) for this table type, # and this record is not in the selected children, then continue - if (frappe.flags.selected_children + if ( + frappe.flags.selected_children and (df.fieldname in frappe.flags.selected_children) - and source_d.name not in frappe.flags.selected_children[df.fieldname]): + and source_d.name not in frappe.flags.selected_children[df.fieldname] + ): continue target_child_doctype = table_map["doctype"] @@ -111,11 +128,11 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, # does row exist for a parentfield? if target_parentfield not in row_exists_for_parentfield: - row_exists_for_parentfield[target_parentfield] = (True - if target_doc.get(target_parentfield) else False) + row_exists_for_parentfield[target_parentfield] = ( + True if target_doc.get(target_parentfield) else False + ) - if table_map.get("add_if_empty") and \ - row_exists_for_parentfield.get(target_parentfield): + if table_map.get("add_if_empty") and row_exists_for_parentfield.get(target_parentfield): continue if table_map.get("filter") and table_map.get("filter")(source_d): @@ -128,29 +145,46 @@ def get_mapped_doc(from_doctype, from_docname, table_maps, target_doc=None, target_doc.set_onload("load_after_mapping", True) - if (apply_strict_user_permissions - and not ignore_permissions and not target_doc.has_permission("create")): + if ( + apply_strict_user_permissions + and not ignore_permissions + and not target_doc.has_permission("create") + ): target_doc.raise_no_permission_to("create") return target_doc + def map_doc(source_doc, target_doc, table_map, source_parent=None): if table_map.get("validation"): for key, condition in table_map["validation"].items(): if condition[0] == "=" and source_doc.get(key) != condition[1]: - frappe.throw(_("Cannot map because following condition fails:") + f" {key}={cstr(condition[1])}") + frappe.throw( + _("Cannot map because following condition fails:") + f" {key}={cstr(condition[1])}" + ) map_fields(source_doc, target_doc, table_map, source_parent) if "postprocess" in table_map: table_map["postprocess"](source_doc, target_doc, source_parent) + def map_fields(source_doc, target_doc, table_map, source_parent): - no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] - + [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)] + no_copy_fields = set( + [ + d.fieldname + for d in source_doc.meta.get("fields") + if (d.no_copy == 1 or d.fieldtype in table_fields) + ] + + [ + d.fieldname + for d in target_doc.meta.get("fields") + if (d.no_copy == 1 or d.fieldtype in table_fields) + ] + list(default_fields) + list(child_table_fields) - + list(table_map.get("field_no_map", []))) + + list(table_map.get("field_no_map", [])) + ) for df in target_doc.meta.get("fields"): if df.fieldname not in no_copy_fields: @@ -192,6 +226,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent): if target_doc.get(df.fieldname): map_fetch_fields(target_doc, df, no_copy_fields) + def map_fetch_fields(target_doc, df, no_copy_fields): linked_doc = None @@ -200,8 +235,9 @@ def map_fetch_fields(target_doc, df, no_copy_fields): if not (fetch_df.fieldtype == "Read Only" or fetch_df.read_only): continue - if ((not target_doc.get(fetch_df.fieldname) or fetch_df.fieldtype == "Read Only") - and fetch_df.fieldname not in no_copy_fields): + if ( + not target_doc.get(fetch_df.fieldname) or fetch_df.fieldtype == "Read Only" + ) and fetch_df.fieldname not in no_copy_fields: source_fieldname = fetch_df.fetch_from.split(".")[1] if not linked_doc: @@ -215,6 +251,7 @@ def map_fetch_fields(target_doc, df, no_copy_fields): if val not in (None, ""): target_doc.set(fetch_df.fieldname, val) + def map_child_doc(source_d, target_parent, table_map, source_parent=None): target_child_doctype = table_map["doctype"] target_parentfield = target_parent.get_parentfield_of_doctype(target_child_doctype) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index a3d167fb9b..c363e63f4f 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -3,7 +3,7 @@ # metadata -''' +""" Load metadata (DocType) class Example: @@ -13,7 +13,7 @@ Example: print("DocType" table has field "first_name") -''' +""" import json import os from datetime import datetime @@ -45,19 +45,22 @@ def get_meta(doctype, cached=True): meta = Meta(meta) else: meta = Meta(doctype) - frappe.cache().hset('meta', doctype, meta.as_dict()) + frappe.cache().hset("meta", doctype, meta.as_dict()) frappe.local.meta_cache[doctype] = meta return frappe.local.meta_cache[doctype] else: return load_meta(doctype) + def load_meta(doctype): return Meta(doctype) + def get_table_columns(doctype): return frappe.db.get_table_columns(doctype) + def load_doctype_from_file(doctype): fname = frappe.scrub(doctype) with open(frappe.get_app_path("frappe", "core", "doctype", fname, fname + ".json"), "r") as f: @@ -75,10 +78,19 @@ def load_doctype_from_file(doctype): return txt + class Meta(Document): _metaclass = True default_fields = list(default_fields)[1:] - special_doctypes = ("DocField", "DocPerm", "DocType", "Module Def", 'DocType Action', 'DocType Link', 'DocType State') + special_doctypes = ( + "DocField", + "DocPerm", + "DocType", + "Module Def", + "DocType Action", + "DocType Link", + "DocType State", + ) standard_set_once_fields = [ frappe._dict(fieldname="creation", fieldtype="Datetime"), frappe._dict(fieldname="owner", fieldtype="Data"), @@ -101,7 +113,7 @@ class Meta(Document): try: super(Meta, self).load_from_db() except frappe.DoesNotExistError: - if self.doctype=="DocType" and self.name in self.special_doctypes: + if self.doctype == "DocType" and self.name in self.special_doctypes: self.__dict__.update(load_doctype_from_file(self.name)) else: raise @@ -119,21 +131,22 @@ class Meta(Document): self.set_custom_permissions() self.add_custom_links_and_actions() - def as_dict(self, no_nulls = False): + def as_dict(self, no_nulls=False): def serialize(doc): out = {} for key in doc.__dict__: value = doc.__dict__.get(key) if isinstance(value, (list, tuple)): - if len(value) > 0 and hasattr(value[0], '__dict__'): + if len(value) > 0 and hasattr(value[0], "__dict__"): value = [serialize(d) for d in value] else: # non standard list object, skip continue - if (isinstance(value, (str, int, float, datetime, list, tuple)) - or (not no_nulls and value is None)): + if isinstance(value, (str, int, float, datetime, list, tuple)) or ( + not no_nulls and value is None + ): out[key] = value # set empty lists for unset table fields @@ -146,19 +159,20 @@ class Meta(Document): return serialize(self) def get_link_fields(self): - return self.get("fields", {"fieldtype": "Link", "options":["!=", "[Select]"]}) + return self.get("fields", {"fieldtype": "Link", "options": ["!=", "[Select]"]}) def get_data_fields(self): return self.get("fields", {"fieldtype": "Data"}) def get_dynamic_link_fields(self): - if not hasattr(self, '_dynamic_link_fields'): + if not hasattr(self, "_dynamic_link_fields"): self._dynamic_link_fields = self.get("fields", {"fieldtype": "Dynamic Link"}) return self._dynamic_link_fields def get_select_fields(self): - return self.get("fields", {"fieldtype": "Select", "options":["not in", - ["[Select]", "Loading..."]]}) + return self.get( + "fields", {"fieldtype": "Select", "options": ["not in", ["[Select]", "Loading..."]]} + ) def get_image_fields(self): return self.get("fields", {"fieldtype": "Attach Image"}) @@ -167,7 +181,7 @@ class Meta(Document): return self.get("fields", {"fieldtype": "Code"}) def get_set_only_once_fields(self): - '''Return fields with `set_only_once` set''' + """Return fields with `set_only_once` set""" if not hasattr(self, "_set_only_once_fields"): self._set_only_once_fields = self.get("fields", {"set_only_once": 1}) fieldnames = [d.fieldname for d in self._set_only_once_fields] @@ -180,18 +194,18 @@ class Meta(Document): def get_table_fields(self): if not hasattr(self, "_table_fields"): - if self.name!="DocType": - self._table_fields = self.get('fields', {"fieldtype": ['in', table_fields]}) + if self.name != "DocType": + self._table_fields = self.get("fields", {"fieldtype": ["in", table_fields]}) else: self._table_fields = DOCTYPE_TABLE_FIELDS return self._table_fields def get_global_search_fields(self): - '''Returns list of fields with `in_global_search` set and `name` if set''' + """Returns list of fields with `in_global_search` set and `name` if set""" fields = self.get("fields", {"in_global_search": 1, "fieldtype": ["not in", no_value_fields]}) - if getattr(self, 'show_name_in_global_search', None): - fields.append(frappe._dict(fieldtype='Data', fieldname='name', label='Name')) + if getattr(self, "show_name_in_global_search", None): + fields.append(frappe._dict(fieldtype="Data", fieldname="name", label="Name")) return fields @@ -201,8 +215,9 @@ class Meta(Document): if self.name in self.special_doctypes and table_exists: self._valid_columns = get_table_columns(self.name) else: - self._valid_columns = self.default_fields + \ - [df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes] + self._valid_columns = self.default_fields + [ + df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes + ] if self.istable: self._valid_columns += list(child_table_fields) @@ -218,7 +233,7 @@ class Meta(Document): }.get(fieldname) def get_field(self, fieldname): - '''Return docfield from meta''' + """Return docfield from meta""" if not self._fields: for f in self.get("fields"): self._fields[f.fieldname] = f @@ -226,23 +241,23 @@ class Meta(Document): return self._fields.get(fieldname) def has_field(self, fieldname): - '''Returns True if fieldname exists''' + """Returns True if fieldname exists""" return True if self.get_field(fieldname) else False def get_label(self, fieldname): - '''Get label of the given fieldname''' + """Get label of the given fieldname""" df = self.get_field(fieldname) if df: label = df.label else: label = { - 'name': _('ID'), - 'owner': _('Created By'), - 'modified_by': _('Modified By'), - 'creation': _('Created On'), - 'modified': _('Last Modified On'), - '_assign': _('Assigned To') - }.get(fieldname) or _('No Label') + "name": _("ID"), + "owner": _("Created By"), + "modified_by": _("Modified By"), + "creation": _("Created On"), + "modified": _("Last Modified On"), + "_assign": _("Assigned To"), + }.get(fieldname) or _("No Label") return label def get_options(self, fieldname): @@ -269,11 +284,11 @@ class Meta(Document): return search_fields def get_fields_to_fetch(self, link_fieldname=None): - '''Returns a list of docfield objects for fields whose values + """Returns a list of docfield objects for fields whose values are to be fetched and updated for a particular link field These fields are of type Data, Link, Text, Readonly and their - fetch_from property is set as `link_fieldname`.`source_fieldname`''' + fetch_from property is set as `link_fieldname`.`source_fieldname`""" out = [] @@ -281,45 +296,46 @@ class Meta(Document): link_fields = [df.fieldname for df in self.get_link_fields()] for df in self.fields: - if df.fieldtype not in no_value_fields and getattr(df, 'fetch_from', None): + if df.fieldtype not in no_value_fields and getattr(df, "fetch_from", None): if link_fieldname: - if df.fetch_from.startswith(link_fieldname + '.'): + if df.fetch_from.startswith(link_fieldname + "."): out.append(df) else: - if '.' in df.fetch_from: - fieldname = df.fetch_from.split('.', 1)[0] + if "." in df.fetch_from: + fieldname = df.fetch_from.split(".", 1)[0] if fieldname in link_fields: out.append(df) return out def get_list_fields(self): - list_fields = ["name"] + [d.fieldname \ - for d in self.fields if (d.in_list_view and d.fieldtype in data_fieldtypes)] + list_fields = ["name"] + [ + d.fieldname for d in self.fields if (d.in_list_view and d.fieldtype in data_fieldtypes) + ] if self.title_field and self.title_field not in list_fields: list_fields.append(self.title_field) return list_fields def get_custom_fields(self): - return [d for d in self.fields if d.get('is_custom_field')] + return [d for d in self.fields if d.get("is_custom_field")] def get_title_field(self): - '''Return the title field of this doctype, - explict via `title_field`, or `title` or `name`''' - title_field = getattr(self, 'title_field', None) - if not title_field and self.has_field('title'): - title_field = 'title' + """Return the title field of this doctype, + explict via `title_field`, or `title` or `name`""" + title_field = getattr(self, "title_field", None) + if not title_field and self.has_field("title"): + title_field = "title" if not title_field: - title_field = 'name' + title_field = "name" return title_field def get_translatable_fields(self): - '''Return all fields that are translation enabled''' + """Return all fields that are translation enabled""" return [d.fieldname for d in self.fields if d.translatable] def is_translatable(self, fieldname): - '''Return true of false given a field''' + """Return true of false given a field""" field = self.get_field(fieldname) return field and field.translatable @@ -327,13 +343,18 @@ class Meta(Document): return get_workflow_name(self.name) def add_custom_fields(self): - if not frappe.db.table_exists('Custom Field'): + if not frappe.db.table_exists("Custom Field"): return - custom_fields = frappe.db.sql(""" + custom_fields = frappe.db.sql( + """ SELECT * FROM `tabCustom Field` WHERE dt = %s AND docstatus < 2 - """, (self.name,), as_dict=1, update={"is_custom_field": 1}) + """, + (self.name,), + as_dict=1, + update={"is_custom_field": 1}, + ) self.extend("fields", custom_fields) @@ -343,53 +364,64 @@ class Meta(Document): of the doctype or its child properties like fields, links etc. This method applies the customized properties over the standard meta object """ - if not frappe.db.table_exists('Property Setter'): + if not frappe.db.table_exists("Property Setter"): return - property_setters = frappe.db.sql("""select * from `tabProperty Setter` where - doc_type=%s""", (self.name,), as_dict=1) + property_setters = frappe.db.sql( + """select * from `tabProperty Setter` where + doc_type=%s""", + (self.name,), + as_dict=1, + ) - if not property_setters: return + if not property_setters: + return for ps in property_setters: - if ps.doctype_or_field=='DocType': + if ps.doctype_or_field == "DocType": self.set(ps.property, cast(ps.property_type, ps.value)) - elif ps.doctype_or_field=='DocField': + elif ps.doctype_or_field == "DocField": for d in self.fields: if d.fieldname == ps.field_name: d.set(ps.property, cast(ps.property_type, ps.value)) break - elif ps.doctype_or_field=='DocType Link': + elif ps.doctype_or_field == "DocType Link": for d in self.links: if d.name == ps.row_name: d.set(ps.property, cast(ps.property_type, ps.value)) break - elif ps.doctype_or_field=='DocType Action': + elif ps.doctype_or_field == "DocType Action": for d in self.actions: if d.name == ps.row_name: d.set(ps.property, cast(ps.property_type, ps.value)) break - elif ps.doctype_or_field=='DocType State': + elif ps.doctype_or_field == "DocType State": for d in self.states: if d.name == ps.row_name: d.set(ps.property, cast(ps.property_type, ps.value)) break def add_custom_links_and_actions(self): - for doctype, fieldname in (('DocType Link', 'links'), ('DocType Action', 'actions'), ('DocType State', 'states')): + for doctype, fieldname in ( + ("DocType Link", "links"), + ("DocType Action", "actions"), + ("DocType State", "states"), + ): # ignore_ddl because the `custom` column was added later via a patch - for d in frappe.get_all(doctype, fields='*', filters=dict(parent=self.name, custom=1), ignore_ddl=True): + for d in frappe.get_all( + doctype, fields="*", filters=dict(parent=self.name, custom=1), ignore_ddl=True + ): self.append(fieldname, d) # set the fields in order if specified # order is saved as `links_order` - order = json.loads(self.get('{}_order'.format(fieldname)) or '[]') + order = json.loads(self.get("{}_order".format(fieldname)) or "[]") if order: - name_map = {d.name:d for d in self.get(fieldname)} + name_map = {d.name: d for d in self.get(fieldname)} new_list = [] for name in order: if name in name_map: @@ -419,7 +451,7 @@ class Meta(Document): custom_fields.pop(custom_fields.index(c)) # standard fields - newlist += [df for df in self.get('fields') if not df.get('is_custom_field')] + newlist += [df for df in self.get("fields") if not df.get("is_custom_field")] newlist_fieldnames = [df.fieldname for df in newlist] for i in range(2): @@ -444,23 +476,23 @@ class Meta(Document): self.fields = newlist def set_custom_permissions(self): - '''Reset `permissions` with Custom DocPerm if exists''' + """Reset `permissions` with Custom DocPerm if exists""" if frappe.flags.in_patch or frappe.flags.in_install: return - if not self.istable and self.name not in ('DocType', 'DocField', 'DocPerm', - 'Custom DocPerm'): - custom_perms = frappe.get_all('Custom DocPerm', fields='*', - filters=dict(parent=self.name), update=dict(doctype='Custom DocPerm')) + if not self.istable and self.name not in ("DocType", "DocField", "DocPerm", "Custom DocPerm"): + custom_perms = frappe.get_all( + "Custom DocPerm", + fields="*", + filters=dict(parent=self.name), + update=dict(doctype="Custom DocPerm"), + ) if custom_perms: self.permissions = [Document(d) for d in custom_perms] def get_fieldnames_with_value(self, with_field_meta=False): def is_value_field(docfield): - return not ( - docfield.get("is_virtual") - or docfield.fieldtype in no_value_fields - ) + return not (docfield.get("is_virtual") or docfield.fieldtype in no_value_fields) if with_field_meta: return [df for df in self.fields if is_value_field(df)] @@ -468,19 +500,18 @@ class Meta(Document): return [df.fieldname for df in self.fields if is_value_field(df)] def get_fields_to_check_permissions(self, user_permission_doctypes): - fields = self.get("fields", { - "fieldtype":"Link", - "parent": self.name, - "ignore_user_permissions":("!=", 1), - "options":("in", user_permission_doctypes) - }) + fields = self.get( + "fields", + { + "fieldtype": "Link", + "parent": self.name, + "ignore_user_permissions": ("!=", 1), + "options": ("in", user_permission_doctypes), + }, + ) if self.name in user_permission_doctypes: - fields.append(frappe._dict({ - "label":"Name", - "fieldname":"name", - "options": self.name - })) + fields.append(frappe._dict({"label": "Name", "fieldname": "name", "options": self.name})) return fields @@ -494,7 +525,7 @@ class Meta(Document): return self.high_permlevel_fields - def get_permlevel_access(self, permission_type='read', parenttype=None): + def get_permlevel_access(self, permission_type="read", parenttype=None): has_access_to = [] roles = frappe.get_roles() for perm in self.get_permissions(parenttype): @@ -509,22 +540,22 @@ class Meta(Document): # use parent permissions permissions = frappe.get_meta(parenttype).permissions else: - permissions = self.get('permissions', []) + permissions = self.get("permissions", []) return permissions def get_dashboard_data(self): - '''Returns dashboard setup related to this doctype. + """Returns dashboard setup related to this doctype. This method will return the `data` property in the `[doctype]_dashboard.py` file in the doctype's folder, along with any overrides or extensions implemented in other Frappe applications via hooks. - ''' + """ data = frappe._dict() if not self.custom: try: - module = load_doctype_module(self.name, suffix='_dashboard') - if hasattr(module, 'get_data'): + module = load_doctype_module(self.name, suffix="_dashboard") + if hasattr(module, "get_data"): data = frappe._dict(module.get_data()) except ImportError: pass @@ -538,10 +569,10 @@ class Meta(Document): return data def add_doctype_links(self, data): - '''add `links` child table in standard link dashboard format''' + """add `links` child table in standard link dashboard format""" dashboard_links = [] - if getattr(self, 'links', None): + if getattr(self, "links", None): dashboard_links.extend(self.links) if not data.transactions: @@ -566,16 +597,15 @@ class Meta(Document): doctype = link.parent_doctype or link.link_doctype # group found if link.group and _(group.label) == _(link.group): - if doctype not in group.get('items'): - group.get('items').append(doctype) + if doctype not in group.get("items"): + group.get("items").append(doctype) link.added = True if not link.added: # group not found, make a new group - data.transactions.append(dict( - label = link.group, - items = [link.parent_doctype or link.link_doctype] - )) + data.transactions.append( + dict(label=link.group, items=[link.parent_doctype or link.link_doctype]) + ) if not link.is_child_table: if link.link_fieldname != data.fieldname: @@ -588,26 +618,28 @@ class Meta(Document): data.fieldname = link.link_fieldname data.internal_links[link.parent_doctype] = [link.table_fieldname, link.link_fieldname] - def get_row_template(self): - return self.get_web_template(suffix='_row') + return self.get_web_template(suffix="_row") def get_list_template(self): - return self.get_web_template(suffix='_list') + return self.get_web_template(suffix="_list") - def get_web_template(self, suffix=''): - '''Returns the relative path of the row template for this doctype''' + def get_web_template(self, suffix=""): + """Returns the relative path of the row template for this doctype""" module_name = frappe.scrub(self.module) doctype = frappe.scrub(self.name) - template_path = frappe.get_module_path(module_name, 'doctype', - doctype, 'templates', doctype + suffix + '.html') + template_path = frappe.get_module_path( + module_name, "doctype", doctype, "templates", doctype + suffix + ".html" + ) if os.path.exists(template_path): - return '{module_name}/doctype/{doctype_name}/templates/{doctype_name}{suffix}.html'.format( - module_name = module_name, doctype_name = doctype, suffix=suffix) + return "{module_name}/doctype/{doctype_name}/templates/{doctype_name}{suffix}.html".format( + module_name=module_name, doctype_name=doctype, suffix=suffix + ) return None def is_nested_set(self): - return self.has_field('lft') and self.has_field('rgt') + return self.has_field("lft") and self.has_field("rgt") + DOCTYPE_TABLE_FIELDS = [ frappe._dict({"fieldname": "fields", "options": "DocField"}), @@ -619,18 +651,24 @@ DOCTYPE_TABLE_FIELDS = [ ####### + def is_single(doctype): try: return frappe.db.get_value("DocType", doctype, "issingle") except IndexError: - raise Exception('Cannot determine whether %s is single' % doctype) + raise Exception("Cannot determine whether %s is single" % doctype) + def get_parent_dt(dt): - parent_dt = frappe.db.get_all('DocField', 'parent', dict(fieldtype=['in', frappe.model.table_fields], options=dt), limit=1) - return parent_dt and parent_dt[0].parent or '' + parent_dt = frappe.db.get_all( + "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=dt), limit=1 + ) + return parent_dt and parent_dt[0].parent or "" + def set_fieldname(field_id, fieldname): - frappe.db.set_value('DocField', field_id, 'fieldname', fieldname) + frappe.db.set_value("DocField", field_id, "fieldname", fieldname) + def get_field_currency(df, doc=None): """get currency based on DocField options and fieldvalue in doc""" @@ -645,14 +683,19 @@ def get_field_currency(df, doc=None): if not getattr(frappe.local, "field_currency", None): frappe.local.field_currency = frappe._dict() - if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or - (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))): + if not ( + frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) + or ( + doc.get("parent") + and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname) + ) + ): ref_docname = doc.get("parent") or doc.name if ":" in cstr(df.get("options")): split_opts = df.get("options").split(":") - if len(split_opts)==3 and doc.get(split_opts[1]): + if len(split_opts) == 3 and doc.get(split_opts[1]): currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2]) else: currency = doc.get(df.get("options")) @@ -665,11 +708,15 @@ def get_field_currency(df, doc=None): currency = frappe.db.get_value(doc.parenttype, doc.parent, df.get("options")) if currency: - frappe.local.field_currency.setdefault((doc.doctype, ref_docname), frappe._dict())\ - .setdefault(df.fieldname, currency) + frappe.local.field_currency.setdefault((doc.doctype, ref_docname), frappe._dict()).setdefault( + df.fieldname, currency + ) + + return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or ( + doc.get("parent") + and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname) + ) - return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \ - (doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname)) def get_field_precision(df, doc=None, currency=None): """get precision based on DocField options and fieldvalue in doc""" @@ -692,21 +739,12 @@ def get_field_precision(df, doc=None, currency=None): def get_default_df(fieldname): if fieldname in (default_fields + child_table_fields): if fieldname in ("creation", "modified"): - return frappe._dict( - fieldname = fieldname, - fieldtype = "Datetime" - ) + return frappe._dict(fieldname=fieldname, fieldtype="Datetime") elif fieldname in ("idx", "docstatus"): - return frappe._dict( - fieldname = fieldname, - fieldtype = "Int" - ) + return frappe._dict(fieldname=fieldname, fieldtype="Int") - return frappe._dict( - fieldname = fieldname, - fieldtype = "Data" - ) + return frappe._dict(fieldname=fieldname, fieldtype="Data") def trim_tables(doctype=None, dry_run=False, quiet=False): @@ -729,7 +767,9 @@ def trim_tables(doctype=None, dry_run=False, quiet=False): if quiet: continue click.secho(f"Ignoring missing table for DocType: {doctype}", fg="yellow", err=True) - click.secho(f"Consider removing record in the DocType table for {doctype}", fg="yellow", err=True) + click.secho( + f"Consider removing record in the DocType table for {doctype}", fg="yellow", err=True + ) except Exception as e: if quiet: continue @@ -739,14 +779,12 @@ def trim_tables(doctype=None, dry_run=False, quiet=False): def trim_table(doctype, dry_run=True): - frappe.cache().hdel('table_columns', f"tab{doctype}") + frappe.cache().hdel("table_columns", f"tab{doctype}") ignore_fields = default_fields + optional_fields + child_table_fields columns = frappe.db.get_table_columns(doctype) fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value() is_internal = lambda f: f not in ignore_fields and not f.startswith("_") - columns_to_remove = [ - f for f in list(set(columns) - set(fields)) if is_internal(f) - ] + columns_to_remove = [f for f in list(set(columns) - set(fields)) if is_internal(f)] DROPPED_COLUMNS = columns_to_remove[:] if columns_to_remove and not dry_run: diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 013e5a19db..eb755851fb 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -1,14 +1,15 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from typing import Optional, TYPE_CHECKING, Union +import re +from typing import TYPE_CHECKING, Optional, Union + import frappe from frappe import _ from frappe.database.sequence import get_next_val, set_next_val -from frappe.utils import now_datetime, cint, cstr -import re from frappe.model import log_types from frappe.query_builder import DocType +from frappe.utils import cint, cstr, now_datetime if TYPE_CHECKING: from frappe.model.meta import Meta @@ -70,20 +71,22 @@ def set_new_name(doc): if not doc.name: doc.name = make_autoname("hash", doc.doctype) - doc.name = validate_name( - doc.doctype, - doc.name, - meta.get_field("name_case") - ) + doc.name = validate_name(doc.doctype, doc.name, meta.get_field("name_case")) + def is_autoincremented(doctype: str, meta: "Meta" = None): if doctype in log_types: - if frappe.local.autoincremented_status_map.get(frappe.local.site) is None or \ - frappe.local.autoincremented_status_map[frappe.local.site] == -1: - if frappe.db.sql( - f"""select data_type FROM information_schema.columns + if ( + frappe.local.autoincremented_status_map.get(frappe.local.site) is None + or frappe.local.autoincremented_status_map[frappe.local.site] == -1 + ): + if ( + frappe.db.sql( + f"""select data_type FROM information_schema.columns where column_name = 'name' and table_name = 'tab{doctype}'""" - )[0][0] == "bigint": + )[0][0] + == "bigint" + ): frappe.local.autoincremented_status_map[frappe.local.site] = 1 return True else: @@ -104,6 +107,7 @@ def is_autoincremented(doctype: str, meta: "Meta" = None): return False + def set_name_from_naming_options(autoname, doc): """ Get a name based on the autoname field option @@ -122,20 +126,26 @@ def set_name_from_naming_options(autoname, doc): elif "#" in autoname: doc.name = make_autoname(autoname, doc=doc) + def set_naming_from_document_naming_rule(doc): - ''' + """ Evaluate rules based on "Document Naming Series" doctype - ''' + """ if doc.doctype in log_types: return # ignore_ddl if naming is not yet bootstrapped - for d in frappe.get_all('Document Naming Rule', - dict(document_type=doc.doctype, disabled=0), order_by='priority desc', ignore_ddl=True): - frappe.get_cached_doc('Document Naming Rule', d.name).apply(doc) + for d in frappe.get_all( + "Document Naming Rule", + dict(document_type=doc.doctype, disabled=0), + order_by="priority desc", + ignore_ddl=True, + ): + frappe.get_cached_doc("Document Naming Rule", d.name).apply(doc) if doc.name: break + def set_name_by_naming_series(doc): """Sets name by the `naming_series` property""" if not doc.naming_series: @@ -144,25 +154,26 @@ def set_name_by_naming_series(doc): if not doc.naming_series: frappe.throw(frappe._("Naming Series mandatory")) - doc.name = make_autoname(doc.naming_series+".#####", "", doc) + doc.name = make_autoname(doc.naming_series + ".#####", "", doc) + def make_autoname(key="", doctype="", doc=""): """ - Creates an autoname from the given key: + Creates an autoname from the given key: - **Autoname rules:** + **Autoname rules:** - * The key is separated by '.' - * '####' represents a series. The string before this part becomes the prefix: - Example: ABC.#### creates a series ABC0001, ABC0002 etc - * 'MM' represents the current month - * 'YY' and 'YYYY' represent the current year + * The key is separated by '.' + * '####' represents a series. The string before this part becomes the prefix: + Example: ABC.#### creates a series ABC0001, ABC0002 etc + * 'MM' represents the current month + * 'YY' and 'YYYY' represent the current year - *Example:* + *Example:* - * DE/./.YY./.MM./.##### will create a series like - DE/09/01/0001 where 09 is the year, 01 is the month and 0001 is the series + * DE/./.YY./.MM./.##### will create a series like + DE/09/01/0001 where 09 is the year, 01 is the month and 0001 is the series """ if key == "hash": return frappe.generate_hash(doctype, 10) @@ -176,40 +187,40 @@ def make_autoname(key="", doctype="", doc=""): frappe.throw(error_message) - parts = key.split('.') + parts = key.split(".") n = parse_naming_series(parts, doctype, doc) return n -def parse_naming_series(parts, doctype='', doc=''): - n = '' +def parse_naming_series(parts, doctype="", doc=""): + n = "" if isinstance(parts, str): - parts = parts.split('.') + parts = parts.split(".") series_set = False today = now_datetime() for e in parts: - part = '' - if e.startswith('#'): + part = "" + if e.startswith("#"): if not series_set: digits = len(e) part = getseries(n, digits) series_set = True - elif e == 'YY': - part = today.strftime('%y') - elif e == 'MM': - part = today.strftime('%m') - elif e == 'DD': + elif e == "YY": + part = today.strftime("%y") + elif e == "MM": + part = today.strftime("%m") + elif e == "DD": part = today.strftime("%d") - elif e == 'YYYY': - part = today.strftime('%Y') - elif e == 'WW': + elif e == "YYYY": + part = today.strftime("%Y") + elif e == "WW": part = determine_consecutive_week_number(today) - elif e == 'timestamp': + elif e == "timestamp": part = str(today) - elif e == 'FY': + elif e == "FY": part = frappe.defaults.get_user_default("fiscal_year") - elif e.startswith('{') and doc: - e = e.replace('{', '').replace('}', '') + elif e.startswith("{") and doc: + e = e.replace("{", "").replace("}", "") part = doc.get(e) elif doc and doc.get(e): part = doc.get(e) @@ -226,12 +237,12 @@ def determine_consecutive_week_number(datetime): """Determines the consecutive calendar week""" m = datetime.month # ISO 8601 calandar week - w = datetime.strftime('%V') + w = datetime.strftime("%V") # Ensure consecutiveness for the first and last days of a year if m == 1 and int(w) >= 52: - w = '00' + w = "00" elif m == 12 and int(w) <= 1: - w = '53' + w = "53" return w @@ -239,12 +250,7 @@ def getseries(key, digits): # series created ? # Using frappe.qb as frappe.get_values does not allow order_by=None series = DocType("Series") - current = ( - frappe.qb.from_(series) - .where(series.name == key) - .for_update() - .select("current") - ).run() + current = (frappe.qb.from_(series).where(series.name == key).for_update().select("current")).run() if current and current[0][0] is not None: current = current[0][0] @@ -255,7 +261,7 @@ def getseries(key, digits): # no, create it frappe.db.sql("INSERT INTO `tabSeries` (`name`, `current`) VALUES (%s, 1)", (key,)) current = 1 - return ('%0'+str(digits)+'d') % current + return ("%0" + str(digits) + "d") % current def revert_series_if_last(key, name, doc=None): @@ -273,15 +279,15 @@ def revert_series_if_last(key, name, doc=None): 2. If hash doesn't exit in hashes, we get the hash from prefix, then update name and prefix accordingly. *Example:* - 1. key = SINV-.YYYY.- - * If key doesn't have hash it will add hash at the end - * prefix will be SINV-YYYY based on this will get current index from Series table. - 2. key = SINV-.####.-2021 - * now prefix = SINV-#### and hashes = 2021 (hash doesn't exist) - * will search hash in key then accordingly get prefix = SINV- - 3. key = ####.-2021 - * prefix = #### and hashes = 2021 (hash doesn't exist) - * will search hash in key then accordingly get prefix = "" + 1. key = SINV-.YYYY.- + * If key doesn't have hash it will add hash at the end + * prefix will be SINV-YYYY based on this will get current index from Series table. + 2. key = SINV-.####.-2021 + * now prefix = SINV-#### and hashes = 2021 (hash doesn't exist) + * will search hash in key then accordingly get prefix = SINV- + 3. key = ####.-2021 + * prefix = #### and hashes = 2021 (hash doesn't exist) + * will search hash in key then accordingly get prefix = "" """ if ".#" in key: prefix, hashes = key.rsplit(".", 1) @@ -295,19 +301,16 @@ def revert_series_if_last(key, name, doc=None): else: prefix = key - if '.' in prefix: - prefix = parse_naming_series(prefix.split('.'), doc=doc) + if "." in prefix: + prefix = parse_naming_series(prefix.split("."), doc=doc) count = cint(name.replace(prefix, "")) series = DocType("Series") current = ( - frappe.qb.from_(series) - .where(series.name == prefix) - .for_update() - .select("current") + frappe.qb.from_(series).where(series.name == prefix).for_update().select("current") ).run() - if current and current[0][0]==count: + if current and current[0][0] == count: frappe.db.sql("UPDATE `tabSeries` SET `current` = `current` - 1 WHERE `name`=%s", prefix) @@ -334,8 +337,10 @@ def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = Non frappe.throw(_("Invalid name type (integer) for varchar name column"), frappe.NameError) - if name.startswith("New "+doctype): - frappe.throw(_("There were some errors setting the name, please contact the administrator"), frappe.NameError) + if name.startswith("New " + doctype): + frappe.throw( + _("There were some errors setting the name, please contact the administrator"), frappe.NameError + ) if case == "Title Case": name = name.title() if case == "UPPER CASE": @@ -348,7 +353,9 @@ def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = Non special_characters = "<>" if re.findall("[{0}]+".format(special_characters), name): message = ", ".join("'{0}'".format(c) for c in special_characters) - frappe.throw(_("Name cannot contain special characters like {0}").format(message), frappe.NameError) + frappe.throw( + _("Name cannot contain special characters like {0}").format(message), frappe.NameError + ) return name @@ -362,14 +369,15 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator) if exists: - last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}` + last = frappe.db.sql( + """SELECT `{fieldname}` FROM `tab{doctype}` WHERE `{fieldname}` {regex_character} %s ORDER BY length({fieldname}) DESC, `{fieldname}` DESC LIMIT 1""".format( - doctype=doctype, - fieldname=fieldname, - regex_character=frappe.db.REGEX_CHARACTER), - regex) + doctype=doctype, fieldname=fieldname, regex_character=frappe.db.REGEX_CHARACTER + ), + regex, + ) if last: count = str(cint(last[0][0].rsplit(separator, 1)[1]) + 1) @@ -401,6 +409,7 @@ def _field_autoname(autoname, doc, skip_slicing=None): name = (cstr(doc.get(fieldname)) or "").strip() return name + def _prompt_autoname(autoname, doc): """ Generate a name using Prompt option. This simply means the user will have to set the name manually. @@ -410,6 +419,7 @@ def _prompt_autoname(autoname, doc): if not doc.name: frappe.throw(_("Please set the document name")) + def _format_autoname(autoname, doc): """ Generate autoname by replacing all instances of braced params (fields, date params ('DD', 'MM', 'YY'), series) @@ -419,7 +429,7 @@ def _format_autoname(autoname, doc): """ first_colon_index = autoname.find(":") - autoname_value = autoname[first_colon_index + 1:] + autoname_value = autoname[first_colon_index + 1 :] def get_param_value_for_match(match): param = match.group() diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index b4a53e3131..dee364ae8d 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -23,10 +23,10 @@ def update_document_title( title: Optional[str] = None, name: Optional[str] = None, merge: bool = False, - **kwargs + **kwargs, ) -> str: """ - Update title from header in form view + Update title from header in form view """ # to maintain backwards API compatibility @@ -43,7 +43,9 @@ def update_document_title( title_field = doc.meta.get_title_field() - title_updated = updated_title and (title_field != "name") and (updated_title != doc.get(title_field)) + title_updated = ( + updated_title and (title_field != "name") and (updated_title != doc.get(title_field)) + ) name_updated = updated_name and (updated_name != doc.name) if name_updated: @@ -64,6 +66,7 @@ def update_document_title( return docname + def rename_doc( doctype: str, old: str, @@ -84,8 +87,10 @@ def rename_doc( frappe.errprint(_("Failed: {0} to {1} because {1} already exists.").format(old, new)) return - if old==new: - frappe.errprint(_("Ignored: {0} to {1} no changes made because old and new name are the same.").format(old, new)) + if old == new: + frappe.errprint( + _("Ignored: {0} to {1} no changes made because old and new name are the same.").format(old, new) + ) return force = cint(force) @@ -112,7 +117,7 @@ def rename_doc( # save the user settings in the db update_user_settings(old, new, link_fields) - if doctype=='DocType': + if doctype == "DocType": rename_doctype(doctype, old, new) update_customizations(old, new) @@ -134,13 +139,18 @@ def rename_doc( rename_password(doctype, old, new) # update user_permissions - frappe.db.sql("""UPDATE `tabDefaultValue` SET `defvalue`=%s WHERE `parenttype`='User Permission' - AND `defkey`=%s AND `defvalue`=%s""", (new, doctype, old)) + frappe.db.sql( + """UPDATE `tabDefaultValue` SET `defvalue`=%s WHERE `parenttype`='User Permission' + AND `defkey`=%s AND `defvalue`=%s""", + (new, doctype, old), + ) if merge: - new_doc.add_comment('Edit', _("merged {0} into {1}").format(frappe.bold(old), frappe.bold(new))) + new_doc.add_comment("Edit", _("merged {0} into {1}").format(frappe.bold(old), frappe.bold(new))) else: - new_doc.add_comment('Edit', _("renamed from {0} to {1}").format(frappe.bold(old), frappe.bold(new))) + new_doc.add_comment( + "Edit", _("renamed from {0} to {1}").format(frappe.bold(old), frappe.bold(new)) + ) if merge: frappe.delete_doc(doctype, old) @@ -148,54 +158,69 @@ def rename_doc( new_doc.clear_cache() frappe.clear_cache() if rebuild_search: - frappe.enqueue('frappe.utils.global_search.rebuild_for_doctype', doctype=doctype) + frappe.enqueue("frappe.utils.global_search.rebuild_for_doctype", doctype=doctype) if show_alert: - frappe.msgprint(_('Document renamed from {0} to {1}').format(bold(old), bold(new)), alert=True, indicator='green') + frappe.msgprint( + _("Document renamed from {0} to {1}").format(bold(old), bold(new)), + alert=True, + indicator="green", + ) return new + def update_assignments(old: str, new: str, doctype: str) -> None: - old_assignments = frappe.parse_json(frappe.db.get_value(doctype, old, '_assign')) or [] - new_assignments = frappe.parse_json(frappe.db.get_value(doctype, new, '_assign')) or [] + old_assignments = frappe.parse_json(frappe.db.get_value(doctype, old, "_assign")) or [] + new_assignments = frappe.parse_json(frappe.db.get_value(doctype, new, "_assign")) or [] common_assignments = list(set(old_assignments).intersection(new_assignments)) for user in common_assignments: # delete todos linked to old doc - todos = frappe.db.get_all('ToDo', + todos = frappe.db.get_all( + "ToDo", { - 'owner': user, - 'reference_type': doctype, - 'reference_name': old, + "owner": user, + "reference_type": doctype, + "reference_name": old, }, - ['name', 'description'] + ["name", "description"], ) for todo in todos: - frappe.delete_doc('ToDo', todo.name) + frappe.delete_doc("ToDo", todo.name) unique_assignments = list(set(old_assignments + new_assignments)) - frappe.db.set_value(doctype, new, '_assign', frappe.as_json(unique_assignments, indent=0)) + frappe.db.set_value(doctype, new, "_assign", frappe.as_json(unique_assignments, indent=0)) + def update_user_settings(old: str, new: str, link_fields: List[Dict]) -> None: - ''' - Update the user settings of all the linked doctypes while renaming. - ''' + """ + Update the user settings of all the linked doctypes while renaming. + """ # store the user settings data from the redis to db sync_user_settings() - if not link_fields: return + if not link_fields: + return # find the user settings for the linked doctypes linked_doctypes = {d.parent for d in link_fields if not d.issingle} - user_settings_details = frappe.db.sql('''SELECT `user`, `doctype`, `data` + user_settings_details = frappe.db.sql( + """SELECT `user`, `doctype`, `data` FROM `__UserSettings` WHERE `data` like %s - AND `doctype` IN ('{doctypes}')'''.format(doctypes="', '".join(linked_doctypes)), (old), as_dict=1) + AND `doctype` IN ('{doctypes}')""".format( + doctypes="', '".join(linked_doctypes) + ), + (old), + as_dict=1, + ) # create the dict using the doctype name as key and values as list of the user settings from collections import defaultdict + user_settings_dict = defaultdict(list) for user_setting in user_settings_details: user_settings_dict[user_setting.doctype].append(user_setting) @@ -209,53 +234,63 @@ def update_user_settings(old: str, new: str, link_fields: List[Dict]) -> None: else: continue + def update_customizations(old: str, new: str) -> None: frappe.db.set_value("Custom DocPerm", {"parent": old}, "parent", new, update_modified=False) + def update_attachments(doctype: str, old: str, new: str) -> None: try: if old != "File Data" and doctype != "DocType": - frappe.db.sql("""update `tabFile` set attached_to_name=%s - where attached_to_name=%s and attached_to_doctype=%s""", (new, old, doctype)) + frappe.db.sql( + """update `tabFile` set attached_to_name=%s + where attached_to_name=%s and attached_to_doctype=%s""", + (new, old, doctype), + ) except frappe.db.ProgrammingError as e: if not frappe.db.is_column_missing(e): raise + def rename_versions(doctype: str, old: str, new: str) -> None: - frappe.db.sql("""UPDATE `tabVersion` SET `docname`=%s WHERE `ref_doctype`=%s AND `docname`=%s""", - (new, doctype, old)) + frappe.db.sql( + """UPDATE `tabVersion` SET `docname`=%s WHERE `ref_doctype`=%s AND `docname`=%s""", + (new, doctype, old), + ) + def rename_eps_records(doctype: str, old: str, new: str) -> None: epl = frappe.qb.DocType("Energy Point Log") - (frappe.qb.update(epl) + ( + frappe.qb.update(epl) .set(epl.reference_name, new) - .where( - (epl.reference_doctype == doctype) - & (epl.reference_name == old) - ) + .where((epl.reference_doctype == doctype) & (epl.reference_name == old)) ).run() + def rename_parent_and_child(doctype: str, old: str, new: str, meta: "Meta") -> None: # rename the doc - frappe.db.sql("UPDATE `tab{0}` SET `name`={1} WHERE `name`={1}".format(doctype, '%s'), (new, old)) + frappe.db.sql("UPDATE `tab{0}` SET `name`={1} WHERE `name`={1}".format(doctype, "%s"), (new, old)) update_autoname_field(doctype, new, meta) update_child_docs(old, new, meta) + def update_autoname_field(doctype: str, new: str, meta: "Meta") -> None: # update the value of the autoname field on rename of the docname - if meta.get('autoname'): - field = meta.get('autoname').split(':') + if meta.get("autoname"): + field = meta.get("autoname").split(":") if field and field[0] == "field": - frappe.db.sql("UPDATE `tab{0}` SET `{1}`={2} WHERE `name`={2}".format(doctype, field[1], '%s'), (new, new)) + frappe.db.sql( + "UPDATE `tab{0}` SET `{1}`={2} WHERE `name`={2}".format(doctype, field[1], "%s"), (new, new) + ) -def validate_rename(doctype: str, new: str, meta: "Meta", merge: bool, force: bool, ignore_permissions: bool) -> str: + +def validate_rename( + doctype: str, new: str, meta: "Meta", merge: bool, force: bool, ignore_permissions: bool +) -> str: # using for update so that it gets locked and someone else cannot edit it while this rename is going on! exists = ( - frappe.qb.from_(doctype) - .where(Field("name") == new) - .for_update() - .select("name") - .run(pluck=True) + frappe.qb.from_(doctype).where(Field("name") == new).for_update().select("name").run(pluck=True) ) exists = exists[0] if exists else None @@ -269,7 +304,9 @@ def validate_rename(doctype: str, new: str, meta: "Meta", merge: bool, force: bo if (not merge) and exists: frappe.throw(_("Another {0} with name {1} exists, select another name").format(doctype, new)) - if not (ignore_permissions or frappe.permissions.has_permission(doctype, "write", raise_exception=False)): + if not ( + ignore_permissions or frappe.permissions.has_permission(doctype, "write", raise_exception=False) + ): frappe.throw(_("You need write permission to rename")) if not (force or ignore_permissions) and not meta.allow_rename: @@ -280,6 +317,7 @@ def validate_rename(doctype: str, new: str, meta: "Meta", merge: bool, force: bo return new + def rename_doctype(doctype: str, old: str, new: str) -> None: # change options for fieldtype Table, Table MultiSelect and Link fields_with_options = ("Link",) + frappe.model.table_fields @@ -295,19 +333,22 @@ def rename_doctype(doctype: str, old: str, new: str) -> None: # change parenttype for fieldtype Table update_parenttype_values(old, new) + def update_child_docs(old: str, new: str, meta: "Meta") -> None: # update "parent" for df in meta.get_table_fields(): - frappe.db.sql("update `tab%s` set parent=%s where parent=%s" \ - % (df.options, '%s', '%s'), (new, old)) + frappe.db.sql( + "update `tab%s` set parent=%s where parent=%s" % (df.options, "%s", "%s"), (new, old) + ) + def update_link_field_values(link_fields: List[Dict], old: str, new: str, doctype: str) -> None: for field in link_fields: - if field['issingle']: + if field["issingle"]: try: - single_doc = frappe.get_doc(field['parent']) - if single_doc.get(field['fieldname'])==old: - single_doc.set(field['fieldname'], new) + single_doc = frappe.get_doc(field["parent"]) + if single_doc.get(field["fieldname"]) == old: + single_doc.set(field["fieldname"], new) # update single docs using ORM rather then query # as single docs also sometimes sets defaults! single_doc.flags.ignore_mandatory = True @@ -317,7 +358,7 @@ def update_link_field_values(link_fields: List[Dict], old: str, new: str, doctyp # or no longer exists pass else: - parent = field['parent'] + parent = field["parent"] docfield = field["fieldname"] # Handles the case where one of the link fields belongs to @@ -333,8 +374,9 @@ def update_link_field_values(link_fields: List[Dict], old: str, new: str, doctyp frappe.db.set_value(parent, {docfield: old}, docfield, new, update_modified=False) # update cached link_fields as per new - if doctype=='DocType' and field['parent'] == old: - field['parent'] = new + if doctype == "DocType" and field["parent"] == old: + field["parent"] = new + def get_link_fields(doctype: str) -> List[Dict]: # get link fields from tabDocField @@ -342,28 +384,37 @@ def get_link_fields(doctype: str) -> List[Dict]: frappe.flags.link_fields = {} if doctype not in frappe.flags.link_fields: - link_fields = frappe.db.sql("""\ + link_fields = frappe.db.sql( + """\ select parent, fieldname, (select issingle from tabDocType dt where dt.name = df.parent) as issingle from tabDocField df where - df.options=%s and df.fieldtype='Link'""", (doctype,), as_dict=1) + df.options=%s and df.fieldtype='Link'""", + (doctype,), + as_dict=1, + ) # get link fields from tabCustom Field - custom_link_fields = frappe.db.sql("""\ + custom_link_fields = frappe.db.sql( + """\ select dt as parent, fieldname, (select issingle from tabDocType dt where dt.name = df.dt) as issingle from `tabCustom Field` df where - df.options=%s and df.fieldtype='Link'""", (doctype,), as_dict=1) + df.options=%s and df.fieldtype='Link'""", + (doctype,), + as_dict=1, + ) # add custom link fields list to link fields list link_fields += custom_link_fields # remove fields whose options have been changed using property setter - property_setter_link_fields = frappe.db.sql("""\ + property_setter_link_fields = frappe.db.sql( + """\ select ps.doc_type as parent, ps.field_name as fieldname, (select issingle from tabDocType dt where dt.name = ps.doc_type) as issingle @@ -371,7 +422,10 @@ def get_link_fields(doctype: str) -> List[Dict]: where ps.property_type='options' and ps.field_name is not null and - ps.value=%s""", (doctype,), as_dict=1) + ps.value=%s""", + (doctype,), + as_dict=1, + ) link_fields += property_setter_link_fields @@ -379,6 +433,7 @@ def get_link_fields(doctype: str) -> List[Dict]: return frappe.flags.link_fields[doctype] + def update_options_for_fieldtype(fieldtype: str, old: str, new: str) -> None: if frappe.conf.developer_mode: for name in frappe.get_all("DocField", filters={"options": old}, pluck="parent"): @@ -391,45 +446,68 @@ def update_options_for_fieldtype(fieldtype: str, old: str, new: str) -> None: if save: doctype.save() else: - frappe.db.sql("""update `tabDocField` set options=%s - where fieldtype=%s and options=%s""", (new, fieldtype, old)) + frappe.db.sql( + """update `tabDocField` set options=%s + where fieldtype=%s and options=%s""", + (new, fieldtype, old), + ) - frappe.db.sql("""update `tabCustom Field` set options=%s - where fieldtype=%s and options=%s""", (new, fieldtype, old)) + frappe.db.sql( + """update `tabCustom Field` set options=%s + where fieldtype=%s and options=%s""", + (new, fieldtype, old), + ) + + frappe.db.sql( + """update `tabProperty Setter` set value=%s + where property='options' and value=%s""", + (new, old), + ) - frappe.db.sql("""update `tabProperty Setter` set value=%s - where property='options' and value=%s""", (new, old)) def get_select_fields(old: str, new: str) -> List[Dict]: """ - get select type fields where doctype's name is hardcoded as - new line separated list + get select type fields where doctype's name is hardcoded as + new line separated list """ # get link fields from tabDocField - select_fields = frappe.db.sql(""" + select_fields = frappe.db.sql( + """ select parent, fieldname, (select issingle from tabDocType dt where dt.name = df.parent) as issingle from tabDocField df where df.parent != %s and df.fieldtype = 'Select' and - df.options like {0} """.format(frappe.db.escape('%' + old + '%')), (new,), as_dict=1) + df.options like {0} """.format( + frappe.db.escape("%" + old + "%") + ), + (new,), + as_dict=1, + ) # get link fields from tabCustom Field - custom_select_fields = frappe.db.sql(""" + custom_select_fields = frappe.db.sql( + """ select dt as parent, fieldname, (select issingle from tabDocType dt where dt.name = df.dt) as issingle from `tabCustom Field` df where df.dt != %s and df.fieldtype = 'Select' and - df.options like {0} """ .format(frappe.db.escape('%' + old + '%')), (new,), as_dict=1) + df.options like {0} """.format( + frappe.db.escape("%" + old + "%") + ), + (new,), + as_dict=1, + ) # add custom link fields list to link fields list select_fields += custom_select_fields # remove fields whose options have been changed using property setter - property_setter_select_fields = frappe.db.sql(""" + property_setter_select_fields = frappe.db.sql( + """ select ps.doc_type as parent, ps.field_name as fieldname, (select issingle from tabDocType dt where dt.name = ps.doc_type) as issingle @@ -438,88 +516,110 @@ def get_select_fields(old: str, new: str) -> List[Dict]: ps.doc_type != %s and ps.property_type='options' and ps.field_name is not null and - ps.value like {0} """.format(frappe.db.escape('%' + old + '%')), (new,), as_dict=1) + ps.value like {0} """.format( + frappe.db.escape("%" + old + "%") + ), + (new,), + as_dict=1, + ) select_fields += property_setter_select_fields return select_fields + def update_select_field_values(old: str, new: str): - frappe.db.sql(""" + frappe.db.sql( + """ update `tabDocField` set options=replace(options, %s, %s) where parent != %s and fieldtype = 'Select' and - (options like {0} or options like {1})""" - .format(frappe.db.escape('%' + '\n' + old + '%'), frappe.db.escape('%' + old + '\n' + '%')), (old, new, new)) + (options like {0} or options like {1})""".format( + frappe.db.escape("%" + "\n" + old + "%"), frappe.db.escape("%" + old + "\n" + "%") + ), + (old, new, new), + ) - frappe.db.sql(""" + frappe.db.sql( + """ update `tabCustom Field` set options=replace(options, %s, %s) where dt != %s and fieldtype = 'Select' and - (options like {0} or options like {1})""" - .format(frappe.db.escape('%' + '\n' + old + '%'), frappe.db.escape('%' + old + '\n' + '%')), (old, new, new)) + (options like {0} or options like {1})""".format( + frappe.db.escape("%" + "\n" + old + "%"), frappe.db.escape("%" + old + "\n" + "%") + ), + (old, new, new), + ) - frappe.db.sql(""" + frappe.db.sql( + """ update `tabProperty Setter` set value=replace(value, %s, %s) where doc_type != %s and field_name is not null and property='options' and - (value like {0} or value like {1})""" - .format(frappe.db.escape('%' + '\n' + old + '%'), frappe.db.escape('%' + old + '\n' + '%')), (old, new, new)) + (value like {0} or value like {1})""".format( + frappe.db.escape("%" + "\n" + old + "%"), frappe.db.escape("%" + old + "\n" + "%") + ), + (old, new, new), + ) + def update_parenttype_values(old: str, new: str): - child_doctypes = frappe.db.get_all('DocField', - fields=['options', 'fieldname'], - filters={ - 'parent': new, - 'fieldtype': ['in', frappe.model.table_fields] - } + child_doctypes = frappe.db.get_all( + "DocField", + fields=["options", "fieldname"], + filters={"parent": new, "fieldtype": ["in", frappe.model.table_fields]}, ) - custom_child_doctypes = frappe.db.get_all('Custom Field', - fields=['options', 'fieldname'], - filters={ - 'dt': new, - 'fieldtype': ['in', frappe.model.table_fields] - } + custom_child_doctypes = frappe.db.get_all( + "Custom Field", + fields=["options", "fieldname"], + filters={"dt": new, "fieldtype": ["in", frappe.model.table_fields]}, ) child_doctypes += custom_child_doctypes - fields = [d['fieldname'] for d in child_doctypes] + fields = [d["fieldname"] for d in child_doctypes] property_setter_child_doctypes = frappe.get_all( "Property Setter", - filters={ - "doc_type": new, - "property": "options", - "field_name": ("in", fields) - }, - pluck="value" + filters={"doc_type": new, "property": "options", "field_name": ("in", fields)}, + pluck="value", ) - child_doctypes = list(d['options'] for d in child_doctypes) + child_doctypes = list(d["options"] for d in child_doctypes) child_doctypes += property_setter_child_doctypes for doctype in child_doctypes: frappe.db.sql(f"update `tab{doctype}` set parenttype=%s where parenttype=%s", (new, old)) + def rename_dynamic_links(doctype: str, old: str, new: str): for df in get_dynamic_link_map().get(doctype, []): # dynamic link in single, just one value to check if frappe.get_meta(df.parent).issingle: refdoc = frappe.db.get_singles_dict(df.parent) - if refdoc.get(df.options)==doctype and refdoc.get(df.fieldname)==old: + if refdoc.get(df.options) == doctype and refdoc.get(df.fieldname) == old: - frappe.db.sql("""update tabSingles set value=%s where - field=%s and value=%s and doctype=%s""", (new, df.fieldname, old, df.parent)) + frappe.db.sql( + """update tabSingles set value=%s where + field=%s and value=%s and doctype=%s""", + (new, df.fieldname, old, df.parent), + ) else: # because the table hasn't been renamed yet! parent = df.parent if df.parent != new else old - frappe.db.sql("""update `tab{parent}` set {fieldname}=%s - where {options}=%s and {fieldname}=%s""".format(parent = parent, - fieldname=df.fieldname, options=df.options), (new, doctype, old)) - -def bulk_rename(doctype: str, rows: Optional[List[List]] = None, via_console: bool = False) -> Optional[List[str]]: + frappe.db.sql( + """update `tab{parent}` set {fieldname}=%s + where {options}=%s and {fieldname}=%s""".format( + parent=parent, fieldname=df.fieldname, options=df.options + ), + (new, doctype, old), + ) + + +def bulk_rename( + doctype: str, rows: Optional[List[List]] = None, via_console: bool = False +) -> Optional[List[str]]: """Bulk rename documents :param doctype: DocType to be renamed @@ -553,13 +653,17 @@ def bulk_rename(doctype: str, rows: Optional[List[List]] = None, via_console: bo else: rename_log.append(msg) - frappe.enqueue('frappe.utils.global_search.rebuild_for_doctype', doctype=doctype) + frappe.enqueue("frappe.utils.global_search.rebuild_for_doctype", doctype=doctype) if not via_console: return rename_log -def update_linked_doctypes(doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: Optional[List] = None) -> None: + +def update_linked_doctypes( + doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: Optional[List] = None +) -> None: from frappe.model.utils.rename_doc import update_linked_doctypes + show_deprecation_warning("update_linked_doctypes") return update_linked_doctypes( @@ -571,16 +675,19 @@ def update_linked_doctypes(doctype: str, docname: str, linked_to: str, value: st ) -def get_fetch_fields(doctype: str, linked_to: str, ignore_doctypes: Optional[List] = None) -> List[Dict]: +def get_fetch_fields( + doctype: str, linked_to: str, ignore_doctypes: Optional[List] = None +) -> List[Dict]: from frappe.model.utils.rename_doc import get_fetch_fields + show_deprecation_warning("get_fetch_fields") - return get_fetch_fields( - doctype=doctype, linked_to=linked_to, ignore_doctypes=ignore_doctypes - ) + return get_fetch_fields(doctype=doctype, linked_to=linked_to, ignore_doctypes=ignore_doctypes) + def show_deprecation_warning(funct: str) -> None: from click import secho + message = ( f"Function frappe.model.rename_doc.{funct} has been deprecated and " "moved to the frappe.model.utils.rename_doc" diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 109260d0fe..a56d1f267f 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -4,8 +4,9 @@ Sync's doctype and docfields from txt files to database perms will get synced only if none exist """ -import frappe import os + +import frappe from frappe.modules.import_file import import_file_by_path from frappe.modules.patch_handler import block_user from frappe.utils import update_progress_bar @@ -30,14 +31,27 @@ def sync_for(app_name, force=0, reset_permissions=False): FRAPPE_PATH = frappe.get_app_path("frappe") - for core_module in ["docfield", "docperm", "doctype_action", "doctype_link", "doctype_state", "role", "has_role", "doctype"]: + for core_module in [ + "docfield", + "docperm", + "doctype_action", + "doctype_link", + "doctype_state", + "role", + "has_role", + "doctype", + ]: files.append(os.path.join(FRAPPE_PATH, "core", "doctype", core_module, f"{core_module}.json")) for custom_module in ["custom_field", "property_setter"]: - files.append(os.path.join(FRAPPE_PATH, "custom", "doctype", custom_module, f"{custom_module}.json")) + files.append( + os.path.join(FRAPPE_PATH, "custom", "doctype", custom_module, f"{custom_module}.json") + ) for website_module in ["web_form", "web_template", "web_form_field", "portal_menu_item"]: - files.append(os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json")) + files.append( + os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") + ) for data_migration_module in [ "data_migration_mapping_detail", @@ -45,7 +59,15 @@ def sync_for(app_name, force=0, reset_permissions=False): "data_migration_plan_mapping", "data_migration_plan", ]: - files.append(os.path.join(FRAPPE_PATH, "data_migration", "doctype", data_migration_module, f"{data_migration_module}.json")) + files.append( + os.path.join( + FRAPPE_PATH, + "data_migration", + "doctype", + data_migration_module, + f"{data_migration_module}.json", + ) + ) for desk_module in [ "number_card", @@ -70,7 +92,9 @@ def sync_for(app_name, force=0, reset_permissions=False): if l: for i, doc_path in enumerate(files): - import_file_by_path(doc_path, force=force, ignore_version=True, reset_permissions=reset_permissions) + import_file_by_path( + doc_path, force=force, ignore_version=True, reset_permissions=reset_permissions + ) frappe.db.commit() diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py index 4cdca5e394..a0dd0d89e8 100644 --- a/frappe/model/utils/__init__.py +++ b/frappe/model/utils/__init__.py @@ -1,92 +1,104 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import io +import re + import frappe from frappe import _ -from frappe.utils import cstr from frappe.build import html_to_js_template -import re -import io +from frappe.utils import cstr STANDARD_FIELD_CONVERSION_MAP = { - 'name': 'Link', - 'owner': 'Data', - 'idx': 'Int', - 'creation': 'Data', - 'modified': 'Data', - 'modified_by': 'Data', - '_user_tags': 'Data', - '_liked_by': 'Data', - '_comments': 'Text', - '_assign': 'Text', - 'docstatus': 'Int' + "name": "Link", + "owner": "Data", + "idx": "Int", + "creation": "Data", + "modified": "Data", + "modified_by": "Data", + "_user_tags": "Data", + "_liked_by": "Data", + "_comments": "Text", + "_assign": "Text", + "docstatus": "Int", } """ Model utilities, unclassified functions """ + def set_default(doc, key): """Set is_default property of given doc and unset all others filtered by given key.""" if not doc.is_default: frappe.db.set(doc, "is_default", 1) - frappe.db.sql("""update `tab%s` set `is_default`=0 - where `%s`=%s and name!=%s""" % (doc.doctype, key, "%s", "%s"), - (doc.get(key), doc.name)) + frappe.db.sql( + """update `tab%s` set `is_default`=0 + where `%s`=%s and name!=%s""" + % (doc.doctype, key, "%s", "%s"), + (doc.get(key), doc.name), + ) + def set_field_property(filters, key, value): - '''utility set a property in all fields of a particular type''' - docs = [frappe.get_doc('DocType', d.parent) for d in \ - frappe.get_all("DocField", fields=['parent'], filters=filters)] + """utility set a property in all fields of a particular type""" + docs = [ + frappe.get_doc("DocType", d.parent) + for d in frappe.get_all("DocField", fields=["parent"], filters=filters) + ] for d in docs: - d.get('fields', filters)[0].set(key, value) + d.get("fields", filters)[0].set(key, value) d.save() - print('Updated {0}'.format(d.name)) + print("Updated {0}".format(d.name)) frappe.db.commit() -class InvalidIncludePath(frappe.ValidationError): pass + +class InvalidIncludePath(frappe.ValidationError): + pass + def render_include(content): - '''render {% raw %}{% include "app/path/filename" %}{% endraw %} in js file''' + """render {% raw %}{% include "app/path/filename" %}{% endraw %} in js file""" content = cstr(content) # try 5 levels of includes for i in range(5): if "{% include" in content: - paths = re.findall(r'''{% include\s['"](.*)['"]\s%}''', content) + paths = re.findall(r"""{% include\s['"](.*)['"]\s%}""", content) if not paths: - frappe.throw(_('Invalid include path'), InvalidIncludePath) + frappe.throw(_("Invalid include path"), InvalidIncludePath) for path in paths: - app, app_path = path.split('/', 1) - with io.open(frappe.get_app_path(app, app_path), 'r', encoding = 'utf-8') as f: + app, app_path = path.split("/", 1) + with io.open(frappe.get_app_path(app, app_path), "r", encoding="utf-8") as f: include = f.read() - if path.endswith('.html'): + if path.endswith(".html"): include = html_to_js_template(path, include) - content = re.sub(r'''{{% include\s['"]{0}['"]\s%}}'''.format(path), include, content) + content = re.sub(r"""{{% include\s['"]{0}['"]\s%}}""".format(path), include, content) else: break return content + def get_fetch_values(doctype, fieldname, value): - '''Returns fetch value dict for the given object + """Returns fetch value dict for the given object :param doctype: Target doctype :param fieldname: Link fieldname selected :param value: Value selected - ''' + """ out = {} meta = frappe.get_meta(doctype) link_df = meta.get_field(fieldname) for df in meta.get_fields_to_fetch(fieldname): # example shipping_address.gistin - link_field, source_fieldname = df.fetch_from.split('.', 1) + link_field, source_fieldname = df.fetch_from.split(".", 1) out[df.fieldname] = frappe.db.get_value(link_df.options, value, source_fieldname) return out diff --git a/frappe/model/utils/link_count.py b/frappe/model/utils/link_count.py index 404b6ec855..25dfe58139 100644 --- a/frappe/model/utils/link_count.py +++ b/frappe/model/utils/link_count.py @@ -3,23 +3,24 @@ import frappe -ignore_doctypes = ("DocType", "Print Format", "Role", "Module Def", "Communication", - "ToDo") +ignore_doctypes = ("DocType", "Print Format", "Role", "Module Def", "Communication", "ToDo") + def notify_link_count(doctype, name): - '''updates link count for given document''' - if hasattr(frappe.local, 'link_count'): + """updates link count for given document""" + if hasattr(frappe.local, "link_count"): if (doctype, name) in frappe.local.link_count: frappe.local.link_count[(doctype, name)] += 1 else: frappe.local.link_count[(doctype, name)] = 1 + def flush_local_link_count(): - '''flush from local before ending request''' - if not getattr(frappe.local, 'link_count', None): + """flush from local before ending request""" + if not getattr(frappe.local, "link_count", None): return - link_count = frappe.cache().get_value('_link_count') + link_count = frappe.cache().get_value("_link_count") if not link_count: link_count = {} @@ -29,20 +30,24 @@ def flush_local_link_count(): else: link_count[key] = frappe.local.link_count[key] - frappe.cache().set_value('_link_count', link_count) + frappe.cache().set_value("_link_count", link_count) + def update_link_count(): - '''increment link count in the `idx` column for the given document''' - link_count = frappe.cache().get_value('_link_count') + """increment link count in the `idx` column for the given document""" + link_count = frappe.cache().get_value("_link_count") if link_count: for key, count in link_count.items(): if key[0] not in ignore_doctypes: try: - frappe.db.sql('update `tab{0}` set idx = idx + {1} where name=%s'.format(key[0], count), - key[1], auto_commit=1) + frappe.db.sql( + "update `tab{0}` set idx = idx + {1} where name=%s".format(key[0], count), + key[1], + auto_commit=1, + ) except Exception as e: - if not frappe.db.is_table_missing(e): # table not found, single + if not frappe.db.is_table_missing(e): # table not found, single raise e # reset the count - frappe.cache().delete_value('_link_count') + frappe.cache().delete_value("_link_count") diff --git a/frappe/model/utils/rename_doc.py b/frappe/model/utils/rename_doc.py index f7afbd0cf2..00e2d78d5f 100644 --- a/frappe/model/utils/rename_doc.py +++ b/frappe/model/utils/rename_doc.py @@ -8,7 +8,9 @@ import frappe from frappe.model.rename_doc import get_link_fields -def update_linked_doctypes(doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: Optional[List] = None): +def update_linked_doctypes( + doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: Optional[List] = None +): """ linked_doctype_info_list = list formed by get_fetch_fields() function docname = Master DocType's name in which modification are made @@ -20,26 +22,28 @@ def update_linked_doctypes(doctype: str, docname: str, linked_to: str, value: st frappe.db.set_value( d.doctype, { - d.master_fieldname : docname, - d.linked_to_fieldname : ("!=", value), + d.master_fieldname: docname, + d.linked_to_fieldname: ("!=", value), }, d.linked_to_fieldname, value, ) -def get_fetch_fields(doctype: str, linked_to: str, ignore_doctypes: Optional[List] = None) -> List[Dict]: +def get_fetch_fields( + doctype: str, linked_to: str, ignore_doctypes: Optional[List] = None +) -> List[Dict]: """ doctype = Master DocType in which the changes are being made linked_to = DocType name of the field thats being updated in Master This function fetches list of all DocType where both doctype and linked_to is found as link fields. Forms a list of dict in the form - - [{doctype: , master_fieldname: , linked_to_fieldname: ] + [{doctype: , master_fieldname: , linked_to_fieldname: ] where - doctype = DocType where changes need to be made - master_fieldname = Fieldname where options = doctype - linked_to_fieldname = Fieldname where options = linked_to + doctype = DocType where changes need to be made + master_fieldname = Fieldname where options = doctype + linked_to_fieldname = Fieldname where options = linked_to """ out = [] diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py index c9c454b7e8..56e69455ef 100644 --- a/frappe/model/utils/rename_field.py +++ b/frappe/model/utils/rename_field.py @@ -1,10 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import json + +import frappe from frappe.model import no_value_fields, table_fields +from frappe.model.utils.user_settings import sync_user_settings, update_user_settings_data from frappe.utils.password import rename_password_field -from frappe.model.utils.user_settings import update_user_settings_data, sync_user_settings def rename_field(doctype, old_fieldname, new_fieldname): @@ -23,19 +24,23 @@ def rename_field(doctype, old_fieldname, new_fieldname): if new_field.fieldtype in table_fields: # change parentfield of table mentioned in options - frappe.db.sql("""update `tab%s` set parentfield=%s - where parentfield=%s""" % (new_field.options.split("\n")[0], "%s", "%s"), - (new_fieldname, old_fieldname)) + frappe.db.sql( + """update `tab%s` set parentfield=%s + where parentfield=%s""" + % (new_field.options.split("\n")[0], "%s", "%s"), + (new_fieldname, old_fieldname), + ) elif new_field.fieldtype not in no_value_fields: if meta.issingle: - frappe.db.sql("""update `tabSingles` set field=%s + frappe.db.sql( + """update `tabSingles` set field=%s where doctype=%s and field=%s""", - (new_fieldname, doctype, old_fieldname)) + (new_fieldname, doctype, old_fieldname), + ) else: # copy field value - frappe.db.sql("""update `tab%s` set `%s`=`%s`""" % \ - (doctype, new_fieldname, old_fieldname)) + frappe.db.sql("""update `tab%s` set `%s`=`%s`""" % (doctype, new_fieldname, old_fieldname)) update_reports(doctype, old_fieldname, new_fieldname) update_users_report_view_settings(doctype, old_fieldname, new_fieldname) @@ -43,35 +48,38 @@ def rename_field(doctype, old_fieldname, new_fieldname): if new_field.fieldtype == "Password": rename_password_field(doctype, old_fieldname, new_fieldname) - # update in property setter update_property_setters(doctype, old_fieldname, new_fieldname) # update in user settings update_user_settings(doctype, old_fieldname, new_fieldname) + def update_reports(doctype, old_fieldname, new_fieldname): def _get_new_sort_by(report_dict, report, key): sort_by = report_dict.get(key) or "" if sort_by: sort_by = sort_by.split(".") if len(sort_by) > 1: - if sort_by[0]==doctype and sort_by[1]==old_fieldname: + if sort_by[0] == doctype and sort_by[1] == old_fieldname: sort_by = doctype + "." + new_fieldname report_dict["updated"] = True - elif report.ref_doctype == doctype and sort_by[0]==old_fieldname: + elif report.ref_doctype == doctype and sort_by[0] == old_fieldname: sort_by = doctype + "." + new_fieldname report_dict["updated"] = True if isinstance(sort_by, list): - sort_by = '.'.join(sort_by) + sort_by = ".".join(sort_by) return sort_by - reports = frappe.db.sql("""select name, ref_doctype, json from tabReport + reports = frappe.db.sql( + """select name, ref_doctype, json from tabReport where report_type = 'Report Builder' and ifnull(is_standard, 'No') = 'No' and json like %s and json like %s""", - ('%%%s%%' % old_fieldname , '%%%s%%' % doctype), as_dict=True) + ("%%%s%%" % old_fieldname, "%%%s%%" % doctype), + as_dict=True, + ) for r in reports: report_dict = json.loads(r.json) @@ -101,48 +109,68 @@ def update_reports(doctype, old_fieldname, new_fieldname): new_sort_by_next = _get_new_sort_by(report_dict, r, "sort_by_next") if report_dict.get("updated"): - new_val = json.dumps({ - "filters": new_filters, - "columns": new_columns, - "sort_by": new_sort_by, - "sort_order": report_dict.get("sort_order"), - "sort_by_next": new_sort_by_next, - "sort_order_next": report_dict.get("sort_order_next") - }) + new_val = json.dumps( + { + "filters": new_filters, + "columns": new_columns, + "sort_by": new_sort_by, + "sort_order": report_dict.get("sort_order"), + "sort_by_next": new_sort_by_next, + "sort_order_next": report_dict.get("sort_order_next"), + } + ) frappe.db.sql("""update `tabReport` set `json`=%s where name=%s""", (new_val, r.name)) + def update_users_report_view_settings(doctype, ref_fieldname, new_fieldname): - user_report_cols = frappe.db.sql("""select defkey, defvalue from `tabDefaultValue` where - defkey like '_list_settings:%'""") + user_report_cols = frappe.db.sql( + """select defkey, defvalue from `tabDefaultValue` where + defkey like '_list_settings:%'""" + ) for key, value in user_report_cols: new_columns = [] columns_modified = False for field, field_doctype in json.loads(value): if field == ref_fieldname and field_doctype == doctype: new_columns.append([new_fieldname, field_doctype]) - columns_modified=True + columns_modified = True else: new_columns.append([field, field_doctype]) if columns_modified: - frappe.db.sql("""update `tabDefaultValue` set defvalue=%s - where defkey=%s""" % ('%s', '%s'), (json.dumps(new_columns), key)) + frappe.db.sql( + """update `tabDefaultValue` set defvalue=%s + where defkey=%s""" + % ("%s", "%s"), + (json.dumps(new_columns), key), + ) + def update_property_setters(doctype, old_fieldname, new_fieldname): - frappe.db.sql("""update `tabProperty Setter` set field_name = %s - where doc_type=%s and field_name=%s""", (new_fieldname, doctype, old_fieldname)) + frappe.db.sql( + """update `tabProperty Setter` set field_name = %s + where doc_type=%s and field_name=%s""", + (new_fieldname, doctype, old_fieldname), + ) - frappe.db.sql('''update `tabCustom Field` set insert_after=%s - where insert_after=%s and dt=%s''', (new_fieldname, old_fieldname, doctype)) + frappe.db.sql( + """update `tabCustom Field` set insert_after=%s + where insert_after=%s and dt=%s""", + (new_fieldname, old_fieldname, doctype), + ) def update_user_settings(doctype, old_fieldname, new_fieldname): # store the user settings data from the redis to db sync_user_settings() - user_settings = frappe.db.sql(''' select user, doctype, data from `__UserSettings` - where doctype=%s and data like "%%%s%%"''', (doctype, old_fieldname), as_dict=1) + user_settings = frappe.db.sql( + ''' select user, doctype, data from `__UserSettings` + where doctype=%s and data like "%%%s%%"''', + (doctype, old_fieldname), + as_dict=1, + ) for user_setting in user_settings: update_user_settings_data(user_setting, "docfield", old_fieldname, new_fieldname) diff --git a/frappe/model/utils/user_settings.py b/frappe/model/utils/user_settings.py index ad378ab93f..a6ae1a818e 100644 --- a/frappe/model/utils/user_settings.py +++ b/frappe/model/utils/user_settings.py @@ -1,39 +1,41 @@ - # Settings saved per user basis # such as page_limit, filters, last_view -import frappe, json +import json + +import frappe from frappe import safe_decode # dict for mapping the index and index type for the filters of different views -filter_dict = { - "doctype": 0, - "docfield": 1, - "operator": 2, - "value": 3 -} +filter_dict = {"doctype": 0, "docfield": 1, "operator": 2, "value": 3} + def get_user_settings(doctype, for_update=False): - user_settings = frappe.cache().hget('_user_settings', - '{0}::{1}'.format(doctype, frappe.session.user)) + user_settings = frappe.cache().hget( + "_user_settings", "{0}::{1}".format(doctype, frappe.session.user) + ) if user_settings is None: - user_settings = frappe.db.sql('''select data from `__UserSettings` - where `user`=%s and `doctype`=%s''', (frappe.session.user, doctype)) - user_settings = user_settings and user_settings[0][0] or '{}' + user_settings = frappe.db.sql( + """select data from `__UserSettings` + where `user`=%s and `doctype`=%s""", + (frappe.session.user, doctype), + ) + user_settings = user_settings and user_settings[0][0] or "{}" if not for_update: update_user_settings(doctype, user_settings, True) - return user_settings or '{}' + return user_settings or "{}" + def update_user_settings(doctype, user_settings, for_update=False): - '''update user settings in cache''' + """update user settings in cache""" if for_update: current = json.loads(user_settings) else: - current = json.loads(get_user_settings(doctype, for_update = True)) + current = json.loads(get_user_settings(doctype, for_update=True)) if isinstance(current, str): # corrupt due to old code, remove this in a future release @@ -41,41 +43,50 @@ def update_user_settings(doctype, user_settings, for_update=False): current.update(user_settings) - frappe.cache().hset('_user_settings', '{0}::{1}'.format(doctype, frappe.session.user), - json.dumps(current)) + frappe.cache().hset( + "_user_settings", "{0}::{1}".format(doctype, frappe.session.user), json.dumps(current) + ) + def sync_user_settings(): - '''Sync from cache to database (called asynchronously via the browser)''' - for key, data in frappe.cache().hgetall('_user_settings').items(): + """Sync from cache to database (called asynchronously via the browser)""" + for key, data in frappe.cache().hgetall("_user_settings").items(): key = safe_decode(key) - doctype, user = key.split('::') # WTF? - frappe.db.multisql({ - 'mariadb': """INSERT INTO `__UserSettings`(`user`, `doctype`, `data`) + doctype, user = key.split("::") # WTF? + frappe.db.multisql( + { + "mariadb": """INSERT INTO `__UserSettings`(`user`, `doctype`, `data`) VALUES (%s, %s, %s) ON DUPLICATE key UPDATE `data`=%s""", - 'postgres': """INSERT INTO `__UserSettings` (`user`, `doctype`, `data`) + "postgres": """INSERT INTO `__UserSettings` (`user`, `doctype`, `data`) VALUES (%s, %s, %s) ON CONFLICT ("user", "doctype") DO UPDATE SET `data`=%s""", - }, (user, doctype, data, data), as_dict=1) + }, + (user, doctype, data, data), + as_dict=1, + ) @frappe.whitelist() def save(doctype, user_settings): - user_settings = json.loads(user_settings or '{}') + user_settings = json.loads(user_settings or "{}") update_user_settings(doctype, user_settings) return user_settings + @frappe.whitelist() def get(doctype): return get_user_settings(doctype) -def update_user_settings_data(user_setting, fieldname, old, new, condition_fieldname=None, condition_values=None): +def update_user_settings_data( + user_setting, fieldname, old, new, condition_fieldname=None, condition_values=None +): data = user_setting.get("data") if data: update = False data = json.loads(data) - for view in ['List', 'Gantt', 'Kanban', 'Calendar', 'Image', 'Inbox', 'Report']: + for view in ["List", "Gantt", "Kanban", "Calendar", "Image", "Inbox", "Report"]: view_settings = data.get(view) if view_settings and view_settings.get("filters"): view_filters = view_settings.get("filters") @@ -86,9 +97,12 @@ def update_user_settings_data(user_setting, fieldname, old, new, condition_field view_filter[filter_dict[fieldname]] = new update = True if update: - frappe.db.sql("update __UserSettings set data=%s where doctype=%s and user=%s", - (json.dumps(data), user_setting.doctype, user_setting.user)) + frappe.db.sql( + "update __UserSettings set data=%s where doctype=%s and user=%s", + (json.dumps(data), user_setting.doctype, user_setting.user), + ) # clear that user settings from the redis cache - frappe.cache().hset('_user_settings', '{0}::{1}'.format(user_setting.doctype, - user_setting.user), None) + frappe.cache().hset( + "_user_settings", "{0}::{1}".format(user_setting.doctype, user_setting.user), None + ) diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 1b26cc2c3a..0edffaf2fb 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -4,25 +4,36 @@ import json import frappe from frappe import _ -from frappe.utils import cint from frappe.model.docstatus import DocStatus +from frappe.utils import cint + + +class WorkflowStateError(frappe.ValidationError): + pass + + +class WorkflowTransitionError(frappe.ValidationError): + pass + + +class WorkflowPermissionError(frappe.ValidationError): + pass -class WorkflowStateError(frappe.ValidationError): pass -class WorkflowTransitionError(frappe.ValidationError): pass -class WorkflowPermissionError(frappe.ValidationError): pass def get_workflow_name(doctype): - workflow_name = frappe.cache().hget('workflow', doctype) + workflow_name = frappe.cache().hget("workflow", doctype) if workflow_name is None: - workflow_name = frappe.db.get_value("Workflow", {"document_type": doctype, - "is_active": 1}, "name") - frappe.cache().hset('workflow', doctype, workflow_name or '') + workflow_name = frappe.db.get_value( + "Workflow", {"document_type": doctype, "is_active": 1}, "name" + ) + frappe.cache().hset("workflow", doctype, workflow_name or "") return workflow_name + @frappe.whitelist() -def get_transitions(doc, workflow = None, raise_exception=False): - '''Return list of possible transitions for the given doc''' +def get_transitions(doc, workflow=None, raise_exception=False): + """Return list of possible transitions for the given doc""" doc = frappe.get_doc(frappe.parse_json(doc)) if doc.is_new(): @@ -30,7 +41,7 @@ def get_transitions(doc, workflow = None, raise_exception=False): doc.load_from_db() - frappe.has_permission(doc, 'read', throw=True) + frappe.has_permission(doc, "read", throw=True) roles = frappe.get_roles() if not workflow: @@ -41,7 +52,7 @@ def get_transitions(doc, workflow = None, raise_exception=False): if raise_exception: raise WorkflowStateError else: - frappe.throw(_('Workflow State not set'), WorkflowStateError) + frappe.throw(_("Workflow State not set"), WorkflowStateError) transitions = [] for transition in workflow.transitions: @@ -51,6 +62,7 @@ def get_transitions(doc, workflow = None, raise_exception=False): transitions.append(transition.as_dict()) return transitions + def get_workflow_safe_globals(): # access to frappe.db.get_value, frappe.db.get_list, and date time utils. return dict( @@ -66,15 +78,19 @@ def get_workflow_safe_globals(): ) ) + def is_transition_condition_satisfied(transition, doc): if not transition.condition: return True else: - return frappe.safe_eval(transition.condition, get_workflow_safe_globals(), dict(doc=doc.as_dict())) + return frappe.safe_eval( + transition.condition, get_workflow_safe_globals(), dict(doc=doc.as_dict()) + ) + @frappe.whitelist() def apply_workflow(doc, action): - '''Allow workflow action on the current doc''' + """Allow workflow action on the current doc""" doc = frappe.get_doc(frappe.parse_json(doc)) workflow = get_workflow(doc.doctype) transitions = get_transitions(doc, workflow) @@ -112,33 +128,35 @@ def apply_workflow(doc, action): elif doc.docstatus.is_submitted() and new_docstatus == DocStatus.cancelled(): doc.cancel() else: - frappe.throw(_('Illegal Document Status for {0}').format(next_state.state)) + frappe.throw(_("Illegal Document Status for {0}").format(next_state.state)) - doc.add_comment('Workflow', _(next_state.state)) + doc.add_comment("Workflow", _(next_state.state)) return doc + @frappe.whitelist() def can_cancel_document(doctype): workflow = get_workflow(doctype) for state_doc in workflow.states: - if state_doc.doc_status == '2': + if state_doc.doc_status == "2": for transition in workflow.transitions: if transition.next_state == state_doc.state: return False return True return True + def validate_workflow(doc): - '''Validate Workflow State and Transition for the current user. + """Validate Workflow State and Transition for the current user. - Check if user is allowed to edit in current state - Check if user is allowed to transition to the next state (if changed) - ''' + """ workflow = get_workflow(doc.doctype) current_state = None - if getattr(doc, '_doc_before_save', None): + if getattr(doc, "_doc_before_save", None): current_state = doc._doc_before_save.get(workflow.workflow_state_field) next_state = doc.get(workflow.workflow_state_field) @@ -151,7 +169,11 @@ def validate_workflow(doc): state_row = [d for d in workflow.states if d.state == current_state] if not state_row: - frappe.throw(_('{0} is not a valid Workflow State. Please update your Workflow and try again.').format(frappe.bold(current_state))) + frappe.throw( + _("{0} is not a valid Workflow State. Please update your Workflow and try again.").format( + frappe.bold(current_state) + ) + ) state_row = state_row[0] # if transitioning, check if user is allowed to transition @@ -162,36 +184,46 @@ def validate_workflow(doc): if not doc._doc_before_save: # transitioning directly to a state other than the first # e.g from data import - frappe.throw(_('Workflow State transition not allowed from {0} to {1}').format(bold_current, bold_next), - WorkflowPermissionError) + frappe.throw( + _("Workflow State transition not allowed from {0} to {1}").format(bold_current, bold_next), + WorkflowPermissionError, + ) transitions = get_transitions(doc._doc_before_save) transition = [d for d in transitions if d.next_state == next_state] if not transition: - frappe.throw(_('Workflow State transition not allowed from {0} to {1}').format(bold_current, bold_next), - WorkflowPermissionError) + frappe.throw( + _("Workflow State transition not allowed from {0} to {1}").format(bold_current, bold_next), + WorkflowPermissionError, + ) + def get_workflow(doctype): - return frappe.get_doc('Workflow', get_workflow_name(doctype)) + return frappe.get_doc("Workflow", get_workflow_name(doctype)) + def has_approval_access(user, doc, transition): - return (user == 'Administrator' - or transition.get('allow_self_approval') - or user != doc.get('owner')) + return ( + user == "Administrator" or transition.get("allow_self_approval") or user != doc.get("owner") + ) + def get_workflow_state_field(workflow_name): - return get_workflow_field_value(workflow_name, 'workflow_state_field') + return get_workflow_field_value(workflow_name, "workflow_state_field") + def send_email_alert(workflow_name): - return get_workflow_field_value(workflow_name, 'send_email_alert') + return get_workflow_field_value(workflow_name, "send_email_alert") + def get_workflow_field_value(workflow_name, field): - value = frappe.cache().hget('workflow_' + workflow_name, field) + value = frappe.cache().hget("workflow_" + workflow_name, field) if value is None: value = frappe.db.get_value("Workflow", workflow_name, field) - frappe.cache().hset('workflow_' + workflow_name, field, value) + frappe.cache().hset("workflow_" + workflow_name, field, value) return value + @frappe.whitelist() def bulk_workflow_approval(docnames, doctype, action): from collections import defaultdict @@ -208,7 +240,7 @@ def bulk_workflow_approval(docnames, doctype, action): for (idx, docname) in enumerate(docnames, 1): message_dict = {} try: - show_progress(docnames, _('Applying: {0}').format(action), idx, docname) + show_progress(docnames, _("Applying: {0}").format(action), idx, docname) apply_workflow(frappe.get_doc(doctype, docname), action) frappe.db.commit() except Exception as e: @@ -221,7 +253,10 @@ def bulk_workflow_approval(docnames, doctype, action): failed_transactions[docname].append(message_dict) frappe.db.rollback() - frappe.log_error(frappe.get_traceback(), "Workflow {0} threw an error for {1} {2}".format(action, doctype, docname)) + frappe.log_error( + frappe.get_traceback(), + "Workflow {0} threw an error for {1} {2}".format(action, doctype, docname), + ) finally: if not message_dict: if frappe.message_log: @@ -240,13 +275,14 @@ def bulk_workflow_approval(docnames, doctype, action): if failed_transactions and successful_transactions: indicator = "orange" elif failed_transactions: - indicator = "red" + indicator = "red" else: indicator = "green" print_workflow_log(failed_transactions, _("Failed Transactions"), doctype, indicator) print_workflow_log(successful_transactions, _("Successful Transactions"), doctype, indicator) + def print_workflow_log(messages, title, doctype, indicator): if messages.keys(): msg = "

{0}

".format(title) @@ -255,8 +291,10 @@ def print_workflow_log(messages, title, doctype, indicator): if len(messages[doc]): html = "
{0}".format(frappe.utils.get_link_to_form(doctype, doc)) for log in messages[doc]: - if log.get('message'): - html += "
{0}
".format(log.get('message')) + if log.get("message"): + html += "
{0}
".format( + log.get("message") + ) html += "
" else: html = "
{0}
".format(doc) @@ -264,6 +302,7 @@ def print_workflow_log(messages, title, doctype, indicator): frappe.msgprint(msg, title=_("Workflow Status"), indicator=indicator, is_minimizable=True) + @frappe.whitelist() def get_common_transition_actions(docs, doctype): common_actions = [] @@ -271,10 +310,13 @@ def get_common_transition_actions(docs, doctype): docs = json.loads(docs) try: for (i, doc) in enumerate(docs, 1): - if not doc.get('doctype'): - doc['doctype'] = doctype - actions = [t.get('action') for t in get_transitions(doc, raise_exception=True) \ - if has_approval_access(frappe.session.user, doc, t)] + if not doc.get("doctype"): + doc["doctype"] = doctype + actions = [ + t.get("action") + for t in get_transitions(doc, raise_exception=True) + if has_approval_access(frappe.session.user, doc, t) + ] if not actions: return [] common_actions = actions if i == 1 else set(common_actions).intersection(actions) @@ -285,17 +327,15 @@ def get_common_transition_actions(docs, doctype): return list(common_actions) + def show_progress(docnames, message, i, description): n = len(docnames) if n >= 5: - frappe.publish_progress( - float(i) * 100 / n, - title = message, - description = description - ) + frappe.publish_progress(float(i) * 100 / n, title=message, description=description) + def set_workflow_state_on_action(doc, workflow_name, action): - workflow = frappe.get_doc('Workflow', workflow_name) + workflow = frappe.get_doc("Workflow", workflow_name) workflow_state_field = workflow.workflow_state_field # If workflow state of doc is already correct, don't set workflow state @@ -303,11 +343,7 @@ def set_workflow_state_on_action(doc, workflow_name, action): if state.state == doc.get(workflow_state_field) and doc.docstatus == cint(state.doc_status): return - action_map = { - 'update_after_submit': '1', - 'submit': '1', - 'cancel': '2' - } + action_map = {"update_after_submit": "1", "submit": "1", "cancel": "2"} docstatus = action_map[action] for state in workflow.states: if state.doc_status == docstatus: diff --git a/frappe/modules/__init__.py b/frappe/modules/__init__.py index 33411f8d74..16281fe0b6 100644 --- a/frappe/modules/__init__.py +++ b/frappe/modules/__init__.py @@ -1,2 +1 @@ - -from .utils import * \ No newline at end of file +from .utils import * diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py index 45e008fa04..8eac7a9229 100644 --- a/frappe/modules/export_file.py +++ b/frappe/modules/export_file.py @@ -1,16 +1,20 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, os +import os + +import frappe import frappe.model -from frappe.modules import scrub, get_module_path, scrub_dt_dn +from frappe.modules import get_module_path, scrub, scrub_dt_dn + def export_doc(doc): write_document_file(doc) + def export_to_files(record_list=None, record_module=None, verbose=0, create_init=None): """ - Export record_list to files. record_list is a list of lists ([doctype, docname, folder name],) , + Export record_list to files. record_list is a list of lists ([doctype, docname, folder name],) , """ if frappe.flags.in_import: return @@ -18,7 +22,13 @@ def export_to_files(record_list=None, record_module=None, verbose=0, create_init if record_list: for record in record_list: folder_name = record[2] if len(record) == 3 else None - write_document_file(frappe.get_doc(record[0], record[1]), record_module, create_init=create_init, folder_name=folder_name) + write_document_file( + frappe.get_doc(record[0], record[1]), + record_module, + create_init=create_init, + folder_name=folder_name, + ) + def write_document_file(doc, record_module=None, create_init=True, folder_name=None): doc_export = doc.as_dict(no_nulls=True) @@ -37,9 +47,10 @@ def write_document_file(doc, record_module=None, create_init=True, folder_name=N write_code_files(folder, fname, doc, doc_export) # write the data file - with open(os.path.join(folder, fname + ".json"), 'w+') as txtfile: + with open(os.path.join(folder, fname + ".json"), "w+") as txtfile: txtfile.write(frappe.as_json(doc_export)) + def strip_default_fields(doc, doc_export): # strip out default fields from children if doc.doctype == "DocType" and doc.migration_hash: @@ -47,37 +58,40 @@ def strip_default_fields(doc, doc_export): for df in doc.meta.get_table_fields(): for d in doc_export.get(df.fieldname): - for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields): + for fieldname in frappe.model.default_fields + frappe.model.child_table_fields: if fieldname in d: del d[fieldname] return doc_export + def write_code_files(folder, fname, doc, doc_export): - '''Export code files and strip from values''' - if hasattr(doc, 'get_code_fields'): + """Export code files and strip from values""" + if hasattr(doc, "get_code_fields"): for key, extn in doc.get_code_fields().items(): if doc.get(key): - with open(os.path.join(folder, fname + "." + extn), 'w+') as txtfile: + with open(os.path.join(folder, fname + "." + extn), "w+") as txtfile: txtfile.write(doc.get(key)) # remove from exporting del doc_export[key] + def get_module_name(doc): - if doc.doctype == 'Module Def': + if doc.doctype == "Module Def": module = doc.name - elif doc.doctype=="Workflow": + elif doc.doctype == "Workflow": module = frappe.db.get_value("DocType", doc.document_type, "module") - elif hasattr(doc, 'module'): + elif hasattr(doc, "module"): module = doc.module else: module = frappe.db.get_value("DocType", doc.doctype, "module") return module + def create_folder(module, dt, dn, create_init): - if frappe.db.get_value('Module Def', module, 'custom'): + if frappe.db.get_value("Module Def", module, "custom"): module_path = get_custom_module_path(module) else: module_path = get_module_path(module) @@ -95,10 +109,11 @@ def create_folder(module, dt, dn, create_init): return folder + def get_custom_module_path(module): - package = frappe.db.get_value('Module Def', module, 'package') + package = frappe.db.get_value("Module Def", module, "package") if not package: - frappe.throw('Package must be set for custom Module {module}'.format(module=module)) + frappe.throw("Package must be set for custom Module {module}".format(module=module)) path = os.path.join(get_package_path(package), scrub(module)) if not os.path.exists(path): @@ -106,17 +121,21 @@ def get_custom_module_path(module): return path + def get_package_path(package): - path = os.path.join(frappe.get_site_path('packages'), frappe.db.get_value('Package', package, 'package_name')) + path = os.path.join( + frappe.get_site_path("packages"), frappe.db.get_value("Package", package, "package_name") + ) if not os.path.exists(path): os.makedirs(path) return path + def create_init_py(module_path, dt, dn): def create_if_not_exists(path): - initpy = os.path.join(path, '__init__.py') + initpy = os.path.join(path, "__init__.py") if not os.path.exists(initpy): - open(initpy, 'w').close() + open(initpy, "w").close() create_if_not_exists(os.path.join(module_path)) create_if_not_exists(os.path.join(module_path, dt)) diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index f9c7b55a99..9cfbe163a5 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -15,10 +15,10 @@ def calculate_hash(path: str) -> str: """Calculate md5 hash of the file in binary mode Args: - path (str): Path to the file to be hashed + path (str): Path to the file to be hashed Returns: - str: The calculated hash + str: The calculated hash """ hash_md5 = hashlib.md5() with open(path, "rb") as f: @@ -43,16 +43,24 @@ def import_files(module, dt=None, dn=None, force=False, pre_process=None, reset_ if type(module) is list: out = [] for m in module: - out.append(import_file(m[0], m[1], m[2], force=force, pre_process=pre_process, reset_permissions=reset_permissions)) + out.append( + import_file( + m[0], m[1], m[2], force=force, pre_process=pre_process, reset_permissions=reset_permissions + ) + ) return out else: - return import_file(module, dt, dn, force=force, pre_process=pre_process, reset_permissions=reset_permissions) + return import_file( + module, dt, dn, force=force, pre_process=pre_process, reset_permissions=reset_permissions + ) def import_file(module, dt, dn, force=False, pre_process=None, reset_permissions=False): """Sync a file from txt if modifed, return false if not updated""" path = get_file_path(module, dt, dn) - ret = import_file_by_path(path, force, pre_process=pre_process, reset_permissions=reset_permissions) + ret = import_file_by_path( + path, force, pre_process=pre_process, reset_permissions=reset_permissions + ) return ret @@ -64,7 +72,14 @@ def get_file_path(module, dt, dn): return path -def import_file_by_path(path: str,force: bool = False,data_import: bool = False,pre_process = None,ignore_version: bool = None,reset_permissions: bool = False): +def import_file_by_path( + path: str, + force: bool = False, + data_import: bool = False, + pre_process=None, + ignore_version: bool = None, + reset_permissions: bool = False, +): """Import file from the given path Some conditions decide if a file should be imported or not. @@ -72,9 +87,9 @@ def import_file_by_path(path: str,force: bool = False,data_import: bool = False, - Check if `force` is true. Import the file. If not, move ahead. - Get `db_modified_timestamp`(value of the modified field in the database for the file). - If the return is `none,` this file doesn't exist in the DB, so Import the file. If not, move ahead. + If the return is `none,` this file doesn't exist in the DB, so Import the file. If not, move ahead. - Check if there is a hash in DB for that file. If there is, Calculate the Hash of the file to import and compare it with the one in DB if they are not equal. - Import the file. If Hash doesn't exist, move ahead. + Import the file. If Hash doesn't exist, move ahead. - Check if `db_modified_timestamp` is older than the timestamp in the file; if it is, we import the file. If timestamp comparison happens for doctypes, that means the Hash for it doesn't exist. @@ -82,15 +97,15 @@ def import_file_by_path(path: str,force: bool = False,data_import: bool = False, So in the subsequent imports, we can use hashes to compare. As a precautionary measure, the timestamp is updated to the current time as well. Args: - path (str): Path to the file. - force (bool, optional): Load the file without checking any conditions. Defaults to False. - data_import (bool, optional): [description]. Defaults to False. - pre_process ([type], optional): Any preprocesing that may need to take place on the doc. Defaults to None. - ignore_version (bool, optional): ignore current version. Defaults to None. - reset_permissions (bool, optional): reset permissions for the file. Defaults to False. + path (str): Path to the file. + force (bool, optional): Load the file without checking any conditions. Defaults to False. + data_import (bool, optional): [description]. Defaults to False. + pre_process ([type], optional): Any preprocesing that may need to take place on the doc. Defaults to None. + ignore_version (bool, optional): ignore current version. Defaults to None. + reset_permissions (bool, optional): reset permissions for the file. Defaults to False. Returns: - [bool]: True if import takes place. False if it wasn't imported. + [bool]: True if import takes place. False if it wasn't imported. """ frappe.flags.dt = frappe.flags.dt or [] try: @@ -141,11 +156,7 @@ def import_file_by_path(path: str,force: bool = False,data_import: bool = False, if doc["doctype"] == "DocType": doctype_table = DocType("DocType") - frappe.qb.update( - doctype_table - ).set( - doctype_table.migration_hash, calculated_hash - ).where( + frappe.qb.update(doctype_table).set(doctype_table.migration_hash, calculated_hash).where( doctype_table.name == doc["name"] ).run() @@ -187,31 +198,35 @@ def update_modified(original_modified, doc): if doc["doctype"] == doc["name"] and doc["name"] != "DocType": singles_table = DocType("Singles") - frappe.qb.update( - singles_table - ).set( - singles_table.value,original_modified - ).where( + frappe.qb.update(singles_table).set(singles_table.value, original_modified).where( singles_table["field"] == "modified", # singles_table.field is a method of pypika Selectable - ).where( - singles_table.doctype == doc["name"] - ).run() + ).where(singles_table.doctype == doc["name"]).run() else: - doctype_table = DocType(doc['doctype']) + doctype_table = DocType(doc["doctype"]) - frappe.qb.update(doctype_table - ).set( - doctype_table.modified, original_modified - ).where( + frappe.qb.update(doctype_table).set(doctype_table.modified, original_modified).where( doctype_table.name == doc["name"] ).run() -def import_doc(docdict, force=False, data_import=False, pre_process=None, ignore_version=None, reset_permissions=False, path=None): + +def import_doc( + docdict, + force=False, + data_import=False, + pre_process=None, + ignore_version=None, + reset_permissions=False, + path=None, +): frappe.flags.in_import = True docdict["__islocal"] = 1 controller = get_controller(docdict["doctype"]) - if controller and hasattr(controller, "prepare_for_import") and callable(getattr(controller, "prepare_for_import")): + if ( + controller + and hasattr(controller, "prepare_for_import") + and callable(getattr(controller, "prepare_for_import")) + ): controller.prepare_for_import(docdict) doc = frappe.get_doc(docdict) diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py index 0a23d5b0f4..ae6d9a6de2 100644 --- a/frappe/modules/patch_handler.py +++ b/frappe/modules/patch_handler.py @@ -60,14 +60,14 @@ def run_all(skip_failing: bool = False, patch_type: Optional[PatchType] = None) def run_patch(patch): try: - if not run_single(patchmodule = patch): - print(patch + ': failed: STOPPED') + if not run_single(patchmodule=patch): + print(patch + ": failed: STOPPED") raise PatchError(patch) except Exception: if not skip_failing: raise else: - print('Failed to execute patch') + print("Failed to execute patch") patches = get_all_patches(patch_type=patch_type) @@ -77,9 +77,10 @@ def run_all(skip_failing: bool = False, patch_type: Optional[PatchType] = None) # patches to be run in the end for patch in frappe.flags.final_patches: - patch = patch.replace('finally:', '') + patch = patch.replace("finally:", "") run_patch(patch) + def get_all_patches(patch_type: Optional[PatchType] = None) -> List[str]: if patch_type and not isinstance(patch_type, PatchType): @@ -91,12 +92,13 @@ def get_all_patches(patch_type: Optional[PatchType] = None) -> List[str]: return patches + def get_patches_from_app(app: str, patch_type: Optional[PatchType] = None) -> List[str]: - """ Get patches from an app's patches.txt + """Get patches from an app's patches.txt - patches.txt can be: - 1. ini like file with section for different patch_type - 2. plain text file with each line representing a patch. + patches.txt can be: + 1. ini like file with section for different patch_type + 2. plain text file with each line representing a patch. """ patches_txt = frappe.get_pymodule_path(app, "patches.txt") @@ -115,8 +117,9 @@ def get_patches_from_app(app: str, patch_type: Optional[PatchType] = None) -> Li return [] if not patch_type: - return [patch for patch in parser[PatchType.pre_model_sync.value]] + \ - [patch for patch in parser[PatchType.post_model_sync.value]] + return [patch for patch in parser[PatchType.pre_model_sync.value]] + [ + patch for patch in parser[PatchType.post_model_sync.value] + ] if patch_type.value in parser.sections(): return [patch for patch in parser[patch_type.value]] @@ -131,9 +134,12 @@ def get_patches_from_app(app: str, patch_type: Optional[PatchType] = None) -> Li return [] + def reload_doc(args): import frappe.modules - run_single(method = frappe.modules.reload_doc, methodargs = args) + + run_single(method=frappe.modules.reload_doc, methodargs=args) + def run_single(patchmodule=None, method=None, methodargs=None, force=False): from frappe import conf @@ -146,6 +152,7 @@ def run_single(patchmodule=None, method=None, methodargs=None, force=False): else: return True + def execute_patch(patchmodule, method=None, methodargs=None): """execute the patch""" block_user(True) @@ -163,7 +170,9 @@ def execute_patch(patchmodule, method=None, methodargs=None): if docstring: docstring = "\n" + indent(dedent(docstring), "\t") - print(f"Executing {patchmodule or methodargs} in {frappe.local.site} ({frappe.db.cur_db_name}){docstring}") + print( + f"Executing {patchmodule or methodargs} in {frappe.local.site} ({frappe.db.cur_db_name}){docstring}" + ) start_time = time.time() frappe.db.begin() @@ -194,28 +203,32 @@ def execute_patch(patchmodule, method=None, methodargs=None): return True + def update_patch_log(patchmodule): """update patch_file in patch log""" frappe.get_doc({"doctype": "Patch Log", "patch": patchmodule}).insert(ignore_permissions=True) + def executed(patchmodule): """return True if is executed""" - if patchmodule.startswith('finally:'): + if patchmodule.startswith("finally:"): # patches are saved without the finally: tag - patchmodule = patchmodule.replace('finally:', '') + patchmodule = patchmodule.replace("finally:", "") return frappe.db.get_value("Patch Log", {"patch": patchmodule}) + def block_user(block, msg=None): """stop/start execution till patch is run""" frappe.local.flags.in_patch = block frappe.db.begin() if not msg: msg = "Patches are being executed in the system. Please try again in a few moments." - frappe.db.set_global('__session_status', block and 'stop' or None) - frappe.db.set_global('__session_status_message', block and msg or None) + frappe.db.set_global("__session_status", block and "stop" or None) + frappe.db.set_global("__session_status_message", block and msg or None) frappe.db.commit() + def check_session_stopped(): - if frappe.db.get_global("__session_status")=='stop': + if frappe.db.get_global("__session_status") == "stop": frappe.msgprint(frappe.db.get_global("__session_status_message")) - raise frappe.SessionStopped('Session Stopped') + raise frappe.SessionStopped("Session Stopped") diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index 0383327b68..60eaefddc5 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -3,82 +3,96 @@ """ Utilities for using modules """ -import frappe, os, json +import json +import os + +import frappe import frappe.utils from frappe import _ from frappe.utils import cint + def export_module_json(doc, is_standard, module): """Make a folder for the given doc and add its json file (make it a standard - object that will be synced)""" - if (not frappe.flags.in_import and getattr(frappe.get_conf(),'developer_mode', 0) - and is_standard): + object that will be synced)""" + if not frappe.flags.in_import and getattr(frappe.get_conf(), "developer_mode", 0) and is_standard: from frappe.modules.export_file import export_to_files # json - export_to_files(record_list=[[doc.doctype, doc.name]], record_module=module, - create_init=is_standard) + export_to_files( + record_list=[[doc.doctype, doc.name]], record_module=module, create_init=is_standard + ) - path = os.path.join(frappe.get_module_path(module), scrub(doc.doctype), - scrub(doc.name), scrub(doc.name)) + path = os.path.join( + frappe.get_module_path(module), scrub(doc.doctype), scrub(doc.name), scrub(doc.name) + ) return path + def get_doc_module(module, doctype, name): """Get custom module for given document""" module_name = "{app}.{module}.{doctype}.{name}.{name}".format( - app = frappe.local.module_app[scrub(module)], - doctype = scrub(doctype), - module = scrub(module), - name = scrub(name) + app=frappe.local.module_app[scrub(module)], + doctype=scrub(doctype), + module=scrub(module), + name=scrub(name), ) return frappe.get_module(module_name) + @frappe.whitelist() def export_customizations(module, doctype, sync_on_migrate=0, with_permissions=0): """Export Custom Field and Property Setter for the current document to the app folder. - This will be synced with bench migrate""" + This will be synced with bench migrate""" sync_on_migrate = cint(sync_on_migrate) with_permissions = cint(with_permissions) if not frappe.get_conf().developer_mode: - raise Exception('Not developer mode') + raise Exception("Not developer mode") - custom = {'custom_fields': [], 'property_setters': [], 'custom_perms': [],'links':[], - 'doctype': doctype, 'sync_on_migrate': sync_on_migrate} + custom = { + "custom_fields": [], + "property_setters": [], + "custom_perms": [], + "links": [], + "doctype": doctype, + "sync_on_migrate": sync_on_migrate, + } def add(_doctype): - custom['custom_fields'] += frappe.get_all('Custom Field', - fields='*', filters={'dt': _doctype}) - custom['property_setters'] += frappe.get_all('Property Setter', - fields='*', filters={'doc_type': _doctype}) - custom['links'] += frappe.get_all('DocType Link', - fields='*', filters={'parent': _doctype}) + custom["custom_fields"] += frappe.get_all("Custom Field", fields="*", filters={"dt": _doctype}) + custom["property_setters"] += frappe.get_all( + "Property Setter", fields="*", filters={"doc_type": _doctype} + ) + custom["links"] += frappe.get_all("DocType Link", fields="*", filters={"parent": _doctype}) add(doctype) if with_permissions: - custom['custom_perms'] = frappe.get_all('Custom DocPerm', - fields='*', filters={'parent': doctype}) + custom["custom_perms"] = frappe.get_all( + "Custom DocPerm", fields="*", filters={"parent": doctype} + ) # also update the custom fields and property setters for all child tables for d in frappe.get_meta(doctype).get_table_fields(): export_customizations(module, d.options, sync_on_migrate, with_permissions) if custom["custom_fields"] or custom["property_setters"] or custom["custom_perms"]: - folder_path = os.path.join(get_module_path(module), 'custom') + folder_path = os.path.join(get_module_path(module), "custom") if not os.path.exists(folder_path): os.makedirs(folder_path) - path = os.path.join(folder_path, scrub(doctype)+ '.json') - with open(path, 'w') as f: + path = os.path.join(folder_path, scrub(doctype) + ".json") + with open(path, "w") as f: f.write(frappe.as_json(custom)) - frappe.msgprint(_('Customizations for {0} exported to:
{1}').format(doctype,path)) + frappe.msgprint(_("Customizations for {0} exported to:
{1}").format(doctype, path)) + def sync_customizations(app=None): - '''Sync custom fields and property setters from custom folder in each app module''' + """Sync custom fields and property setters from custom folder in each app module""" if app: apps = [app] @@ -87,21 +101,21 @@ def sync_customizations(app=None): for app_name in apps: for module_name in frappe.local.app_modules.get(app_name) or []: - folder = frappe.get_app_path(app_name, module_name, 'custom') + folder = frappe.get_app_path(app_name, module_name, "custom") if os.path.exists(folder): for fname in os.listdir(folder): - if fname.endswith('.json'): - with open(os.path.join(folder, fname), 'r') as f: + if fname.endswith(".json"): + with open(os.path.join(folder, fname), "r") as f: data = json.loads(f.read()) - if data.get('sync_on_migrate'): + if data.get("sync_on_migrate"): sync_customizations_for_doctype(data, folder) def sync_customizations_for_doctype(data, folder): - '''Sync doctype customzations for a particular data set''' + """Sync doctype customzations for a particular data set""" from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype - doctype = data['doctype'] + doctype = data["doctype"] update_schema = False def sync(key, custom_doctype, doctype_fieldname): @@ -111,11 +125,11 @@ def sync_customizations_for_doctype(data, folder): def sync_single_doctype(doc_type): def _insert(data): if data.get(doctype_fieldname) == doc_type: - data['doctype'] = custom_doctype + data["doctype"] = custom_doctype doc = frappe.get_doc(data) doc.db_insert() - if custom_doctype != 'Custom Field': + if custom_doctype != "Custom Field": frappe.db.delete(custom_doctype, {doctype_fieldname: doc_type}) for d in data[key]: @@ -135,59 +149,76 @@ def sync_customizations_for_doctype(data, folder): for doc_type in doctypes: # only sync the parent doctype and child doctype if there isn't any other child table json file - if doc_type == doctype or not os.path.exists(os.path.join(folder, frappe.scrub(doc_type)+".json")): + if doc_type == doctype or not os.path.exists( + os.path.join(folder, frappe.scrub(doc_type) + ".json") + ): sync_single_doctype(doc_type) - if data['custom_fields']: - sync('custom_fields', 'Custom Field', 'dt') + if data["custom_fields"]: + sync("custom_fields", "Custom Field", "dt") update_schema = True - if data['property_setters']: - sync('property_setters', 'Property Setter', 'doc_type') + if data["property_setters"]: + sync("property_setters", "Property Setter", "doc_type") - if data.get('custom_perms'): - sync('custom_perms', 'Custom DocPerm', 'parent') + if data.get("custom_perms"): + sync("custom_perms", "Custom DocPerm", "parent") - print('Updating customizations for {0}'.format(doctype)) + print("Updating customizations for {0}".format(doctype)) validate_fields_for_doctype(doctype) - if update_schema and not frappe.db.get_value('DocType', doctype, 'issingle'): + if update_schema and not frappe.db.get_value("DocType", doctype, "issingle"): frappe.db.updatedb(doctype) + def scrub(txt): return frappe.scrub(txt) + def scrub_dt_dn(dt, dn): """Returns in lowercase and code friendly names of doctype and name for certain types""" return scrub(dt), scrub(dn) + def get_module_path(module): """Returns path of the given module""" return frappe.get_module_path(module) + def get_doc_path(module, doctype, name): dt, dn = scrub_dt_dn(doctype, name) return os.path.join(get_module_path(module), dt, dn) + def reload_doc(module, dt=None, dn=None, force=False, reset_permissions=False): from frappe.modules.import_file import import_files + return import_files(module, dt, dn, force=force, reset_permissions=reset_permissions) + def export_doc(doctype, name, module=None): """Write a doc to standard path.""" from frappe.modules.export_file import write_document_file + print(doctype, name) - if not module: module = frappe.db.get_value('DocType', name, 'module') + if not module: + module = frappe.db.get_value("DocType", name, "module") write_document_file(frappe.get_doc(doctype, name), module) + def get_doctype_module(doctype): """Returns **Module Def** name of given doctype.""" + def make_modules_dict(): return dict(frappe.db.sql("select name, module from tabDocType")) + return frappe.cache().get_value("doctype_modules", make_modules_dict)[doctype] + doctype_python_modules = {} + + def load_doctype_module(doctype, module=None, prefix="", suffix=""): """Returns the module object for given doctype.""" if not module: @@ -203,21 +234,27 @@ def load_doctype_module(doctype, module=None, prefix="", suffix=""): if key not in doctype_python_modules: doctype_python_modules[key] = frappe.get_module(module_name) except ImportError as e: - raise ImportError('Module import failed for {0} ({1})'.format(doctype, module_name + ' Error: ' + str(e))) + raise ImportError( + "Module import failed for {0} ({1})".format(doctype, module_name + " Error: " + str(e)) + ) return doctype_python_modules[key] + def get_module_name(doctype, module, prefix="", suffix="", app=None): - return '{app}.{module}.doctype.{doctype}.{prefix}{doctype}{suffix}'.format(\ - app = scrub(app or get_module_app(module)), - module = scrub(module), - doctype = scrub(doctype), + return "{app}.{module}.doctype.{doctype}.{prefix}{doctype}{suffix}".format( + app=scrub(app or get_module_app(module)), + module=scrub(module), + doctype=scrub(doctype), prefix=prefix, - suffix=suffix) + suffix=suffix, + ) + def get_module_app(module): return frappe.local.module_app[scrub(module)] + def get_app_publisher(module): app = frappe.local.module_app[scrub(module)] if not app: @@ -225,14 +262,16 @@ def get_app_publisher(module): app_publisher = frappe.get_hooks(hook="app_publisher", app_name=app)[0] return app_publisher + def make_boilerplate(template, doc, opts=None): target_path = get_doc_path(doc.module, doc.doctype, doc.name) template_name = template.replace("controller", scrub(doc.name)) - if template_name.endswith('._py'): - template_name = template_name[:-4] + '.py' + if template_name.endswith("._py"): + template_name = template_name[:-4] + ".py" target_file_path = os.path.join(target_path, template_name) - if not doc: doc = {} + if not doc: + doc = {} app_publisher = get_app_publisher(doc.module) @@ -240,14 +279,14 @@ def make_boilerplate(template, doc, opts=None): if not opts: opts = {} - base_class = 'Document' - base_class_import = 'from frappe.model.document import Document' - if doc.get('is_tree'): - base_class = 'NestedSet' - base_class_import = 'from frappe.utils.nestedset import NestedSet' + base_class = "Document" + base_class_import = "from frappe.model.document import Document" + if doc.get("is_tree"): + base_class = "NestedSet" + base_class_import = "from frappe.utils.nestedset import NestedSet" - custom_controller = 'pass' - if doc.get('is_virtual'): + custom_controller = "pass" + if doc.get("is_virtual"): custom_controller = """ def db_insert(self): pass @@ -267,16 +306,22 @@ def make_boilerplate(template, doc, opts=None): def get_stats(self, args): pass""" - with open(target_file_path, 'w') as target: - with open(os.path.join(get_module_path("core"), "doctype", scrub(doc.doctype), - "boilerplate", template), 'r') as source: - target.write(frappe.as_unicode( - frappe.utils.cstr(source.read()).format( - app_publisher=app_publisher, - year=frappe.utils.nowdate()[:4], - classname=doc.name.replace(" ", "").replace("-", ""), - base_class_import=base_class_import, - base_class=base_class, - doctype=doc.name, **opts, - custom_controller=custom_controller) - )) + with open(target_file_path, "w") as target: + with open( + os.path.join(get_module_path("core"), "doctype", scrub(doc.doctype), "boilerplate", template), + "r", + ) as source: + target.write( + frappe.as_unicode( + frappe.utils.cstr(source.read()).format( + app_publisher=app_publisher, + year=frappe.utils.nowdate()[:4], + classname=doc.name.replace(" ", "").replace("-", ""), + base_class_import=base_class_import, + base_class=base_class, + doctype=doc.name, + **opts, + custom_controller=custom_controller + ) + ) + ) diff --git a/frappe/monitor.py b/frappe/monitor.py index 6bad03dfe9..74f9e06ef3 100644 --- a/frappe/monitor.py +++ b/frappe/monitor.py @@ -2,14 +2,15 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from datetime import datetime import json -import traceback -import frappe import os +import traceback import uuid +from datetime import datetime + import rq +import frappe MONITOR_REDIS_KEY = "monitor-transactions" MONITOR_MAX_ENTRIES = 1000000 diff --git a/frappe/oauth.py b/frappe/oauth.py index 67d346ad8a..e7fa101bfd 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -4,9 +4,11 @@ import hashlib import re from http import cookies from urllib.parse import unquote, urlparse + import jwt import pytz from oauthlib.openid import RequestValidator + import frappe from frappe.auth import LoginManager @@ -27,9 +29,9 @@ class OAuthWebRequestValidator(RequestValidator): # Is the client allowed to use the supplied redirect_uri? i.e. has # the client previously registered this EXACT redirect uri. - redirect_uris = frappe.db.get_value( - "OAuth Client", client_id, "redirect_uris" - ).split(get_url_delimiter()) + redirect_uris = frappe.db.get_value("OAuth Client", client_id, "redirect_uris").split( + get_url_delimiter() + ) if redirect_uri in redirect_uris: return True @@ -40,9 +42,7 @@ class OAuthWebRequestValidator(RequestValidator): # The redirect used if none has been supplied. # Prefer your clients to pre register a redirect uri rather than # supplying one on each authorization request. - redirect_uri = frappe.db.get_value( - "OAuth Client", client_id, "default_redirect_uri" - ) + redirect_uri = frappe.db.get_value("OAuth Client", client_id, "default_redirect_uri") return redirect_uri def validate_scopes(self, client_id, scopes, client, request, *args, **kwargs): @@ -57,9 +57,7 @@ class OAuthWebRequestValidator(RequestValidator): request.scopes = scopes # Apparently this is possible. return scopes - def validate_response_type( - self, client_id, response_type, client, request, *args, **kwargs - ): + def validate_response_type(self, client_id, response_type, client, request, *args, **kwargs): allowed_response_types = [ # From OAuth Client response_type field client.response_type.lower(), @@ -113,9 +111,7 @@ class OAuthWebRequestValidator(RequestValidator): elif "token" in frappe.form_dict: oc = frappe.get_doc( "OAuth Client", - frappe.db.get_value( - "OAuth Bearer Token", frappe.form_dict["token"], "client" - ), + frappe.db.get_value("OAuth Bearer Token", frappe.form_dict["token"], "client"), ) else: oc = frappe.get_doc( @@ -132,11 +128,7 @@ class OAuthWebRequestValidator(RequestValidator): return generate_json_error_response(e) cookie_dict = get_cookie_dict_from_headers(request) - user_id = ( - unquote(cookie_dict.get("user_id").value) - if "user_id" in cookie_dict - else "Guest" - ) + user_id = unquote(cookie_dict.get("user_id").value) if "user_id" in cookie_dict else "Guest" return frappe.session.user == user_id def authenticate_client_id(self, client_id, request, *args, **kwargs): @@ -162,22 +154,18 @@ class OAuthWebRequestValidator(RequestValidator): checkcodes.append(vcode["name"]) if code in checkcodes: - request.scopes = frappe.db.get_value( - "OAuth Authorization Code", code, "scopes" - ).split(get_url_delimiter()) + request.scopes = frappe.db.get_value("OAuth Authorization Code", code, "scopes").split( + get_url_delimiter() + ) request.user = frappe.db.get_value("OAuth Authorization Code", code, "user") code_challenge_method = frappe.db.get_value( "OAuth Authorization Code", code, "code_challenge_method" ) - code_challenge = frappe.db.get_value( - "OAuth Authorization Code", code, "code_challenge" - ) + code_challenge = frappe.db.get_value("OAuth Authorization Code", code, "code_challenge") if code_challenge and not request.code_verifier: if frappe.db.exists("OAuth Authorization Code", code): - frappe.delete_doc( - "OAuth Authorization Code", code, ignore_permissions=True - ) + frappe.delete_doc("OAuth Authorization Code", code, ignore_permissions=True) frappe.db.commit() return False @@ -197,12 +185,8 @@ class OAuthWebRequestValidator(RequestValidator): return False - def confirm_redirect_uri( - self, client_id, code, redirect_uri, client, *args, **kwargs - ): - saved_redirect_uri = frappe.db.get_value( - "OAuth Client", client_id, "default_redirect_uri" - ) + def confirm_redirect_uri(self, client_id, code, redirect_uri, client, *args, **kwargs): + saved_redirect_uri = frappe.db.get_value("OAuth Client", client_id, "default_redirect_uri") redirect_uris = frappe.db.get_value("OAuth Client", client_id, "redirect_uris") @@ -212,9 +196,7 @@ class OAuthWebRequestValidator(RequestValidator): return saved_redirect_uri == redirect_uri - def validate_grant_type( - self, client_id, grant_type, client, request, *args, **kwargs - ): + def validate_grant_type(self, client_id, grant_type, client, request, *args, **kwargs): # Clients should only be allowed to use one type of grant. # In this case, it must be "authorization_code" or "refresh_token" return grant_type in ["authorization_code", "refresh_token", "password"] @@ -270,17 +252,14 @@ class OAuthWebRequestValidator(RequestValidator): ) token_expiration_utc = token_expiration_local.astimezone(pytz.utc) is_token_valid = ( - frappe.utils.datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - < token_expiration_utc + frappe.utils.datetime.datetime.utcnow().replace(tzinfo=pytz.utc) < token_expiration_utc ) and otoken.status != "Revoked" - client_scopes = frappe.db.get_value( - "OAuth Client", otoken.client, "scopes" - ).split(get_url_delimiter()) + client_scopes = frappe.db.get_value("OAuth Client", otoken.client, "scopes").split( + get_url_delimiter() + ) are_scopes_valid = True for scp in scopes: - are_scopes_valid = ( - are_scopes_valid and True if scp in client_scopes else False - ) + are_scopes_valid = are_scopes_valid and True if scp in client_scopes else False return is_token_valid and are_scopes_valid @@ -291,9 +270,7 @@ class OAuthWebRequestValidator(RequestValidator): # return its scopes, these will be passed on to the refreshed # access token if the client did not specify a scope during the # request. - obearer_token = frappe.get_doc( - "OAuth Bearer Token", {"refresh_token": refresh_token} - ) + obearer_token = frappe.get_doc("OAuth Bearer Token", {"refresh_token": refresh_token}) return obearer_token.scopes def revoke_token(self, token, token_type_hint, request, *args, **kwargs): @@ -309,9 +286,7 @@ class OAuthWebRequestValidator(RequestValidator): if token_type_hint == "access_token": frappe.db.set_value("OAuth Bearer Token", token, "status", "Revoked") elif token_type_hint == "refresh_token": - frappe.db.set_value( - "OAuth Bearer Token", {"refresh_token": token}, "status", "Revoked" - ) + frappe.db.set_value("OAuth Bearer Token", {"refresh_token": token}, "status", "Revoked") else: frappe.db.set_value("OAuth Bearer Token", token, "status", "Revoked") frappe.db.commit() @@ -574,9 +549,7 @@ def calculate_at_hash(access_token, hash_alg): def delete_oauth2_data(): # Delete Invalid Authorization Code and Revoked Token commit_code, commit_token = False, False - code_list = frappe.get_all( - "OAuth Authorization Code", filters={"validity": "Invalid"} - ) + code_list = frappe.get_all("OAuth Authorization Code", filters={"validity": "Invalid"}) token_list = frappe.get_all("OAuth Bearer Token", filters={"status": "Revoked"}) if len(code_list) > 0: commit_code = True diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py index 1a6892d30d..f5367e9dc6 100644 --- a/frappe/parallel_test_runner.py +++ b/frappe/parallel_test_runner.py @@ -4,17 +4,20 @@ import re import sys import time import unittest + import click -import frappe import requests -from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config) +import frappe + +from .test_runner import SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config click_ctx = click.get_current_context(True) if click_ctx: click_ctx.color = True -class ParallelTestRunner(): + +class ParallelTestRunner: def __init__(self, app, site, build_number=1, total_builds=1): self.app = app self.site = site @@ -39,15 +42,15 @@ class ParallelTestRunner(): for fn in frappe.get_hooks("before_tests", app_name=self.app): frappe.get_attr(fn)() - test_module = frappe.get_module(f'{self.app}.tests') + test_module = frappe.get_module(f"{self.app}.tests") if hasattr(test_module, "global_test_dependencies"): for doctype in test_module.global_test_dependencies: make_test_records(doctype) elapsed = time.time() - start_time - elapsed = click.style(f' ({elapsed:.03}s)', fg='red') - click.echo(f'Before Test {elapsed}') + elapsed = click.style(f" ({elapsed:.03}s)", fg="red") + click.echo(f"Before Test {elapsed}") def run_tests(self): self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2) @@ -58,9 +61,10 @@ class ParallelTestRunner(): self.print_result() def run_tests_for_file(self, file_info): - if not file_info: return + if not file_info: + return - frappe.set_user('Administrator') + frappe.set_user("Administrator") path, filename = file_info module = self.get_module(path, filename) self.create_test_dependency_records(module, path, filename) @@ -76,10 +80,10 @@ class ParallelTestRunner(): if os.path.basename(os.path.dirname(path)) == "doctype": # test_data_migration_connector.py > data_migration_connector.json - test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json") + test_record_filename = re.sub("^test_", "", filename).replace(".py", ".json") test_record_file_path = os.path.join(path, test_record_filename) if os.path.exists(test_record_file_path): - with open(test_record_file_path, 'r') as f: + with open(test_record_file_path, "r") as f: doc = json.loads(f.read()) doctype = doc["name"] make_test_records(doctype) @@ -87,12 +91,12 @@ class ParallelTestRunner(): def get_module(self, path, filename): app_path = frappe.get_pymodule_path(self.app) relative_path = os.path.relpath(path, app_path) - if relative_path == '.': + if relative_path == ".": module_name = self.app else: - relative_path = relative_path.replace('/', '.') + relative_path = relative_path.replace("/", ".") module_name = os.path.splitext(filename)[0] - module_name = f'{self.app}.{relative_path}.{module_name}' + module_name = f"{self.app}.{relative_path}.{module_name}" return frappe.get_module(module_name) @@ -100,14 +104,14 @@ class ParallelTestRunner(): self.test_result.printErrors() click.echo(self.test_result) if self.test_result.failures or self.test_result.errors: - if os.environ.get('CI'): + if os.environ.get("CI"): sys.exit(1) def get_test_file_list(self): test_list = get_all_tests(self.app) split_size = frappe.utils.ceil(len(test_list) / self.total_builds) # [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2 - test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)] + test_chunks = [test_list[x : x + split_size] for x in range(0, len(test_list), split_size)] return test_chunks[self.build_number - 1] @@ -116,18 +120,18 @@ class ParallelTestResult(unittest.TextTestResult): self._started_at = time.time() super(unittest.TextTestResult, self).startTest(test) test_class = unittest.util.strclass(test.__class__) - if not hasattr(self, 'current_test_class') or self.current_test_class != test_class: + if not hasattr(self, "current_test_class") or self.current_test_class != test_class: click.echo(f"\n{unittest.util.strclass(test.__class__)}") self.current_test_class = test_class def getTestMethodName(self, test): - return test._testMethodName if hasattr(test, '_testMethodName') else str(test) + return test._testMethodName if hasattr(test, "_testMethodName") else str(test) def addSuccess(self, test): super(unittest.TextTestResult, self).addSuccess(test) elapsed = time.time() - self._started_at threshold_passed = elapsed >= SLOW_TEST_THRESHOLD - elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else '' + elapsed = click.style(f" ({elapsed:.03}s)", fg="red") if threshold_passed else "" click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}") def addError(self, test, err): @@ -151,9 +155,9 @@ class ParallelTestResult(unittest.TextTestResult): click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}") def printErrors(self): - click.echo('\n') - self.printErrorList(' ERROR ', self.errors, 'red') - self.printErrorList(' FAIL ', self.failures, 'red') + click.echo("\n") + self.printErrorList(" ERROR ", self.errors, "red") + self.printErrorList(" FAIL ", self.failures, "red") def printErrorList(self, flavour, errors, color): for test, err in errors: @@ -165,10 +169,11 @@ class ParallelTestResult(unittest.TextTestResult): def __str__(self): return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}" + def get_all_tests(app): test_file_list = [] for path, folders, files in os.walk(frappe.get_pymodule_path(app)): - for dontwalk in ('locals', '.git', 'public', '__pycache__'): + for dontwalk in ("locals", ".git", "public", "__pycache__"): if dontwalk in folders: folders.remove(dontwalk) @@ -181,61 +186,61 @@ def get_all_tests(app): continue for filename in files: - if filename.startswith("test_") and filename.endswith(".py") \ - and filename != 'test_runner.py': + if filename.startswith("test_") and filename.endswith(".py") and filename != "test_runner.py": test_file_list.append([path, filename]) return test_file_list class ParallelTestWithOrchestrator(ParallelTestRunner): - ''' - This can be used to balance-out test time across multiple instances - This is dependent on external orchestrator which returns next test to run - - orchestrator endpoints - - register-instance (, , test_spec_list) - - get-next-test-spec (, ) - - test-completed (, ) - ''' + """ + This can be used to balance-out test time across multiple instances + This is dependent on external orchestrator which returns next test to run + + orchestrator endpoints + - register-instance (, , test_spec_list) + - get-next-test-spec (, ) + - test-completed (, ) + """ + def __init__(self, app, site): - self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL') + self.orchestrator_url = os.environ.get("ORCHESTRATOR_URL") if not self.orchestrator_url: - click.echo('ORCHESTRATOR_URL environment variable not found!') - click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator') + click.echo("ORCHESTRATOR_URL environment variable not found!") + click.echo("Pass public URL after hosting https://github.com/frappe/test-orchestrator") sys.exit(1) - self.ci_build_id = os.environ.get('CI_BUILD_ID') - self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10) + self.ci_build_id = os.environ.get("CI_BUILD_ID") + self.ci_instance_id = os.environ.get("CI_INSTANCE_ID") or frappe.generate_hash(length=10) if not self.ci_build_id: - click.echo('CI_BUILD_ID environment variable not found!') + click.echo("CI_BUILD_ID environment variable not found!") sys.exit(1) ParallelTestRunner.__init__(self, app, site) def run_tests(self): - self.test_status = 'ongoing' + self.test_status = "ongoing" self.register_instance() super().run_tests() def get_test_file_list(self): - while self.test_status == 'ongoing': + while self.test_status == "ongoing": yield self.get_next_test() def register_instance(self): test_spec_list = get_all_tests(self.app) - response_data = self.call_orchestrator('register-instance', data={ - 'test_spec_list': test_spec_list - }) - self.is_master = response_data.get('is_master') + response_data = self.call_orchestrator( + "register-instance", data={"test_spec_list": test_spec_list} + ) + self.is_master = response_data.get("is_master") def get_next_test(self): - response_data = self.call_orchestrator('get-next-test-spec') - self.test_status = response_data.get('status') - return response_data.get('next_test') + response_data = self.call_orchestrator("get-next-test-spec") + self.test_status = response_data.get("status") + return response_data.get("next_test") def print_result(self): - self.call_orchestrator('test-completed') + self.call_orchestrator("test-completed") return super().print_result() def call_orchestrator(self, endpoint, data=None): @@ -244,15 +249,15 @@ class ParallelTestWithOrchestrator(ParallelTestRunner): # add repo token header # build id in header headers = { - 'CI-BUILD-ID': self.ci_build_id, - 'CI-INSTANCE-ID': self.ci_instance_id, - 'REPO-TOKEN': '2948288382838DE' + "CI-BUILD-ID": self.ci_build_id, + "CI-INSTANCE-ID": self.ci_instance_id, + "REPO-TOKEN": "2948288382838DE", } - url = f'{self.orchestrator_url}/{endpoint}' + url = f"{self.orchestrator_url}/{endpoint}" res = requests.get(url, json=data, headers=headers) res.raise_for_status() response_data = {} - if 'application/json' in res.headers.get('content-type'): + if "application/json" in res.headers.get("content-type"): response_data = res.json() return response_data diff --git a/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py b/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py index 24f915c512..25154ac2c8 100644 --- a/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py +++ b/frappe/patches/v10_0/enable_chat_by_default_within_system_settings.py @@ -1,13 +1,13 @@ - import frappe + def execute(): - frappe.reload_doctype('System Settings') - doc = frappe.get_single('System Settings') + frappe.reload_doctype("System Settings") + doc = frappe.get_single("System Settings") doc.enable_chat = 1 # Changes prescribed by Nabin Hait (nabin@frappe.io) - doc.flags.ignore_mandatory = True + doc.flags.ignore_mandatory = True doc.flags.ignore_permissions = True - doc.save() \ No newline at end of file + doc.save() diff --git a/frappe/patches/v10_0/enhance_security.py b/frappe/patches/v10_0/enhance_security.py index 4f6ca4faa1..247e8ed7f4 100644 --- a/frappe/patches/v10_0/enhance_security.py +++ b/frappe/patches/v10_0/enhance_security.py @@ -1,7 +1,7 @@ import frappe - from frappe.utils import cint + def execute(): """ The motive of this patch is to increase the overall security in frappe framework @@ -14,7 +14,7 @@ def execute(): Security is something we take very seriously at frappe, and hence we chose to make security tighter by default. """ - doc = frappe.get_single('System Settings') + doc = frappe.get_single("System Settings") # Enforce a Password Policy if cint(doc.enable_password_policy) == 0: diff --git a/frappe/patches/v10_0/increase_single_table_column_length.py b/frappe/patches/v10_0/increase_single_table_column_length.py index e578d192fc..21b5a790ab 100644 --- a/frappe/patches/v10_0/increase_single_table_column_length.py +++ b/frappe/patches/v10_0/increase_single_table_column_length.py @@ -1,9 +1,9 @@ - """ Run this after updating country_info.json and or """ import frappe + def execute(): for col in ("field", "doctype"): frappe.db.sql_ddl("alter table `tabSingles` modify column `{0}` varchar(255)".format(col)) diff --git a/frappe/patches/v10_0/migrate_passwords_passlib.py b/frappe/patches/v10_0/migrate_passwords_passlib.py index d0b36efbaa..b18581ee3e 100644 --- a/frappe/patches/v10_0/migrate_passwords_passlib.py +++ b/frappe/patches/v10_0/migrate_passwords_passlib.py @@ -1,19 +1,22 @@ - import frappe from frappe.utils.password import LegacyPassword def execute(): - all_auths = frappe.db.sql("""SELECT `name`, `password`, `salt` FROM `__Auth` + all_auths = frappe.db.sql( + """SELECT `name`, `password`, `salt` FROM `__Auth` WHERE doctype='User' AND `fieldname`='password'""", - as_dict=True) + as_dict=True, + ) for auth in all_auths: if auth.salt and auth.salt != "": - pwd = LegacyPassword.hash(auth.password, salt=auth.salt.encode('UTF-8')) - frappe.db.sql("""UPDATE `__Auth` SET `password`=%(pwd)s, `salt`=NULL + pwd = LegacyPassword.hash(auth.password, salt=auth.salt.encode("UTF-8")) + frappe.db.sql( + """UPDATE `__Auth` SET `password`=%(pwd)s, `salt`=NULL WHERE `doctype`='User' AND `fieldname`='password' AND `name`=%(user)s""", - {'pwd': pwd, 'user': auth.name}) + {"pwd": pwd, "user": auth.name}, + ) frappe.reload_doctype("User") diff --git a/frappe/patches/v10_0/modify_naming_series_table.py b/frappe/patches/v10_0/modify_naming_series_table.py index ca6114eb55..2b0e5f9f6d 100644 --- a/frappe/patches/v10_0/modify_naming_series_table.py +++ b/frappe/patches/v10_0/modify_naming_series_table.py @@ -1,8 +1,10 @@ -''' +""" Modify the Integer 10 Digits Value to BigInt 20 Digit value to generate long Naming Series -''' +""" import frappe + + def execute(): - frappe.db.sql(""" ALTER TABLE `tabSeries` MODIFY current BIGINT """) + frappe.db.sql(""" ALTER TABLE `tabSeries` MODIFY current BIGINT """) diff --git a/frappe/patches/v10_0/modify_smallest_currency_fraction.py b/frappe/patches/v10_0/modify_smallest_currency_fraction.py index 9469d546ce..b86642e5f6 100644 --- a/frappe/patches/v10_0/modify_smallest_currency_fraction.py +++ b/frappe/patches/v10_0/modify_smallest_currency_fraction.py @@ -3,5 +3,6 @@ import frappe + def execute(): - frappe.db.set_value('Currency', 'USD', 'smallest_currency_fraction_value', '0.01') \ No newline at end of file + frappe.db.set_value("Currency", "USD", "smallest_currency_fraction_value", "0.01") diff --git a/frappe/patches/v10_0/refactor_social_login_keys.py b/frappe/patches/v10_0/refactor_social_login_keys.py index a3f08939ec..7beaa4e53b 100644 --- a/frappe/patches/v10_0/refactor_social_login_keys.py +++ b/frappe/patches/v10_0/refactor_social_login_keys.py @@ -1,6 +1,7 @@ import frappe from frappe.utils import cstr + def execute(): # Update Social Logins in User run_patch() @@ -8,7 +9,7 @@ def execute(): # Create Social Login Key(s) from Social Login Keys frappe.reload_doc("integrations", "doctype", "social_login_key", force=True) - if not frappe.db.exists('DocType', 'Social Login Keys'): + if not frappe.db.exists("DocType", "Social Login Keys"): return social_login_keys = frappe.get_doc("Social Login Keys", "Social Login Keys") @@ -29,7 +30,9 @@ def execute(): frappe_login_key.base_url = social_login_keys.get("frappe_server_url") frappe_login_key.client_id = social_login_keys.get("frappe_client_id") frappe_login_key.client_secret = social_login_keys.get("frappe_client_secret") - if not (frappe_login_key.client_secret and frappe_login_key.client_id and frappe_login_key.base_url): + if not ( + frappe_login_key.client_secret and frappe_login_key.client_id and frappe_login_key.base_url + ): frappe_login_key.enable_social_login = 0 frappe_login_key.save() @@ -55,28 +58,40 @@ def execute(): frappe.delete_doc("DocType", "Social Login Keys") + def run_patch(): frappe.reload_doc("core", "doctype", "user", force=True) frappe.reload_doc("core", "doctype", "user_social_login", force=True) - users = frappe.get_all("User", fields=["*"], filters={"name":("not in", ["Administrator", "Guest"])}) + users = frappe.get_all( + "User", fields=["*"], filters={"name": ("not in", ["Administrator", "Guest"])} + ) for user in users: idx = 0 if user.frappe_userid: - insert_user_social_login(user.name, user.modified_by, 'frappe', idx, userid=user.frappe_userid) + insert_user_social_login(user.name, user.modified_by, "frappe", idx, userid=user.frappe_userid) idx += 1 if user.fb_userid or user.fb_username: - insert_user_social_login(user.name, user.modified_by, 'facebook', idx, userid=user.fb_userid, username=user.fb_username) + insert_user_social_login( + user.name, user.modified_by, "facebook", idx, userid=user.fb_userid, username=user.fb_username + ) idx += 1 if user.github_userid or user.github_username: - insert_user_social_login(user.name, user.modified_by, 'github', idx, userid=user.github_userid, username=user.github_username) + insert_user_social_login( + user.name, + user.modified_by, + "github", + idx, + userid=user.github_userid, + username=user.github_username, + ) idx += 1 if user.google_userid: - insert_user_social_login(user.name, user.modified_by, 'google', idx, userid=user.google_userid) + insert_user_social_login(user.name, user.modified_by, "google", idx, userid=user.google_userid) idx += 1 @@ -94,7 +109,7 @@ def insert_user_social_login(user, modified_by, provider, idx, userid=None, user "User", "social_logins", cstr(idx), - provider + provider, ] if userid: @@ -105,26 +120,40 @@ def insert_user_social_login(user, modified_by, provider, idx, userid=None, user source_cols.append("username") values.append(username) - query = """INSERT INTO `tabUser Social Login` (`{source_cols}`) VALUES ({values}) """.format( - source_cols = "`, `".join(source_cols), - values= ", ".join([frappe.db.escape(d) for d in values]) + source_cols="`, `".join(source_cols), values=", ".join([frappe.db.escape(d) for d in values]) ) frappe.db.sql(query) + def get_provider_field_map(): - return frappe._dict({ - "frappe": ["frappe_userid"], - "facebook": ["fb_userid", "fb_username"], - "github": ["github_userid", "github_username"], - "google": ["google_userid"], - }) + return frappe._dict( + { + "frappe": ["frappe_userid"], + "facebook": ["fb_userid", "fb_username"], + "github": ["github_userid", "github_username"], + "google": ["google_userid"], + } + ) + def get_provider_fields(provider): - return get_provider_field_map().get(provider) + return get_provider_field_map().get(provider) + def get_standard_cols(): - return ["name", "creation", "modified", "owner", "modified_by", "parent", "parenttype", "parentfield", "idx", "provider"] + return [ + "name", + "creation", + "modified", + "owner", + "modified_by", + "parent", + "parenttype", + "parentfield", + "idx", + "provider", + ] diff --git a/frappe/patches/v10_0/reload_countries_and_currencies.py b/frappe/patches/v10_0/reload_countries_and_currencies.py index 8d019a4855..927fb10a7c 100644 --- a/frappe/patches/v10_0/reload_countries_and_currencies.py +++ b/frappe/patches/v10_0/reload_countries_and_currencies.py @@ -1,4 +1,3 @@ - """ Run this after updating country_info.json and or """ diff --git a/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py b/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py index 54839cfe02..7dc4601ce4 100644 --- a/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py +++ b/frappe/patches/v10_0/remove_custom_field_for_disabled_domain.py @@ -1,6 +1,6 @@ - import frappe + def execute(): frappe.reload_doc("core", "doctype", "domain") frappe.reload_doc("core", "doctype", "has_domain") diff --git a/frappe/patches/v10_0/set_default_locking_time.py b/frappe/patches/v10_0/set_default_locking_time.py index 11993e1163..7c9f3ceff1 100644 --- a/frappe/patches/v10_0/set_default_locking_time.py +++ b/frappe/patches/v10_0/set_default_locking_time.py @@ -3,6 +3,7 @@ import frappe + def execute(): frappe.reload_doc("core", "doctype", "system_settings") - frappe.db.set_value('System Settings', None, "allow_login_after_fail", 60) \ No newline at end of file + frappe.db.set_value("System Settings", None, "allow_login_after_fail", 60) diff --git a/frappe/patches/v10_0/set_no_copy_to_workflow_state.py b/frappe/patches/v10_0/set_no_copy_to_workflow_state.py index eb469b8452..72b1c87ff2 100644 --- a/frappe/patches/v10_0/set_no_copy_to_workflow_state.py +++ b/frappe/patches/v10_0/set_no_copy_to_workflow_state.py @@ -1,10 +1,11 @@ - import frappe + def execute(): - for dt in frappe.get_all("Workflow", fields=['name', 'document_type', 'workflow_state_field']): - fieldname = frappe.db.get_value("Custom Field", filters={'dt': dt.document_type, - 'fieldname': dt.workflow_state_field}) + for dt in frappe.get_all("Workflow", fields=["name", "document_type", "workflow_state_field"]): + fieldname = frappe.db.get_value( + "Custom Field", filters={"dt": dt.document_type, "fieldname": dt.workflow_state_field} + ) if fieldname: custom_field = frappe.get_doc("Custom Field", fieldname) diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index 7e84c5ae24..d652efcef7 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -8,22 +8,19 @@ from frappe.utils import cint # for custom doctypes and user may not be able to # see previous customization + def execute(): - custom_doctypes = frappe.get_all('DocType', filters={ - 'custom': 1 - }) + custom_doctypes = frappe.get_all("DocType", filters={"custom": 1}) for doctype in custom_doctypes: - property_setters = frappe.get_all('Property Setter', filters={ - 'doc_type': doctype.name, - 'doctype_or_field': 'DocField' - }, fields=['name', 'property', 'value', 'property_type', 'field_name']) - - custom_fields = frappe.get_all('Custom Field', - filters={'dt': doctype.name}, - fields=['*'] + property_setters = frappe.get_all( + "Property Setter", + filters={"doc_type": doctype.name, "doctype_or_field": "DocField"}, + fields=["name", "property", "value", "property_type", "field_name"], ) + custom_fields = frappe.get_all("Custom Field", filters={"dt": doctype.name}, fields=["*"]) + property_setter_map = {} for prop in property_setters: @@ -35,19 +32,19 @@ def execute(): for df in meta.fields: ps = property_setter_map.get(df.fieldname, None) if ps: - value = cint(ps.value) if ps.property_type == 'Int' else ps.value + value = cint(ps.value) if ps.property_type == "Int" else ps.value df.set(ps.property, value) for cf in custom_fields: - cf.pop('parenttype') - cf.pop('parentfield') - cf.pop('parent') - cf.pop('name') + cf.pop("parenttype") + cf.pop("parentfield") + cf.pop("parent") + cf.pop("name") field = meta.get_field(cf.fieldname) if field: field.update(cf) else: - df = frappe.new_doc('DocField', meta, 'fields') + df = frappe.new_doc("DocField", meta, "fields") df.update(cf) meta.fields.append(df) frappe.db.delete("Custom Field", {"name": cf.name}) diff --git a/frappe/patches/v11_0/change_email_signature_fieldtype.py b/frappe/patches/v11_0/change_email_signature_fieldtype.py index 7c57aa044e..8ca13437e6 100644 --- a/frappe/patches/v11_0/change_email_signature_fieldtype.py +++ b/frappe/patches/v11_0/change_email_signature_fieldtype.py @@ -3,11 +3,14 @@ import frappe + def execute(): - signatures = frappe.db.get_list('User', {'email_signature': ['!=', '']},['name', 'email_signature']) - frappe.reload_doc('core', 'doctype', 'user') + signatures = frappe.db.get_list( + "User", {"email_signature": ["!=", ""]}, ["name", "email_signature"] + ) + frappe.reload_doc("core", "doctype", "user") for d in signatures: - signature = d.get('email_signature') - signature = signature.replace('\n', '
') - signature = '
' + signature + '
' - frappe.db.set_value('User', d.get('name'), 'email_signature', signature) \ No newline at end of file + signature = d.get("email_signature") + signature = signature.replace("\n", "
") + signature = "
" + signature + "
" + frappe.db.set_value("User", d.get("name"), "email_signature", signature) diff --git a/frappe/patches/v11_0/copy_fetch_data_from_options.py b/frappe/patches/v11_0/copy_fetch_data_from_options.py index e256c7085f..1f141188f9 100644 --- a/frappe/patches/v11_0/copy_fetch_data_from_options.py +++ b/frappe/patches/v11_0/copy_fetch_data_from_options.py @@ -1,32 +1,38 @@ - import frappe + def execute(): frappe.reload_doc("core", "doctype", "docfield", force=True) frappe.reload_doc("custom", "doctype", "custom_field", force=True) frappe.reload_doc("custom", "doctype", "customize_form_field", force=True) frappe.reload_doc("custom", "doctype", "property_setter", force=True) - frappe.db.sql(''' + frappe.db.sql( + """ update `tabDocField` set fetch_from = options, options='' where options like '%.%' and (fetch_from is NULL OR fetch_from='') and fieldtype in ('Data', 'Read Only', 'Text', 'Small Text', 'Text Editor', 'Code', 'Link', 'Check') and fieldname!='naming_series' - ''') + """ + ) - frappe.db.sql(''' + frappe.db.sql( + """ update `tabCustom Field` set fetch_from = options, options='' where options like '%.%' and (fetch_from is NULL OR fetch_from='') and fieldtype in ('Data', 'Read Only', 'Text', 'Small Text', 'Text Editor', 'Code', 'Link', 'Check') and fieldname!='naming_series' - ''') + """ + ) - frappe.db.sql(''' + frappe.db.sql( + """ update `tabProperty Setter` set property="fetch_from", name=concat(doc_type, '-', field_name, '-', property) where property="options" and value like '%.%' and property_type in ('Data', 'Read Only', 'Text', 'Small Text', 'Text Editor', 'Code', 'Link', 'Check') and field_name!='naming_series' - ''') \ No newline at end of file + """ + ) diff --git a/frappe/patches/v11_0/create_contact_for_user.py b/frappe/patches/v11_0/create_contact_for_user.py index 5a483b630e..b921787422 100644 --- a/frappe/patches/v11_0/create_contact_for_user.py +++ b/frappe/patches/v11_0/create_contact_for_user.py @@ -1,25 +1,28 @@ +import re import frappe from frappe.core.doctype.user.user import create_contact -import re + def execute(): - """ Create Contact for each User if not present """ - frappe.reload_doc('integrations', 'doctype', 'google_contacts') - frappe.reload_doc('contacts', 'doctype', 'contact') - frappe.reload_doc('core', 'doctype', 'dynamic_link') + """Create Contact for each User if not present""" + frappe.reload_doc("integrations", "doctype", "google_contacts") + frappe.reload_doc("contacts", "doctype", "contact") + frappe.reload_doc("core", "doctype", "dynamic_link") contact_meta = frappe.get_meta("Contact") if contact_meta.has_field("phone_nos") and contact_meta.has_field("email_ids"): - frappe.reload_doc('contacts', 'doctype', 'contact_phone') - frappe.reload_doc('contacts', 'doctype', 'contact_email') + frappe.reload_doc("contacts", "doctype", "contact_phone") + frappe.reload_doc("contacts", "doctype", "contact_email") - users = frappe.get_all('User', filters={"name": ('not in', 'Administrator, Guest')}, fields=["*"]) + users = frappe.get_all("User", filters={"name": ("not in", "Administrator, Guest")}, fields=["*"]) for user in users: - if frappe.db.exists("Contact", {"email_id": user.email}) or frappe.db.exists("Contact Email", {"email_id": user.email}): + if frappe.db.exists("Contact", {"email_id": user.email}) or frappe.db.exists( + "Contact Email", {"email_id": user.email} + ): continue if user.first_name: - user.first_name = re.sub("[<>]+", '', frappe.safe_decode(user.first_name)) + user.first_name = re.sub("[<>]+", "", frappe.safe_decode(user.first_name)) if user.last_name: - user.last_name = re.sub("[<>]+", '', frappe.safe_decode(user.last_name)) + user.last_name = re.sub("[<>]+", "", frappe.safe_decode(user.last_name)) create_contact(user, ignore_links=True, ignore_mandatory=True) diff --git a/frappe/patches/v11_0/delete_all_prepared_reports.py b/frappe/patches/v11_0/delete_all_prepared_reports.py index 77f041e3ee..097be97053 100644 --- a/frappe/patches/v11_0/delete_all_prepared_reports.py +++ b/frappe/patches/v11_0/delete_all_prepared_reports.py @@ -1,8 +1,8 @@ - import frappe + def execute(): - if frappe.db.table_exists('Prepared Report'): + if frappe.db.table_exists("Prepared Report"): frappe.reload_doc("core", "doctype", "prepared_report") prepared_reports = frappe.get_all("Prepared Report") for report in prepared_reports: diff --git a/frappe/patches/v11_0/delete_duplicate_user_permissions.py b/frappe/patches/v11_0/delete_duplicate_user_permissions.py index 518c1f7714..b986c6f825 100644 --- a/frappe/patches/v11_0/delete_duplicate_user_permissions.py +++ b/frappe/patches/v11_0/delete_duplicate_user_permissions.py @@ -1,13 +1,20 @@ - import frappe + def execute(): - duplicateRecords = frappe.db.sql("""select count(name) as `count`, allow, user, for_value + duplicateRecords = frappe.db.sql( + """select count(name) as `count`, allow, user, for_value from `tabUser Permission` group by allow, user, for_value - having count(*) > 1 """, as_dict=1) + having count(*) > 1 """, + as_dict=1, + ) for record in duplicateRecords: - frappe.db.sql("""delete from `tabUser Permission` - where allow=%s and user=%s and for_value=%s limit {0}""" - .format(record.count - 1), (record.allow, record.user, record.for_value)) + frappe.db.sql( + """delete from `tabUser Permission` + where allow=%s and user=%s and for_value=%s limit {0}""".format( + record.count - 1 + ), + (record.allow, record.user, record.for_value), + ) diff --git a/frappe/patches/v11_0/drop_column_apply_user_permissions.py b/frappe/patches/v11_0/drop_column_apply_user_permissions.py index 629d5a5da4..bfc4aee72c 100644 --- a/frappe/patches/v11_0/drop_column_apply_user_permissions.py +++ b/frappe/patches/v11_0/drop_column_apply_user_permissions.py @@ -1,15 +1,14 @@ - import frappe + def execute(): - column = 'apply_user_permissions' - to_remove = ['DocPerm', 'Custom DocPerm'] + column = "apply_user_permissions" + to_remove = ["DocPerm", "Custom DocPerm"] for doctype in to_remove: if frappe.db.table_exists(doctype): if column in frappe.db.get_table_columns(doctype): frappe.db.sql("alter table `tab{0}` drop column {1}".format(doctype, column)) - frappe.reload_doc('core', 'doctype', 'docperm', force=True) - frappe.reload_doc('core', 'doctype', 'custom_docperm', force=True) - + frappe.reload_doc("core", "doctype", "docperm", force=True) + frappe.reload_doc("core", "doctype", "custom_docperm", force=True) diff --git a/frappe/patches/v11_0/fix_order_by_in_reports_json.py b/frappe/patches/v11_0/fix_order_by_in_reports_json.py index 096e0e7654..3dfec0954f 100644 --- a/frappe/patches/v11_0/fix_order_by_in_reports_json.py +++ b/frappe/patches/v11_0/fix_order_by_in_reports_json.py @@ -1,26 +1,35 @@ +import json + +import frappe -import frappe, json def execute(): - reports_data = frappe.get_all('Report', - filters={'json': ['not like', '%%%"order_by": "`tab%%%'], - 'report_type': 'Report Builder', 'is_standard': 'No'}, fields=['name']) + reports_data = frappe.get_all( + "Report", + filters={ + "json": ["not like", '%%%"order_by": "`tab%%%'], + "report_type": "Report Builder", + "is_standard": "No", + }, + fields=["name"], + ) for d in reports_data: - doc = frappe.get_doc('Report', d.get('name')) + doc = frappe.get_doc("Report", d.get("name")) - if not doc.get('json'): continue + if not doc.get("json"): + continue - json_data = json.loads(doc.get('json')) + json_data = json.loads(doc.get("json")) parts = [] - if ('order_by' in json_data) and ('.' in json_data.get('order_by')): - parts = json_data.get('order_by').split('.') + if ("order_by" in json_data) and ("." in json_data.get("order_by")): + parts = json_data.get("order_by").split(".") - sort_by = parts[1].split(' ') + sort_by = parts[1].split(" ") - json_data['order_by'] = '`tab{0}`.`{1}`'.format(doc.ref_doctype, sort_by[0]) - json_data['order_by'] += ' {0}'.format(sort_by[1]) if len(sort_by) > 1 else '' + json_data["order_by"] = "`tab{0}`.`{1}`".format(doc.ref_doctype, sort_by[0]) + json_data["order_by"] += " {0}".format(sort_by[1]) if len(sort_by) > 1 else "" doc.json = json.dumps(json_data) doc.save() diff --git a/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py b/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py index a099b89b40..277235ce04 100644 --- a/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py +++ b/frappe/patches/v11_0/make_all_prepared_report_attachments_private.py @@ -1,12 +1,18 @@ - import frappe def execute(): - if frappe.db.count("File", filters={"attached_to_doctype": "Prepared Report", "is_private": 0}) > 10000: + if ( + frappe.db.count("File", filters={"attached_to_doctype": "Prepared Report", "is_private": 0}) + > 10000 + ): frappe.db.auto_commit_on_many_writes = True - files = frappe.get_all("File", fields=["name", "attached_to_name"], filters={"attached_to_doctype": "Prepared Report", "is_private": 0}) + files = frappe.get_all( + "File", + fields=["name", "attached_to_name"], + filters={"attached_to_doctype": "Prepared Report", "is_private": 0}, + ) for file_dict in files: # For some reason Prepared Report doc might not exist, check if it exists first if frappe.db.exists("Prepared Report", file_dict.attached_to_name): @@ -23,4 +29,3 @@ def execute(): if frappe.db.auto_commit_on_many_writes: frappe.db.auto_commit_on_many_writes = False - diff --git a/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py b/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py index e5b18368db..c867502fde 100644 --- a/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py +++ b/frappe/patches/v11_0/migrate_report_settings_for_new_listview.py @@ -1,34 +1,34 @@ +import json -import frappe, json +import frappe -def execute(): - ''' - Migrate JSON field of Report according to changes in New ListView - Rename key columns to fields - Rename key add_total_row to add_totals_row - Convert sort_by and sort_order to order_by - ''' - - reports = frappe.get_all('Report', { 'report_type': 'Report Builder' }) - for report_name in reports: - settings = frappe.db.get_value('Report', report_name, 'json') - if not settings: - continue +def execute(): + """ + Migrate JSON field of Report according to changes in New ListView + Rename key columns to fields + Rename key add_total_row to add_totals_row + Convert sort_by and sort_order to order_by + """ - settings = frappe._dict(json.loads(settings)) + reports = frappe.get_all("Report", {"report_type": "Report Builder"}) - # columns -> fields - settings.fields = settings.columns or [] - settings.pop('columns', None) + for report_name in reports: + settings = frappe.db.get_value("Report", report_name, "json") + if not settings: + continue - # sort_by + order_by -> order_by - settings.order_by = (settings.sort_by or 'modified') + ' ' + (settings.order_by or 'desc') + settings = frappe._dict(json.loads(settings)) - # add_total_row -> add_totals_row - settings.add_totals_row = settings.add_total_row - settings.pop('add_total_row', None) + # columns -> fields + settings.fields = settings.columns or [] + settings.pop("columns", None) - frappe.db.set_value('Report', report_name, 'json', json.dumps(settings)) + # sort_by + order_by -> order_by + settings.order_by = (settings.sort_by or "modified") + " " + (settings.order_by or "desc") + # add_total_row -> add_totals_row + settings.add_totals_row = settings.add_total_row + settings.pop("add_total_row", None) + frappe.db.set_value("Report", report_name, "json", json.dumps(settings)) diff --git a/frappe/patches/v11_0/multiple_references_in_events.py b/frappe/patches/v11_0/multiple_references_in_events.py index 9fa5968d8e..3eddea92bb 100644 --- a/frappe/patches/v11_0/multiple_references_in_events.py +++ b/frappe/patches/v11_0/multiple_references_in_events.py @@ -1,19 +1,24 @@ - import frappe + def execute(): - frappe.reload_doctype('Event') + frappe.reload_doctype("Event") # Rename "Cancel" to "Cancelled" frappe.db.sql("""UPDATE tabEvent set event_type='Cancelled' where event_type='Cancel'""") # Move references to Participants table - events = frappe.db.sql("""SELECT name, ref_type, ref_name FROM tabEvent WHERE ref_type!=''""", as_dict=True) + events = frappe.db.sql( + """SELECT name, ref_type, ref_name FROM tabEvent WHERE ref_type!=''""", as_dict=True + ) for event in events: if event.ref_type and event.ref_name: try: - e = frappe.get_doc('Event', event.name) - e.append('event_participants', {"reference_doctype": event.ref_type, "reference_docname": event.ref_name}) - e.flags.ignore_mandatory = True + e = frappe.get_doc("Event", event.name) + e.append( + "event_participants", + {"reference_doctype": event.ref_type, "reference_docname": event.ref_name}, + ) + e.flags.ignore_mandatory = True e.flags.ignore_permissions = True e.save() except Exception: - frappe.log_error(frappe.get_traceback()) \ No newline at end of file + frappe.log_error(frappe.get_traceback()) diff --git a/frappe/patches/v11_0/reload_and_rename_view_log.py b/frappe/patches/v11_0/reload_and_rename_view_log.py index fa0432c4e2..67a18f3a30 100644 --- a/frappe/patches/v11_0/reload_and_rename_view_log.py +++ b/frappe/patches/v11_0/reload_and_rename_view_log.py @@ -1,8 +1,8 @@ - import frappe + def execute(): - if frappe.db.table_exists('View log'): + if frappe.db.table_exists("View log"): # for mac users direct renaming would not work since mysql for mac saves table name in lower case # so while renaming `tabView log` to `tabView Log` we get "Table 'tabView Log' already exists" error # more info https://stackoverflow.com/a/44753093/5955589 , @@ -13,10 +13,10 @@ def execute(): # deleting old View log table frappe.db.sql("DROP table `tabView log`") - frappe.delete_doc('DocType', 'View log') + frappe.delete_doc("DocType", "View log") # reloading view log doctype to create `tabView Log` table - frappe.reload_doc('core', 'doctype', 'view_log') + frappe.reload_doc("core", "doctype", "view_log") # Move the data to newly created `tabView Log` table frappe.db.sql("INSERT INTO `tabView Log` SELECT * FROM `ViewLogTemp`") @@ -25,4 +25,4 @@ def execute(): # Delete temporary table frappe.db.sql("DROP table `ViewLogTemp`") else: - frappe.reload_doc('core', 'doctype', 'view_log') + frappe.reload_doc("core", "doctype", "view_log") diff --git a/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py b/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py index ff5cf3fc5e..17a2856173 100644 --- a/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py +++ b/frappe/patches/v11_0/remove_doctype_user_permissions_for_page_and_report.py @@ -3,5 +3,6 @@ import frappe + def execute(): - frappe.delete_doc_if_exists("DocType", "User Permission for Page and Report") \ No newline at end of file + frappe.delete_doc_if_exists("DocType", "User Permission for Page and Report") diff --git a/frappe/patches/v11_0/remove_skip_for_doctype.py b/frappe/patches/v11_0/remove_skip_for_doctype.py index 6e66c75f68..e7c1d71a0a 100644 --- a/frappe/patches/v11_0/remove_skip_for_doctype.py +++ b/frappe/patches/v11_0/remove_skip_for_doctype.py @@ -1,4 +1,3 @@ - import frappe from frappe.desk.form.linked_with import get_linked_doctypes from frappe.patches.v11_0.replicate_old_user_permissions import get_doctypes_to_skip @@ -13,8 +12,9 @@ from frappe.query_builder import Field # if the user permission is applicable for all doctypes, then only # one record is created + def execute(): - frappe.reload_doctype('User Permission') + frappe.reload_doctype("User Permission") # to check if we need to migrate from skip_for_doctype has_skip_for_doctype = frappe.db.has_column("User Permission", "skip_for_doctype") @@ -24,15 +24,15 @@ def execute(): user_permissions_to_delete = [] - for user_permission in frappe.get_all('User Permission', fields=['*']): + for user_permission in frappe.get_all("User Permission", fields=["*"]): skip_for_doctype = [] # while migrating from v11 -> v11 if has_skip_for_doctype: if not user_permission.skip_for_doctype: continue - skip_for_doctype = user_permission.skip_for_doctype.split('\n') - else: # while migrating from v10 -> v11 + skip_for_doctype = user_permission.skip_for_doctype.split("\n") + else: # while migrating from v10 -> v11 if skip_for_doctype_map.get((user_permission.allow, user_permission.user)) is None: skip_for_doctype = get_doctypes_to_skip(user_permission.allow, user_permission.user) # cache skip for doctype for same user and doctype @@ -58,27 +58,35 @@ def execute(): for doctype in applicable_for_doctypes: if doctype: # Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified) - new_user_permissions_list.append(( - frappe.generate_hash("", 10), - user_permission.user, - user_permission.allow, - user_permission.for_value, - doctype, - 0, - user_permission.creation, - user_permission.modified - )) + new_user_permissions_list.append( + ( + frappe.generate_hash("", 10), + user_permission.user, + user_permission.allow, + user_permission.for_value, + doctype, + 0, + user_permission.creation, + user_permission.modified, + ) + ) else: # No skip_for_doctype found! Just update apply_to_all_doctypes. - frappe.db.set_value('User Permission', user_permission.name, 'apply_to_all_doctypes', 1) + frappe.db.set_value("User Permission", user_permission.name, "apply_to_all_doctypes", 1) if new_user_permissions_list: frappe.qb.into("User Permission").columns( - "name", "user", "allow", "for_value", "applicable_for", "apply_to_all_doctypes", "creation", "modified" + "name", + "user", + "allow", + "for_value", + "applicable_for", + "apply_to_all_doctypes", + "creation", + "modified", ).insert(*new_user_permissions_list).run() if user_permissions_to_delete: frappe.db.delete( - "User Permission", - filters=(Field("name").isin(tuple(user_permissions_to_delete))) + "User Permission", filters=(Field("name").isin(tuple(user_permissions_to_delete))) ) diff --git a/frappe/patches/v11_0/rename_email_alert_to_notification.py b/frappe/patches/v11_0/rename_email_alert_to_notification.py index 365b76ea48..7f0032d639 100644 --- a/frappe/patches/v11_0/rename_email_alert_to_notification.py +++ b/frappe/patches/v11_0/rename_email_alert_to_notification.py @@ -1,12 +1,14 @@ - import frappe from frappe.model.rename_doc import rename_doc + def execute(): - if frappe.db.table_exists("Email Alert Recipient") and not frappe.db.table_exists("Notification Recipient"): - rename_doc('DocType', 'Email Alert Recipient', 'Notification Recipient') - frappe.reload_doc('email', 'doctype', 'notification_recipient') + if frappe.db.table_exists("Email Alert Recipient") and not frappe.db.table_exists( + "Notification Recipient" + ): + rename_doc("DocType", "Email Alert Recipient", "Notification Recipient") + frappe.reload_doc("email", "doctype", "notification_recipient") if frappe.db.table_exists("Email Alert") and not frappe.db.table_exists("Notification"): - rename_doc('DocType', 'Email Alert', 'Notification') - frappe.reload_doc('email', 'doctype', 'notification') + rename_doc("DocType", "Email Alert", "Notification") + frappe.reload_doc("email", "doctype", "notification") diff --git a/frappe/patches/v11_0/rename_google_maps_doctype.py b/frappe/patches/v11_0/rename_google_maps_doctype.py index 8091154b9c..354c19a7ed 100644 --- a/frappe/patches/v11_0/rename_google_maps_doctype.py +++ b/frappe/patches/v11_0/rename_google_maps_doctype.py @@ -1,7 +1,9 @@ - import frappe from frappe.model.rename_doc import rename_doc + def execute(): - if frappe.db.exists("DocType","Google Maps") and not frappe.db.exists("DocType","Google Maps Settings"): - rename_doc('DocType', 'Google Maps', 'Google Maps Settings') + if frappe.db.exists("DocType", "Google Maps") and not frappe.db.exists( + "DocType", "Google Maps Settings" + ): + rename_doc("DocType", "Google Maps", "Google Maps Settings") diff --git a/frappe/patches/v11_0/rename_standard_reply_to_email_template.py b/frappe/patches/v11_0/rename_standard_reply_to_email_template.py index 2906085738..dabdb96633 100644 --- a/frappe/patches/v11_0/rename_standard_reply_to_email_template.py +++ b/frappe/patches/v11_0/rename_standard_reply_to_email_template.py @@ -1,8 +1,8 @@ - import frappe from frappe.model.rename_doc import rename_doc + def execute(): if frappe.db.table_exists("Standard Reply") and not frappe.db.table_exists("Email Template"): - rename_doc('DocType', 'Standard Reply', 'Email Template') - frappe.reload_doc('email', 'doctype', 'email_template') + rename_doc("DocType", "Standard Reply", "Email Template") + frappe.reload_doc("email", "doctype", "email_template") diff --git a/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py b/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py index 9a48104611..b8fa85217a 100644 --- a/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py +++ b/frappe/patches/v11_0/rename_workflow_action_to_workflow_action_master.py @@ -1,9 +1,10 @@ - import frappe from frappe.model.rename_doc import rename_doc def execute(): - if frappe.db.table_exists("Workflow Action") and not frappe.db.table_exists("Workflow Action Master"): - rename_doc('DocType', 'Workflow Action', 'Workflow Action Master') - frappe.reload_doc('workflow', 'doctype', 'workflow_action_master') + if frappe.db.table_exists("Workflow Action") and not frappe.db.table_exists( + "Workflow Action Master" + ): + rename_doc("DocType", "Workflow Action", "Workflow Action Master") + frappe.reload_doc("workflow", "doctype", "workflow_action_master") diff --git a/frappe/patches/v11_0/replicate_old_user_permissions.py b/frappe/patches/v11_0/replicate_old_user_permissions.py index 50a81b5ce7..999a5d7698 100644 --- a/frappe/patches/v11_0/replicate_old_user_permissions.py +++ b/frappe/patches/v11_0/replicate_old_user_permissions.py @@ -1,41 +1,51 @@ +import json import frappe -import json -from frappe.utils import cint from frappe.permissions import get_valid_perms +from frappe.utils import cint + def execute(): frappe.reload_doctype("User Permission") - user_permissions = frappe.get_all('User Permission', fields=['allow', 'name', 'user']) + user_permissions = frappe.get_all("User Permission", fields=["allow", "name", "user"]) doctype_to_skip_map = {} for permission in user_permissions: if (permission.allow, permission.user) not in doctype_to_skip_map: - doctype_to_skip_map[(permission.allow, permission.user)] = get_doctypes_to_skip(permission.allow, permission.user) + doctype_to_skip_map[(permission.allow, permission.user)] = get_doctypes_to_skip( + permission.allow, permission.user + ) - if not doctype_to_skip_map: return + if not doctype_to_skip_map: + return for key, doctype_to_skip in doctype_to_skip_map.items(): - if not doctype_to_skip: continue - if not frappe.db.has_column("User Permission", "applicable_for") \ - and frappe.db.has_column("User Permission", "skip_for_doctype"): - doctype_to_skip = '\n'.join(doctype_to_skip) - frappe.db.sql(""" + if not doctype_to_skip: + continue + if not frappe.db.has_column("User Permission", "applicable_for") and frappe.db.has_column( + "User Permission", "skip_for_doctype" + ): + doctype_to_skip = "\n".join(doctype_to_skip) + frappe.db.sql( + """ update `tabUser Permission` set skip_for_doctype = %s where user=%s and allow=%s - """, (doctype_to_skip, key[1], key[0])) + """, + (doctype_to_skip, key[1], key[0]), + ) def get_doctypes_to_skip(doctype, user): - ''' Returns doctypes to be skipped from user permission check''' + """Returns doctypes to be skipped from user permission check""" doctypes_to_skip = [] valid_perms = get_user_valid_perms(user) or [] for perm in valid_perms: parent_doctype = perm.parent try: linked_doctypes = get_linked_doctypes(parent_doctype) - if doctype not in linked_doctypes: continue + if doctype not in linked_doctypes: + continue except frappe.DoesNotExistError: # if doctype not found (may be due to rename) it should not be considered for skip continue @@ -44,40 +54,47 @@ def get_doctypes_to_skip(doctype, user): # add doctype to skip list if any of the perm does not apply user permission doctypes_to_skip.append(parent_doctype) - elif parent_doctype not in doctypes_to_skip: user_permission_doctypes = get_user_permission_doctypes(perm) # "No doctypes present" indicates that user permission will be applied to each link field - if not user_permission_doctypes: continue + if not user_permission_doctypes: + continue - elif doctype in user_permission_doctypes: continue + elif doctype in user_permission_doctypes: + continue - else: doctypes_to_skip.append(parent_doctype) + else: + doctypes_to_skip.append(parent_doctype) # to remove possible duplicates doctypes_to_skip = list(set(doctypes_to_skip)) return doctypes_to_skip + # store user's valid perms to avoid repeated query user_valid_perm = {} + def get_user_valid_perms(user): if not user_valid_perm.get(user): user_valid_perm[user] = get_valid_perms(user=user) return user_valid_perm.get(user) + def get_user_permission_doctypes(perm): try: - return json.loads(perm.user_permission_doctypes or '[]') + return json.loads(perm.user_permission_doctypes or "[]") except ValueError: return [] + def get_linked_doctypes(doctype): from frappe.permissions import get_linked_doctypes + linked_doctypes = get_linked_doctypes(doctype) child_doctypes = [d.options for d in frappe.get_meta(doctype).get_table_fields()] for child_dt in child_doctypes: linked_doctypes += get_linked_doctypes(child_dt) - return linked_doctypes \ No newline at end of file + return linked_doctypes diff --git a/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py b/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py index 63ae5f949f..fc1997bee1 100644 --- a/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py +++ b/frappe/patches/v11_0/set_allow_self_approval_in_workflow.py @@ -1,6 +1,6 @@ - import frappe + def execute(): frappe.reload_doc("workflow", "doctype", "workflow_transition") - frappe.db.sql("update `tabWorkflow Transition` set allow_self_approval=1") \ No newline at end of file + frappe.db.sql("update `tabWorkflow Transition` set allow_self_approval=1") diff --git a/frappe/patches/v11_0/set_default_letter_head_source.py b/frappe/patches/v11_0/set_default_letter_head_source.py index 3639524e7d..139d8cdd07 100644 --- a/frappe/patches/v11_0/set_default_letter_head_source.py +++ b/frappe/patches/v11_0/set_default_letter_head_source.py @@ -1,7 +1,8 @@ import frappe + def execute(): - frappe.reload_doctype('Letter Head') + frappe.reload_doctype("Letter Head") - # source of all existing letter heads must be HTML - frappe.db.sql("update `tabLetter Head` set source = 'HTML'") + # source of all existing letter heads must be HTML + frappe.db.sql("update `tabLetter Head` set source = 'HTML'") diff --git a/frappe/patches/v11_0/set_dropbox_file_backup.py b/frappe/patches/v11_0/set_dropbox_file_backup.py index 27492b3ab2..c9dec31414 100644 --- a/frappe/patches/v11_0/set_dropbox_file_backup.py +++ b/frappe/patches/v11_0/set_dropbox_file_backup.py @@ -1,9 +1,9 @@ - -from frappe.utils import cint import frappe +from frappe.utils import cint + def execute(): frappe.reload_doctype("Dropbox Settings") check_dropbox_enabled = cint(frappe.db.get_value("Dropbox Settings", None, "enabled")) - if check_dropbox_enabled == 1: - frappe.db.set_value("Dropbox Settings", None, 'file_backup', 1) + if check_dropbox_enabled == 1: + frappe.db.set_value("Dropbox Settings", None, "file_backup", 1) diff --git a/frappe/patches/v11_0/set_missing_creation_and_modified_value_for_user_permissions.py b/frappe/patches/v11_0/set_missing_creation_and_modified_value_for_user_permissions.py index 84d6d6c994..f6594c3efb 100644 --- a/frappe/patches/v11_0/set_missing_creation_and_modified_value_for_user_permissions.py +++ b/frappe/patches/v11_0/set_missing_creation_and_modified_value_for_user_permissions.py @@ -1,6 +1,9 @@ import frappe + def execute(): - frappe.db.sql('''UPDATE `tabUser Permission` + frappe.db.sql( + """UPDATE `tabUser Permission` SET `modified`=NOW(), `creation`=NOW() - WHERE `creation` IS NULL''') \ No newline at end of file + WHERE `creation` IS NULL""" + ) diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index 901ab66bfd..019ecef67c 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -1,18 +1,23 @@ - import frappe from frappe.utils.password import get_decrypted_password + def execute(): - publishable_key = frappe.db.sql("select value from tabSingles where doctype='Stripe Settings' and field='publishable_key'") + publishable_key = frappe.db.sql( + "select value from tabSingles where doctype='Stripe Settings' and field='publishable_key'" + ) if publishable_key: - secret_key = get_decrypted_password('Stripe Settings', 'Stripe Settings', - fieldname='secret_key', raise_exception=False) + secret_key = get_decrypted_password( + "Stripe Settings", "Stripe Settings", fieldname="secret_key", raise_exception=False + ) if secret_key: - frappe.reload_doc('integrations', 'doctype', 'stripe_settings') + frappe.reload_doc("integrations", "doctype", "stripe_settings") frappe.db.commit() settings = frappe.new_doc("Stripe Settings") - settings.gateway_name = frappe.db.get_value("Global Defaults", None, "default_company") or "Stripe Settings" + settings.gateway_name = ( + frappe.db.get_value("Global Defaults", None, "default_company") or "Stripe Settings" + ) settings.publishable_key = publishable_key settings.secret_key = secret_key settings.save(ignore_permissions=True) diff --git a/frappe/patches/v11_0/update_list_user_settings.py b/frappe/patches/v11_0/update_list_user_settings.py index 1b179d8cdf..146b29346c 100644 --- a/frappe/patches/v11_0/update_list_user_settings.py +++ b/frappe/patches/v11_0/update_list_user_settings.py @@ -1,28 +1,35 @@ +import json + +import frappe +from frappe.model.utils.user_settings import sync_user_settings, update_user_settings -import frappe, json -from frappe.model.utils.user_settings import update_user_settings, sync_user_settings def execute(): - """ Update list_view's order by property from __UserSettings """ + """Update list_view's order by property from __UserSettings""" users = frappe.db.sql("select distinct(user) from `__UserSettings`", as_dict=True) for user in users: # get user_settings for each user - settings = frappe.db.sql("select * from `__UserSettings` \ - where user={0}".format(frappe.db.escape(user.user)), as_dict=True) + settings = frappe.db.sql( + "select * from `__UserSettings` \ + where user={0}".format( + frappe.db.escape(user.user) + ), + as_dict=True, + ) # traverse through each doctype's settings for a user for d in settings: - data = json.loads(d['data']) - if data and ('List' in data) and ('order_by' in data['List']) and data['List']['order_by']: + data = json.loads(d["data"]) + if data and ("List" in data) and ("order_by" in data["List"]) and data["List"]["order_by"]: # convert order_by to sort_order & sort_by and delete order_by - order_by = data['List']['order_by'] - if '`' in order_by and '.' in order_by: - order_by = order_by.replace('`', '').split('.')[1] + order_by = data["List"]["order_by"] + if "`" in order_by and "." in order_by: + order_by = order_by.replace("`", "").split(".")[1] - data['List']['sort_by'], data['List']['sort_order'] = order_by.split(' ') - data['List'].pop('order_by') - update_user_settings(d['doctype'], json.dumps(data), for_update=True) + data["List"]["sort_by"], data["List"]["sort_order"] = order_by.split(" ") + data["List"].pop("order_by") + update_user_settings(d["doctype"], json.dumps(data), for_update=True) sync_user_settings() diff --git a/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py b/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py index dfdab6baf5..19f976084b 100644 --- a/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py +++ b/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py @@ -1,13 +1,16 @@ -import frappe import json +import frappe + + def execute(): - if not frappe.db.table_exists('Dashboard Chart'): + if not frappe.db.table_exists("Dashboard Chart"): return - charts_to_modify = frappe.db.get_all('Dashboard Chart', - fields = ['name', 'filters_json', 'document_type'], - filters = {'chart_type': ['not in', ['Report', 'Custom']]} + charts_to_modify = frappe.db.get_all( + "Dashboard Chart", + fields=["name", "filters_json", "document_type"], + filters={"chart_type": ["not in", ["Report", "Custom"]]}, ) for chart in charts_to_modify: @@ -22,7 +25,7 @@ def execute(): if isinstance(filter_value, list): new_filters.append([doctype, key, filter_value[0], filter_value[1], 0]) else: - new_filters.append([doctype, key, '=', filter_value, 0]) + new_filters.append([doctype, key, "=", filter_value, 0]) new_filters_json = json.dumps(new_filters) - frappe.db.set_value('Dashboard Chart', chart.name, 'filters_json', new_filters_json) \ No newline at end of file + frappe.db.set_value("Dashboard Chart", chart.name, "filters_json", new_filters_json) diff --git a/frappe/patches/v12_0/copy_to_parent_for_tags.py b/frappe/patches/v12_0/copy_to_parent_for_tags.py index 1135f9076b..ae3702a0d5 100644 --- a/frappe/patches/v12_0/copy_to_parent_for_tags.py +++ b/frappe/patches/v12_0/copy_to_parent_for_tags.py @@ -1,6 +1,7 @@ import frappe + def execute(): frappe.db.sql("UPDATE `tabTag Link` SET parenttype=document_type") - frappe.db.sql("UPDATE `tabTag Link` SET parent=document_name") \ No newline at end of file + frappe.db.sql("UPDATE `tabTag Link` SET parent=document_name") diff --git a/frappe/patches/v12_0/create_notification_settings_for_user.py b/frappe/patches/v12_0/create_notification_settings_for_user.py index 6edfd88872..eb6858967b 100644 --- a/frappe/patches/v12_0/create_notification_settings_for_user.py +++ b/frappe/patches/v12_0/create_notification_settings_for_user.py @@ -1,11 +1,13 @@ - import frappe -from frappe.desk.doctype.notification_settings.notification_settings import create_notification_settings +from frappe.desk.doctype.notification_settings.notification_settings import ( + create_notification_settings, +) + def execute(): - frappe.reload_doc('desk', 'doctype', 'notification_settings') - frappe.reload_doc('desk', 'doctype', 'notification_subscribed_document') + frappe.reload_doc("desk", "doctype", "notification_settings") + frappe.reload_doc("desk", "doctype", "notification_subscribed_document") - users = frappe.db.get_all('User', fields=['name']) + users = frappe.db.get_all("User", fields=["name"]) for user in users: - create_notification_settings(user.name) \ No newline at end of file + create_notification_settings(user.name) diff --git a/frappe/patches/v12_0/delete_duplicate_indexes.py b/frappe/patches/v12_0/delete_duplicate_indexes.py index ea73d49efc..6a6b0b3204 100644 --- a/frappe/patches/v12_0/delete_duplicate_indexes.py +++ b/frappe/patches/v12_0/delete_duplicate_indexes.py @@ -1,11 +1,13 @@ -import frappe from pymysql import InternalError +import frappe + # This patch deletes all the duplicate indexes created for same column # The patch only checks for indexes with UNIQUE constraints + def execute(): - if frappe.db.db_type != 'mariadb': + if frappe.db.db_type != "mariadb": return all_tables = frappe.db.get_tables() @@ -14,7 +16,8 @@ def execute(): for table in all_tables: indexes_to_keep_map = frappe._dict() indexes_to_delete = [] - index_info = frappe.db.sql(""" + index_info = frappe.db.sql( + """ SELECT column_name, index_name, @@ -24,7 +27,10 @@ def execute(): AND column_name!='name' AND non_unique=0 ORDER BY index_name; - """, table, as_dict=1) + """, + table, + as_dict=1, + ) for index in index_info: if not indexes_to_keep_map.get(index.column_name): diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index c1bf46b14a..4fecb45b25 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -1,5 +1,5 @@ - import frappe + def execute(): frappe.db.delete("DocType", {"name": "Feedback Request"}) diff --git a/frappe/patches/v12_0/delete_gsuite_if_exists.py b/frappe/patches/v12_0/delete_gsuite_if_exists.py index 7379ac9cdf..1fb3a8c2d0 100644 --- a/frappe/patches/v12_0/delete_gsuite_if_exists.py +++ b/frappe/patches/v12_0/delete_gsuite_if_exists.py @@ -1,8 +1,9 @@ import frappe + def execute(): - ''' - Remove GSuite Template and GSuite Settings - ''' + """ + Remove GSuite Template and GSuite Settings + """ frappe.delete_doc_if_exists("DocType", "GSuite Settings") - frappe.delete_doc_if_exists("DocType", "GSuite Templates") \ No newline at end of file + frappe.delete_doc_if_exists("DocType", "GSuite Templates") diff --git a/frappe/patches/v12_0/fix_email_id_formatting.py b/frappe/patches/v12_0/fix_email_id_formatting.py index 03f606e0cc..9cc2f49966 100644 --- a/frappe/patches/v12_0/fix_email_id_formatting.py +++ b/frappe/patches/v12_0/fix_email_id_formatting.py @@ -1,44 +1,61 @@ import frappe + def execute(): fix_communications() fix_show_as_cc_email_queue() fix_email_queue_recipients() + def fix_communications(): - for communication in frappe.db.sql('''select name, recipients, cc, bcc from tabCommunication + for communication in frappe.db.sql( + """select name, recipients, cc, bcc from tabCommunication where creation > '2020-06-01' and communication_medium='Email' and communication_type='Communication' and (cc like '%<%' or bcc like '%<%' or recipients like '%<%') - ''', as_dict=1): + """, + as_dict=1, + ): + + communication["recipients"] = format_email_id(communication.recipients) + communication["cc"] = format_email_id(communication.cc) + communication["bcc"] = format_email_id(communication.bcc) - communication['recipients'] = format_email_id(communication.recipients) - communication['cc'] = format_email_id(communication.cc) - communication['bcc'] = format_email_id(communication.bcc) + frappe.db.sql( + """update `tabCommunication` set recipients=%s,cc=%s,bcc=%s + where name =%s """, + (communication["recipients"], communication["cc"], communication["bcc"], communication["name"]), + ) - frappe.db.sql('''update `tabCommunication` set recipients=%s,cc=%s,bcc=%s - where name =%s ''', (communication['recipients'], communication['cc'], - communication['bcc'], communication['name'])) def fix_show_as_cc_email_queue(): - for queue in frappe.get_all("Email Queue", {'creation': ['>', '2020-06-01'], - 'status': 'Not Sent', 'show_as_cc': ['like', '%<%']}, - ['name', 'show_as_cc']): + for queue in frappe.get_all( + "Email Queue", + {"creation": [">", "2020-06-01"], "status": "Not Sent", "show_as_cc": ["like", "%<%"]}, + ["name", "show_as_cc"], + ): + + frappe.db.set_value( + "Email Queue", queue["name"], "show_as_cc", format_email_id(queue["show_as_cc"]) + ) - frappe.db.set_value('Email Queue', queue['name'], - 'show_as_cc', format_email_id(queue['show_as_cc'])) def fix_email_queue_recipients(): - for recipient in frappe.db.sql('''select recipient, name from + for recipient in frappe.db.sql( + """select recipient, name from `tabEmail Queue Recipient` where recipient like '%<%' - and status='Not Sent' and creation > '2020-06-01' ''', as_dict=1): + and status='Not Sent' and creation > '2020-06-01' """, + as_dict=1, + ): + + frappe.db.set_value( + "Email Queue Recipient", recipient["name"], "recipient", format_email_id(recipient["recipient"]) + ) - frappe.db.set_value('Email Queue Recipient', recipient['name'], - 'recipient', format_email_id(recipient['recipient'])) def format_email_id(email): - if email and ('<' in email and '>' in email): - return email.replace('>', '>').replace('<', '<') + if email and ("<" in email and ">" in email): + return email.replace(">", ">").replace("<", "<") return email diff --git a/frappe/patches/v12_0/fix_public_private_files.py b/frappe/patches/v12_0/fix_public_private_files.py index 065360a78e..e1ad2f1862 100644 --- a/frappe/patches/v12_0/fix_public_private_files.py +++ b/frappe/patches/v12_0/fix_public_private_files.py @@ -1,24 +1,26 @@ import frappe + def execute(): - files = frappe.get_all('File', - fields=['is_private', 'file_url', 'name'], - filters={'is_folder': 0}) + files = frappe.get_all( + "File", fields=["is_private", "file_url", "name"], filters={"is_folder": 0} + ) for file in files: file_url = file.file_url or "" if file.is_private: - if not file_url.startswith('/private/files/'): + if not file_url.startswith("/private/files/"): generate_file(file.name) else: - if file_url.startswith('/private/files/'): + if file_url.startswith("/private/files/"): generate_file(file.name) + def generate_file(file_name): try: - file_doc = frappe.get_doc('File', file_name) + file_doc = frappe.get_doc("File", file_name) # private - new_doc = frappe.new_doc('File') + new_doc = frappe.new_doc("File") new_doc.is_private = file_doc.is_private new_doc.file_name = file_doc.file_name # to create copy of file in right location diff --git a/frappe/patches/v12_0/init_desk_settings.py b/frappe/patches/v12_0/init_desk_settings.py index fceb44b924..5ec9764e8f 100644 --- a/frappe/patches/v12_0/init_desk_settings.py +++ b/frappe/patches/v12_0/init_desk_settings.py @@ -1,8 +1,10 @@ import json + import frappe from frappe.config import get_modules_from_all_apps_for_user from frappe.desk.moduleview import get_onboard_items + def execute(): """Reset the initial customizations for desk, with modules, indices and links.""" frappe.reload_doc("core", "doctype", "user") diff --git a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py index 12680609d5..1a369b4e12 100644 --- a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py +++ b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py @@ -1,18 +1,22 @@ import frappe + def execute(): frappe.reload_doc("contacts", "doctype", "contact_email") frappe.reload_doc("contacts", "doctype", "contact_phone") frappe.reload_doc("contacts", "doctype", "contact") - contact_details = frappe.db.sql(""" + contact_details = frappe.db.sql( + """ SELECT `name`, `email_id`, `phone`, `mobile_no`, `modified_by`, `creation`, `modified` FROM `tabContact` where not exists (select * from `tabContact Email` where `tabContact Email`.parent=`tabContact`.name and `tabContact Email`.email_id=`tabContact`.email_id) - """, as_dict=True) + """, + as_dict=True, + ) email_values = [] phone_values = [] @@ -20,71 +24,87 @@ def execute(): phone_counter = 1 is_primary = 1 if contact_detail.email_id: - email_values.append(( - 1, - frappe.generate_hash(contact_detail.email_id, 10), - contact_detail.email_id, - 'email_ids', - 'Contact', - contact_detail.name, - 1, - contact_detail.creation, - contact_detail.modified, - contact_detail.modified_by - )) + email_values.append( + ( + 1, + frappe.generate_hash(contact_detail.email_id, 10), + contact_detail.email_id, + "email_ids", + "Contact", + contact_detail.name, + 1, + contact_detail.creation, + contact_detail.modified, + contact_detail.modified_by, + ) + ) if contact_detail.phone: is_primary_phone = 1 if phone_counter == 1 else 0 - phone_values.append(( - phone_counter, - frappe.generate_hash(contact_detail.email_id, 10), - contact_detail.phone, - 'phone_nos', - 'Contact', - contact_detail.name, - is_primary_phone, - 0, - contact_detail.creation, - contact_detail.modified, - contact_detail.modified_by - )) + phone_values.append( + ( + phone_counter, + frappe.generate_hash(contact_detail.email_id, 10), + contact_detail.phone, + "phone_nos", + "Contact", + contact_detail.name, + is_primary_phone, + 0, + contact_detail.creation, + contact_detail.modified, + contact_detail.modified_by, + ) + ) phone_counter += 1 if contact_detail.mobile_no: is_primary_mobile_no = 1 if phone_counter == 1 else 0 - phone_values.append(( - phone_counter, - frappe.generate_hash(contact_detail.email_id, 10), - contact_detail.mobile_no, - 'phone_nos', - 'Contact', - contact_detail.name, - 0, - is_primary_mobile_no, - contact_detail.creation, - contact_detail.modified, - contact_detail.modified_by - )) + phone_values.append( + ( + phone_counter, + frappe.generate_hash(contact_detail.email_id, 10), + contact_detail.mobile_no, + "phone_nos", + "Contact", + contact_detail.name, + 0, + is_primary_mobile_no, + contact_detail.creation, + contact_detail.modified, + contact_detail.modified_by, + ) + ) - if email_values and (count%10000 == 0 or count == len(contact_details)-1): - frappe.db.sql(""" + if email_values and (count % 10000 == 0 or count == len(contact_details) - 1): + frappe.db.sql( + """ INSERT INTO `tabContact Email` (`idx`, `name`, `email_id`, `parentfield`, `parenttype`, `parent`, `is_primary`, `creation`, `modified`, `modified_by`) VALUES {} - """.format(", ".join(['%s'] * len(email_values))), tuple(email_values)) + """.format( + ", ".join(["%s"] * len(email_values)) + ), + tuple(email_values), + ) email_values = [] - if phone_values and (count%10000 == 0 or count == len(contact_details)-1): - frappe.db.sql(""" + if phone_values and (count % 10000 == 0 or count == len(contact_details) - 1): + frappe.db.sql( + """ INSERT INTO `tabContact Phone` (`idx`, `name`, `phone`, `parentfield`, `parenttype`, `parent`, `is_primary_phone`, `is_primary_mobile_no`, `creation`, `modified`, `modified_by`) VALUES {} - """.format(", ".join(['%s'] * len(phone_values))), tuple(phone_values)) + """.format( + ", ".join(["%s"] * len(phone_values)) + ), + tuple(phone_values), + ) phone_values = [] frappe.db.add_index("Contact Phone", ["phone"]) - frappe.db.add_index("Contact Email", ["email_id"]) \ No newline at end of file + frappe.db.add_index("Contact Email", ["email_id"]) diff --git a/frappe/patches/v12_0/move_form_attachments_to_attachments_folder.py b/frappe/patches/v12_0/move_form_attachments_to_attachments_folder.py index c96741cc3b..47154c437a 100644 --- a/frappe/patches/v12_0/move_form_attachments_to_attachments_folder.py +++ b/frappe/patches/v12_0/move_form_attachments_to_attachments_folder.py @@ -1,9 +1,12 @@ import frappe + def execute(): - frappe.db.sql(''' + frappe.db.sql( + """ UPDATE tabFile SET folder = 'Home/Attachments' WHERE ifnull(attached_to_doctype, '') != '' AND folder = 'Home' - ''') + """ + ) diff --git a/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py b/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py index 85be3f7feb..2207edd958 100644 --- a/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py +++ b/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py @@ -1,14 +1,18 @@ import frappe + def execute(): - communications = frappe.db.sql(""" + communications = frappe.db.sql( + """ SELECT `tabCommunication`.name, `tabCommunication`.creation, `tabCommunication`.modified, `tabCommunication`.modified_by,`tabCommunication`.timeline_doctype, `tabCommunication`.timeline_name, `tabCommunication`.link_doctype, `tabCommunication`.link_name FROM `tabCommunication` WHERE `tabCommunication`.communication_medium='Email' - """, as_dict=True) + """, + as_dict=True, + ) name = 1000000000 values = [] @@ -17,25 +21,45 @@ def execute(): counter = 1 if communication.timeline_doctype and communication.timeline_name: name += 1 - values.append("""({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format( - counter, str(name), frappe.db.escape(communication.name), frappe.db.escape(communication.timeline_doctype), - frappe.db.escape(communication.timeline_name), communication.creation, communication.modified, communication.modified_by - )) + values.append( + """({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format( + counter, + str(name), + frappe.db.escape(communication.name), + frappe.db.escape(communication.timeline_doctype), + frappe.db.escape(communication.timeline_name), + communication.creation, + communication.modified, + communication.modified_by, + ) + ) counter += 1 if communication.link_doctype and communication.link_name: name += 1 - values.append("""({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format( - counter, str(name), frappe.db.escape(communication.name), frappe.db.escape(communication.link_doctype), - frappe.db.escape(communication.link_name), communication.creation, communication.modified, communication.modified_by - )) + values.append( + """({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format( + counter, + str(name), + frappe.db.escape(communication.name), + frappe.db.escape(communication.link_doctype), + frappe.db.escape(communication.link_name), + communication.creation, + communication.modified, + communication.modified_by, + ) + ) if values and (count % 10000 == 0 or count == len(communications) - 1): - frappe.db.sql(""" + frappe.db.sql( + """ INSERT INTO `tabCommunication Link` (`idx`, `name`, `parentfield`, `parenttype`, `parent`, `link_doctype`, `link_name`, `creation`, `modified`, `modified_by`) VALUES {0} - """.format(", ".join([d for d in values]))) + """.format( + ", ".join([d for d in values]) + ) + ) values = [] diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 9c9a79ccbf..5727ab7b48 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -1,13 +1,12 @@ import frappe + def execute(): - frappe.reload_doc('core', 'doctype', 'doctype_link') - frappe.reload_doc('core', 'doctype', 'doctype_action') - frappe.reload_doc('core', 'doctype', 'doctype') - frappe.model.delete_fields({ - 'DocType': ['hide_heading', 'image_view', 'read_only_onload'] - }, delete=1) + frappe.reload_doc("core", "doctype", "doctype_link") + frappe.reload_doc("core", "doctype", "doctype_action") + frappe.reload_doc("core", "doctype", "doctype") + frappe.model.delete_fields( + {"DocType": ["hide_heading", "image_view", "read_only_onload"]}, delete=1 + ) - frappe.db.delete("Property Setter", { - "property": "read_only_onload" - }) \ No newline at end of file + frappe.db.delete("Property Setter", {"property": "read_only_onload"}) diff --git a/frappe/patches/v12_0/remove_example_email_thread_notify.py b/frappe/patches/v12_0/remove_example_email_thread_notify.py index 94959b6077..4072065b8f 100644 --- a/frappe/patches/v12_0/remove_example_email_thread_notify.py +++ b/frappe/patches/v12_0/remove_example_email_thread_notify.py @@ -3,6 +3,8 @@ import frappe def execute(): # remove all example.com email user accounts from notifications - frappe.db.sql("""UPDATE `tabUser` + frappe.db.sql( + """UPDATE `tabUser` SET thread_notify=0, send_me_a_copy=0 - WHERE email like '%@example.com'""") + WHERE email like '%@example.com'""" + ) diff --git a/frappe/patches/v12_0/remove_feedback_rating.py b/frappe/patches/v12_0/remove_feedback_rating.py index 5184bb7b79..2d9c296efc 100644 --- a/frappe/patches/v12_0/remove_feedback_rating.py +++ b/frappe/patches/v12_0/remove_feedback_rating.py @@ -1,9 +1,10 @@ import frappe + def execute(): - ''' - Deprecate Feedback Trigger and Rating. This feature was not customizable. - Now can be achieved via custom Web Forms - ''' - frappe.delete_doc('DocType', 'Feedback Trigger') - frappe.delete_doc('DocType', 'Feedback Rating') + """ + Deprecate Feedback Trigger and Rating. This feature was not customizable. + Now can be achieved via custom Web Forms + """ + frappe.delete_doc("DocType", "Feedback Trigger") + frappe.delete_doc("DocType", "Feedback Rating") diff --git a/frappe/patches/v12_0/remove_gcalendar_gmaps.py b/frappe/patches/v12_0/remove_gcalendar_gmaps.py index 84c400f6a8..1177441130 100644 --- a/frappe/patches/v12_0/remove_gcalendar_gmaps.py +++ b/frappe/patches/v12_0/remove_gcalendar_gmaps.py @@ -1,10 +1,11 @@ import frappe + def execute(): - ''' - Remove GCalendar and GCalendar Settings - Remove Google Maps Settings as its been merged with Delivery Trips - ''' - frappe.delete_doc_if_exists('DocType', 'GCalendar Account') - frappe.delete_doc_if_exists('DocType', 'GCalendar Settings') - frappe.delete_doc_if_exists('DocType', 'Google Maps Settings') + """ + Remove GCalendar and GCalendar Settings + Remove Google Maps Settings as its been merged with Delivery Trips + """ + frappe.delete_doc_if_exists("DocType", "GCalendar Account") + frappe.delete_doc_if_exists("DocType", "GCalendar Settings") + frappe.delete_doc_if_exists("DocType", "Google Maps Settings") diff --git a/frappe/patches/v12_0/rename_events_repeat_on.py b/frappe/patches/v12_0/rename_events_repeat_on.py index b0dface644..5276689a2a 100644 --- a/frappe/patches/v12_0/rename_events_repeat_on.py +++ b/frappe/patches/v12_0/rename_events_repeat_on.py @@ -1,18 +1,37 @@ import frappe from frappe.utils import get_datetime + def execute(): weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] - weekly_events = frappe.get_list("Event", filters={"repeat_this_event": 1, "repeat_on": "Every Week"}, fields=["name", "starts_on"]) + weekly_events = frappe.get_list( + "Event", + filters={"repeat_this_event": 1, "repeat_on": "Every Week"}, + fields=["name", "starts_on"], + ) frappe.reload_doc("desk", "doctype", "event") # Initially Daily Events had option to choose days, but now Weekly does, so just changing from Daily -> Weekly does the job - frappe.db.sql("""UPDATE `tabEvent` SET `tabEvent`.repeat_on='Weekly' WHERE `tabEvent`.repeat_on='Every Day'""") - frappe.db.sql("""UPDATE `tabEvent` SET `tabEvent`.repeat_on='Weekly' WHERE `tabEvent`.repeat_on='Every Week'""") - frappe.db.sql("""UPDATE `tabEvent` SET `tabEvent`.repeat_on='Monthly' WHERE `tabEvent`.repeat_on='Every Month'""") - frappe.db.sql("""UPDATE `tabEvent` SET `tabEvent`.repeat_on='Yearly' WHERE `tabEvent`.repeat_on='Every Year'""") + frappe.db.sql( + """UPDATE `tabEvent` SET `tabEvent`.repeat_on='Weekly' WHERE `tabEvent`.repeat_on='Every Day'""" + ) + frappe.db.sql( + """UPDATE `tabEvent` SET `tabEvent`.repeat_on='Weekly' WHERE `tabEvent`.repeat_on='Every Week'""" + ) + frappe.db.sql( + """UPDATE `tabEvent` SET `tabEvent`.repeat_on='Monthly' WHERE `tabEvent`.repeat_on='Every Month'""" + ) + frappe.db.sql( + """UPDATE `tabEvent` SET `tabEvent`.repeat_on='Yearly' WHERE `tabEvent`.repeat_on='Every Year'""" + ) for weekly_event in weekly_events: # Set WeekDay based on the starts_on so that event can repeat Weekly - frappe.db.set_value('Event', weekly_event.name, weekdays[get_datetime(weekly_event.starts_on).weekday()], 1, update_modified=False) + frappe.db.set_value( + "Event", + weekly_event.name, + weekdays[get_datetime(weekly_event.starts_on).weekday()], + 1, + update_modified=False, + ) diff --git a/frappe/patches/v12_0/rename_uploaded_files_with_proper_name.py b/frappe/patches/v12_0/rename_uploaded_files_with_proper_name.py index 854a381e1c..5907040b6a 100644 --- a/frappe/patches/v12_0/rename_uploaded_files_with_proper_name.py +++ b/frappe/patches/v12_0/rename_uploaded_files_with_proper_name.py @@ -1,14 +1,17 @@ -import frappe import os +import frappe + + def execute(): - file_names_with_url = frappe.get_all("File", filters={ - "is_folder": 0, - "file_name": ["like", "%/%"] - }, fields=['name', 'file_name', 'file_url']) + file_names_with_url = frappe.get_all( + "File", + filters={"is_folder": 0, "file_name": ["like", "%/%"]}, + fields=["name", "file_name", "file_url"], + ) for f in file_names_with_url: - filename = f.file_name.rsplit('/', 1)[-1] + filename = f.file_name.rsplit("/", 1)[-1] if not f.file_url: f.file_url = f.file_name @@ -16,16 +19,15 @@ def execute(): try: if not file_exists(f.file_url): continue - frappe.db.set_value('File', f.name, { - "file_name": filename, - "file_url": f.file_url - }, update_modified=False) + frappe.db.set_value( + "File", f.name, {"file_name": filename, "file_url": f.file_url}, update_modified=False + ) except Exception: continue + def file_exists(file_path): file_path = frappe.utils.get_files_path( - file_path.rsplit('/', 1)[-1], - is_private=file_path.startswith('/private') + file_path.rsplit("/", 1)[-1], is_private=file_path.startswith("/private") ) return os.path.exists(file_path) diff --git a/frappe/patches/v12_0/replace_null_values_in_tables.py b/frappe/patches/v12_0/replace_null_values_in_tables.py index 023a255a6d..1dc8d964a1 100644 --- a/frappe/patches/v12_0/replace_null_values_in_tables.py +++ b/frappe/patches/v12_0/replace_null_values_in_tables.py @@ -1,21 +1,30 @@ -import frappe import re +import frappe + + def execute(): - fields = frappe.db.sql(""" + fields = frappe.db.sql( + """ SELECT COLUMN_NAME , TABLE_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE DATA_TYPE IN ('INT', 'FLOAT', 'DECIMAL') AND IS_NULLABLE = 'YES' - """, as_dict=1) + """, + as_dict=1, + ) update_column_table_map = {} for field in fields: update_column_table_map.setdefault(field.TABLE_NAME, []) - update_column_table_map[field.TABLE_NAME].append("`{fieldname}`=COALESCE(`{fieldname}`, 0)".format(fieldname=field.COLUMN_NAME)) + update_column_table_map[field.TABLE_NAME].append( + "`{fieldname}`=COALESCE(`{fieldname}`, 0)".format(fieldname=field.COLUMN_NAME) + ) for table in frappe.db.get_tables(): - if update_column_table_map.get(table) and frappe.db.exists("DocType", re.sub('^tab', '', table)): - frappe.db.sql("""UPDATE `{table}` SET {columns}""" - .format(table=table, columns=", ".join(update_column_table_map.get(table)))) - + if update_column_table_map.get(table) and frappe.db.exists("DocType", re.sub("^tab", "", table)): + frappe.db.sql( + """UPDATE `{table}` SET {columns}""".format( + table=table, columns=", ".join(update_column_table_map.get(table)) + ) + ) diff --git a/frappe/patches/v12_0/reset_home_settings.py b/frappe/patches/v12_0/reset_home_settings.py index e4b9de6cb2..c498b8b14b 100644 --- a/frappe/patches/v12_0/reset_home_settings.py +++ b/frappe/patches/v12_0/reset_home_settings.py @@ -1,9 +1,12 @@ import frappe + def execute(): - frappe.reload_doc('core', 'doctype', 'user') - frappe.db.sql(''' + frappe.reload_doc("core", "doctype", "user") + frappe.db.sql( + """ UPDATE `tabUser` SET `home_settings` = '' WHERE `user_type` = 'System User' - ''') + """ + ) diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index 5aaadd00e8..d06cc04969 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -1,5 +1,6 @@ import frappe -from frappe.query_builder.functions import GroupConcat, Coalesce +from frappe.query_builder.functions import Coalesce, GroupConcat + def execute(): frappe.reload_doc("desk", "doctype", "todo") @@ -23,5 +24,5 @@ def execute(): doc.reference_name, "_assign", frappe.as_json(assignments), - update_modified=False - ) \ No newline at end of file + update_modified=False, + ) diff --git a/frappe/patches/v12_0/set_correct_url_in_files.py b/frappe/patches/v12_0/set_correct_url_in_files.py index 4613f88694..dca42a3c04 100644 --- a/frappe/patches/v12_0/set_correct_url_in_files.py +++ b/frappe/patches/v12_0/set_correct_url_in_files.py @@ -1,39 +1,41 @@ -import frappe import os +import frappe + + def execute(): - files = frappe.get_all('File', - fields = ['name', 'file_name', 'file_url'], - filters = { - 'is_folder': 0, - 'file_url': ['!=', ''], - }) + files = frappe.get_all( + "File", + fields=["name", "file_name", "file_url"], + filters={ + "is_folder": 0, + "file_url": ["!=", ""], + }, + ) - private_file_path = frappe.get_site_path('private', 'files') - public_file_path = frappe.get_site_path('public', 'files') + private_file_path = frappe.get_site_path("private", "files") + public_file_path = frappe.get_site_path("public", "files") for file in files: file_path = file.file_url - file_name = file_path.split('/')[-1] + file_name = file_path.split("/")[-1] - if not file_path.startswith(('/private/', '/files/')): + if not file_path.startswith(("/private/", "/files/")): continue - file_is_private = file_path.startswith('/private/files/') + file_is_private = file_path.startswith("/private/files/") full_path = frappe.utils.get_files_path(file_name, is_private=file_is_private) if not os.path.exists(full_path): if file_is_private: public_file_url = os.path.join(public_file_path, file_name) if os.path.exists(public_file_url): - frappe.db.set_value('File', file.name, { - 'file_url': '/files/{0}'.format(file_name), - 'is_private': 0 - }) + frappe.db.set_value( + "File", file.name, {"file_url": "/files/{0}".format(file_name), "is_private": 0} + ) else: private_file_url = os.path.join(private_file_path, file_name) if os.path.exists(private_file_url): - frappe.db.set_value('File', file.name, { - 'file_url': '/private/files/{0}'.format(file_name), - 'is_private': 1 - }) + frappe.db.set_value( + "File", file.name, {"file_url": "/private/files/{0}".format(file_name), "is_private": 1} + ) diff --git a/frappe/patches/v12_0/set_default_incoming_email_port.py b/frappe/patches/v12_0/set_default_incoming_email_port.py index eb225c683d..822ce06f70 100644 --- a/frappe/patches/v12_0/set_default_incoming_email_port.py +++ b/frappe/patches/v12_0/set_default_incoming_email_port.py @@ -1,11 +1,12 @@ import frappe from frappe.email.utils import get_port + def execute(): - ''' - 1. Set default incoming email port in email domain - 2. Set default incoming email port in all email account (for those account where domain is missing) - ''' + """ + 1. Set default incoming email port in email domain + 2. Set default incoming email port in all email account (for those account where domain is missing) + """ frappe.reload_doc("email", "doctype", "email_domain", force=True) frappe.reload_doc("email", "doctype", "email_account", force=True) @@ -14,19 +15,31 @@ def execute(): def setup_incoming_email_port_in_email_domains(): - email_domains = frappe.db.get_all("Email Domain", ['incoming_port', 'use_imap', 'use_ssl', 'name']) + email_domains = frappe.db.get_all( + "Email Domain", ["incoming_port", "use_imap", "use_ssl", "name"] + ) for domain in email_domains: if not domain.incoming_port: incoming_port = get_port(domain) - frappe.db.set_value("Email Domain", domain.name, 'incoming_port', incoming_port, update_modified=False) + frappe.db.set_value( + "Email Domain", domain.name, "incoming_port", incoming_port, update_modified=False + ) + + # update incoming email port in all + frappe.db.sql( + """update `tabEmail Account` set incoming_port=%s where domain = %s""", + (domain.incoming_port, domain.name), + ) - #update incoming email port in all - frappe.db.sql('''update `tabEmail Account` set incoming_port=%s where domain = %s''', (domain.incoming_port, domain.name)) def setup_incoming_email_port_in_email_accounts(): - email_accounts = frappe.db.get_all("Email Account", ['incoming_port', 'use_imap', 'use_ssl', 'name', 'enable_incoming']) + email_accounts = frappe.db.get_all( + "Email Account", ["incoming_port", "use_imap", "use_ssl", "name", "enable_incoming"] + ) for account in email_accounts: if account.enable_incoming and not account.incoming_port: incoming_port = get_port(account) - frappe.db.set_value("Email Account", account.name, 'incoming_port', incoming_port, update_modified=False) + frappe.db.set_value( + "Email Account", account.name, "incoming_port", incoming_port, update_modified=False + ) diff --git a/frappe/patches/v12_0/set_default_password_reset_limit.py b/frappe/patches/v12_0/set_default_password_reset_limit.py index e403b5251e..9e29e75c36 100644 --- a/frappe/patches/v12_0/set_default_password_reset_limit.py +++ b/frappe/patches/v12_0/set_default_password_reset_limit.py @@ -6,4 +6,4 @@ import frappe def execute(): frappe.reload_doc("core", "doctype", "system_settings", force=1) - frappe.db.set_value('System Settings', None, "password_reset_limit", 3) + frappe.db.set_value("System Settings", None, "password_reset_limit", 3) diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index 83a903fc2d..488c0da618 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -1,24 +1,26 @@ import frappe + def execute(): - #if current = 0, simply delete the key as it'll be recreated on first entry + # if current = 0, simply delete the key as it'll be recreated on first entry frappe.db.delete("Series", {"current": 0}) - duplicate_keys = frappe.db.sql(''' + duplicate_keys = frappe.db.sql( + """ SELECT name, max(current) as current from `tabSeries` group by name having count(name) > 1 - ''', as_dict=True) + """, + as_dict=True, + ) for row in duplicate_keys: - frappe.db.delete("Series", { - "name": row.name - }) + frappe.db.delete("Series", {"name": row.name}) if row.current: - frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) + frappe.db.sql("insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)", row) frappe.db.commit() - frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') + frappe.db.sql("ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)") diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index 11e02965f1..ad168f7c22 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -1,15 +1,17 @@ import frappe + def execute(): frappe.reload_doctype("Comment") - if frappe.db.count('Communication', filters = dict(communication_type = 'Comment')) > 20000: + if frappe.db.count("Communication", filters=dict(communication_type="Comment")) > 20000: frappe.db.auto_commit_on_many_writes = True - for comment in frappe.get_all('Communication', fields = ['*'], - filters = dict(communication_type = 'Comment')): + for comment in frappe.get_all( + "Communication", fields=["*"], filters=dict(communication_type="Comment") + ): - new_comment = frappe.new_doc('Comment') + new_comment = frappe.new_doc("Comment") new_comment.comment_type = comment.comment_type new_comment.comment_email = comment.sender new_comment.comment_by = comment.sender_full_name @@ -29,6 +31,4 @@ def execute(): frappe.db.auto_commit_on_many_writes = False # clean up - frappe.db.delete("Communication", { - "communication_type": "Comment" - }) + frappe.db.delete("Communication", {"communication_type": "Comment"}) diff --git a/frappe/patches/v12_0/setup_email_linking.py b/frappe/patches/v12_0/setup_email_linking.py index 9e939e1245..56166f89dc 100644 --- a/frappe/patches/v12_0/setup_email_linking.py +++ b/frappe/patches/v12_0/setup_email_linking.py @@ -1,4 +1,5 @@ from frappe.desk.page.setup_wizard.install_fixtures import setup_email_linking + def execute(): - setup_email_linking() \ No newline at end of file + setup_email_linking() diff --git a/frappe/patches/v12_0/setup_tags.py b/frappe/patches/v12_0/setup_tags.py index d663cb2e0e..46482a102b 100644 --- a/frappe/patches/v12_0/setup_tags.py +++ b/frappe/patches/v12_0/setup_tags.py @@ -1,5 +1,6 @@ import frappe + def execute(): frappe.delete_doc_if_exists("DocType", "Tag Category") frappe.delete_doc_if_exists("DocType", "Tag Doc Category") @@ -15,7 +16,9 @@ def execute(): if not frappe.db.count(doctype.name) or not frappe.db.has_column(doctype.name, "_user_tags"): continue - for _user_tags in frappe.db.sql("select `name`, `_user_tags` from `tab{0}`".format(doctype.name), as_dict=True): + for _user_tags in frappe.db.sql( + "select `name`, `_user_tags` from `tab{0}`".format(doctype.name), as_dict=True + ): if not _user_tags.get("_user_tags"): continue @@ -23,11 +26,22 @@ def execute(): if not tag: continue - tag_list.append((tag.strip(), time, time, 'Administrator')) + tag_list.append((tag.strip(), time, time, "Administrator")) tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10) - tag_links.append((tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, 'Administrator')) - - - frappe.db.bulk_insert("Tag", fields=["name", "creation", "modified", "modified_by"], values=set(tag_list), ignore_duplicates=True) - frappe.db.bulk_insert("Tag Link", fields=["name", "document_type", "document_name", "tag", "creation", "modified", "modified_by"], values=set(tag_links), ignore_duplicates=True) + tag_links.append( + (tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator") + ) + + frappe.db.bulk_insert( + "Tag", + fields=["name", "creation", "modified", "modified_by"], + values=set(tag_list), + ignore_duplicates=True, + ) + frappe.db.bulk_insert( + "Tag Link", + fields=["name", "document_type", "document_name", "tag", "creation", "modified", "modified_by"], + values=set(tag_links), + ignore_duplicates=True, + ) diff --git a/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py b/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py index 3a3dcec315..55208ee8eb 100644 --- a/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py +++ b/frappe/patches/v12_0/update_auto_repeat_status_and_not_submittable.py @@ -1,9 +1,9 @@ - import frappe from frappe.custom.doctype.custom_field.custom_field import create_custom_field + def execute(): - #auto repeat is not submittable in v12 + # auto repeat is not submittable in v12 frappe.reload_doc("automation", "doctype", "Auto Repeat") frappe.db.sql("update `tabDocPerm` set submit=0, cancel=0, amend=0 where parent='Auto Repeat'") frappe.db.sql("update `tabAuto Repeat` set docstatus=0 where docstatus=1 or docstatus=2") @@ -11,16 +11,24 @@ def execute(): for entry in frappe.get_all("Auto Repeat"): doc = frappe.get_doc("Auto Repeat", entry.name) - #create custom field for allow auto repeat + # create custom field for allow auto repeat fields = frappe.get_meta(doc.reference_doctype).fields insert_after = fields[len(fields) - 1].fieldname - df = dict(fieldname="auto_repeat", label="Auto Repeat", fieldtype="Link", insert_after=insert_after, - options="Auto Repeat", hidden=1, print_hide=1, read_only=1) + df = dict( + fieldname="auto_repeat", + label="Auto Repeat", + fieldtype="Link", + insert_after=insert_after, + options="Auto Repeat", + hidden=1, + print_hide=1, + read_only=1, + ) create_custom_field(doc.reference_doctype, df) if doc.status in ["Draft", "Stopped", "Cancelled"]: doc.disabled = 1 doc.flags.ignore_links = 1 - #updates current status as Active, Disabled or Completed on validate - doc.save() \ No newline at end of file + # updates current status as Active, Disabled or Completed on validate + doc.save() diff --git a/frappe/patches/v12_0/update_global_search.py b/frappe/patches/v12_0/update_global_search.py index 8042a2ee68..2f07322ab9 100644 --- a/frappe/patches/v12_0/update_global_search.py +++ b/frappe/patches/v12_0/update_global_search.py @@ -1,7 +1,8 @@ import frappe from frappe.desk.page.setup_wizard.install_fixtures import update_global_search_doctypes + def execute(): frappe.reload_doc("desk", "doctype", "global_search_doctype") frappe.reload_doc("desk", "doctype", "global_search_settings") - update_global_search_doctypes() \ No newline at end of file + update_global_search_doctypes() diff --git a/frappe/patches/v12_0/update_print_format_type.py b/frappe/patches/v12_0/update_print_format_type.py index 577dc68d94..f85d32ee9a 100644 --- a/frappe/patches/v12_0/update_print_format_type.py +++ b/frappe/patches/v12_0/update_print_format_type.py @@ -1,13 +1,18 @@ import frappe + def execute(): - frappe.db.sql(''' + frappe.db.sql( + """ UPDATE `tabPrint Format` SET `print_format_type` = 'Jinja' WHERE `print_format_type` in ('Server', 'Client') - ''') - frappe.db.sql(''' + """ + ) + frappe.db.sql( + """ UPDATE `tabPrint Format` SET `print_format_type` = 'JS' WHERE `print_format_type` = 'Js' - ''') + """ + ) diff --git a/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py b/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py index cb960e84bb..32473481b8 100644 --- a/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py +++ b/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py @@ -1,14 +1,12 @@ import frappe + def execute(): - web_pages = frappe.get_all('Web Page', ['name', 'description']) + web_pages = frappe.get_all("Web Page", ["name", "description"]) for web_page in web_pages: if web_page.description and web_page.route: - doc = frappe.new_doc('Website Route Meta') + doc = frappe.new_doc("Website Route Meta") doc.name = web_page.route - doc.append('meta_tags', { - 'key': 'description', - 'value': web_page.description - }) + doc.append("meta_tags", {"key": "description", "value": web_page.description}) doc.save() diff --git a/frappe/patches/v12_0/website_meta_tag_parent.py b/frappe/patches/v12_0/website_meta_tag_parent.py index 7cc84b1283..8920189826 100644 --- a/frappe/patches/v12_0/website_meta_tag_parent.py +++ b/frappe/patches/v12_0/website_meta_tag_parent.py @@ -1,9 +1,12 @@ import frappe + def execute(): # convert all /path to path - frappe.db.sql(''' + frappe.db.sql( + """ UPDATE `tabWebsite Meta Tag` SET parent = SUBSTR(parent, 2) WHERE parent like '/%' - ''') + """ + ) diff --git a/frappe/patches/v13_0/add_standard_navbar_items.py b/frappe/patches/v13_0/add_standard_navbar_items.py index 4473cb8c07..b06494c53a 100644 --- a/frappe/patches/v13_0/add_standard_navbar_items.py +++ b/frappe/patches/v13_0/add_standard_navbar_items.py @@ -1,9 +1,9 @@ - import frappe from frappe.utils.install import add_standard_navbar_items + def execute(): # Add standard navbar items for ERPNext in Navbar Settings - frappe.reload_doc('core', 'doctype', 'navbar_settings') - frappe.reload_doc('core', 'doctype', 'navbar_item') - add_standard_navbar_items() \ No newline at end of file + frappe.reload_doc("core", "doctype", "navbar_settings") + frappe.reload_doc("core", "doctype", "navbar_item") + add_standard_navbar_items() diff --git a/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py b/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py index b5542c9c8a..00f5aa23d2 100644 --- a/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py +++ b/frappe/patches/v13_0/add_switch_theme_to_navbar_settings.py @@ -1,21 +1,24 @@ - import frappe + def execute(): navbar_settings = frappe.get_single("Navbar Settings") - if frappe.db.exists('Navbar Item', {'item_label': 'Toggle Theme'}): + if frappe.db.exists("Navbar Item", {"item_label": "Toggle Theme"}): return for navbar_item in navbar_settings.settings_dropdown[6:]: navbar_item.idx = navbar_item.idx + 1 - navbar_settings.append('settings_dropdown', { - 'item_label': 'Toggle Theme', - 'item_type': 'Action', - 'action': 'new frappe.ui.ThemeSwitcher().show()', - 'is_standard': 1, - 'idx': 7 - }) + navbar_settings.append( + "settings_dropdown", + { + "item_label": "Toggle Theme", + "item_type": "Action", + "action": "new frappe.ui.ThemeSwitcher().show()", + "is_standard": 1, + "idx": 7, + }, + ) - navbar_settings.save() \ No newline at end of file + navbar_settings.save() diff --git a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py index bd3367377c..f59786dfdf 100644 --- a/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py +++ b/frappe/patches/v13_0/add_toggle_width_in_navbar_settings.py @@ -1,21 +1,24 @@ - import frappe + def execute(): navbar_settings = frappe.get_single("Navbar Settings") - if frappe.db.exists('Navbar Item', {'item_label': 'Toggle Full Width'}): + if frappe.db.exists("Navbar Item", {"item_label": "Toggle Full Width"}): return for navbar_item in navbar_settings.settings_dropdown[5:]: navbar_item.idx = navbar_item.idx + 1 - navbar_settings.append('settings_dropdown', { - 'item_label': 'Toggle Full Width', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.toggle_full_width()', - 'is_standard': 1, - 'idx': 6 - }) + navbar_settings.append( + "settings_dropdown", + { + "item_label": "Toggle Full Width", + "item_type": "Action", + "action": "frappe.ui.toolbar.toggle_full_width()", + "is_standard": 1, + "idx": 6, + }, + ) - navbar_settings.save() \ No newline at end of file + navbar_settings.save() diff --git a/frappe/patches/v13_0/cleanup_desk_cards.py b/frappe/patches/v13_0/cleanup_desk_cards.py index b6fab66475..988e98a647 100644 --- a/frappe/patches/v13_0/cleanup_desk_cards.py +++ b/frappe/patches/v13_0/cleanup_desk_cards.py @@ -1,9 +1,11 @@ -import frappe from json import loads + +import frappe from frappe.desk.doctype.workspace.workspace import get_link_type, get_report_type + def execute(): - frappe.reload_doc('desk', 'doctype', 'workspace') + frappe.reload_doc("desk", "doctype", "workspace") pages = frappe.db.sql("Select `name` from `tabDesk Page`") # pages = frappe.get_all("Workspace", filters={"is_standard": 0}, pluck="name") @@ -13,6 +15,7 @@ def execute(): frappe.delete_doc("DocType", "Desk Card") + def rebuild_links(page): # Empty links table @@ -32,36 +35,41 @@ def rebuild_links(page): else: links = card.links - doc.append('links', { - "label": card.label, - "type": "Card Break", - "icon": card.icon, - "hidden": card.hidden or False - }) + doc.append( + "links", + {"label": card.label, "type": "Card Break", "icon": card.icon, "hidden": card.hidden or False}, + ) for link in links: - if not frappe.db.exists(get_link_type(link.get('type')), link.get('name')): + if not frappe.db.exists(get_link_type(link.get("type")), link.get("name")): continue - doc.append('links', { - "label": link.get('label') or link.get('name'), - "type": "Link", - "link_type": get_link_type(link.get('type')), - "link_to": link.get('name'), - "onboard": link.get('onboard'), - "dependencies": ', '.join(link.get('dependencies', [])), - "is_query_report": get_report_type(link.get('name')) if link.get('type').lower() == "report" else 0 - }) + doc.append( + "links", + { + "label": link.get("label") or link.get("name"), + "type": "Link", + "link_type": get_link_type(link.get("type")), + "link_to": link.get("name"), + "onboard": link.get("onboard"), + "dependencies": ", ".join(link.get("dependencies", [])), + "is_query_report": get_report_type(link.get("name")) + if link.get("type").lower() == "report" + else 0, + }, + ) try: doc.save(ignore_permissions=True) except frappe.LinkValidationError: print(doc.as_dict()) + def get_doc_from_db(page): - result = frappe.db.sql("SELECT * FROM `tabDesk Page` WHERE name=%s", [page], as_dict=True) + result = frappe.db.sql("SELECT * FROM `tabDesk Page` WHERE name=%s", [page], as_dict=True) if result: return result[0].update({"doctype": "Workspace"}) + def get_all_cards(page): - return frappe.db.get_all("Desk Card", filters={"parent": page}, fields=['*'], order_by="idx") \ No newline at end of file + return frappe.db.get_all("Desk Card", filters={"parent": page}, fields=["*"], order_by="idx") diff --git a/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py b/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py index 9a075a22cc..4ac6b918b5 100644 --- a/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py +++ b/frappe/patches/v13_0/create_custom_dashboards_cards_and_charts.py @@ -2,42 +2,45 @@ import frappe from frappe.model.naming import append_number_if_name_exists from frappe.utils.dashboard import get_dashboards_with_link + def execute(): - if not frappe.db.table_exists('Dashboard Chart')\ - or not frappe.db.table_exists('Number Card')\ - or not frappe.db.table_exists('Dashboard'): + if ( + not frappe.db.table_exists("Dashboard Chart") + or not frappe.db.table_exists("Number Card") + or not frappe.db.table_exists("Dashboard") + ): return - frappe.reload_doc('desk', 'doctype', 'dashboard_chart') - frappe.reload_doc('desk', 'doctype', 'number_card') - frappe.reload_doc('desk', 'doctype', 'dashboard') + frappe.reload_doc("desk", "doctype", "dashboard_chart") + frappe.reload_doc("desk", "doctype", "number_card") + frappe.reload_doc("desk", "doctype", "dashboard") - modified_charts = get_modified_docs('Dashboard Chart') - modified_cards = get_modified_docs('Number Card') - modified_dashboards = [doc.name for doc in get_modified_docs('Dashboard')] + modified_charts = get_modified_docs("Dashboard Chart") + modified_cards = get_modified_docs("Number Card") + modified_dashboards = [doc.name for doc in get_modified_docs("Dashboard")] for chart in modified_charts: - modified_dashboards += get_dashboards_with_link(chart.name, 'Dashboard Chart') - rename_modified_doc(chart.name, 'Dashboard Chart') + modified_dashboards += get_dashboards_with_link(chart.name, "Dashboard Chart") + rename_modified_doc(chart.name, "Dashboard Chart") for card in modified_cards: - modified_dashboards += get_dashboards_with_link(card.name, 'Number Card') - rename_modified_doc(card.name, 'Number Card') + modified_dashboards += get_dashboards_with_link(card.name, "Number Card") + rename_modified_doc(card.name, "Number Card") modified_dashboards = list(set(modified_dashboards)) for dashboard in modified_dashboards: - rename_modified_doc(dashboard, 'Dashboard') + rename_modified_doc(dashboard, "Dashboard") + def get_modified_docs(doctype): - return frappe.get_all(doctype, - filters = { - 'owner': 'Administrator', - 'modified_by': ['!=', 'Administrator'] - }) + return frappe.get_all( + doctype, filters={"owner": "Administrator", "modified_by": ["!=", "Administrator"]} + ) + def rename_modified_doc(docname, doctype): - new_name = docname + ' Custom' + new_name = docname + " Custom" try: frappe.rename_doc(doctype, docname, new_name) except frappe.ValidationError: diff --git a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py b/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py index 2d9e232da5..9cb081e15a 100644 --- a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py +++ b/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py @@ -3,6 +3,7 @@ import frappe + def execute(): if frappe.db.exists("DocType", "Event Producer"): frappe.db.sql("""UPDATE `tabEvent Producer` SET api_key='', api_secret=''""") diff --git a/frappe/patches/v13_0/email_unsubscribe.py b/frappe/patches/v13_0/email_unsubscribe.py index 69ed1be948..d5fe2ae32e 100644 --- a/frappe/patches/v13_0/email_unsubscribe.py +++ b/frappe/patches/v13_0/email_unsubscribe.py @@ -1,13 +1,14 @@ import frappe + def execute(): email_unsubscribe = [ {"email": "admin@example.com", "global_unsubscribe": 1}, - {"email": "guest@example.com", "global_unsubscribe": 1} + {"email": "guest@example.com", "global_unsubscribe": 1}, ] for unsubscribe in email_unsubscribe: if not frappe.get_all("Email Unsubscribe", filters=unsubscribe): doc = frappe.new_doc("Email Unsubscribe") doc.update(unsubscribe) - doc.insert(ignore_permissions=True) \ No newline at end of file + doc.insert(ignore_permissions=True) diff --git a/frappe/patches/v13_0/enable_custom_script.py b/frappe/patches/v13_0/enable_custom_script.py index de027ab97a..9ff9732a4e 100644 --- a/frappe/patches/v13_0/enable_custom_script.py +++ b/frappe/patches/v13_0/enable_custom_script.py @@ -3,9 +3,12 @@ import frappe + def execute(): """Enable all the existing Client script""" - frappe.db.sql(""" + frappe.db.sql( + """ UPDATE `tabClient Script` SET enabled=1 - """) \ No newline at end of file + """ + ) diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py index 6e8e0d7fc5..9b905a9bbb 100644 --- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -14,6 +14,6 @@ def execute(): try: doc.generate_bootstrap_theme() doc.save() - except: # noqa: E722 - print('Ignoring....') + except: # noqa: E722 + print("Ignoring....") print(frappe.get_traceback()) diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index deb7d7e98a..8933c99009 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,4 +1,5 @@ import frappe + def execute(): frappe.db.change_column_type("__Auth", column="password", type="TEXT") diff --git a/frappe/patches/v13_0/jinja_hook.py b/frappe/patches/v13_0/jinja_hook.py index e1c9175576..b7e0a501ee 100644 --- a/frappe/patches/v13_0/jinja_hook.py +++ b/frappe/patches/v13_0/jinja_hook.py @@ -1,12 +1,17 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe from click import secho +import frappe + + def execute(): - if frappe.get_hooks('jenv'): + if frappe.get_hooks("jenv"): print() - secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow') - secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow') + secho( + 'WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', + fg="yellow", + ) + secho("https://github.com/frappe/frappe/wiki/Migrating-to-Version-13", fg="yellow") print() diff --git a/frappe/patches/v13_0/make_user_type.py b/frappe/patches/v13_0/make_user_type.py index 0fd5b98e9d..1e77dd4f54 100644 --- a/frappe/patches/v13_0/make_user_type.py +++ b/frappe/patches/v13_0/make_user_type.py @@ -1,12 +1,12 @@ import frappe from frappe.utils.install import create_user_type -def execute(): - frappe.reload_doc('core', 'doctype', 'role') - frappe.reload_doc('core', 'doctype', 'user_document_type') - frappe.reload_doc('core', 'doctype', 'user_type_module') - frappe.reload_doc('core', 'doctype', 'user_select_document_type') - frappe.reload_doc('core', 'doctype', 'user_type') +def execute(): + frappe.reload_doc("core", "doctype", "role") + frappe.reload_doc("core", "doctype", "user_document_type") + frappe.reload_doc("core", "doctype", "user_type_module") + frappe.reload_doc("core", "doctype", "user_select_document_type") + frappe.reload_doc("core", "doctype", "user_type") create_user_type() diff --git a/frappe/patches/v13_0/migrate_translation_column_data.py b/frappe/patches/v13_0/migrate_translation_column_data.py index 7c83f93081..831c27bc75 100644 --- a/frappe/patches/v13_0/migrate_translation_column_data.py +++ b/frappe/patches/v13_0/migrate_translation_column_data.py @@ -1,5 +1,8 @@ import frappe + def execute(): - frappe.reload_doctype('Translation') - frappe.db.sql("UPDATE `tabTranslation` SET `translated_text`=`target_name`, `source_text`=`source_name`, `contributed`=0") + frappe.reload_doctype("Translation") + frappe.db.sql( + "UPDATE `tabTranslation` SET `translated_text`=`target_name`, `source_text`=`source_name`, `contributed`=0" + ) diff --git a/frappe/patches/v13_0/queryreport_columns.py b/frappe/patches/v13_0/queryreport_columns.py index ed22ce4441..3081823db6 100644 --- a/frappe/patches/v13_0/queryreport_columns.py +++ b/frappe/patches/v13_0/queryreport_columns.py @@ -1,21 +1,18 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import json +import frappe + + def execute(): """Convert Query Report json to support other content""" - records = frappe.get_all('Report', - filters={ - "json": ["!=", ""] - }, - fields=["name", "json"] - ) + records = frappe.get_all("Report", filters={"json": ["!=", ""]}, fields=["name", "json"]) for record in records: jstr = record["json"] data = json.loads(jstr) if isinstance(data, list): # double escape braces jstr = f'{{"columns":{jstr}}}' - frappe.db.update('Report', record["name"], "json", jstr) + frappe.db.update("Report", record["name"], "json", jstr) diff --git a/frappe/patches/v13_0/remove_chat.py b/frappe/patches/v13_0/remove_chat.py index 1804c7693f..685df37366 100644 --- a/frappe/patches/v13_0/remove_chat.py +++ b/frappe/patches/v13_0/remove_chat.py @@ -1,6 +1,8 @@ -import frappe import click +import frappe + + def execute(): frappe.delete_doc_if_exists("DocType", "Chat Message") frappe.delete_doc_if_exists("DocType", "Chat Message Attachment") @@ -14,4 +16,4 @@ def execute(): "Chat Module is moved to a separate app and is removed from Frappe in version-13.\n" "Please install the app to continue using the chat feature: https://github.com/frappe/chat", fg="yellow", - ) \ No newline at end of file + ) diff --git a/frappe/patches/v13_0/remove_custom_link.py b/frappe/patches/v13_0/remove_custom_link.py index f38bb642f0..8803b66b98 100644 --- a/frappe/patches/v13_0/remove_custom_link.py +++ b/frappe/patches/v13_0/remove_custom_link.py @@ -1,15 +1,18 @@ import frappe + def execute(): - ''' + """ Remove the doctype "Custom Link" that was used to add Custom Links to the Dashboard since this is now managed by Customize Form. Update `parent` property to the DocType and delte the doctype - ''' - frappe.reload_doctype('DocType Link') - if frappe.db.has_table('Custom Link'): - for custom_link in frappe.get_all('Custom Link', ['name', 'document_type']): - frappe.db.sql('update `tabDocType Link` set custom=1, parent=%s where parent=%s', - (custom_link.document_type, custom_link.name)) + """ + frappe.reload_doctype("DocType Link") + if frappe.db.has_table("Custom Link"): + for custom_link in frappe.get_all("Custom Link", ["name", "document_type"]): + frappe.db.sql( + "update `tabDocType Link` set custom=1, parent=%s where parent=%s", + (custom_link.document_type, custom_link.name), + ) - frappe.delete_doc('DocType', 'Custom Link') \ No newline at end of file + frappe.delete_doc("DocType", "Custom Link") diff --git a/frappe/patches/v13_0/remove_duplicate_navbar_items.py b/frappe/patches/v13_0/remove_duplicate_navbar_items.py index b6c6033f64..593a529efc 100644 --- a/frappe/patches/v13_0/remove_duplicate_navbar_items.py +++ b/frappe/patches/v13_0/remove_duplicate_navbar_items.py @@ -1,12 +1,12 @@ - import frappe + def execute(): navbar_settings = frappe.get_single("Navbar Settings") duplicate_items = [] for navbar_item in navbar_settings.settings_dropdown: - if navbar_item.item_label == 'Toggle Full Width': + if navbar_item.item_label == "Toggle Full Width": duplicate_items.append(navbar_item) if len(duplicate_items) > 1: diff --git a/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py b/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py index 90e4b3c5c6..9bddcfe3db 100644 --- a/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py +++ b/frappe/patches/v13_0/remove_invalid_options_for_data_fields.py @@ -7,11 +7,9 @@ from frappe.model import data_field_options def execute(): - custom_field = frappe.qb.DocType('Custom Field') - (frappe.qb - .update(custom_field) + custom_field = frappe.qb.DocType("Custom Field") + ( + frappe.qb.update(custom_field) .set(custom_field.options, None) - .where( - (custom_field.fieldtype == "Data") - & (custom_field.options.notin(data_field_options))) + .where((custom_field.fieldtype == "Data") & (custom_field.options.notin(data_field_options))) ).run() diff --git a/frappe/patches/v13_0/remove_tailwind_from_page_builder.py b/frappe/patches/v13_0/remove_tailwind_from_page_builder.py index b26d2bef4a..44a1f5940a 100644 --- a/frappe/patches/v13_0/remove_tailwind_from_page_builder.py +++ b/frappe/patches/v13_0/remove_tailwind_from_page_builder.py @@ -9,4 +9,3 @@ def execute(): # remove unused templates frappe.delete_doc("Web Template", "Navbar with Links on Right", force=1) frappe.delete_doc("Web Template", "Footer Horizontal", force=1) - diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 826edfb951..ae0fc6552b 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -9,14 +9,12 @@ def execute(): While making Twilio as a standaone app, we missed to delete Twilio records from DB through migration. Adding the missing patch. """ - frappe.delete_doc_if_exists('DocType', 'Twilio Number Group') + frappe.delete_doc_if_exists("DocType", "Twilio Number Group") if twilio_settings_doctype_in_integrations(): - frappe.delete_doc_if_exists('DocType', 'Twilio Settings') - frappe.db.delete("Singles", { - "doctype": "Twilio Settings" - }) + frappe.delete_doc_if_exists("DocType", "Twilio Settings") + frappe.db.delete("Singles", {"doctype": "Twilio Settings"}) + def twilio_settings_doctype_in_integrations() -> bool: - """Check Twilio Settings doctype exists in integrations module or not. - """ - return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) + """Check Twilio Settings doctype exists in integrations module or not.""" + return frappe.db.exists("DocType", {"name": "Twilio Settings", "module": "Integrations"}) diff --git a/frappe/patches/v13_0/remove_web_view.py b/frappe/patches/v13_0/remove_web_view.py index 7c9109fd03..4695c45558 100644 --- a/frappe/patches/v13_0/remove_web_view.py +++ b/frappe/patches/v13_0/remove_web_view.py @@ -1,6 +1,7 @@ import frappe + def execute(): frappe.delete_doc_if_exists("DocType", "Web View") frappe.delete_doc_if_exists("DocType", "Web View Component") - frappe.delete_doc_if_exists("DocType", "CSS Class") \ No newline at end of file + frappe.delete_doc_if_exists("DocType", "CSS Class") diff --git a/frappe/patches/v13_0/rename_desk_page_to_workspace.py b/frappe/patches/v13_0/rename_desk_page_to_workspace.py index 6483fc380c..e4d26a5b7a 100644 --- a/frappe/patches/v13_0/rename_desk_page_to_workspace.py +++ b/frappe/patches/v13_0/rename_desk_page_to_workspace.py @@ -1,21 +1,22 @@ import frappe from frappe.model.rename_doc import rename_doc + def execute(): if frappe.db.exists("DocType", "Desk Page"): - if frappe.db.exists('DocType', 'Workspace'): + if frappe.db.exists("DocType", "Workspace"): # this patch was not added initially, so this page might still exist - frappe.delete_doc('DocType', 'Desk Page') + frappe.delete_doc("DocType", "Desk Page") else: frappe.flags.ignore_route_conflict_validation = True - rename_doc('DocType', 'Desk Page', 'Workspace') + rename_doc("DocType", "Desk Page", "Workspace") frappe.flags.ignore_route_conflict_validation = False - rename_doc('DocType', 'Desk Chart', 'Workspace Chart', ignore_if_exists=True) - rename_doc('DocType', 'Desk Shortcut', 'Workspace Shortcut', ignore_if_exists=True) - rename_doc('DocType', 'Desk Link', 'Workspace Link', ignore_if_exists=True) + rename_doc("DocType", "Desk Chart", "Workspace Chart", ignore_if_exists=True) + rename_doc("DocType", "Desk Shortcut", "Workspace Shortcut", ignore_if_exists=True) + rename_doc("DocType", "Desk Link", "Workspace Link", ignore_if_exists=True) - frappe.reload_doc('desk', 'doctype', 'workspace', force=True) - frappe.reload_doc('desk', 'doctype', 'workspace_link', force=True) - frappe.reload_doc('desk', 'doctype', 'workspace_chart', force=True) - frappe.reload_doc('desk', 'doctype', 'workspace_shortcut', force=True) + frappe.reload_doc("desk", "doctype", "workspace", force=True) + frappe.reload_doc("desk", "doctype", "workspace_link", force=True) + frappe.reload_doc("desk", "doctype", "workspace_chart", force=True) + frappe.reload_doc("desk", "doctype", "workspace_shortcut", force=True) diff --git a/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py b/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py index 4da0f8164a..61f80fdb2f 100644 --- a/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py +++ b/frappe/patches/v13_0/rename_is_custom_field_in_dashboard_chart.py @@ -1,11 +1,12 @@ import frappe from frappe.model.utils.rename_field import rename_field + def execute(): - if not frappe.db.table_exists('Dashboard Chart'): + if not frappe.db.table_exists("Dashboard Chart"): return - frappe.reload_doc('desk', 'doctype', 'dashboard_chart') + frappe.reload_doc("desk", "doctype", "dashboard_chart") - if frappe.db.has_column('Dashboard Chart', 'is_custom'): - rename_field('Dashboard Chart', 'is_custom', 'use_report_chart') \ No newline at end of file + if frappe.db.has_column("Dashboard Chart", "is_custom"): + rename_field("Dashboard Chart", "is_custom", "use_report_chart") diff --git a/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py b/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py index db3ab1b32a..2147a2da94 100644 --- a/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py +++ b/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py @@ -5,15 +5,18 @@ import frappe def execute(): - if frappe.db.table_exists('List View Setting'): - if not frappe.db.table_exists('List View Settings'): + if frappe.db.table_exists("List View Setting"): + if not frappe.db.table_exists("List View Settings"): frappe.reload_doc("desk", "doctype", "List View Settings") - existing_list_view_settings = frappe.get_all('List View Settings', as_list=True) - for list_view_setting in frappe.get_all('List View Setting', fields = ['disable_count', 'disable_sidebar_stats', 'disable_auto_refresh', 'name']): - name = list_view_setting.pop('name') + existing_list_view_settings = frappe.get_all("List View Settings", as_list=True) + for list_view_setting in frappe.get_all( + "List View Setting", + fields=["disable_count", "disable_sidebar_stats", "disable_auto_refresh", "name"], + ): + name = list_view_setting.pop("name") if name not in [x[0] for x in existing_list_view_settings]: - list_view_setting['doctype'] = 'List View Settings' + list_view_setting["doctype"] = "List View Settings" list_view_settings = frappe.get_doc(list_view_setting) # setting name here is necessary because autoname is set as prompt list_view_settings.name = name diff --git a/frappe/patches/v13_0/rename_notification_fields.py b/frappe/patches/v13_0/rename_notification_fields.py index 2f314df9c1..74674eb6f7 100644 --- a/frappe/patches/v13_0/rename_notification_fields.py +++ b/frappe/patches/v13_0/rename_notification_fields.py @@ -4,12 +4,13 @@ import frappe from frappe.model.utils.rename_field import rename_field + def execute(): """ - Change notification recipient fields from email to receiver fields + Change notification recipient fields from email to receiver fields """ frappe.reload_doc("Email", "doctype", "Notification Recipient") frappe.reload_doc("Email", "doctype", "Notification") rename_field("Notification Recipient", "email_by_document_field", "receiver_by_document_field") - rename_field("Notification Recipient", "email_by_role", "receiver_by_role") \ No newline at end of file + rename_field("Notification Recipient", "email_by_role", "receiver_by_role") diff --git a/frappe/patches/v13_0/rename_onboarding.py b/frappe/patches/v13_0/rename_onboarding.py index cd910195ad..f4e8445320 100644 --- a/frappe/patches/v13_0/rename_onboarding.py +++ b/frappe/patches/v13_0/rename_onboarding.py @@ -3,7 +3,7 @@ import frappe + def execute(): if frappe.db.exists("DocType", "Onboarding"): frappe.rename_doc("DocType", "Onboarding", "Module Onboarding", ignore_if_exists=True) - diff --git a/frappe/patches/v13_0/replace_field_target_with_open_in_new_tab.py b/frappe/patches/v13_0/replace_field_target_with_open_in_new_tab.py index 21b2d8ef03..a133d7f6fe 100644 --- a/frappe/patches/v13_0/replace_field_target_with_open_in_new_tab.py +++ b/frappe/patches/v13_0/replace_field_target_with_open_in_new_tab.py @@ -3,9 +3,8 @@ import frappe def execute(): doctype = "Top Bar Item" - if not frappe.db.table_exists(doctype) \ - or not frappe.db.has_column(doctype, "target"): + if not frappe.db.table_exists(doctype) or not frappe.db.has_column(doctype, "target"): return frappe.reload_doc("website", "doctype", "top_bar_item") - frappe.db.set_value(doctype, {"target": 'target = "_blank"'}, 'open_in_new_tab', 1) + frappe.db.set_value(doctype, {"target": 'target = "_blank"'}, "open_in_new_tab", 1) diff --git a/frappe/patches/v13_0/replace_old_data_import.py b/frappe/patches/v13_0/replace_old_data_import.py index 7d2692a433..009c2e4f10 100644 --- a/frappe/patches/v13_0/replace_old_data_import.py +++ b/frappe/patches/v13_0/replace_old_data_import.py @@ -5,7 +5,8 @@ import frappe def execute(): - if not frappe.db.table_exists("Data Import"): return + if not frappe.db.table_exists("Data Import"): + return meta = frappe.get_meta("Data Import") # if Data Import is the new one, return early diff --git a/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py b/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py index 80c6105440..11015033b9 100644 --- a/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py +++ b/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py @@ -1,21 +1,21 @@ import frappe + def execute(): - frappe.reload_doc('desk', 'doctype', 'dashboard_chart') + frappe.reload_doc("desk", "doctype", "dashboard_chart") - if not frappe.db.table_exists('Dashboard Chart'): + if not frappe.db.table_exists("Dashboard Chart"): return users_with_permission = frappe.get_all( "Has Role", fields=["parent"], - filters={"role": ['in', ['System Manager', 'Dashboard Manager']], "parenttype": "User"}, + filters={"role": ["in", ["System Manager", "Dashboard Manager"]], "parenttype": "User"}, distinct=True, ) users = [item.parent for item in users_with_permission] - charts = frappe.db.get_all('Dashboard Chart', filters={'owner': ['in', users]}) + charts = frappe.db.get_all("Dashboard Chart", filters={"owner": ["in", users]}) for chart in charts: - frappe.db.set_value('Dashboard Chart', chart.name, 'is_public', 1) - + frappe.db.set_value("Dashboard Chart", chart.name, "is_public", 1) diff --git a/frappe/patches/v13_0/set_first_day_of_the_week.py b/frappe/patches/v13_0/set_first_day_of_the_week.py index cfb694bbf1..165ec3c42b 100644 --- a/frappe/patches/v13_0/set_first_day_of_the_week.py +++ b/frappe/patches/v13_0/set_first_day_of_the_week.py @@ -1,7 +1,8 @@ import frappe + def execute(): frappe.reload_doctype("System Settings") # setting first_day_of_the_week value as "Monday" to avoid breaking change # because before the configuration was introduced, system used to consider "Monday" as start of the week - frappe.db.set_value("System Settings", "System Settings", "first_day_of_the_week", "Monday") \ No newline at end of file + frappe.db.set_value("System Settings", "System Settings", "first_day_of_the_week", "Monday") diff --git a/frappe/patches/v13_0/set_path_for_homepage_in_web_page_view.py b/frappe/patches/v13_0/set_path_for_homepage_in_web_page_view.py index d4fe2d2f36..a297eeb64c 100644 --- a/frappe/patches/v13_0/set_path_for_homepage_in_web_page_view.py +++ b/frappe/patches/v13_0/set_path_for_homepage_in_web_page_view.py @@ -1,5 +1,6 @@ import frappe + def execute(): - frappe.reload_doc('website', 'doctype', 'web_page_view', force=True) + frappe.reload_doc("website", "doctype", "web_page_view", force=True) frappe.db.sql("""UPDATE `tabWeb Page View` set path='/' where path=''""") diff --git a/frappe/patches/v13_0/set_read_times.py b/frappe/patches/v13_0/set_read_times.py index 6957139372..987ed0f1b9 100644 --- a/frappe/patches/v13_0/set_read_times.py +++ b/frappe/patches/v13_0/set_read_times.py @@ -1,13 +1,18 @@ -import frappe -from frappe.utils import strip_html_tags, markdown from math import ceil +import frappe +from frappe.utils import markdown, strip_html_tags + + def execute(): frappe.reload_doc("website", "doctype", "blog_post") for blog in frappe.get_all("Blog Post"): blog = frappe.get_doc("Blog Post", blog.name) - frappe.db.set_value("Blog Post", blog.name, "read_time", get_read_time(blog), update_modified=False) + frappe.db.set_value( + "Blog Post", blog.name, "read_time", get_read_time(blog), update_modified=False + ) + def get_read_time(blog): content = blog.content or blog.content_html @@ -15,4 +20,4 @@ def get_read_time(blog): content = markdown(blog.content_md) total_words = len(strip_html_tags(content or "").split()) - return ceil(total_words/250) \ No newline at end of file + return ceil(total_words / 250) diff --git a/frappe/patches/v13_0/set_route_for_blog_category.py b/frappe/patches/v13_0/set_route_for_blog_category.py index 7ea26bc2c0..e9d15e0cc3 100644 --- a/frappe/patches/v13_0/set_route_for_blog_category.py +++ b/frappe/patches/v13_0/set_route_for_blog_category.py @@ -1,5 +1,6 @@ import frappe + def execute(): categories = frappe.get_list("Blog Category") for category in categories: diff --git a/frappe/patches/v13_0/set_social_icons.py b/frappe/patches/v13_0/set_social_icons.py index b1c63b48e0..25b144ed53 100644 --- a/frappe/patches/v13_0/set_social_icons.py +++ b/frappe/patches/v13_0/set_social_icons.py @@ -1,9 +1,10 @@ import frappe + def execute(): providers = frappe.get_all("Social Login Key") for provider in providers: doc = frappe.get_doc("Social Login Key", provider) doc.set_icon() - doc.save() \ No newline at end of file + doc.save() diff --git a/frappe/patches/v13_0/set_unique_for_page_view.py b/frappe/patches/v13_0/set_unique_for_page_view.py index 2a084e52e3..1a674f6697 100644 --- a/frappe/patches/v13_0/set_unique_for_page_view.py +++ b/frappe/patches/v13_0/set_unique_for_page_view.py @@ -1,6 +1,9 @@ import frappe + def execute(): - frappe.reload_doc('website', 'doctype', 'web_page_view', force=True) + frappe.reload_doc("website", "doctype", "web_page_view", force=True) site_url = frappe.utils.get_site_url(frappe.local.site) - frappe.db.sql("""UPDATE `tabWeb Page View` set is_unique=1 where referrer LIKE '%{0}%'""".format(site_url)) + frappe.db.sql( + """UPDATE `tabWeb Page View` set is_unique=1 where referrer LIKE '%{0}%'""".format(site_url) + ) diff --git a/frappe/patches/v13_0/site_wise_logging.py b/frappe/patches/v13_0/site_wise_logging.py index 6f04e0c9dd..5807a85c33 100644 --- a/frappe/patches/v13_0/site_wise_logging.py +++ b/frappe/patches/v13_0/site_wise_logging.py @@ -1,10 +1,11 @@ import os + import frappe def execute(): site = frappe.local.site - log_folder = os.path.join(site, 'logs') + log_folder = os.path.join(site, "logs") if not os.path.exists(log_folder): - os.mkdir(log_folder) \ No newline at end of file + os.mkdir(log_folder) diff --git a/frappe/patches/v13_0/update_date_filters_in_user_settings.py b/frappe/patches/v13_0/update_date_filters_in_user_settings.py index fa13a6789e..a2ac9e94fb 100644 --- a/frappe/patches/v13_0/update_date_filters_in_user_settings.py +++ b/frappe/patches/v13_0/update_date_filters_in_user_settings.py @@ -1,20 +1,27 @@ +import json + +import frappe +from frappe.model.utils.user_settings import sync_user_settings, update_user_settings -import frappe, json -from frappe.model.utils.user_settings import update_user_settings, sync_user_settings def execute(): users = frappe.db.sql("select distinct(user) from `__UserSettings`", as_dict=True) for user in users: - user_settings = frappe.db.sql(''' + user_settings = frappe.db.sql( + """ select * from `__UserSettings` where user='{user}' - '''.format(user = user.user), as_dict=True) + """.format( + user=user.user + ), + as_dict=True, + ) for setting in user_settings: - data = frappe.parse_json(setting.get('data')) + data = frappe.parse_json(setting.get("data")) if data: for key in data: update_user_setting_filters(data, key, setting) @@ -24,31 +31,26 @@ def execute(): def update_user_setting_filters(data, key, user_setting): timespan_map = { - '1 week': 'week', - '1 month': 'month', - '3 months': 'quarter', - '6 months': '6 months', - '1 year': 'year', + "1 week": "week", + "1 month": "month", + "3 months": "quarter", + "6 months": "6 months", + "1 year": "year", } - period_map = { - 'Previous': 'last', - 'Next': 'next' - } + period_map = {"Previous": "last", "Next": "next"} if data.get(key): update = False if isinstance(data.get(key), dict): - filters = data.get(key).get('filters') + filters = data.get(key).get("filters") if filters and isinstance(filters, list): for f in filters: - if f[2] == 'Next' or f[2] == 'Previous': + if f[2] == "Next" or f[2] == "Previous": update = True - f[3] = period_map[f[2]] + ' ' + timespan_map[f[3]] - f[2] = 'Timespan' + f[3] = period_map[f[2]] + " " + timespan_map[f[3]] + f[2] = "Timespan" if update: - data[key]['filters'] = filters - update_user_settings(user_setting['doctype'], json.dumps(data), for_update=True) - - + data[key]["filters"] = filters + update_user_settings(user_setting["doctype"], json.dumps(data), for_update=True) diff --git a/frappe/patches/v13_0/update_duration_options.py b/frappe/patches/v13_0/update_duration_options.py index 48f0dc0969..56ad4fe698 100644 --- a/frappe/patches/v13_0/update_duration_options.py +++ b/frappe/patches/v13_0/update_duration_options.py @@ -3,25 +3,30 @@ import frappe + def execute(): - frappe.reload_doc('core', 'doctype', 'DocField') + frappe.reload_doc("core", "doctype", "DocField") - if frappe.db.has_column('DocField', 'show_days'): - frappe.db.sql(""" + if frappe.db.has_column("DocField", "show_days"): + frappe.db.sql( + """ UPDATE tabDocField SET hide_days = 1 WHERE show_days = 0 - """) - frappe.db.sql_ddl('alter table tabDocField drop column show_days') + """ + ) + frappe.db.sql_ddl("alter table tabDocField drop column show_days") - if frappe.db.has_column('DocField', 'show_seconds'): - frappe.db.sql(""" + if frappe.db.has_column("DocField", "show_seconds"): + frappe.db.sql( + """ UPDATE tabDocField SET hide_seconds = 1 WHERE show_seconds = 0 - """) - frappe.db.sql_ddl('alter table tabDocField drop column show_seconds') + """ + ) + frappe.db.sql_ddl("alter table tabDocField drop column show_seconds") - frappe.clear_cache(doctype='DocField') \ No newline at end of file + frappe.clear_cache(doctype="DocField") diff --git a/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py b/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py index ff58f99c2f..d48fe217c8 100644 --- a/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py +++ b/frappe/patches/v13_0/update_icons_in_customized_desk_pages.py @@ -1,10 +1,13 @@ - import frappe + def execute(): - if not frappe.db.exists('Desk Page'): return + if not frappe.db.exists("Desk Page"): + return - pages = frappe.get_all("Desk Page", filters={ "is_standard": False }, fields=["name", "extends", "for_user"]) + pages = frappe.get_all( + "Desk Page", filters={"is_standard": False}, fields=["name", "extends", "for_user"] + ) default_icon = {} for page in pages: if page.extends and page.for_user: @@ -12,4 +15,4 @@ def execute(): default_icon[page.extends] = frappe.db.get_value("Desk Page", page.extends, "icon") icon = default_icon.get(page.extends) - frappe.db.set_value("Desk Page", page.name, "icon", icon) \ No newline at end of file + frappe.db.set_value("Desk Page", page.name, "icon", icon) diff --git a/frappe/patches/v13_0/update_newsletter_content_type.py b/frappe/patches/v13_0/update_newsletter_content_type.py index 39758c8257..2004019750 100644 --- a/frappe/patches/v13_0/update_newsletter_content_type.py +++ b/frappe/patches/v13_0/update_newsletter_content_type.py @@ -3,9 +3,12 @@ import frappe + def execute(): - frappe.reload_doc('email', 'doctype', 'Newsletter') - frappe.db.sql(""" + frappe.reload_doc("email", "doctype", "Newsletter") + frappe.db.sql( + """ UPDATE tabNewsletter SET content_type = 'Rich Text' - """) + """ + ) diff --git a/frappe/patches/v13_0/update_notification_channel_if_empty.py b/frappe/patches/v13_0/update_notification_channel_if_empty.py index 43cf813c74..5ebab68f27 100644 --- a/frappe/patches/v13_0/update_notification_channel_if_empty.py +++ b/frappe/patches/v13_0/update_notification_channel_if_empty.py @@ -3,12 +3,15 @@ import frappe + def execute(): frappe.reload_doc("Email", "doctype", "Notification") - notifications = frappe.get_all('Notification', {'is_standard': 1}, {'name', 'channel'}) + notifications = frappe.get_all("Notification", {"is_standard": 1}, {"name", "channel"}) for notification in notifications: if not notification.channel: - frappe.db.set_value("Notification", notification.name, "channel", "Email", update_modified=False) + frappe.db.set_value( + "Notification", notification.name, "channel", "Email", update_modified=False + ) frappe.db.commit() diff --git a/frappe/patches/v13_0/web_template_set_module.py b/frappe/patches/v13_0/web_template_set_module.py index d200f4e0da..6fd0f8666f 100644 --- a/frappe/patches/v13_0/web_template_set_module.py +++ b/frappe/patches/v13_0/web_template_set_module.py @@ -3,14 +3,15 @@ import frappe + def execute(): """Set default module for standard Web Template, if none.""" - frappe.reload_doc('website', 'doctype', 'Web Template Field') - frappe.reload_doc('website', 'doctype', 'web_template') + frappe.reload_doc("website", "doctype", "Web Template Field") + frappe.reload_doc("website", "doctype", "web_template") - standard_templates = frappe.get_list('Web Template', {'standard': 1}) + standard_templates = frappe.get_list("Web Template", {"standard": 1}) for template in standard_templates: - doc = frappe.get_doc('Web Template', template.name) + doc = frappe.get_doc("Web Template", template.name) if not doc.module: - doc.module = 'Website' + doc.module = "Website" doc.save() diff --git a/frappe/patches/v13_0/website_theme_custom_scss.py b/frappe/patches/v13_0/website_theme_custom_scss.py index 2acb73e11a..d1a5e11228 100644 --- a/frappe/patches/v13_0/website_theme_custom_scss.py +++ b/frappe/patches/v13_0/website_theme_custom_scss.py @@ -1,13 +1,14 @@ import frappe + def execute(): - frappe.reload_doc('website', 'doctype', 'website_theme_ignore_app') - frappe.reload_doc('website', 'doctype', 'color') - frappe.reload_doc('website', 'doctype', 'website_theme', force=True) + frappe.reload_doc("website", "doctype", "website_theme_ignore_app") + frappe.reload_doc("website", "doctype", "color") + frappe.reload_doc("website", "doctype", "website_theme", force=True) - for theme in frappe.get_all('Website Theme'): - doc = frappe.get_doc('Website Theme', theme.name) - if not doc.get('custom_scss') and doc.theme_scss: + for theme in frappe.get_all("Website Theme"): + doc = frappe.get_doc("Website Theme", theme.name) + if not doc.get("custom_scss") and doc.theme_scss: # move old theme to new theme doc.custom_scss = doc.theme_scss @@ -16,9 +17,12 @@ def execute(): doc.save() + def setup_color_record(color): - frappe.get_doc({ - "doctype": "Color", - "__newname": color, - "color": color, - }).save() + frappe.get_doc( + { + "doctype": "Color", + "__newname": color, + "color": color, + } + ).save() diff --git a/frappe/patches/v14_0/copy_mail_data.py b/frappe/patches/v14_0/copy_mail_data.py index 8ef9cfaf1f..6b976ba6fb 100644 --- a/frappe/patches/v14_0/copy_mail_data.py +++ b/frappe/patches/v14_0/copy_mail_data.py @@ -1,21 +1,27 @@ from __future__ import unicode_literals + import frappe def execute(): # patch for all Email Account with the flag use_imap - for email_account in frappe.get_list("Email Account", filters={"enable_incoming": 1, "use_imap": 1}): + for email_account in frappe.get_list( + "Email Account", filters={"enable_incoming": 1, "use_imap": 1} + ): # get all data from Email Account doc = frappe.get_doc("Email Account", email_account.name) imap_list = [folder.folder_name for folder in doc.imap_folder] # and append the old data to the child table if doc.uidvalidity or doc.uidnext and "INBOX" not in imap_list: - doc.append("imap_folder", { - "folder_name": "INBOX", - "append_to": doc.append_to, - "uid_validity": doc.uidvalidity, - "uidnext": doc.uidnext, - }) + doc.append( + "imap_folder", + { + "folder_name": "INBOX", + "append_to": doc.append_to, + "uid_validity": doc.uidvalidity, + "uidnext": doc.uidnext, + }, + ) doc.save() diff --git a/frappe/patches/v14_0/drop_data_import_legacy.py b/frappe/patches/v14_0/drop_data_import_legacy.py index 2037930c9f..16d366a845 100644 --- a/frappe/patches/v14_0/drop_data_import_legacy.py +++ b/frappe/patches/v14_0/drop_data_import_legacy.py @@ -1,6 +1,7 @@ -import frappe import click +import frappe + def execute(): doctype = "Data Import Legacy" diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py index a8fa5b2ba1..6dc34a784b 100644 --- a/frappe/patches/v14_0/remove_db_aggregation.py +++ b/frappe/patches/v14_0/remove_db_aggregation.py @@ -8,20 +8,23 @@ def execute(): """Replace temporarily available Database Aggregate APIs on frappe (develop) APIs changed: - * frappe.db.max => frappe.qb.max - * frappe.db.min => frappe.qb.min - * frappe.db.sum => frappe.qb.sum - * frappe.db.avg => frappe.qb.avg + * frappe.db.max => frappe.qb.max + * frappe.db.min => frappe.qb.min + * frappe.db.sum => frappe.qb.sum + * frappe.db.avg => frappe.qb.avg """ ServerScript = DocType("Server Script") - server_scripts = frappe.qb.from_(ServerScript).where( - ServerScript.script.like("%frappe.db.max(%") - | ServerScript.script.like("%frappe.db.min(%") - | ServerScript.script.like("%frappe.db.sum(%") - | ServerScript.script.like("%frappe.db.avg(%") - ).select( - "name", "script" - ).run(as_dict=True) + server_scripts = ( + frappe.qb.from_(ServerScript) + .where( + ServerScript.script.like("%frappe.db.max(%") + | ServerScript.script.like("%frappe.db.min(%") + | ServerScript.script.like("%frappe.db.sum(%") + | ServerScript.script.like("%frappe.db.avg(%") + ) + .select("name", "script") + .run(as_dict=True) + ) for server_script in server_scripts: name, script = server_script["name"], server_script["script"] diff --git a/frappe/patches/v14_0/remove_post_and_post_comment.py b/frappe/patches/v14_0/remove_post_and_post_comment.py index 3a93139961..ee3c41d459 100644 --- a/frappe/patches/v14_0/remove_post_and_post_comment.py +++ b/frappe/patches/v14_0/remove_post_and_post_comment.py @@ -1,5 +1,6 @@ import frappe + def execute(): frappe.delete_doc_if_exists("DocType", "Post") frappe.delete_doc_if_exists("DocType", "Post Comment") diff --git a/frappe/patches/v14_0/reset_creation_datetime.py b/frappe/patches/v14_0/reset_creation_datetime.py index 54eb6c65af..5ba1bc9529 100644 --- a/frappe/patches/v14_0/reset_creation_datetime.py +++ b/frappe/patches/v14_0/reset_creation_datetime.py @@ -1,7 +1,8 @@ import glob import json -import frappe import os + +import frappe from frappe.query_builder import DocType as _DocType @@ -12,9 +13,7 @@ def execute(): os.path.join("..", "apps", "frappe", "frappe", "**", "doctype", "**", "*.json") ) - frappe_modules = frappe.get_all( - "Module Def", filters={"app_name": "frappe"}, pluck="name" - ) + frappe_modules = frappe.get_all("Module Def", filters={"app_name": "frappe"}, pluck="name") site_doctypes = frappe.get_all( "DocType", filters={"module": ("in", frappe_modules), "custom": False}, @@ -36,6 +35,6 @@ def execute(): continue if file_schema.creation != _site_schema[0].creation: - frappe.qb.update(DocType).set( - DocType.creation, file_schema.creation - ).where(DocType.name == file_schema.name).run() + frappe.qb.update(DocType).set(DocType.creation, file_schema.creation).where( + DocType.name == file_schema.name + ).run() diff --git a/frappe/patches/v14_0/save_ratings_in_fraction.py b/frappe/patches/v14_0/save_ratings_in_fraction.py index c933179b44..cedaf1b70d 100644 --- a/frappe/patches/v14_0/save_ratings_in_fraction.py +++ b/frappe/patches/v14_0/save_ratings_in_fraction.py @@ -31,9 +31,7 @@ def execute(): frappe.db.change_column_type(doctype_name, column=field, type=RATING_FIELD_TYPE, nullable=True) # update data: int => decimal - frappe.qb.update(doctype).set( - doctype[field], doctype[field] / 5 - ).run() + frappe.qb.update(doctype).set(doctype[field], doctype[field] / 5).run() # commit to flush updated rows frappe.db.commit() diff --git a/frappe/patches/v14_0/update_auto_account_deletion_duration.py b/frappe/patches/v14_0/update_auto_account_deletion_duration.py index 74957066e6..6b01816092 100644 --- a/frappe/patches/v14_0/update_auto_account_deletion_duration.py +++ b/frappe/patches/v14_0/update_auto_account_deletion_duration.py @@ -1,5 +1,6 @@ import frappe + def execute(): days = frappe.db.get_single_value("Website Settings", "auto_account_deletion") frappe.db.set_value("Website Settings", None, "auto_account_deletion", days * 24) diff --git a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py index ff03604754..f8d6f236cd 100644 --- a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py +++ b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py @@ -2,20 +2,22 @@ # MIT License. See license.txt from __future__ import unicode_literals + import frappe + def execute(): indicator_map = { - 'blue': 'Blue', - 'orange': 'Orange', - 'red': 'Red', - 'green': 'Green', - 'darkgrey': 'Gray', - 'gray': 'Gray', - 'purple': 'Purple', - 'yellow': 'Yellow', - 'lightblue': 'Light Blue', + "blue": "Blue", + "orange": "Orange", + "red": "Red", + "green": "Green", + "darkgrey": "Gray", + "gray": "Gray", + "purple": "Purple", + "yellow": "Yellow", + "lightblue": "Light Blue", } - for d in frappe.db.get_all('Kanban Board Column', fields=['name', 'indicator']): - color_name = indicator_map.get(d.indicator, 'Gray') - frappe.db.set_value('Kanban Board Column', d.name, 'indicator', color_name) + for d in frappe.db.get_all("Kanban Board Column", fields=["name", "indicator"]): + color_name = indicator_map.get(d.indicator, "Gray") + frappe.db.set_value("Kanban Board Column", d.name, "indicator", color_name) diff --git a/frappe/patches/v14_0/update_is_system_generated_flag.py b/frappe/patches/v14_0/update_is_system_generated_flag.py index 657e02aebc..79ed3f9571 100644 --- a/frappe/patches/v14_0/update_is_system_generated_flag.py +++ b/frappe/patches/v14_0/update_is_system_generated_flag.py @@ -1,17 +1,20 @@ import frappe + def execute(): # assuming all customization generated by Admin is system generated customization custom_field = frappe.qb.DocType("Custom Field") ( frappe.qb.update(custom_field) .set(custom_field.is_system_generated, True) - .where(custom_field.owner == 'Administrator').run() + .where(custom_field.owner == "Administrator") + .run() ) property_setter = frappe.qb.DocType("Property Setter") ( frappe.qb.update(property_setter) .set(property_setter.is_system_generated, True) - .where(property_setter.owner == 'Administrator').run() + .where(property_setter.owner == "Administrator") + .run() ) diff --git a/frappe/patches/v14_0/update_workspace2.py b/frappe/patches/v14_0/update_workspace2.py index a4b057b989..c6586f46a1 100644 --- a/frappe/patches/v14_0/update_workspace2.py +++ b/frappe/patches/v14_0/update_workspace2.py @@ -1,63 +1,77 @@ -import frappe import json + +import frappe from frappe import _ + def execute(): - frappe.reload_doc('desk', 'doctype', 'workspace', force=True) + frappe.reload_doc("desk", "doctype", "workspace", force=True) - for seq, workspace in enumerate(frappe.get_all('Workspace', order_by='name asc')): - doc = frappe.get_doc('Workspace', workspace.name) + for seq, workspace in enumerate(frappe.get_all("Workspace", order_by="name asc")): + doc = frappe.get_doc("Workspace", workspace.name) content = create_content(doc) update_workspace(doc, seq, content) frappe.db.commit() + def create_content(doc): content = [] if doc.onboarding: - content.append({"type":"onboarding","data":{"onboarding_name":doc.onboarding,"col":12}}) + content.append({"type": "onboarding", "data": {"onboarding_name": doc.onboarding, "col": 12}}) if doc.charts: invalid_links = [] for c in doc.charts: if c.get_invalid_links()[0]: invalid_links.append(c) else: - content.append({"type":"chart","data":{"chart_name":c.label,"col":12}}) + content.append({"type": "chart", "data": {"chart_name": c.label, "col": 12}}) for l in invalid_links: del doc.charts[doc.charts.index(l)] if doc.shortcuts: invalid_links = [] if doc.charts: - content.append({"type":"spacer","data":{"col":12}}) - content.append({"type":"header","data":{"text":doc.shortcuts_label or _("Your Shortcuts"),"level":4,"col":12}}) + content.append({"type": "spacer", "data": {"col": 12}}) + content.append( + { + "type": "header", + "data": {"text": doc.shortcuts_label or _("Your Shortcuts"), "level": 4, "col": 12}, + } + ) for s in doc.shortcuts: if s.get_invalid_links()[0]: invalid_links.append(s) else: - content.append({"type":"shortcut","data":{"shortcut_name":s.label,"col":4}}) + content.append({"type": "shortcut", "data": {"shortcut_name": s.label, "col": 4}}) for l in invalid_links: del doc.shortcuts[doc.shortcuts.index(l)] if doc.links: invalid_links = [] - content.append({"type":"spacer","data":{"col":12}}) - content.append({"type":"header","data":{"text":doc.cards_label or _("Reports & Masters"),"level":4,"col":12}}) + content.append({"type": "spacer", "data": {"col": 12}}) + content.append( + { + "type": "header", + "data": {"text": doc.cards_label or _("Reports & Masters"), "level": 4, "col": 12}, + } + ) for l in doc.links: - if l.type == 'Card Break': - content.append({"type":"card","data":{"card_name":l.label,"col":4}}) + if l.type == "Card Break": + content.append({"type": "card", "data": {"card_name": l.label, "col": 4}}) if l.get_invalid_links()[0]: invalid_links.append(l) for l in invalid_links: del doc.links[doc.links.index(l)] return content + def update_workspace(doc, seq, content): if not doc.title and not doc.content and not doc.is_standard and not doc.public: doc.sequence_id = seq + 1 doc.content = json.dumps(content) doc.public = 0 if doc.for_user else 1 doc.title = doc.extends or doc.label - doc.extends = '' - doc.category = '' - doc.onboarding = '' + doc.extends = "" + doc.category = "" + doc.onboarding = "" doc.extends_another_page = 0 doc.is_default = 0 doc.is_standard = 0 @@ -66,4 +80,4 @@ def update_workspace(doc, seq, content): doc.pin_to_top = 0 doc.pin_to_bottom = 0 doc.hide_custom = 0 - doc.save(ignore_permissions=True) \ No newline at end of file + doc.save(ignore_permissions=True) diff --git a/frappe/permissions.py b/frappe/permissions.py index a6c17fb59f..8980d2e63e 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -5,44 +5,72 @@ import copy import frappe import frappe.share from frappe import _, msgprint -from frappe.utils import cint from frappe.query_builder import DocType +from frappe.utils import cint -rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", - "print", "email", "report", "import", "export", "set_user_permissions", "share") +rights = ( + "select", + "read", + "write", + "create", + "delete", + "submit", + "cancel", + "amend", + "print", + "email", + "report", + "import", + "export", + "set_user_permissions", + "share", +) def check_admin_or_system_manager(user=None): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - if ("System Manager" not in frappe.get_roles(user)) and (user!="Administrator"): + if ("System Manager" not in frappe.get_roles(user)) and (user != "Administrator"): frappe.throw(_("Not permitted"), frappe.PermissionError) + def print_has_permission_check_logs(func): def inner(*args, **kwargs): - frappe.flags['has_permission_check_logs'] = [] + frappe.flags["has_permission_check_logs"] = [] result = func(*args, **kwargs) - self_perm_check = True if not kwargs.get('user') else kwargs.get('user') == frappe.session.user - raise_exception = False if kwargs.get('raise_exception') is False else True + self_perm_check = True if not kwargs.get("user") else kwargs.get("user") == frappe.session.user + raise_exception = False if kwargs.get("raise_exception") is False else True # print only if access denied # and if user is checking his own permission if not result and self_perm_check and raise_exception: - msgprint(('
').join(frappe.flags.get('has_permission_check_logs', []))) - frappe.flags.pop('has_permission_check_logs', None) + msgprint(("
").join(frappe.flags.get("has_permission_check_logs", []))) + frappe.flags.pop("has_permission_check_logs", None) return result + return inner + @print_has_permission_check_logs -def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, raise_exception=True, parent_doctype=None): +def has_permission( + doctype, + ptype="read", + doc=None, + verbose=False, + user=None, + raise_exception=True, + parent_doctype=None, +): """Returns True if user has permission `ptype` for given `doctype`. If `doc` is passed, it also checks user, share and owner permissions. Note: if Table DocType is passed, it always returns True. """ - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - if not doc and hasattr(doctype, 'doctype'): + if not doc and hasattr(doctype, "doctype"): # first argument can be doc or doctype doc = doctype doctype = doc.doctype @@ -51,8 +79,9 @@ def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, ra return True if frappe.is_table(doctype): - return has_child_table_permission(doctype, ptype, doc, verbose, - user, raise_exception, parent_doctype) + return has_child_table_permission( + doctype, ptype, doc, verbose, user, raise_exception, parent_doctype + ) meta = frappe.get_meta(doctype) @@ -60,13 +89,16 @@ def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, ra if isinstance(doc, str): doc = frappe.get_doc(meta.name, doc) perm = get_doc_permissions(doc, user=user, ptype=ptype).get(ptype) - if not perm: push_perm_check_log(_('User {0} does not have access to this document').format(frappe.bold(user))) + if not perm: + push_perm_check_log( + _("User {0} does not have access to this document").format(frappe.bold(user)) + ) else: - if ptype=="submit" and not cint(meta.is_submittable): + if ptype == "submit" and not cint(meta.is_submittable): push_perm_check_log(_("Document Type is not submittable")) return False - if ptype=="import" and not cint(meta.allow_import): + if ptype == "import" and not cint(meta.allow_import): push_perm_check_log(_("Document Type is not importable")) return False @@ -74,12 +106,17 @@ def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, ra perm = role_permissions.get(ptype) if not perm: - push_perm_check_log(_('User {0} does not have doctype access via role permission for document {1}').format(frappe.bold(user), frappe.bold(doctype))) + push_perm_check_log( + _("User {0} does not have doctype access via role permission for document {1}").format( + frappe.bold(user), frappe.bold(doctype) + ) + ) def false_if_not_shared(): if ptype in ("read", "write", "share", "submit", "email", "print"): - shared = frappe.share.get_shared(doctype, user, - ["read" if ptype in ("email", "print") else ptype]) + shared = frappe.share.get_shared( + doctype, user, ["read" if ptype in ("email", "print") else ptype] + ) if doc: doc_name = get_doc_name(doc) @@ -99,11 +136,14 @@ def has_permission(doctype, ptype="read", doc=None, verbose=False, user=None, ra return bool(perm) + def get_doc_permissions(doc, user=None, ptype=None): """Returns a dict of evaluated permissions for given `doc` like `{"read":1, "write":1}`""" - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - if frappe.is_table(doc.doctype): return {"read": 1, "write": 1} + if frappe.is_table(doc.doctype): + return {"read": 1, "write": 1} meta = frappe.get_meta(doc.doctype) @@ -111,7 +151,7 @@ def get_doc_permissions(doc, user=None, ptype=None): return (doc.get("owner") or "").lower() == user.lower() if has_controller_permissions(doc, ptype, user=user) is False: - push_perm_check_log('Not allowed via controller permission check') + push_perm_check_log("Not allowed via controller permission check") return {ptype: 0} permissions = copy.deepcopy(get_role_permissions(meta, user=user, is_owner=is_user_owner())) @@ -123,7 +163,7 @@ def get_doc_permissions(doc, user=None, ptype=None): permissions["import"] = 0 # Override with `if_owner` perms irrespective of user - if permissions.get('has_if_owner_enabled'): + if permissions.get("has_if_owner_enabled"): # apply owner permissions on top of existing permissions # some access might be only for the owner # eg. everyone might have read access but only owner can delete @@ -134,80 +174,86 @@ def get_doc_permissions(doc, user=None, ptype=None): # replace with owner permissions permissions = permissions.get("if_owner", {}) # if_owner does not come with create rights... - permissions['create'] = 0 + permissions["create"] = 0 else: permissions = {} return permissions + def get_role_permissions(doctype_meta, user=None, is_owner=None): """ Returns dict of evaluated role permissions like - { - "read": 1, - "write": 0, - // if "if_owner" is enabled - "if_owner": - { - "read": 1, - "write": 0 - } - } + { + "read": 1, + "write": 0, + // if "if_owner" is enabled + "if_owner": + { + "read": 1, + "write": 0 + } + } """ if isinstance(doctype_meta, str): - doctype_meta = frappe.get_meta(doctype_meta) # assuming doctype name was passed + doctype_meta = frappe.get_meta(doctype_meta) # assuming doctype name was passed - if not user: user = frappe.session.user + if not user: + user = frappe.session.user cache_key = (doctype_meta.name, user) - if user == 'Administrator': + if user == "Administrator": return allow_everything() if not frappe.local.role_permissions.get(cache_key): - perms = frappe._dict( - if_owner={} - ) + perms = frappe._dict(if_owner={}) roles = frappe.get_roles(user) def is_perm_applicable(perm): - return perm.role in roles and cint(perm.permlevel)==0 + return perm.role in roles and cint(perm.permlevel) == 0 def has_permission_without_if_owner_enabled(ptype): - return any(p.get(ptype, 0) and not p.get('if_owner', 0) for p in applicable_permissions) + return any(p.get(ptype, 0) and not p.get("if_owner", 0) for p in applicable_permissions) - applicable_permissions = list(filter(is_perm_applicable, getattr(doctype_meta, 'permissions', []))) - has_if_owner_enabled = any(p.get('if_owner', 0) for p in applicable_permissions) - perms['has_if_owner_enabled'] = has_if_owner_enabled + applicable_permissions = list( + filter(is_perm_applicable, getattr(doctype_meta, "permissions", [])) + ) + has_if_owner_enabled = any(p.get("if_owner", 0) for p in applicable_permissions) + perms["has_if_owner_enabled"] = has_if_owner_enabled for ptype in rights: pvalue = any(p.get(ptype, 0) for p in applicable_permissions) # check if any perm object allows perm type perms[ptype] = cint(pvalue) if ( - pvalue - and has_if_owner_enabled - and not has_permission_without_if_owner_enabled(ptype) - and ptype != 'create' + pvalue + and has_if_owner_enabled + and not has_permission_without_if_owner_enabled(ptype) + and ptype != "create" ): - perms['if_owner'][ptype] = cint(pvalue and is_owner) + perms["if_owner"][ptype] = cint(pvalue and is_owner) # has no access if not owner # only provide select or read access so that user is able to at-least access list # (and the documents will be filtered based on owner sin further checks) - perms[ptype] = 1 if ptype in ('select', 'read') else 0 + perms[ptype] = 1 if ptype in ("select", "read") else 0 frappe.local.role_permissions[cache_key] = perms return frappe.local.role_permissions[cache_key] + def get_user_permissions(user): from frappe.core.doctype.user_permission.user_permission import get_user_permissions + return get_user_permissions(user) + def has_user_permission(doc, user=None): - '''Returns True if User is allowed to view considering User Permissions''' + """Returns True if User is allowed to view considering User Permissions""" from frappe.core.doctype.user_permission.user_permission import get_user_permissions + user_permissions = get_user_permissions(user) if not user_permissions: @@ -215,13 +261,13 @@ def has_user_permission(doc, user=None): return True # user can create own role permissions, so nothing applies - if get_role_permissions('User Permission', user=user).get('write'): + if get_role_permissions("User Permission", user=user).get("write"): return True - apply_strict_user_permissions = frappe.get_system_settings('apply_strict_user_permissions') + apply_strict_user_permissions = frappe.get_system_settings("apply_strict_user_permissions") - doctype = doc.get('doctype') - docname = doc.get('name') + doctype = doc.get("doctype") + docname = doc.get("name") # STEP 1: --------------------- # check user permissions on self @@ -233,7 +279,7 @@ def has_user_permission(doc, user=None): # only check if allowed_docs is not empty if allowed_docs and docname not in allowed_docs: # no user permissions for this doc specified - push_perm_check_log(_('Not allowed for {0}: {1}').format(_(doctype), docname)) + push_perm_check_log(_("Not allowed for {0}: {1}").format(_(doctype), docname)) return False # STEP 2: --------------------------------- @@ -250,7 +296,8 @@ def has_user_permission(doc, user=None): # check all link fields for user permissions for field in meta.get_link_fields(): - if field.ignore_user_permissions: continue + if field.ignore_user_permissions: + continue # empty value, do you still want to apply user permissions? if not d.get(field.fieldname) and not apply_strict_user_permissions: @@ -266,14 +313,16 @@ def has_user_permission(doc, user=None): if allowed_docs and d.get(field.fieldname) not in allowed_docs: # restricted for this link field, and no matching values found # make the right message and exit - if d.get('parentfield'): + if d.get("parentfield"): # "Not allowed for Company = Restricted Company in Row 3. Restricted field: reference_type" - msg = _('Not allowed for {0}: {1} in Row {2}. Restricted field: {3}').format( - _(field.options), d.get(field.fieldname), d.idx, field.fieldname) + msg = _("Not allowed for {0}: {1} in Row {2}. Restricted field: {3}").format( + _(field.options), d.get(field.fieldname), d.idx, field.fieldname + ) else: # "Not allowed for Company = Restricted Company. Restricted field: reference_type" - msg = _('Not allowed for {0}: {1}. Restricted field: {2}').format( - _(field.options), d.get(field.fieldname), field.fieldname) + msg = _("Not allowed for {0}: {1}. Restricted field: {2}").format( + _(field.options), d.get(field.fieldname), field.fieldname + ) push_perm_check_log(msg) @@ -290,9 +339,11 @@ def has_user_permission(doc, user=None): return True + def has_controller_permissions(doc, ptype, user=None): """Returns controller permissions if defined. None if not defined""" - if not user: user = frappe.session.user + if not user: + user = frappe.session.user methods = frappe.get_hooks("has_permission").get(doc.doctype, []) @@ -307,15 +358,19 @@ def has_controller_permissions(doc, ptype, user=None): # controller permissions could not decide on True or False return None + def get_doctypes_with_read(): - return list({p.parent if type(p.parent) == str else p.parent.encode('UTF8') for p in get_valid_perms()}) + return list( + {p.parent if type(p.parent) == str else p.parent.encode("UTF8") for p in get_valid_perms()} + ) + def get_valid_perms(doctype=None, user=None): - '''Get valid permissions for the current user from DocPerm and Custom DocPerm''' + """Get valid permissions for the current user from DocPerm and Custom DocPerm""" roles = get_roles(user) perms = get_perms_for(roles) - custom_perms = get_perms_for(roles, 'Custom DocPerm') + custom_perms = get_perms_for(roles, "Custom DocPerm") doctypes_with_custom_perms = get_doctypes_with_custom_docperms() for p in perms: @@ -327,10 +382,11 @@ def get_valid_perms(doctype=None, user=None): else: return custom_perms + def get_all_perms(role): - '''Returns valid permissions for a given role''' - perms = frappe.get_all('DocPerm', fields='*', filters=dict(role=role)) - custom_perms = frappe.get_all('Custom DocPerm', fields='*', filters=dict(role=role)) + """Returns valid permissions for a given role""" + perms = frappe.get_all("DocPerm", fields="*", filters=dict(role=role)) + custom_perms = frappe.get_all("Custom DocPerm", fields="*", filters=dict(role=role)) doctypes_with_custom_perms = frappe.get_all("Custom DocPerm", pluck="parent", distinct=True) for p in perms: @@ -338,52 +394,56 @@ def get_all_perms(role): custom_perms.append(p) return custom_perms + def get_roles(user=None, with_standard=True): """get roles of current user""" if not user: user = frappe.session.user - if user=='Guest': - return ['Guest'] + if user == "Guest": + return ["Guest"] def get(): - if user == 'Administrator': - return frappe.get_all("Role", pluck="name") # return all available roles + if user == "Administrator": + return frappe.get_all("Role", pluck="name") # return all available roles else: table = DocType("Has Role") - roles = frappe.qb.from_(table).where( - (table.parent == user) & (table.role.notin(["All", "Guest"])) - ).select(table.role).run(pluck=True) - return roles + ['All', 'Guest'] + roles = ( + frappe.qb.from_(table) + .where((table.parent == user) & (table.role.notin(["All", "Guest"]))) + .select(table.role) + .run(pluck=True) + ) + return roles + ["All", "Guest"] roles = frappe.cache().hget("roles", user, get) # filter standard if required if not with_standard: - roles = filter(lambda x: x not in ['All', 'Guest', 'Administrator'], roles) + roles = filter(lambda x: x not in ["All", "Guest", "Administrator"], roles) return roles + def get_doctype_roles(doctype, access_type="read"): """Returns a list of roles that are allowed to access passed doctype.""" meta = frappe.get_meta(doctype) return [d.role for d in meta.get("permissions") if d.get(access_type)] -def get_perms_for(roles, perm_doctype='DocPerm'): - '''Get perms for given roles''' - filters = { - 'permlevel': 0, - 'docstatus': 0, - 'role': ['in', roles] - } - return frappe.db.get_all(perm_doctype, fields=['*'], filters=filters) + +def get_perms_for(roles, perm_doctype="DocPerm"): + """Get perms for given roles""" + filters = {"permlevel": 0, "docstatus": 0, "role": ["in", roles]} + return frappe.db.get_all(perm_doctype, fields=["*"], filters=filters) + def get_doctypes_with_custom_docperms(): - '''Returns all the doctypes with Custom Docperms''' + """Returns all the doctypes with Custom Docperms""" - doctypes = frappe.db.get_all('Custom DocPerm', fields=['parent'], distinct=1) + doctypes = frappe.db.get_all("Custom DocPerm", fields=["parent"], distinct=1) return [d.parent for d in doctypes] + def can_set_user_permissions(doctype, docname=None): # System Manager can always set user permissions if frappe.session.user == "Administrator" or "System Manager" in frappe.get_roles(): @@ -396,46 +456,61 @@ def can_set_user_permissions(doctype, docname=None): return False # check if current user has a role that can set permission - if get_role_permissions(meta).set_user_permissions!=1: + if get_role_permissions(meta).set_user_permissions != 1: return False return True + def set_user_permission_if_allowed(doctype, name, user, with_message=False): - if get_role_permissions(frappe.get_meta(doctype), user).set_user_permissions!=1: + if get_role_permissions(frappe.get_meta(doctype), user).set_user_permissions != 1: add_user_permission(doctype, name, user) -def add_user_permission(doctype, name, user, ignore_permissions=False, applicable_for=None, - is_default=0, hide_descendants=0): - '''Add user permission''' + +def add_user_permission( + doctype, + name, + user, + ignore_permissions=False, + applicable_for=None, + is_default=0, + hide_descendants=0, +): + """Add user permission""" from frappe.core.doctype.user_permission.user_permission import user_permission_exists if not user_permission_exists(user, doctype, name, applicable_for): if not frappe.db.exists(doctype, name): frappe.throw(_("{0} {1} not found").format(_(doctype), name), frappe.DoesNotExistError) - frappe.get_doc(dict( - doctype='User Permission', - user=user, - allow=doctype, - for_value=name, - is_default=is_default, - applicable_for=applicable_for, - hide_descendants=hide_descendants, - )).insert(ignore_permissions=ignore_permissions) + frappe.get_doc( + dict( + doctype="User Permission", + user=user, + allow=doctype, + for_value=name, + is_default=is_default, + applicable_for=applicable_for, + hide_descendants=hide_descendants, + ) + ).insert(ignore_permissions=ignore_permissions) + def remove_user_permission(doctype, name, user): - user_permission_name = frappe.db.get_value('User Permission', - dict(user=user, allow=doctype, for_value=name)) - frappe.delete_doc('User Permission', user_permission_name) + user_permission_name = frappe.db.get_value( + "User Permission", dict(user=user, allow=doctype, for_value=name) + ) + frappe.delete_doc("User Permission", user_permission_name) + def clear_user_permissions_for_doctype(doctype, user=None): - filters = {'allow': doctype} + filters = {"allow": doctype} if user: - filters['user'] = user - user_permissions_for_doctype = frappe.db.get_all('User Permission', filters=filters) + filters["user"] = user + user_permissions_for_doctype = frappe.db.get_all("User Permission", filters=filters) for d in user_permissions_for_doctype: - frappe.delete_doc('User Permission', d.name) + frappe.delete_doc("User Permission", d.name) + def can_import(doctype, raise_exception=False): if not ("System Manager" in frappe.get_roles() or has_permission(doctype, "import")): @@ -445,24 +520,25 @@ def can_import(doctype, raise_exception=False): return False return True + def can_export(doctype, raise_exception=False): if "System Manager" in frappe.get_roles(): return True else: role_permissions = frappe.permissions.get_role_permissions(doctype) - has_access = role_permissions.get('export') or \ - role_permissions.get('if_owner').get('export') + has_access = role_permissions.get("export") or role_permissions.get("if_owner").get("export") if not has_access and raise_exception: raise frappe.PermissionError(_("You are not allowed to export {} doctype").format(doctype)) return has_access + def update_permission_property(doctype, role, permlevel, ptype, value=None, validate=True): - '''Update a property in Custom Perm''' + """Update a property in Custom Perm""" from frappe.core.doctype.doctype.doctype import validate_permissions_for_doctype + out = setup_custom_perms(doctype) - name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, - permlevel=permlevel)) + name = frappe.get_value("Custom DocPerm", dict(parent=doctype, role=role, permlevel=permlevel)) table = DocType("Custom DocPerm") frappe.qb.update(table).set(ptype, value).where(table.name == name).run() @@ -471,127 +547,167 @@ def update_permission_property(doctype, role, permlevel, ptype, value=None, vali return out + def setup_custom_perms(parent): - '''if custom permssions are not setup for the current doctype, set them up''' - if not frappe.db.exists('Custom DocPerm', dict(parent=parent)): + """if custom permssions are not setup for the current doctype, set them up""" + if not frappe.db.exists("Custom DocPerm", dict(parent=parent)): copy_perms(parent) return True + def add_permission(doctype, role, permlevel=0, ptype=None): - '''Add a new permission rule to the given doctype - for the given Role and Permission Level''' + """Add a new permission rule to the given doctype + for the given Role and Permission Level""" from frappe.core.doctype.doctype.doctype import validate_permissions_for_doctype + setup_custom_perms(doctype) - if frappe.db.get_value('Custom DocPerm', dict(parent=doctype, role=role, - permlevel=permlevel, if_owner=0)): + if frappe.db.get_value( + "Custom DocPerm", dict(parent=doctype, role=role, permlevel=permlevel, if_owner=0) + ): return if not ptype: - ptype = 'read' - - custom_docperm = frappe.get_doc({ - "doctype":"Custom DocPerm", - "__islocal": 1, - "parent": doctype, - "parenttype": "DocType", - "parentfield": "permissions", - "role": role, - "permlevel": permlevel, - ptype: 1, - }) + ptype = "read" + + custom_docperm = frappe.get_doc( + { + "doctype": "Custom DocPerm", + "__islocal": 1, + "parent": doctype, + "parenttype": "DocType", + "parentfield": "permissions", + "role": role, + "permlevel": permlevel, + ptype: 1, + } + ) custom_docperm.save() validate_permissions_for_doctype(doctype) return custom_docperm.name + def copy_perms(parent): - '''Copy all DocPerm in to Custom DocPerm for the given document''' - for d in frappe.get_all('DocPerm', fields='*', filters=dict(parent=parent)): - custom_perm = frappe.new_doc('Custom DocPerm') + """Copy all DocPerm in to Custom DocPerm for the given document""" + for d in frappe.get_all("DocPerm", fields="*", filters=dict(parent=parent)): + custom_perm = frappe.new_doc("Custom DocPerm") custom_perm.update(d) custom_perm.insert(ignore_permissions=True) + def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for + delete_notification_count_for(doctype) frappe.db.delete("Custom DocPerm", {"parent": doctype}) + def get_linked_doctypes(dt): - return list(set([dt] + [d.options for d in - frappe.get_meta(dt).get("fields", { - "fieldtype":"Link", - "ignore_user_permissions":("!=", 1), - "options": ("!=", "[Select]") - }) - ])) + return list( + set( + [dt] + + [ + d.options + for d in frappe.get_meta(dt).get( + "fields", + {"fieldtype": "Link", "ignore_user_permissions": ("!=", 1), "options": ("!=", "[Select]")}, + ) + ] + ) + ) + def get_doc_name(doc): - if not doc: return None + if not doc: + return None return doc if isinstance(doc, str) else doc.name + def allow_everything(): - ''' + """ returns a dict with access to everything eg. {"read": 1, "write": 1, ...} - ''' + """ perm = {ptype: 1 for ptype in rights} return perm + def get_allowed_docs_for_doctype(user_permissions, doctype): - ''' Returns all the docs from the passed user_permissions that are - allowed under provided doctype ''' + """Returns all the docs from the passed user_permissions that are + allowed under provided doctype""" return filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc=False) + def filter_allowed_docs_for_doctype(user_permissions, doctype, with_default_doc=True): - ''' Returns all the docs from the passed user_permissions that are - allowed under provided doctype along with default doc value if with_default_doc is set ''' + """Returns all the docs from the passed user_permissions that are + allowed under provided doctype along with default doc value if with_default_doc is set""" allowed_doc = [] default_doc = None for doc in user_permissions: - if not doc.get('applicable_for') or doc.get('applicable_for') == doctype: - allowed_doc.append(doc.get('doc')) - if doc.get('is_default') or len(user_permissions) == 1 and with_default_doc: - default_doc = doc.get('doc') + if not doc.get("applicable_for") or doc.get("applicable_for") == doctype: + allowed_doc.append(doc.get("doc")) + if doc.get("is_default") or len(user_permissions) == 1 and with_default_doc: + default_doc = doc.get("doc") return (allowed_doc, default_doc) if with_default_doc else allowed_doc + def push_perm_check_log(log): - if frappe.flags.get('has_permission_check_logs') is None: + if frappe.flags.get("has_permission_check_logs") is None: return - frappe.flags.get('has_permission_check_logs').append(_(log)) + frappe.flags.get("has_permission_check_logs").append(_(log)) -def has_child_table_permission(child_doctype, ptype="read", child_doc=None, - verbose=False, user=None, raise_exception=True, parent_doctype=None): + +def has_child_table_permission( + child_doctype, + ptype="read", + child_doc=None, + verbose=False, + user=None, + raise_exception=True, + parent_doctype=None, +): parent_doc = None if child_doc: parent_doctype = child_doc.get("parenttype") - parent_doc = frappe.get_cached_doc({ - "doctype": parent_doctype, - "docname": child_doc.get("parent") - }) + parent_doc = frappe.get_cached_doc( + {"doctype": parent_doctype, "docname": child_doc.get("parent")} + ) if parent_doctype: if not is_parent_valid(child_doctype, parent_doctype): - frappe.throw(_("{0} is not a valid parent DocType for {1}").format( - frappe.bold(parent_doctype), - frappe.bold(child_doctype) - ), title=_("Invalid Parent DocType")) + frappe.throw( + _("{0} is not a valid parent DocType for {1}").format( + frappe.bold(parent_doctype), frappe.bold(child_doctype) + ), + title=_("Invalid Parent DocType"), + ) else: - frappe.throw(_("Please specify a valid parent DocType for {0}").format( - frappe.bold(child_doctype) - ), title=_("Parent DocType Required")) + frappe.throw( + _("Please specify a valid parent DocType for {0}").format(frappe.bold(child_doctype)), + title=_("Parent DocType Required"), + ) - return has_permission(parent_doctype, ptype=ptype, doc=parent_doc, - verbose=verbose, user=user, raise_exception=raise_exception) + return has_permission( + parent_doctype, + ptype=ptype, + doc=parent_doc, + verbose=verbose, + user=user, + raise_exception=raise_exception, + ) def is_parent_valid(child_doctype, parent_doctype): from frappe.core.utils import find + parent_meta = frappe.get_meta(parent_doctype) - child_table_field_exists = find(parent_meta.get_table_fields(), lambda d: d.options == child_doctype) + child_table_field_exists = find( + parent_meta.get_table_fields(), lambda d: d.options == child_doctype + ) return not parent_meta.istable and child_table_field_exists diff --git a/frappe/printing/doctype/letter_head/letter_head.py b/frappe/printing/doctype/letter_head/letter_head.py index 67c0d236e0..98c2fc7c2b 100644 --- a/frappe/printing/doctype/letter_head/letter_head.py +++ b/frappe/printing/doctype/letter_head/letter_head.py @@ -2,14 +2,15 @@ # License: MIT. See LICENSE import frappe -from frappe.utils import is_image, flt -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document +from frappe.utils import flt, is_image + class LetterHead(Document): def before_insert(self): # for better UX, let user set from attachment - self.source = 'Image' + self.source = "Image" def validate(self): self.set_image() @@ -20,24 +21,26 @@ class LetterHead(Document): frappe.throw(_("Letter Head cannot be both disabled and default")) if not self.is_default and not self.disabled: - if not frappe.db.exists('Letter Head', dict(is_default=1)): + if not frappe.db.exists("Letter Head", dict(is_default=1)): self.is_default = 1 def set_image(self): - if self.source=='Image': + if self.source == "Image": if self.image and is_image(self.image): self.image_width = flt(self.image_width) self.image_height = flt(self.image_height) - dimension = 'width' if self.image_width > self.image_height else 'height' - dimension_value = self.get('image_' + dimension) - self.content = f''' + dimension = "width" if self.image_width > self.image_height else "height" + dimension_value = self.get("image_" + dimension) + self.content = f"""
{self.name}
- ''' - frappe.msgprint(frappe._('Header HTML set from attachment {0}').format(self.image), alert = True) + """ + frappe.msgprint(frappe._("Header HTML set from attachment {0}").format(self.image), alert=True) else: - frappe.msgprint(frappe._('Please attach an image file to set HTML'), alert = True, indicator = 'orange') + frappe.msgprint( + frappe._("Please attach an image file to set HTML"), alert=True, indicator="orange" + ) def on_update(self): self.set_as_default() @@ -47,14 +50,14 @@ class LetterHead(Document): def set_as_default(self): from frappe.utils import set_default + if self.is_default: - frappe.db.sql("update `tabLetter Head` set is_default=0 where name != %s", - self.name) + frappe.db.sql("update `tabLetter Head` set is_default=0 where name != %s", self.name) - set_default('letter_head', self.name) + set_default("letter_head", self.name) # update control panel - so it loads new letter directly frappe.db.set_default("default_letter_head_content", self.content) else: - frappe.defaults.clear_default('letter_head', self.name) + frappe.defaults.clear_default("letter_head", self.name) frappe.defaults.clear_default("default_letter_head_content", self.content) diff --git a/frappe/printing/doctype/letter_head/test_letter_head.py b/frappe/printing/doctype/letter_head/test_letter_head.py index 67d307ee8b..9357d15315 100644 --- a/frappe/printing/doctype/letter_head/test_letter_head.py +++ b/frappe/printing/doctype/letter_head/test_letter_head.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestLetterHead(unittest.TestCase): def test_auto_image(self): - letter_head = frappe.get_doc(dict( - doctype = 'Letter Head', - letter_head_name = 'Test', - source = 'Image', - image = '/public/test.png' - )).insert() + letter_head = frappe.get_doc( + dict(doctype="Letter Head", letter_head_name="Test", source="Image", image="/public/test.png") + ).insert() # test if image is automatically set self.assertTrue(letter_head.image in letter_head.content) - diff --git a/frappe/printing/doctype/network_printer_settings/network_printer_settings.py b/frappe/printing/doctype/network_printer_settings/network_printer_settings.py index e42ed818c7..3e0b8f0c2e 100644 --- a/frappe/printing/doctype/network_printer_settings/network_printer_settings.py +++ b/frappe/printing/doctype/network_printer_settings/network_printer_settings.py @@ -2,29 +2,31 @@ # For license information, please see license.txt import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document + class NetworkPrinterSettings(Document): @frappe.whitelist() - def get_printers_list(self,ip="localhost",port=631): + def get_printers_list(self, ip="localhost", port=631): printer_list = [] try: import cups except ImportError: - frappe.throw(_('''This feature can not be used as dependencies are missing. - Please contact your system manager to enable this by installing pycups!''')) + frappe.throw( + _( + """This feature can not be used as dependencies are missing. + Please contact your system manager to enable this by installing pycups!""" + ) + ) return try: cups.setServer(self.server_ip) cups.setPort(self.port) conn = cups.Connection() printers = conn.getPrinters() - for printer_id,printer in printers.items(): - printer_list.append({ - 'value': printer_id, - 'label': printer['printer-make-and-model'] - }) + for printer_id, printer in printers.items(): + printer_list.append({"value": printer_id, "label": printer["printer-make-and-model"]}) except RuntimeError: frappe.throw(_("Failed to connect to server")) @@ -32,6 +34,7 @@ class NetworkPrinterSettings(Document): frappe.throw(_("Failed to connect to server")) return printer_list + @frappe.whitelist() def get_network_printer_settings(): - return frappe.db.get_list('Network Printer Settings', pluck='name') + return frappe.db.get_list("Network Printer Settings", pluck="name") diff --git a/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py b/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py index 86509b239f..17ce16cb82 100644 --- a/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py +++ b/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestNetworkPrinterSettings(unittest.TestCase): pass diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py index 74dc5460d9..e54f46b487 100644 --- a/frappe/printing/doctype/print_format/print_format.py +++ b/frappe/printing/doctype/print_format/print_format.py @@ -2,13 +2,15 @@ # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe import frappe.utils -import json from frappe import _ -from frappe.utils.jinja import validate_template -from frappe.utils.weasyprint import get_html, download_pdf from frappe.model.document import Document +from frappe.utils.jinja import validate_template +from frappe.utils.weasyprint import download_pdf, get_html + class PrintFormat(Document): def onload(self): @@ -26,29 +28,32 @@ class PrintFormat(Document): return download_pdf(self.doc_type, docname, self.name, letterhead) def validate(self): - if (self.standard=="Yes" + if ( + self.standard == "Yes" and not frappe.local.conf.get("developer_mode") - and not (frappe.flags.in_import or frappe.flags.in_test)): + and not (frappe.flags.in_import or frappe.flags.in_test) + ): frappe.throw(frappe._("Standard Print Format cannot be updated")) # old_doc_type is required for clearing item cache - self.old_doc_type = frappe.db.get_value('Print Format', - self.name, 'doc_type') + self.old_doc_type = frappe.db.get_value("Print Format", self.name, "doc_type") self.extract_images() if not self.module: - self.module = frappe.db.get_value('DocType', self.doc_type, 'module') + self.module = frappe.db.get_value("DocType", self.doc_type, "module") - if self.html and self.print_format_type != 'JS': + if self.html and self.print_format_type != "JS": validate_template(self.html) if self.custom_format and self.raw_printing and not self.raw_commands: - frappe.throw(_('{0} are required').format(frappe.bold(_('Raw Commands'))), frappe.MandatoryError) + frappe.throw( + _("{0} are required").format(frappe.bold(_("Raw Commands"))), frappe.MandatoryError + ) if self.custom_format and not self.html and not self.raw_printing: - frappe.throw(_('{0} is required').format(frappe.bold(_('HTML'))), frappe.MandatoryError) + frappe.throw(_("{0} is required").format(frappe.bold(_("HTML"))), frappe.MandatoryError) def extract_images(self): from frappe.core.doctype.file.file import extract_images_from_html @@ -59,12 +64,12 @@ class PrintFormat(Document): if self.format_data: data = json.loads(self.format_data) for df in data: - if df.get('fieldtype') and df['fieldtype'] in ('HTML', 'Custom HTML') and df.get('options'): - df['options'] = extract_images_from_html(self, df['options']) + if df.get("fieldtype") and df["fieldtype"] in ("HTML", "Custom HTML") and df.get("options"): + df["options"] = extract_images_from_html(self, df["options"]) self.format_data = json.dumps(data) def on_update(self): - if hasattr(self, 'old_doc_type') and self.old_doc_type: + if hasattr(self, "old_doc_type") and self.old_doc_type: frappe.clear_cache(doctype=self.old_doc_type) if self.doc_type: frappe.clear_cache(doctype=self.doc_type) @@ -76,21 +81,28 @@ class PrintFormat(Document): frappe.clear_cache(doctype=self.doc_type) # update property setter default_print_format if set - frappe.db.set_value("Property Setter", { - "doctype_or_field": "DocType", - "doc_type": self.doc_type, - "property": "default_print_format", - "value": old, - }, "value", new) + frappe.db.set_value( + "Property Setter", + { + "doctype_or_field": "DocType", + "doc_type": self.doc_type, + "property": "default_print_format", + "value": old, + }, + "value", + new, + ) def export_doc(self): from frappe.modules.utils import export_module_json - export_module_json(self, self.standard == 'Yes', self.module) + + export_module_json(self, self.standard == "Yes", self.module) def on_trash(self): if self.doc_type: frappe.clear_cache(doctype=self.doc_type) + @frappe.whitelist() def make_default(name): """Set print format as default""" @@ -98,21 +110,24 @@ def make_default(name): print_format = frappe.get_doc("Print Format", name) - if (frappe.conf.get('developer_mode') or 0) == 1: + if (frappe.conf.get("developer_mode") or 0) == 1: # developer mode, set it default in doctype doctype = frappe.get_doc("DocType", print_format.doc_type) doctype.default_print_format = name doctype.save() else: # customization - frappe.make_property_setter({ - 'doctype_or_field': "DocType", - 'doctype': print_format.doc_type, - 'property': "default_print_format", - 'value': name, - }) - - frappe.msgprint(frappe._("{0} is now default print format for {1} doctype").format( - frappe.bold(name), - frappe.bold(print_format.doc_type) - )) + frappe.make_property_setter( + { + "doctype_or_field": "DocType", + "doctype": print_format.doc_type, + "property": "default_print_format", + "value": name, + } + ) + + frappe.msgprint( + frappe._("{0} is now default print format for {1} doctype").format( + frappe.bold(name), frappe.bold(print_format.doc_type) + ) + ) diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py index 564a2c750c..fbfeecb3ab 100644 --- a/frappe/printing/doctype/print_format/test_print_format.py +++ b/frappe/printing/doctype/print_format/test_print_format.py @@ -1,22 +1,26 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import unittest import re +import unittest + +import frappe + +test_records = frappe.get_test_records("Print Format") -test_records = frappe.get_test_records('Print Format') class TestPrintFormat(unittest.TestCase): def test_print_user(self, style=None): print_html = frappe.get_print("User", "Administrator", style=style) self.assertTrue("" in print_html) - self.assertTrue(re.findall(r'
[\s]*administrator[\s]*
', print_html)) + self.assertTrue( + re.findall(r'
[\s]*administrator[\s]*
', print_html) + ) return print_html def test_print_user_standard(self): print_html = self.test_print_user("Standard") - self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html)) - self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html)) + self.assertTrue(re.findall(r"\.print-format {[\s]*font-size: 9pt;", print_html)) + self.assertFalse(re.findall(r"th {[\s]*background-color: #eee;[\s]*}", print_html)) self.assertFalse("font-family: serif;" in print_html) def test_print_user_modern(self): diff --git a/frappe/printing/doctype/print_format_field_template/print_format_field_template.py b/frappe/printing/doctype/print_format_field_template/print_format_field_template.py index b66afdb6b1..428979b3ad 100644 --- a/frappe/printing/doctype/print_format_field_template/print_format_field_template.py +++ b/frappe/printing/doctype/print_format_field_template/print_format_field_template.py @@ -2,8 +2,8 @@ # For license information, please see license.txt import frappe -from frappe.model.document import Document from frappe import _ +from frappe.model.document import Document class PrintFormatFieldTemplate(Document): diff --git a/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py b/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py index f0b1329763..b55a492635 100644 --- a/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py +++ b/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestPrintFormatFieldTemplate(unittest.TestCase): pass diff --git a/frappe/printing/doctype/print_heading/print_heading.py b/frappe/printing/doctype/print_heading/print_heading.py index 39c46ad152..c905e68d47 100644 --- a/frappe/printing/doctype/print_heading/print_heading.py +++ b/frappe/printing/doctype/print_heading/print_heading.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class PrintHeading(Document): pass diff --git a/frappe/printing/doctype/print_heading/test_print_heading.py b/frappe/printing/doctype/print_heading/test_print_heading.py index 7eaa1bc6ba..02eddb072f 100644 --- a/frappe/printing/doctype/print_heading/test_print_heading.py +++ b/frappe/printing/doctype/print_heading/test_print_heading.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestPrintHeading(unittest.TestCase): pass diff --git a/frappe/printing/doctype/print_settings/print_settings.py b/frappe/printing/doctype/print_settings/print_settings.py index 3253cea2dc..f52d08e6ec 100644 --- a/frappe/printing/doctype/print_settings/print_settings.py +++ b/frappe/printing/doctype/print_settings/print_settings.py @@ -4,16 +4,13 @@ import frappe from frappe import _ -from frappe.utils import cint - from frappe.model.document import Document +from frappe.utils import cint class PrintSettings(Document): def validate(self): - if self.pdf_page_size == "Custom" and not ( - self.pdf_page_height and self.pdf_page_width - ): + if self.pdf_page_size == "Custom" and not (self.pdf_page_height and self.pdf_page_width): frappe.throw(_("Page height and width cannot be zero")) def on_update(self): diff --git a/frappe/printing/doctype/print_settings/test_print_settings.py b/frappe/printing/doctype/print_settings/test_print_settings.py index 82883eaee5..ba22df4438 100644 --- a/frappe/printing/doctype/print_settings/test_print_settings.py +++ b/frappe/printing/doctype/print_settings/test_print_settings.py @@ -3,5 +3,6 @@ # License: MIT. See LICENSE import unittest + class TestPrintSettings(unittest.TestCase): pass diff --git a/frappe/printing/doctype/print_style/print_style.py b/frappe/printing/doctype/print_style/print_style.py index 7985c006f4..00de829deb 100644 --- a/frappe/printing/doctype/print_style/print_style.py +++ b/frappe/printing/doctype/print_style/print_style.py @@ -5,11 +5,14 @@ import frappe from frappe.model.document import Document + class PrintStyle(Document): def validate(self): - if (self.standard==1 + if ( + self.standard == 1 and not frappe.local.conf.get("developer_mode") - and not (frappe.flags.in_import or frappe.flags.in_test)): + and not (frappe.flags.in_import or frappe.flags.in_test) + ): frappe.throw(frappe._("Standard Print Style cannot be changed. Please duplicate to edit.")) @@ -19,4 +22,5 @@ class PrintStyle(Document): def export_doc(self): # export from frappe.modules.utils import export_module_json - export_module_json(self, self.standard == 1, 'Printing') + + export_module_json(self, self.standard == 1, "Printing") diff --git a/frappe/printing/doctype/print_style/test_print_style.py b/frappe/printing/doctype/print_style/test_print_style.py index cbf5c465d1..ad2b61cc87 100644 --- a/frappe/printing/doctype/print_style/test_print_style.py +++ b/frappe/printing/doctype/print_style/test_print_style.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestPrintStyle(unittest.TestCase): pass diff --git a/frappe/printing/page/print/print.py b/frappe/printing/page/print/print.py index 7ddf68c30d..82ad22656d 100644 --- a/frappe/printing/page/print/print.py +++ b/frappe/printing/page/print/print.py @@ -1,11 +1,12 @@ import frappe + @frappe.whitelist() def get_print_settings_to_show(doctype, docname): doc = frappe.get_doc(doctype, docname) - print_settings = frappe.get_single('Print Settings') + print_settings = frappe.get_single("Print Settings") - if hasattr(doc, 'get_print_settings'): + if hasattr(doc, "get_print_settings"): fields = doc.get_print_settings() or [] else: return [] diff --git a/frappe/printing/page/print_format_builder/print_format_builder.py b/frappe/printing/page/print_format_builder/print_format_builder.py index fae564d3c3..67273120c2 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder.py +++ b/frappe/printing/page/print_format_builder/print_format_builder.py @@ -1,8 +1,9 @@ import frappe + @frappe.whitelist() -def create_custom_format(doctype, name, based_on='Standard', beta=False): - doc = frappe.new_doc('Print Format') +def create_custom_format(doctype, name, based_on="Standard", beta=False): + doc = frappe.new_doc("Print Format") doc.doc_type = doctype doc.name = name beta = frappe.parse_json(beta) @@ -11,7 +12,8 @@ def create_custom_format(doctype, name, based_on='Standard', beta=False): doc.print_format_builder_beta = 1 else: doc.print_format_builder = 1 - doc.format_data = frappe.db.get_value('Print Format', based_on, 'format_data') \ - if based_on != 'Standard' else None + doc.format_data = ( + frappe.db.get_value("Print Format", based_on, "format_data") if based_on != "Standard" else None + ) doc.insert() - return doc \ No newline at end of file + return doc diff --git a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py index e13412cd07..668be8d05a 100644 --- a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py +++ b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py @@ -2,9 +2,11 @@ # MIT License. See license.txt from __future__ import unicode_literals -import frappe + import functools +import frappe + @frappe.whitelist() def get_google_fonts(): diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 5b58e70c4e..1bf9ec97d9 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -5,11 +5,11 @@ from pypika.utils import ignore_copy from frappe.query_builder.terms import ParameterizedFunction, ParameterizedValueWrapper from frappe.query_builder.utils import ( - Column, - DocType, - get_query_builder, - patch_query_aggregation, - patch_query_execute, + Column, + DocType, + get_query_builder, + patch_query_aggregation, + patch_query_execute, ) pypika.terms.ValueWrapper = ParameterizedValueWrapper diff --git a/frappe/query_builder/custom.py b/frappe/query_builder/custom.py index c86132e44d..b0c8f863d3 100644 --- a/frappe/query_builder/custom.py +++ b/frappe/query_builder/custom.py @@ -11,8 +11,8 @@ class GROUP_CONCAT(DistinctOptionFunction): def __init__(self, column: str, alias: Optional[str] = None): """[ Implements the group concat function read more about it at https://www.geeksforgeeks.org/mysql-group_concat-function ] Args: - column (str): [ name of the column you want to concat] - alias (Optional[str], optional): [ is this an alias? ]. Defaults to None. + column (str): [ name of the column you want to concat] + alias (Optional[str], optional): [ is this an alias? ]. Defaults to None. """ super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", column, alias=alias) @@ -22,9 +22,9 @@ class STRING_AGG(DistinctOptionFunction): """[ Implements the group concat function read more about it at https://docs.microsoft.com/en-us/sql/t-sql/functions/string-agg-transact-sql?view=sql-server-ver15 ] Args: - column (str): [ name of the column you want to concat ] - separator (str, optional): [separator to be used]. Defaults to ",". - alias (Optional[str], optional): [description]. Defaults to None. + column (str): [ name of the column you want to concat ] + separator (str, optional): [separator to be used]. Defaults to ",". + alias (Optional[str], optional): [description]. Defaults to None. """ super(STRING_AGG, self).__init__("STRING_AGG", column, separator, alias=alias) @@ -34,7 +34,7 @@ class MATCH(DistinctOptionFunction): """[ Implementation of Match Against read more about it https://dev.mysql.com/doc/refman/8.0/en/fulltext-search.html#function_match ] Args: - column (str):[ column to search in ] + column (str):[ column to search in ] """ alias = kwargs.get("alias") super(MATCH, self).__init__(" MATCH", column, *args, alias=alias) @@ -52,7 +52,7 @@ class MATCH(DistinctOptionFunction): """[ Text that has to be searched against ] Args: - text (str): [ the text string that we match it against ] + text (str): [ the text string that we match it against ] """ self._Against = text @@ -62,7 +62,7 @@ class TO_TSVECTOR(DistinctOptionFunction): """[ Implementation of TO_TSVECTOR read more about it https://www.postgresql.org/docs/9.1/textsearch-controls.html] Args: - column (str): [ column to search in ] + column (str): [ column to search in ] """ alias = kwargs.get("alias") super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", column, *args, alias=alias) @@ -79,7 +79,7 @@ class TO_TSVECTOR(DistinctOptionFunction): """[ Text that has to be searched against ] Args: - text (str): [ the text string that we match it against ] + text (str): [ the text string that we match it against ] """ self._PLAINTO_TSQUERY = text @@ -91,14 +91,14 @@ class ConstantColumn(Term): """[ Returns a pseudo column with a constant value in all the rows] Args: - value (str): [ Value of the column ] + value (str): [ Value of the column ] """ self.value = value def get_sql(self, quote_char: Optional[str] = None, **kwargs: Any) -> str: return format_alias_sql( - format_quotes(self.value, kwargs.get("secondary_quote_char") or ""), - self.alias or self.value, - quote_char=quote_char, - **kwargs - ) + format_quotes(self.value, kwargs.get("secondary_quote_char") or ""), + self.alias or self.value, + quote_char=quote_char, + **kwargs, + ) diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index b0292f2728..ee4ae6c30b 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -1,8 +1,10 @@ from pypika.functions import * -from pypika.terms import Function, CustomFunction, ArithmeticExpression, Arithmetic -from frappe.query_builder.utils import ImportMapper, db_type_is -from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR +from pypika.terms import Arithmetic, ArithmeticExpression, CustomFunction, Function + from frappe.database.query import Query +from frappe.query_builder.custom import GROUP_CONCAT, MATCH, STRING_AGG, TO_TSVECTOR +from frappe.query_builder.utils import ImportMapper, db_type_is + from .utils import Column @@ -11,19 +13,10 @@ class Concat_ws(Function): super(Concat_ws, self).__init__("CONCAT_WS", *terms, **kwargs) -GroupConcat = ImportMapper( - { - db_type_is.MARIADB: GROUP_CONCAT, - db_type_is.POSTGRES: STRING_AGG - } -) +GroupConcat = ImportMapper({db_type_is.MARIADB: GROUP_CONCAT, db_type_is.POSTGRES: STRING_AGG}) + +Match = ImportMapper({db_type_is.MARIADB: MATCH, db_type_is.POSTGRES: TO_TSVECTOR}) -Match = ImportMapper( - { - db_type_is.MARIADB: MATCH, - db_type_is.POSTGRES: TO_TSVECTOR - } -) class _PostgresTimestamp(ArithmeticExpression): def __init__(self, datepart, timepart, alias=None): @@ -32,8 +25,7 @@ class _PostgresTimestamp(ArithmeticExpression): if isinstance(timepart, str): timepart = Cast(timepart, "time") - super().__init__(operator=Arithmetic.add, - left=datepart, right=timepart, alias=alias) + super().__init__(operator=Arithmetic.add, left=datepart, right=timepart, alias=alias) CombineDatetime = ImportMapper( @@ -43,15 +35,19 @@ CombineDatetime = ImportMapper( } ) -DateFormat = ImportMapper({ - db_type_is.MARIADB: CustomFunction("DATE_FORMAT", ["date", "format"]), - db_type_is.POSTGRES: ToChar, -}) +DateFormat = ImportMapper( + { + db_type_is.MARIADB: CustomFunction("DATE_FORMAT", ["date", "format"]), + db_type_is.POSTGRES: ToChar, + } +) + class Cast_(Function): def __init__(self, value, as_type, alias=None): if db_type_is.MARIADB and ( - (hasattr(as_type, "get_sql") and as_type.get_sql().lower() == "varchar") or str(as_type).lower() == "varchar" + (hasattr(as_type, "get_sql") and as_type.get_sql().lower() == "varchar") + or str(as_type).lower() == "varchar" ): # mimics varchar cast in mariadb # as mariadb doesn't have varchar data cast @@ -66,16 +62,17 @@ class Cast_(Function): def get_special_params_sql(self, **kwargs): if self.name.lower() == "cast": - type_sql = self.as_type.get_sql(**kwargs) if hasattr(self.as_type, "get_sql") else str(self.as_type).upper() + type_sql = ( + self.as_type.get_sql(**kwargs) + if hasattr(self.as_type, "get_sql") + else str(self.as_type).upper() + ) return "AS {type}".format(type=type_sql) def _aggregate(function, dt, fieldname, filters, **kwargs): return ( - Query() - .build_conditions(dt, filters) - .select(function(Column(fieldname))) - .run(**kwargs)[0][0] + Query().build_conditions(dt, filters).select(function(Column(fieldname))).run(**kwargs)[0][0] or 0 ) @@ -83,11 +80,14 @@ def _aggregate(function, dt, fieldname, filters, **kwargs): def _max(dt, fieldname, filters=None, **kwargs): return _aggregate(Max, dt, fieldname, filters, **kwargs) + def _min(dt, fieldname, filters=None, **kwargs): return _aggregate(Min, dt, fieldname, filters, **kwargs) + def _avg(dt, fieldname, filters=None, **kwargs): return _aggregate(Avg, dt, fieldname, filters, **kwargs) + def _sum(dt, fieldname, filters=None, **kwargs): return _aggregate(Sum, dt, fieldname, filters, **kwargs) diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index aee6bf029e..8d64d2ddcd 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -18,10 +18,10 @@ class NamedParameterWrapper: """returns SQL for a parameter, while adding the real value in a dict Args: - param_value (Any): Value of the parameter + param_value (Any): Value of the parameter Returns: - str: parameter used in the SQL query + str: parameter used in the SQL query """ param_key = f"%(param{len(self.parameters) + 1})s" self.parameters[param_key[2:-2]] = param_value @@ -31,7 +31,7 @@ class NamedParameterWrapper: """get dict with parameters and values Returns: - Dict[str, Any]: parameter dict + Dict[str, Any]: parameter dict """ return self.parameters @@ -90,22 +90,22 @@ class ParameterizedFunction(Function): if self.schema is not None: function_sql = "{schema}.{function}".format( - schema=self.schema.get_sql( - quote_char=quote_char, dialect=dialect, **kwargs - ), + schema=self.schema.get_sql(quote_char=quote_char, dialect=dialect, **kwargs), function=function_sql, ) if with_alias: - return format_alias_sql( - function_sql, self.alias, quote_char=quote_char, **kwargs - ) + return format_alias_sql(function_sql, self.alias, quote_char=quote_char, **kwargs) return function_sql class subqry(Criterion): - def __init__(self, subq: QueryBuilder, alias: Optional[str] = None,) -> None: + def __init__( + self, + subq: QueryBuilder, + alias: Optional[str] = None, + ) -> None: super().__init__(alias) self.subq = subq diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 32dbfaf3cd..71cfa88db1 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -16,6 +16,7 @@ class db_type_is(Enum): MARIADB = "mariadb" POSTGRES = "postgres" + class ImportMapper: def __init__(self, func_map: Dict[db_type_is, Callable]) -> None: self.func_map = func_map @@ -24,31 +25,36 @@ class ImportMapper: db = db_type_is(frappe.conf.db_type or "mariadb") return self.func_map[db](*args, **kwds) + class BuilderIdentificationFailed(Exception): def __init__(self): super().__init__("Couldn't guess builder") + def get_query_builder(type_of_db: str) -> Union[Postgres, MariaDB]: """[return the query builder object] Args: - type_of_db (str): [string value of the db used] + type_of_db (str): [string value of the db used] Returns: - Query: [Query object] + Query: [Query object] """ db = db_type_is(type_of_db) picks = {db_type_is.MARIADB: MariaDB, db_type_is.POSTGRES: Postgres} return picks[db] + def get_attr(method_string): - modulename = '.'.join(method_string.split('.')[:-1]) - methodname = method_string.split('.')[-1] + modulename = ".".join(method_string.split(".")[:-1]) + methodname = method_string.split(".")[-1] return getattr(import_module(modulename), methodname) + def DocType(*args, **kwargs): return frappe.qb.DocType(*args, **kwargs) + def patch_query_execute(): """Patch the Query Builder with helper execute method This excludes the use of `frappe.db.sql` method while @@ -56,10 +62,9 @@ def patch_query_execute(): """ from frappe.utils.safe_exec import check_safe_sql_query - def execute_query(query, *args, **kwargs): query, params = prepare_query(query) - return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep + return frappe.db.sql(query, params, *args, **kwargs) # nosemgrep def prepare_query(query): import inspect @@ -83,11 +88,11 @@ def patch_query_execute(): # p.s. stack() returns `""` as filename if not a file. pass else: - raise frappe.PermissionError('Only SELECT SQL allowed in scripting') + raise frappe.PermissionError("Only SELECT SQL allowed in scripting") return query, param_collector.get_parameters() query_class = get_attr(str(frappe.qb).split("'")[1]) - builder_class = get_type_hints(query_class._builder).get('return') + builder_class = get_type_hints(query_class._builder).get("return") if not builder_class: raise BuilderIdentificationFailed @@ -97,8 +102,7 @@ def patch_query_execute(): def patch_query_aggregation(): - """Patch aggregation functions to frappe.qb - """ + """Patch aggregation functions to frappe.qb""" from frappe.query_builder.functions import _avg, _max, _min, _sum frappe.qb.max = _max diff --git a/frappe/rate_limiter.py b/frappe/rate_limiter.py index b49c1e500e..6693f8abaf 100644 --- a/frappe/rate_limiter.py +++ b/frappe/rate_limiter.py @@ -4,7 +4,7 @@ from datetime import datetime from functools import wraps -from typing import Union, Callable +from typing import Callable, Union from werkzeug.wrappers import Response @@ -82,7 +82,14 @@ class RateLimiter: if self.rejected: return Response(_("Too Many Requests"), status=429) -def rate_limit(key: str = None, limit: Union[int, Callable] = 5, seconds: int = 24*60*60, methods: Union[str, list] = 'ALL', ip_based: bool = True): + +def rate_limit( + key: str = None, + limit: Union[int, Callable] = 5, + seconds: int = 24 * 60 * 60, + methods: Union[str, list] = "ALL", + ip_based: bool = True, +): """Decorator to rate limit an endpoint. This will limit Number of requests per endpoint to `limit` within `seconds`. @@ -93,18 +100,24 @@ def rate_limit(key: str = None, limit: Union[int, Callable] = 5, seconds: int = :type limit: Callable or Integer :param seconds: window time to allow requests :param methods: Limit the validation for these methods. - `ALL` is a wildcard that applies rate limit on all methods. + `ALL` is a wildcard that applies rate limit on all methods. :type methods: string or list or tuple :param ip_based: flag to allow ip based rate-limiting :type ip_based: Boolean :returns: a decorator function that limit the number of requests per endpoint """ + def ratelimit_decorator(fun): @wraps(fun) def wrapper(*args, **kwargs): # Do not apply rate limits if method is not opted to check - if methods != 'ALL' and frappe.request and frappe.request.method and frappe.request.method.upper() not in methods: + if ( + methods != "ALL" + and frappe.request + and frappe.request.method + and frappe.request.method.upper() not in methods + ): return frappe.call(fun, **frappe.form_dict or kwargs) _limit = limit() if callable(limit) else limit @@ -121,7 +134,7 @@ def rate_limit(key: str = None, limit: Union[int, Callable] = 5, seconds: int = identity = identity or ip or user_key if not identity: - frappe.throw(_('Either key or IP flag is required.')) + frappe.throw(_("Either key or IP flag is required.")) cache_key = f"rl:{frappe.form_dict.cmd}:{identity}" @@ -131,8 +144,12 @@ def rate_limit(key: str = None, limit: Union[int, Callable] = 5, seconds: int = value = frappe.cache().incrby(cache_key, 1) if value > _limit: - frappe.throw(_("You hit the rate limit because of too many requests. Please try after sometime.")) + frappe.throw( + _("You hit the rate limit because of too many requests. Please try after sometime.") + ) return frappe.call(fun, **frappe.form_dict or kwargs) + return wrapper + return ratelimit_decorator diff --git a/frappe/realtime.py b/frappe/realtime.py index dc47599923..e833b8c44d 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -1,22 +1,36 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import frappe -from frappe.utils.data import cstr import os + import redis +import frappe +from frappe.utils.data import cstr + redis_server = None def publish_progress(percent, title=None, doctype=None, docname=None, description=None): - publish_realtime('progress', {'percent': percent, 'title': title, 'description': description}, - user=frappe.session.user, doctype=doctype, docname=docname) - - -def publish_realtime(event=None, message=None, room=None, - user=None, doctype=None, docname=None, task_id=None, - after_commit=False): + publish_realtime( + "progress", + {"percent": percent, "title": title, "description": description}, + user=frappe.session.user, + doctype=doctype, + docname=docname, + ) + + +def publish_realtime( + event=None, + message=None, + room=None, + user=None, + doctype=None, + docname=None, + task_id=None, + after_commit=False, +): """Publish real-time updates :param event: Event name, like `task_progress` etc. that will be handled by the client (default is `task_progress` if within task or `global`) @@ -35,7 +49,7 @@ def publish_realtime(event=None, message=None, room=None, else: event = "global" - if event=='msgprint' and not user: + if event == "msgprint" and not user: user = frappe.session.user if not room: @@ -76,7 +90,7 @@ def emit_via_redis(event, message, room): r = get_redis_server() try: - r.publish('events', frappe.as_json({'event': event, 'message': message, 'room': room})) + r.publish("events", frappe.as_json({"event": event, "message": message, "room": room})) except redis.exceptions.ConnectionError: # print(frappe.get_traceback()) pass @@ -87,45 +101,53 @@ def get_redis_server(): global redis_server if not redis_server: from redis import Redis - redis_server = Redis.from_url(frappe.conf.redis_socketio - or "redis://localhost:12311") + + redis_server = Redis.from_url(frappe.conf.redis_socketio or "redis://localhost:12311") return redis_server @frappe.whitelist(allow_guest=True) def can_subscribe_doc(doctype, docname): - if os.environ.get('CI'): + if os.environ.get("CI"): return True - from frappe.sessions import Session from frappe.exceptions import PermissionError + from frappe.sessions import Session + session = Session(None, resume=True).get_session_data() - if not frappe.has_permission(user=session.user, doctype=doctype, doc=docname, ptype='read'): + if not frappe.has_permission(user=session.user, doctype=doctype, doc=docname, ptype="read"): raise PermissionError() return True + @frappe.whitelist(allow_guest=True) def get_user_info(): from frappe.sessions import Session + session = Session(None, resume=True).get_session_data() return { - 'user': session.user, + "user": session.user, } + def get_doc_room(doctype, docname): - return ''.join([frappe.local.site, ':doc:', doctype, '/', cstr(docname)]) + return "".join([frappe.local.site, ":doc:", doctype, "/", cstr(docname)]) + def get_user_room(user): - return ''.join([frappe.local.site, ':user:', user]) + return "".join([frappe.local.site, ":user:", user]) + def get_site_room(): - return ''.join([frappe.local.site, ':all']) + return "".join([frappe.local.site, ":all"]) + def get_task_progress_room(task_id): return "".join([frappe.local.site, ":task_progress:", task_id]) + def get_chat_room(room): - room = ''.join([frappe.local.site, ":room:", room]) + room = "".join([frappe.local.site, ":room:", room]) return room diff --git a/frappe/recorder.py b/frappe/recorder.py index d4d9a275db..95b78dd085 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from collections import Counter import datetime import inspect import json import re import time -import frappe +from collections import Counter + import sqlparse +import frappe from frappe import _ RECORDER_INTERCEPT_FLAG = "recorder-intercept" @@ -64,6 +65,7 @@ def get_current_stack_frames(): except Exception: pass + def record(): if __debug__: if frappe.cache().get_value(RECORDER_INTERCEPT_FLAG): @@ -98,9 +100,7 @@ class Recorder: "cmd": self.cmd, "time": self.time, "queries": len(self.calls), - "time_queries": float( - "{:0.3f}".format(sum(call["duration"] for call in self.calls)) - ), + "time_queries": float("{:0.3f}".format(sum(call["duration"] for call in self.calls))), "duration": float( "{:0.3f}".format((datetime.datetime.now() - self.time).total_seconds() * 1000) ), diff --git a/frappe/search/__init__.py b/frappe/search/__init__.py index 8e1302c0b4..81df8f4a80 100644 --- a/frappe/search/__init__.py +++ b/frappe/search/__init__.py @@ -2,12 +2,13 @@ # License: MIT. See LICENSE import frappe -from frappe.utils import cint -from frappe.search.website_search import WebsiteSearch from frappe.search.full_text_search import FullTextSearch +from frappe.search.website_search import WebsiteSearch +from frappe.utils import cint + @frappe.whitelist(allow_guest=True) def web_search(query, scope=None, limit=20): limit = cint(limit) ws = WebsiteSearch(index_name="web_routes") - return ws.search(query, scope, limit) \ No newline at end of file + return ws.search(query, scope, limit) diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index 79ccd3c6d5..6a2cfd9f70 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -1,18 +1,18 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from whoosh.fields import ID, TEXT, Schema +from whoosh.index import EmptyIndexError, create_in, open_dir +from whoosh.qparser import FieldsPlugin, MultifieldParser, WildcardPlugin +from whoosh.query import FuzzyTerm, Prefix +from whoosh.writing import AsyncWriter + import frappe from frappe.utils import update_progress_bar -from whoosh.index import create_in, open_dir, EmptyIndexError -from whoosh.fields import TEXT, ID, Schema -from whoosh.qparser import MultifieldParser, FieldsPlugin, WildcardPlugin -from whoosh.query import Prefix, FuzzyTerm -from whoosh.writing import AsyncWriter - class FullTextSearch: - """ Frappe Wrapper for Whoosh """ + """Frappe Wrapper for Whoosh""" def __init__(self, index_name): self.index_name = index_name @@ -37,7 +37,7 @@ class FullTextSearch: return {} def build(self): - """ Build search index for all documents """ + """Build search index for all documents""" self.documents = self.get_items_to_index() self.build_index() @@ -47,8 +47,8 @@ class FullTextSearch: and should only be run as administrator or in a background job. Args: - self (object): FullTextSearch Instance - doc_name (str): name of the document to be updated + self (object): FullTextSearch Instance + doc_name (str): name of the document to be updated """ document = self.get_document_to_index(doc_name) if document: @@ -58,8 +58,8 @@ class FullTextSearch: """Remove document from search index Args: - self (object): FullTextSearch Instance - doc_name (str): name of the document to be removed + self (object): FullTextSearch Instance + doc_name (str): name of the document to be removed """ if not doc_name: return @@ -74,8 +74,8 @@ class FullTextSearch: """Update search index for a document Args: - self (object): FullTextSearch Instance - document (_dict): A dictionary with title, path and content + self (object): FullTextSearch Instance + document (_dict): A dictionary with title, path and content """ ix = self.get_index() @@ -111,12 +111,12 @@ class FullTextSearch: """Search from the current index Args: - text (str): String to search for - scope (str, optional): Scope to limit the search. Defaults to None. - limit (int, optional): Limit number of search results. Defaults to 20. + text (str): String to search for + scope (str, optional): Scope to limit the search. Defaults to None. + limit (int, optional): Limit number of search results. Defaults to 20. Returns: - [List(_dict)]: Search results + [List(_dict)]: Search results """ ix = self.get_index() @@ -131,7 +131,9 @@ class FullTextSearch: fieldboosts[field] = 1.0 / idx with ix.searcher() as searcher: - parser = MultifieldParser(search_fields, ix.schema, termclass=FuzzyTermExtended, fieldboosts=fieldboosts) + parser = MultifieldParser( + search_fields, ix.schema, termclass=FuzzyTermExtended, fieldboosts=fieldboosts + ) parser.remove_plugin_class(FieldsPlugin) parser.remove_plugin_class(WildcardPlugin) query = parser.parse(text) @@ -148,10 +150,15 @@ class FullTextSearch: class FuzzyTermExtended(FuzzyTerm): - def __init__(self, fieldname, text, boost=1.0, maxdist=2, prefixlength=1, - constantscore=True): - super().__init__(fieldname, text, boost=boost, maxdist=maxdist, - prefixlength=prefixlength, constantscore=constantscore) + def __init__(self, fieldname, text, boost=1.0, maxdist=2, prefixlength=1, constantscore=True): + super().__init__( + fieldname, + text, + boost=boost, + maxdist=maxdist, + prefixlength=prefixlength, + constantscore=constantscore, + ) def get_index_path(index_name): diff --git a/frappe/search/test_full_text_search.py b/frappe/search/test_full_text_search.py index 0dbc7e775b..4284b64d9c 100644 --- a/frappe/search/test_full_text_search.py +++ b/frappe/search/test_full_text_search.py @@ -1,10 +1,11 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest + from frappe.search.full_text_search import FullTextSearch -class TestFullTextSearch(unittest.TestCase): +class TestFullTextSearch(unittest.TestCase): def setUp(self): index = get_index() index.build() @@ -13,13 +14,13 @@ class TestFullTextSearch(unittest.TestCase): def test_search_term(self): # Search Wikipedia res = self.index.search("multilingual online encyclopedia") - self.assertEqual(res[0], 'site/wikipedia') + self.assertEqual(res[0], "site/wikipedia") res = self.index.search("Linux kernel") - self.assertEqual(res[0], 'os/linux') + self.assertEqual(res[0], "os/linux") res = self.index.search("Enterprise Resource Planning") - self.assertEqual(res[0], 'sw/erpnext') + self.assertEqual(res[0], "sw/erpnext") def test_search_limit(self): res = self.index.search("CommonSearchTerm") @@ -39,8 +40,8 @@ class TestFullTextSearch(unittest.TestCase): # Search inside scope res = self.index.search("CommonSearchTerm", scope=["os"]) self.assertEqual(len(res), 2) - self.assertTrue('os/linux' in res) - self.assertTrue('os/gnu' in res) + self.assertTrue("os/linux" in res) + self.assertTrue("os/gnu" in res) def test_remove_document_from_index(self): self.index.remove_document_from_index("os/gnu") @@ -49,28 +50,21 @@ class TestFullTextSearch(unittest.TestCase): def test_update_index(self): # Update existing index - self.index.update_index({ - 'name': "sw/erpnext", - 'content': """AwesomeERPNext""" - }) + self.index.update_index({"name": "sw/erpnext", "content": """AwesomeERPNext"""}) res = self.index.search("CommonSearchTerm") - self.assertTrue('sw/erpnext' not in res) + self.assertTrue("sw/erpnext" not in res) res = self.index.search("AwesomeERPNext") self.assertEqual(res[0], "sw/erpnext") # Update new doc - self.index.update_index({ - 'name': "sw/frappebooks", - 'content': """DesktopAccounting""" - }) + self.index.update_index({"name": "sw/frappebooks", "content": """DesktopAccounting"""}) res = self.index.search("DesktopAccounting") self.assertEqual(res[0], "sw/frappebooks") - class TestWrapper(FullTextSearch): def get_items_to_index(self): return get_documents() @@ -88,41 +82,52 @@ class TestWrapper(FullTextSearch): def get_index(): return TestWrapper("test_frappe_index") + def get_documents(): docs = [] - docs.append({ - 'name': "site/wikipedia", - 'content': """Wikipedia is a multilingual online encyclopedia created and maintained + docs.append( + { + "name": "site/wikipedia", + "content": """Wikipedia is a multilingual online encyclopedia created and maintained as an open collaboration project by a community of volunteer editors using a wiki-based editing system. - It is the largest and most popular general reference work on the World Wide Web. CommonSearchTerm""" - }) - - docs.append({ - 'name': "os/linux", - 'content': """Linux is a family of open source Unix-like operating systems based on the + It is the largest and most popular general reference work on the World Wide Web. CommonSearchTerm""", + } + ) + + docs.append( + { + "name": "os/linux", + "content": """Linux is a family of open source Unix-like operating systems based on the Linux kernel, an operating system kernel first released on September 17, 1991, by Linus Torvalds. - Linux is typically packaged in a Linux distribution. CommonSearchTerm""" - }) - - docs.append({ - 'name': "os/gnu", - 'content': """GNU is an operating system and an extensive collection of computer software. + Linux is typically packaged in a Linux distribution. CommonSearchTerm""", + } + ) + + docs.append( + { + "name": "os/gnu", + "content": """GNU is an operating system and an extensive collection of computer software. GNU is composed wholly of free software, most of which is licensed under the GNU Project's own General Public License. GNU is a recursive acronym for "GNU's Not Unix! ", - chosen because GNU's design is Unix-like, but differs from Unix by being free software and containing no Unix code. CommonSearchTerm""" - }) - - docs.append({ - 'name': "sw/erpnext", - 'content': """ERPNext is a free and open-source integrated Enterprise Resource Planning software developed by + chosen because GNU's design is Unix-like, but differs from Unix by being free software and containing no Unix code. CommonSearchTerm""", + } + ) + + docs.append( + { + "name": "sw/erpnext", + "content": """ERPNext is a free and open-source integrated Enterprise Resource Planning software developed by Frappe Technologies Pvt. Ltd. and is built on MariaDB database system using a Python based server-side framework. - ERPNext is a generic ERP software used by manufacturers, distributors and services companies. CommonSearchTerm""" - }) - - docs.append({ - 'name': "sw/frappe", - 'content': """Frappe Framework is a full-stack web framework, that includes everything you need to build and - deploy business applications with Rich Admin Interface. CommonSearchTerm""" - }) + ERPNext is a generic ERP software used by manufacturers, distributors and services companies. CommonSearchTerm""", + } + ) + + docs.append( + { + "name": "sw/frappe", + "content": """Frappe Framework is a full-stack web framework, that includes everything you need to build and + deploy business applications with Rich Admin Interface. CommonSearchTerm""", + } + ) return docs diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 30eadae6f1..e55db9e42e 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -13,13 +13,12 @@ from frappe.website.serve import get_response_content INDEX_NAME = "web_routes" + class WebsiteSearch(FullTextSearch): - """ Wrapper for WebsiteSearch """ + """Wrapper for WebsiteSearch""" def get_schema(self): - return Schema( - title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True) - ) + return Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT(stored=True)) def get_fields_to_search(self): return ["title", "content"] @@ -32,7 +31,7 @@ class WebsiteSearch(FullTextSearch): in www/ and routes from published documents Returns: - self (object): FullTextSearch Instance + self (object): FullTextSearch Instance """ if getattr(self, "_items_to_index", False): @@ -40,10 +39,8 @@ class WebsiteSearch(FullTextSearch): self._items_to_index = [] - routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index) - for i, route in enumerate(routes): update_progress_bar("Retrieving Routes", i, len(routes)) self._items_to_index += [self.get_document_to_index(route)] @@ -56,10 +53,10 @@ class WebsiteSearch(FullTextSearch): """Render a page and parse it using BeautifulSoup Args: - path (str): route of the page to be parsed + path (str): route of the page to be parsed Returns: - document (_dict): A dictionary with title, path and content + document (_dict): A dictionary with title, path and content """ frappe.set_user("Guest") frappe.local.no_cache = True @@ -92,14 +89,14 @@ class WebsiteSearch(FullTextSearch): def slugs_with_web_view(_items_to_index): all_routes = [] - filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} + filters = {"has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} fields = ["name", "is_published_field", "website_search_field"] doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields) for doctype in doctype_with_web_views: if doctype.is_published_field: - fields=["route", doctype.website_search_field] - filters={doctype.is_published_field: 1}, + fields = ["route", doctype.website_search_field] + filters = ({doctype.is_published_field: 1},) if doctype.website_search_field: docs = frappe.get_all(doctype.name, filters=filters, fields=fields + ["title"]) for doc in docs: @@ -113,31 +110,36 @@ def slugs_with_web_view(_items_to_index): return all_routes + def get_static_pages_from_all_apps(): from glob import glob + apps = frappe.get_installed_apps() routes_to_index = [] for app in apps: - path_to_index = frappe.get_app_path(app, 'www') + path_to_index = frappe.get_app_path(app, "www") - files_to_index = glob(path_to_index + '/**/*.html', recursive=True) - files_to_index.extend(glob(path_to_index + '/**/*.md', recursive=True)) + files_to_index = glob(path_to_index + "/**/*.html", recursive=True) + files_to_index.extend(glob(path_to_index + "/**/*.md", recursive=True)) for file in files_to_index: - route = os.path.relpath(file, path_to_index).split('.')[0] - if route.endswith('index'): - route = route.rsplit('index', 1)[0] + route = os.path.relpath(file, path_to_index).split(".")[0] + if route.endswith("index"): + route = route.rsplit("index", 1)[0] routes_to_index.append(route) return routes_to_index + def update_index_for_path(path): ws = WebsiteSearch(INDEX_NAME) return ws.update_index_by_name(path) + def remove_document_from_index(path): ws = WebsiteSearch(INDEX_NAME) return ws.remove_document_from_index(path) + def build_index_for_all_routes(): ws = WebsiteSearch(INDEX_NAME) return ws.build() diff --git a/frappe/sessions.py b/frappe/sessions.py index 4bbcaaa2ae..d701ac24a7 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -6,19 +6,22 @@ Boot session from cache or build Session bootstraps info needed by common client side activities including permission, homepage, default variables, system defaults etc """ -import frappe, json -from frappe import _ -import frappe.utils -from frappe.utils import cint, cstr, get_assets_json -import frappe.model.meta +import json +from urllib.parse import unquote + +import redis + +import frappe import frappe.defaults +import frappe.model.meta import frappe.translate -import redis -from urllib.parse import unquote +import frappe.utils +from frappe import _ from frappe.cache_manager import clear_user_cache -from frappe.query_builder import Order, DocType -from frappe.query_builder.utils import PseudoColumn +from frappe.query_builder import DocType, Order from frappe.query_builder.functions import Now +from frappe.query_builder.utils import PseudoColumn +from frappe.utils import cint, cstr, get_assets_json @frappe.whitelist() @@ -26,16 +29,17 @@ def clear(): frappe.local.session_obj.update(force=True) frappe.local.db.commit() clear_user_cache(frappe.session.user) - frappe.response['message'] = _("Cache Cleared") + frappe.response["message"] = _("Cache Cleared") + def clear_sessions(user=None, keep_current=False, device=None, force=False): - '''Clear other sessions of the current user. Called at login / logout + """Clear other sessions of the current user. Called at login / logout :param user: user name (default: current user) :param keep_current: keep current session (default: false) :param device: delete sessions of this device (default: desktop, mobile) :param force: triggered by the user (default false) - ''' + """ reason = "Logged In From Another Session" if force: @@ -44,13 +48,14 @@ def clear_sessions(user=None, keep_current=False, device=None, force=False): for sid in get_sessions_to_clear(user, keep_current, device): delete_session(sid, reason=reason) + def get_sessions_to_clear(user=None, keep_current=False, device=None): - '''Returns sessions of the current user. Called at login / logout + """Returns sessions of the current user. Called at login / logout :param user: user name (default: current user) :param keep_current: keep current session (default: false) :param device: delete sessions of this device (default: desktop, mobile) - ''' + """ if not user: user = frappe.session.user @@ -62,11 +67,13 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): offset = 0 if user == frappe.session.user: - simultaneous_sessions = frappe.db.get_value('User', user, 'simultaneous_sessions') or 1 + simultaneous_sessions = frappe.db.get_value("User", user, "simultaneous_sessions") or 1 offset = simultaneous_sessions - 1 session = DocType("Sessions") - session_id = frappe.qb.from_(session).where((session.user == user) & (session.device.isin(device))) + session_id = frappe.qb.from_(session).where( + (session.user == user) & (session.device.isin(device)) + ) if keep_current: session_id = session_id.where(session.sid != frappe.session.sid) @@ -79,6 +86,7 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): return query.run(pluck=True) + def delete_session(sid=None, user=None, reason="Session Expired"): from frappe.core.doctype.activity_log.feed import logout_feed @@ -86,24 +94,28 @@ def delete_session(sid=None, user=None, reason="Session Expired"): frappe.cache().hdel("last_db_session_update", sid) if sid and not user: table = DocType("Sessions") - user_details = frappe.qb.from_(table).where( - table.sid == sid - ).select(table.user).run(as_dict=True) - if user_details: user = user_details[0].get("user") + user_details = ( + frappe.qb.from_(table).where(table.sid == sid).select(table.user).run(as_dict=True) + ) + if user_details: + user = user_details[0].get("user") logout_feed(user, reason) frappe.db.delete("Sessions", {"sid": sid}) frappe.db.commit() + def clear_all_sessions(reason=None): """This effectively logs out all users""" frappe.only_for("Administrator") - if not reason: reason = "Deleted All Active Session" + if not reason: + reason = "Deleted All Active Session" for sid in frappe.qb.from_("Sessions").select("sid").run(pluck=True): delete_session(sid, reason=reason) + def get_expired_sessions(): - '''Returns list of expired sessions''' + """Returns list of expired sessions""" sessions = DocType("Sessions") expired = [] @@ -124,24 +136,25 @@ def get_expired_sessions(): return expired + def clear_expired_sessions(): """This function is meant to be called from scheduler""" for sid in get_expired_sessions(): delete_session(sid, reason="Session Expired") + def get(): """get session boot info""" from frappe.boot import get_bootinfo, get_unseen_notes from frappe.utils.change_log import get_change_log bootinfo = None - if not getattr(frappe.conf,'disable_session_cache', None): + if not getattr(frappe.conf, "disable_session_cache", None): # check if cache exists bootinfo = frappe.cache().hget("bootinfo", frappe.session.user) if bootinfo: - bootinfo['from_cache'] = 1 - bootinfo["user"]["recent"] = json.dumps(\ - frappe.cache().hget("user_recent", frappe.session.user)) + bootinfo["from_cache"] = 1 + bootinfo["user"]["recent"] = json.dumps(frappe.cache().hget("user_recent", frappe.session.user)) if not bootinfo: # if not create it @@ -151,10 +164,10 @@ def get(): frappe.cache().ping() except redis.exceptions.ConnectionError: message = _("Redis cache server not running. Please contact Administrator / Tech support") - if 'messages' in bootinfo: - bootinfo['messages'].append(message) + if "messages" in bootinfo: + bootinfo["messages"].append(message) else: - bootinfo['messages'] = [message] + bootinfo["messages"] = [message] # check only when clear cache is done, and don't cache this if frappe.local.request: @@ -173,37 +186,44 @@ def get(): bootinfo["lang"] = frappe.translate.get_user_lang() bootinfo["disable_async"] = frappe.conf.disable_async - bootinfo["setup_complete"] = cint(frappe.db.get_single_value('System Settings', 'setup_complete')) - bootinfo["is_first_startup"] = cint(frappe.db.get_single_value('System Settings', 'is_first_startup')) + bootinfo["setup_complete"] = cint(frappe.db.get_single_value("System Settings", "setup_complete")) + bootinfo["is_first_startup"] = cint( + frappe.db.get_single_value("System Settings", "is_first_startup") + ) - bootinfo['desk_theme'] = frappe.db.get_value("User", frappe.session.user, "desk_theme") or 'Light' + bootinfo["desk_theme"] = frappe.db.get_value("User", frappe.session.user, "desk_theme") or "Light" return bootinfo + @frappe.whitelist() def get_boot_assets_json(): return get_assets_json() + def get_csrf_token(): if not frappe.local.session.data.csrf_token: generate_csrf_token() return frappe.local.session.data.csrf_token + def generate_csrf_token(): frappe.local.session.data.csrf_token = frappe.generate_hash() if not frappe.flags.in_test: frappe.local.session_obj.update(force=True) + class Session: def __init__(self, user, resume=False, full_name=None, user_type=None): - self.sid = cstr(frappe.form_dict.get('sid') or - unquote(frappe.request.cookies.get('sid', 'Guest'))) + self.sid = cstr( + frappe.form_dict.get("sid") or unquote(frappe.request.cookies.get("sid", "Guest")) + ) self.user = user self.device = frappe.form_dict.get("device") or "desktop" self.user_type = user_type self.full_name = full_name - self.data = frappe._dict({'data': frappe._dict({})}) + self.data = frappe._dict({"data": frappe._dict({})}) self.time_diff = None # set local session @@ -219,8 +239,8 @@ class Session: def start(self): """start a new session""" # generate sid - if self.user=='Guest': - sid = 'Guest' + if self.user == "Guest": + sid = "Guest" else: sid = frappe.generate_hash() @@ -229,27 +249,32 @@ class Session: self.data.data.user = self.user self.data.data.session_ip = frappe.local.request_ip if self.user != "Guest": - self.data.data.update({ - "last_updated": frappe.utils.now(), - "session_expiry": get_expiry_period(self.device), - "full_name": self.full_name, - "user_type": self.user_type, - "device": self.device, - "session_country": get_geo_ip_country(frappe.local.request_ip) if frappe.local.request_ip else None, - }) + self.data.data.update( + { + "last_updated": frappe.utils.now(), + "session_expiry": get_expiry_period(self.device), + "full_name": self.full_name, + "user_type": self.user_type, + "device": self.device, + "session_country": get_geo_ip_country(frappe.local.request_ip) + if frappe.local.request_ip + else None, + } + ) # insert session - if self.user!="Guest": + if self.user != "Guest": self.insert_session_record() # update user - user = frappe.get_doc("User", self.data['user']) - user_doctype=frappe.qb.DocType("User") - (frappe.qb.update(user_doctype) + user = frappe.get_doc("User", self.data["user"]) + user_doctype = frappe.qb.DocType("User") + ( + frappe.qb.update(user_doctype) .set(user_doctype.last_login, frappe.utils.now()) .set(user_doctype.last_ip, frappe.local.request_ip) .set(user_doctype.last_active, frappe.utils.now()) - .where(user_doctype.name == self.data['user']) + .where(user_doctype.name == self.data["user"]) ).run() user.run_notifications("before_change") @@ -257,10 +282,12 @@ class Session: frappe.db.commit() def insert_session_record(self): - frappe.db.sql("""insert into `tabSessions` + frappe.db.sql( + """insert into `tabSessions` (`sessiondata`, `user`, `lastupdate`, `sid`, `status`, `device`) values (%s , %s, NOW(), %s, 'Active', %s)""", - (str(self.data['data']), self.data['user'], self.data['sid'], self.device)) + (str(self.data["data"]), self.data["user"], self.data["sid"], self.device), + ) # also add to memcache frappe.cache().hset("session", self.data.sid, self.data) @@ -269,10 +296,11 @@ class Session: """non-login request: load a session""" import frappe from frappe.auth import validate_ip_address + data = self.get_session_record() if data: - self.data.update({'data': data, 'user':data.user, 'sid': self.sid}) + self.data.update({"data": data, "user": data.user, "sid": self.sid}) self.user = data.user validate_ip_address(self.user) self.device = data.device @@ -286,6 +314,7 @@ class Session: def get_session_record(self): """get session record, or return the standard Guest Record""" from frappe.auth import clear_cookies + r = self.get_session_data() if not r: @@ -297,8 +326,8 @@ class Session: return r def get_session_data(self): - if self.sid=="Guest": - return frappe._dict({"user":"Guest"}) + if self.sid == "Guest": + return frappe._dict({"user": "Guest"}) data = self.get_session_data_from_cache() if not data: @@ -312,8 +341,9 @@ class Session: session_data = data.get("data", {}) # set user for correct timezone - self.time_diff = frappe.utils.time_diff_in_seconds(frappe.utils.now(), - session_data.get("last_updated")) + self.time_diff = frappe.utils.time_diff_in_seconds( + frappe.utils.now(), session_data.get("last_updated") + ) expiry = get_expiry_in_seconds(session_data.get("session_expiry")) if self.time_diff > expiry: @@ -325,9 +355,15 @@ class Session: def get_session_data_from_db(self): sessions = DocType("Sessions") - self.device = frappe.db.get_value( - sessions, filters=sessions.sid == self.sid, fieldname="device", order_by=None, - ) or "desktop" + self.device = ( + frappe.db.get_value( + sessions, + filters=sessions.sid == self.sid, + fieldname="device", + order_by=None, + ) + or "desktop" + ) rec = frappe.db.get_values( sessions, filters=(sessions.sid == self.sid) @@ -340,7 +376,7 @@ class Session: ) if rec: - data = frappe._dict(frappe.safe_eval(rec and rec[0][1] or '{}')) + data = frappe._dict(frappe.safe_eval(rec and rec[0][1] or "{}")) data.user = rec[0][0] else: self._delete_session() @@ -358,13 +394,13 @@ class Session: def update(self, force=False): """extend session expiry""" - if (frappe.session['user'] == "Guest" or frappe.form_dict.cmd=="logout"): + if frappe.session["user"] == "Guest" or frappe.form_dict.cmd == "logout": return now = frappe.utils.now() - self.data['data']['last_updated'] = now - self.data['data']['lang'] = str(frappe.lang) + self.data["data"]["last_updated"] = now + self.data["data"]["lang"] = str(frappe.lang) # update session in db last_updated = frappe.cache().hget("last_db_session_update", self.sid) @@ -374,15 +410,17 @@ class Session: updated_in_db = False if force or (time_diff is None) or (time_diff > 600): # update sessions table - frappe.db.sql("""update `tabSessions` set sessiondata=%s, - lastupdate=NOW() where sid=%s""" , (str(self.data['data']), - self.data['sid'])) + frappe.db.sql( + """update `tabSessions` set sessiondata=%s, + lastupdate=NOW() where sid=%s""", + (str(self.data["data"]), self.data["sid"]), + ) # update last active in user table - frappe.db.sql("""update `tabUser` set last_active=%(now)s where name=%(name)s""", { - "now": now, - "name": frappe.session.user - }) + frappe.db.sql( + """update `tabUser` set last_active=%(now)s where name=%(name)s""", + {"now": now, "name": frappe.session.user}, + ) frappe.db.commit() frappe.cache().hset("last_db_session_update", self.sid, now) @@ -394,20 +432,23 @@ class Session: return updated_in_db + def get_expiry_period_for_query(device=None): - if frappe.db.db_type == 'postgres': + if frappe.db.db_type == "postgres": return get_expiry_period(device) else: return get_expiry_in_seconds(device=device) + def get_expiry_in_seconds(expiry=None, device=None): if not expiry: expiry = get_expiry_period(device) parts = expiry.split(":") return (cint(parts[0]) * 3600) + (cint(parts[1]) * 60) + cint(parts[2]) + def get_expiry_period(device="desktop"): - if device=="mobile": + if device == "mobile": key = "session_expiry_mobile" default = "720:00:00" else: @@ -417,17 +458,19 @@ def get_expiry_period(device="desktop"): exp_sec = frappe.defaults.get_global_default(key) or default # incase seconds is missing - if len(exp_sec.split(':')) == 2: - exp_sec = exp_sec + ':00' + if len(exp_sec.split(":")) == 2: + exp_sec = exp_sec + ":00" return exp_sec + def get_geo_from_ip(ip_addr): try: from geolite2 import geolite2 + with geolite2 as f: reader = f.reader() - data = reader.get(ip_addr) + data = reader.get(ip_addr) return frappe._dict(data) except ImportError: @@ -437,6 +480,7 @@ def get_geo_from_ip(ip_addr): except TypeError: return + def get_geo_ip_country(ip_addr): match = get_geo_from_ip(ip_addr) if match: diff --git a/frappe/share.py b/frappe/share.py index 917181e563..01d1412b8d 100644 --- a/frappe/share.py +++ b/frappe/share.py @@ -3,13 +3,19 @@ import frappe from frappe import _ +from frappe.desk.doctype.notification_log.notification_log import ( + enqueue_create_notification, + get_title, + get_title_html, +) from frappe.desk.form.document_follow import follow_document -from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\ - get_title, get_title_html from frappe.utils import cint + @frappe.whitelist() -def add(doctype, name, user=None, read=1, write=0, submit=0, share=0, everyone=0, flags=None, notify=0): +def add( + doctype, name, user=None, read=1, write=0, submit=0, share=0, everyone=0, flags=None, notify=0 +): """Share the given document with a user.""" if not user: user = frappe.session.user @@ -23,23 +29,22 @@ def add(doctype, name, user=None, read=1, write=0, submit=0, share=0, everyone=0 doc = frappe.get_doc("DocShare", share_name) else: doc = frappe.new_doc("DocShare") - doc.update({ - "user": user, - "share_doctype": doctype, - "share_name": name, - "everyone": cint(everyone) - }) + doc.update( + {"user": user, "share_doctype": doctype, "share_name": name, "everyone": cint(everyone)} + ) if flags: doc.flags.update(flags) - doc.update({ - # always add read, since you are adding! - "read": 1, - "write": cint(write), - "submit": cint(submit), - "share": cint(share) - }) + doc.update( + { + # always add read, since you are adding! + "read": 1, + "write": cint(write), + "submit": cint(submit), + "share": cint(share), + } + ) doc.save(ignore_permissions=True) notify_assignment(user, doctype, name, everyone, notify=notify) @@ -49,13 +54,16 @@ def add(doctype, name, user=None, read=1, write=0, submit=0, share=0, everyone=0 return doc + def remove(doctype, name, user, flags=None): - share_name = frappe.db.get_value("DocShare", {"user": user, "share_name": name, - "share_doctype": doctype}) + share_name = frappe.db.get_value( + "DocShare", {"user": user, "share_name": name, "share_doctype": doctype} + ) if share_name: frappe.delete_doc("DocShare", share_name, flags=flags) + @frappe.whitelist() def set_permission(doctype, name, user, permission_to, value=1, everyone=0): """Set share permission.""" @@ -78,7 +86,7 @@ def set_permission(doctype, name, user, permission_to, value=1, everyone=0): if not value: # un-set higher-order permissions too - if permission_to=="read": + if permission_to == "read": share.read = share.write = share.submit = share.share = 0 share.save() @@ -89,15 +97,26 @@ def set_permission(doctype, name, user, permission_to, value=1, everyone=0): return share + @frappe.whitelist() def get_users(doctype, name): """Get list of users with which this document is shared""" - return frappe.db.get_all("DocShare", - fields=["`name`", "`user`", "`read`", "`write`", "`submit`", "`share`", "everyone", "owner", "creation"], - filters=dict( - share_doctype=doctype, - share_name=name - )) + return frappe.db.get_all( + "DocShare", + fields=[ + "`name`", + "`user`", + "`read`", + "`write`", + "`submit`", + "`share`", + "everyone", + "owner", + "creation", + ], + filters=dict(share_doctype=doctype, share_name=name), + ) + def get_shared(doctype, user=None, rights=None): """Get list of shared document names for given user and DocType. @@ -112,43 +131,53 @@ def get_shared(doctype, user=None, rights=None): if not rights: rights = ["read"] - filters = [[right, '=', 1] for right in rights] - filters += [['share_doctype', '=', doctype]] - or_filters = [['user', '=', user]] - if user != 'Guest': - or_filters += [['everyone', '=', 1]] + filters = [[right, "=", 1] for right in rights] + filters += [["share_doctype", "=", doctype]] + or_filters = [["user", "=", user]] + if user != "Guest": + or_filters += [["everyone", "=", 1]] - shared_docs = frappe.db.get_all('DocShare', - fields=['share_name'], - filters=filters, - or_filters=or_filters) + shared_docs = frappe.db.get_all( + "DocShare", fields=["share_name"], filters=filters, or_filters=or_filters + ) return [doc.share_name for doc in shared_docs] + def get_shared_doctypes(user=None): """Return list of doctypes in which documents are shared for the given user.""" if not user: user = frappe.session.user table = frappe.qb.DocType("DocShare") - query = frappe.qb.from_(table).where( - (table.user == user) | (table.everyone == 1) - ).select(table.share_doctype).distinct() + query = ( + frappe.qb.from_(table) + .where((table.user == user) | (table.everyone == 1)) + .select(table.share_doctype) + .distinct() + ) return query.run(pluck=True) + def get_share_name(doctype, name, user, everyone): if cint(everyone): - share_name = frappe.db.get_value("DocShare", {"everyone": 1, "share_name": name, - "share_doctype": doctype}) + share_name = frappe.db.get_value( + "DocShare", {"everyone": 1, "share_name": name, "share_doctype": doctype} + ) else: - share_name = frappe.db.get_value("DocShare", {"user": user, "share_name": name, - "share_doctype": doctype}) + share_name = frappe.db.get_value( + "DocShare", {"user": user, "share_name": name, "share_doctype": doctype} + ) return share_name + def check_share_permission(doctype, name): """Check if the user can share with other users""" if not frappe.has_permission(doctype, ptype="share", doc=name): - frappe.throw(_("No permission to {0} {1} {2}").format("share", doctype, name), frappe.PermissionError) + frappe.throw( + _("No permission to {0} {1} {2}").format("share", doctype, name), frappe.PermissionError + ) + def notify_assignment(shared_by, doctype, doc_name, everyone, notify=0): @@ -160,15 +189,16 @@ def notify_assignment(shared_by, doctype, doc_name, everyone, notify=0): title = get_title(doctype, doc_name) reference_user = get_fullname(frappe.session.user) - notification_message = _('{0} shared a document {1} {2} with you').format( - frappe.bold(reference_user), frappe.bold(doctype), get_title_html(title)) + notification_message = _("{0} shared a document {1} {2} with you").format( + frappe.bold(reference_user), frappe.bold(doctype), get_title_html(title) + ) notification_doc = { - 'type': 'Share', - 'document_type': doctype, - 'subject': notification_message, - 'document_name': doc_name, - 'from_user': frappe.session.user + "type": "Share", + "document_type": doctype, + "subject": notification_message, + "document_name": doc_name, + "from_user": frappe.session.user, } enqueue_create_notification(shared_by, notification_doc) diff --git a/frappe/social/doctype/energy_point_log/energy_point_log.py b/frappe/social/doctype/energy_point_log/energy_point_log.py index 5e6a9df16f..a0c5120fa7 100644 --- a/frappe/social/doctype/energy_point_log/energy_point_log.py +++ b/frappe/social/doctype/energy_point_log/energy_point_log.py @@ -2,118 +2,131 @@ # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -import json +from frappe.desk.doctype.notification_log.notification_log import ( + enqueue_create_notification, + get_title, + get_title_html, +) +from frappe.desk.doctype.notification_settings.notification_settings import ( + is_email_notifications_enabled, + is_email_notifications_enabled_for_type, +) from frappe.model.document import Document -from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\ - get_title, get_title_html -from frappe.desk.doctype.notification_settings.notification_settings\ - import is_email_notifications_enabled_for_type, is_email_notifications_enabled -from frappe.utils import cint, get_fullname, getdate, get_link_to_form +from frappe.utils import cint, get_fullname, get_link_to_form, getdate + class EnergyPointLog(Document): def validate(self): self.map_milestone_reference() - if self.type in ['Appreciation', 'Criticism'] and self.user == self.owner: - frappe.throw(_('You cannot give review points to yourself')) + if self.type in ["Appreciation", "Criticism"] and self.user == self.owner: + frappe.throw(_("You cannot give review points to yourself")) def map_milestone_reference(self): # link energy point to the original reference, if set by milestone - if self.reference_doctype == 'Milestone': - self.reference_doctype, self.reference_name = frappe.db.get_value('Milestone', self.reference_name, - ['reference_type', 'reference_name']) + if self.reference_doctype == "Milestone": + self.reference_doctype, self.reference_name = frappe.db.get_value( + "Milestone", self.reference_name, ["reference_type", "reference_name"] + ) def after_insert(self): alert_dict = get_alert_dict(self) if alert_dict: - frappe.publish_realtime('energy_point_alert', message=alert_dict, user=self.user) + frappe.publish_realtime("energy_point_alert", message=alert_dict, user=self.user) - frappe.cache().hdel('energy_points', self.user) - frappe.publish_realtime('update_points', after_commit=True) + frappe.cache().hdel("energy_points", self.user) + frappe.publish_realtime("update_points", after_commit=True) - if self.type != 'Review' and \ - frappe.get_cached_value('Notification Settings', self.user, 'energy_points_system_notifications'): + if self.type != "Review" and frappe.get_cached_value( + "Notification Settings", self.user, "energy_points_system_notifications" + ): - reference_user = self.user if self.type == 'Auto' else self.owner + reference_user = self.user if self.type == "Auto" else self.owner notification_doc = { - 'type': 'Energy Point', - 'document_type': self.reference_doctype, - 'document_name': self.reference_name, - 'subject': get_notification_message(self), - 'from_user': reference_user, - 'email_content': '
{}
'.format(self.reason) if self.reason else None + "type": "Energy Point", + "document_type": self.reference_doctype, + "document_name": self.reference_name, + "subject": get_notification_message(self), + "from_user": reference_user, + "email_content": "
{}
".format(self.reason) if self.reason else None, } enqueue_create_notification(self.user, notification_doc) def on_trash(self): - if self.type == 'Revert': - reference_log = frappe.get_doc('Energy Point Log', self.revert_of) + if self.type == "Revert": + reference_log = frappe.get_doc("Energy Point Log", self.revert_of) reference_log.reverted = 0 reference_log.save() @frappe.whitelist() def revert(self, reason, ignore_permissions=False): if not ignore_permissions: - frappe.only_for('System Manager') + frappe.only_for("System Manager") - if self.type != 'Auto': - frappe.throw(_('This document cannot be reverted')) + if self.type != "Auto": + frappe.throw(_("This document cannot be reverted")) - if self.get('reverted'): + if self.get("reverted"): return self.reverted = 1 self.save(ignore_permissions=True) - revert_log = frappe.get_doc({ - 'doctype': 'Energy Point Log', - 'points': -(self.points), - 'type': 'Revert', - 'user': self.user, - 'reason': reason, - 'reference_doctype': self.reference_doctype, - 'reference_name': self.reference_name, - 'revert_of': self.name - }).insert(ignore_permissions=True) + revert_log = frappe.get_doc( + { + "doctype": "Energy Point Log", + "points": -(self.points), + "type": "Revert", + "user": self.user, + "reason": reason, + "reference_doctype": self.reference_doctype, + "reference_name": self.reference_name, + "revert_of": self.name, + } + ).insert(ignore_permissions=True) return revert_log + def get_notification_message(doc): owner_name = get_fullname(doc.owner) points = doc.points title = get_title(doc.reference_doctype, doc.reference_name) - if doc.type == 'Auto': - owner_name = frappe.bold('You') + if doc.type == "Auto": + owner_name = frappe.bold("You") if points == 1: - message = _('{0} gained {1} point for {2} {3}') + message = _("{0} gained {1} point for {2} {3}") else: - message = _('{0} gained {1} points for {2} {3}') + message = _("{0} gained {1} points for {2} {3}") message = message.format(owner_name, frappe.bold(points), doc.rule, get_title_html(title)) - elif doc.type == 'Appreciation': + elif doc.type == "Appreciation": if points == 1: - message = _('{0} appreciated your work on {1} with {2} point') + message = _("{0} appreciated your work on {1} with {2} point") else: - message = _('{0} appreciated your work on {1} with {2} points') + message = _("{0} appreciated your work on {1} with {2} points") message = message.format(frappe.bold(owner_name), get_title_html(title), frappe.bold(points)) - elif doc.type == 'Criticism': + elif doc.type == "Criticism": if points == 1: - message = _('{0} criticized your work on {1} with {2} point') + message = _("{0} criticized your work on {1} with {2} point") else: - message = _('{0} criticized your work on {1} with {2} points') + message = _("{0} criticized your work on {1} with {2} points") message = message.format(frappe.bold(owner_name), get_title_html(title), frappe.bold(points)) - elif doc.type == 'Revert': + elif doc.type == "Revert": if points == 1: - message = _('{0} reverted your point on {1}') + message = _("{0} reverted your point on {1}") else: - message = _('{0} reverted your points on {1}') + message = _("{0} reverted your points on {1}") message = message.format(frappe.bold(owner_name), get_title_html(title)) return message + def get_alert_dict(doc): alert_dict = frappe._dict() owner_name = get_fullname(doc.owner) @@ -121,46 +134,38 @@ def get_alert_dict(doc): doc_link = get_link_to_form(doc.reference_doctype, doc.reference_name) points = doc.points bold_points = frappe.bold(doc.points) - if doc.type == 'Auto': + if doc.type == "Auto": if points == 1: - message = _('You gained {0} point') + message = _("You gained {0} point") else: - message = _('You gained {0} points') + message = _("You gained {0} points") alert_dict.message = message.format(bold_points) - alert_dict.indicator = 'green' - elif doc.type == 'Appreciation': + alert_dict.indicator = "green" + elif doc.type == "Appreciation": if points == 1: - message = _('{0} appreciated your work on {1} with {2} point') + message = _("{0} appreciated your work on {1} with {2} point") else: - message = _('{0} appreciated your work on {1} with {2} points') - alert_dict.message = message.format( - owner_name, - doc_link, - bold_points - ) - alert_dict.indicator = 'green' - elif doc.type == 'Criticism': + message = _("{0} appreciated your work on {1} with {2} points") + alert_dict.message = message.format(owner_name, doc_link, bold_points) + alert_dict.indicator = "green" + elif doc.type == "Criticism": if points == 1: - message = _('{0} criticized your work on {1} with {2} point') + message = _("{0} criticized your work on {1} with {2} point") else: - message = _('{0} criticized your work on {1} with {2} points') + message = _("{0} criticized your work on {1} with {2} points") - alert_dict.message = message.format( - owner_name, - doc_link, - bold_points - ) - alert_dict.indicator = 'red' - elif doc.type == 'Revert': + alert_dict.message = message.format(owner_name, doc_link, bold_points) + alert_dict.indicator = "red" + elif doc.type == "Revert": if points == 1: - message = _('{0} reverted your point on {1}') + message = _("{0} reverted your point on {1}") else: - message = _('{0} reverted your points on {1}') + message = _("{0} reverted your points on {1}") alert_dict.message = message.format( owner_name, doc_link, ) - alert_dict.indicator = 'red' + alert_dict.indicator = "red" return alert_dict @@ -168,49 +173,53 @@ def get_alert_dict(doc): def create_energy_points_log(ref_doctype, ref_name, doc, apply_only_once=False): doc = frappe._dict(doc) - log_exists = check_if_log_exists(ref_doctype, - ref_name, doc.rule, None if apply_only_once else doc.user) + log_exists = check_if_log_exists( + ref_doctype, ref_name, doc.rule, None if apply_only_once else doc.user + ) if log_exists: - return frappe.get_doc('Energy Point Log', log_exists) + return frappe.get_doc("Energy Point Log", log_exists) - new_log = frappe.new_doc('Energy Point Log') + new_log = frappe.new_doc("Energy Point Log") new_log.reference_doctype = ref_doctype new_log.reference_name = ref_name new_log.update(doc) new_log.insert(ignore_permissions=True) return new_log + def check_if_log_exists(ref_doctype, ref_name, rule, user=None): - ''''Checks if Energy Point Log already exists''' - filters = frappe._dict({ - 'rule': rule, - 'reference_doctype': ref_doctype, - 'reference_name': ref_name, - 'reverted': 0 - }) + """'Checks if Energy Point Log already exists""" + filters = frappe._dict( + {"rule": rule, "reference_doctype": ref_doctype, "reference_name": ref_name, "reverted": 0} + ) if user: filters.user = user - return frappe.db.exists('Energy Point Log', filters) + return frappe.db.exists("Energy Point Log", filters) + def create_review_points_log(user, points, reason=None, doctype=None, docname=None): - return frappe.get_doc({ - 'doctype': 'Energy Point Log', - 'points': points, - 'type': 'Review', - 'user': user, - 'reason': reason, - 'reference_doctype': doctype, - 'reference_name': docname - }).insert(ignore_permissions=True) + return frappe.get_doc( + { + "doctype": "Energy Point Log", + "points": points, + "type": "Review", + "user": user, + "reason": reason, + "reference_doctype": doctype, + "reference_name": docname, + } + ).insert(ignore_permissions=True) + @frappe.whitelist() def add_review_points(user, points): - frappe.only_for('System Manager') + frappe.only_for("System Manager") create_review_points_log(user, points) + @frappe.whitelist() def get_energy_points(user): # points = frappe.cache().hget('energy_points', user, @@ -219,21 +228,23 @@ def get_energy_points(user): points = get_user_energy_and_review_points(user) return frappe._dict(points.get(user, {})) + @frappe.whitelist() def get_user_energy_and_review_points(user=None, from_date=None, as_dict=True): - conditions = '' - given_points_condition = '' + conditions = "" + given_points_condition = "" values = frappe._dict() if user: - conditions = 'WHERE `user` = %(user)s' + conditions = "WHERE `user` = %(user)s" values.user = user if from_date: - conditions += 'WHERE' if not conditions else 'AND' + conditions += "WHERE" if not conditions else "AND" given_points_condition += "AND `creation` >= %(from_date)s" conditions += " `creation` >= %(from_date)s OR `type`='Review'" values.from_date = from_date - points_list = frappe.db.sql(""" + points_list = frappe.db.sql( + """ SELECT SUM(CASE WHEN `type` != 'Review' THEN `points` ELSE 0 END) AS energy_points, SUM(CASE WHEN `type` = 'Review' THEN `points` ELSE 0 END) AS review_points, @@ -248,33 +259,40 @@ def get_user_energy_and_review_points(user=None, from_date=None, as_dict=True): GROUP BY `user` ORDER BY `energy_points` DESC """.format( - conditions=conditions, - given_points_condition=given_points_condition - ), values=values, as_dict=1) + conditions=conditions, given_points_condition=given_points_condition + ), + values=values, + as_dict=1, + ) if not as_dict: return points_list dict_to_return = frappe._dict() for d in points_list: - dict_to_return[d.pop('user')] = d + dict_to_return[d.pop("user")] = d return dict_to_return + @frappe.whitelist() -def review(doc, points, to_user, reason, review_type='Appreciation'): +def review(doc, points, to_user, reason, review_type="Appreciation"): current_review_points = get_energy_points(frappe.session.user).review_points - doc = doc.as_dict() if hasattr(doc, 'as_dict') else frappe._dict(json.loads(doc)) + doc = doc.as_dict() if hasattr(doc, "as_dict") else frappe._dict(json.loads(doc)) points = abs(cint(points)) if current_review_points < points: - frappe.msgprint(_('You do not have enough review points')) + frappe.msgprint(_("You do not have enough review points")) return - review_doc = create_energy_points_log(doc.doctype, doc.name, { - 'type': review_type, - 'reason': reason, - 'points': points if review_type == 'Appreciation' else -points, - 'user': to_user - }) + review_doc = create_energy_points_log( + doc.doctype, + doc.name, + { + "type": review_type, + "reason": reason, + "points": points if review_type == "Appreciation" else -points, + "user": to_user, + }, + ) # deduct review points from reviewer create_review_points_log( @@ -282,66 +300,81 @@ def review(doc, points, to_user, reason, review_type='Appreciation'): points=-points, reason=reason, doctype=review_doc.doctype, - docname=review_doc.name + docname=review_doc.name, ) return review_doc + @frappe.whitelist() def get_reviews(doctype, docname): - return frappe.get_all('Energy Point Log', filters={ - 'reference_doctype': doctype, - 'reference_name': docname, - 'type': ['in', ('Appreciation', 'Criticism')], - }, fields=['points', 'owner', 'type', 'user', 'reason', 'creation']) + return frappe.get_all( + "Energy Point Log", + filters={ + "reference_doctype": doctype, + "reference_name": docname, + "type": ["in", ("Appreciation", "Criticism")], + }, + fields=["points", "owner", "type", "user", "reason", "creation"], + ) + def send_weekly_summary(): - send_summary('Weekly') + send_summary("Weekly") + def send_monthly_summary(): - send_summary('Monthly') + send_summary("Monthly") + def send_summary(timespan): + from frappe.social.doctype.energy_point_settings.energy_point_settings import ( + is_energy_point_enabled, + ) from frappe.utils.user import get_enabled_system_users - from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled if not is_energy_point_enabled(): return - if not is_email_notifications_enabled_for_type(frappe.session.user, 'Energy Point'): + if not is_email_notifications_enabled_for_type(frappe.session.user, "Energy Point"): return from_date = frappe.utils.add_to_date(None, weeks=-1) - if timespan == 'Monthly': + if timespan == "Monthly": from_date = frappe.utils.add_to_date(None, months=-1) user_points = get_user_energy_and_review_points(from_date=from_date, as_dict=False) # do not send report if no activity found - if not user_points or not user_points[0].energy_points: return + if not user_points or not user_points[0].energy_points: + return from_date = getdate(from_date) to_date = getdate() # select only those users that have energy point email notifications enabled - all_users = [user.email for user in get_enabled_system_users() if - is_email_notifications_enabled_for_type(user.name, 'Energy Point')] + all_users = [ + user.email + for user in get_enabled_system_users() + if is_email_notifications_enabled_for_type(user.name, "Energy Point") + ] frappe.sendmail( - subject = '{} energy points summary'.format(timespan), - recipients = all_users, - template = "energy_points_summary", - args = { - 'top_performer': user_points[0], - 'top_reviewer': max(user_points, key=lambda x:x['given_points']), - 'standings': user_points[:10], # top 10 - 'footer_message': get_footer_message(timespan).format(from_date, to_date), - }, - with_container = 1 - ) + subject="{} energy points summary".format(timespan), + recipients=all_users, + template="energy_points_summary", + args={ + "top_performer": user_points[0], + "top_reviewer": max(user_points, key=lambda x: x["given_points"]), + "standings": user_points[:10], # top 10 + "footer_message": get_footer_message(timespan).format(from_date, to_date), + }, + with_container=1, + ) + def get_footer_message(timespan): - if timespan == 'Monthly': + if timespan == "Monthly": return _("Stats based on last month's performance (from {0} to {1})") else: return _("Stats based on last week's performance (from {0} to {1})") diff --git a/frappe/social/doctype/energy_point_log/test_energy_point_log.py b/frappe/social/doctype/energy_point_log/test_energy_point_log.py index a7d45ef61f..9a244a0a2c 100644 --- a/frappe/social/doctype/energy_point_log/test_energy_point_log.py +++ b/frappe/social/doctype/energy_point_log/test_energy_point_log.py @@ -2,117 +2,122 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import frappe - +from frappe.desk.form.assign_to import add as assign_to from frappe.tests.utils import FrappeTestCase -from .energy_point_log import get_energy_points as _get_energy_points, create_review_points_log, review from frappe.utils.testutils import add_custom_field, clear_custom_fields -from frappe.desk.form.assign_to import add as assign_to + +from .energy_point_log import create_review_points_log +from .energy_point_log import get_energy_points as _get_energy_points +from .energy_point_log import review class TestEnergyPointLog(FrappeTestCase): @classmethod def setUpClass(cls): - settings = frappe.get_single('Energy Point Settings') + settings = frappe.get_single("Energy Point Settings") settings.enabled = 1 settings.save() @classmethod def tearDownClass(cls): - settings = frappe.get_single('Energy Point Settings') + settings = frappe.get_single("Energy Point Settings") settings.enabled = 0 settings.save() def setUp(self): - frappe.cache().delete_value('energy_point_rule_map') + frappe.cache().delete_value("energy_point_rule_map") def tearDown(self): - frappe.set_user('Administrator') + frappe.set_user("Administrator") frappe.db.delete("Energy Point Log") frappe.db.delete("Energy Point Rule") - frappe.cache().delete_value('energy_point_rule_map') + frappe.cache().delete_value("energy_point_rule_map") def test_user_energy_point(self): - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") todo_point_rule = create_energy_point_rule_for_todo() - energy_point_of_user = get_points('test@example.com') + energy_point_of_user = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - points_after_closing_todo = get_points('test@example.com') + points_after_closing_todo = get_points("test@example.com") self.assertEqual(points_after_closing_todo, energy_point_of_user + todo_point_rule.points) created_todo.save() - points_after_double_save = get_points('test@example.com') + points_after_double_save = get_points("test@example.com") # point should not be awarded more than once for same doc self.assertEqual(points_after_double_save, energy_point_of_user + todo_point_rule.points) def test_points_based_on_multiplier_field(self): - frappe.set_user('test@example.com') - add_custom_field('ToDo', 'multiplier', 'Float') + frappe.set_user("test@example.com") + add_custom_field("ToDo", "multiplier", "Float") multiplier_value = 0.51 - todo_point_rule = create_energy_point_rule_for_todo('multiplier') - energy_point_of_user = get_points('test@example.com') + todo_point_rule = create_energy_point_rule_for_todo("multiplier") + energy_point_of_user = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.multiplier = multiplier_value created_todo.save() - points_after_closing_todo = get_points('test@example.com') + points_after_closing_todo = get_points("test@example.com") - self.assertEqual(points_after_closing_todo, - energy_point_of_user + round(todo_point_rule.points * multiplier_value)) + self.assertEqual( + points_after_closing_todo, + energy_point_of_user + round(todo_point_rule.points * multiplier_value), + ) - clear_custom_fields('ToDo') + clear_custom_fields("ToDo") def test_points_based_on_max_points(self): - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") # here multiplier is high # let see if points get capped to max_point limit multiplier_value = 15 max_points = 50 - add_custom_field('ToDo', 'multiplier', 'Float') - todo_point_rule = create_energy_point_rule_for_todo('multiplier', max_points=max_points) - energy_point_of_user = get_points('test@example.com') + add_custom_field("ToDo", "multiplier", "Float") + todo_point_rule = create_energy_point_rule_for_todo("multiplier", max_points=max_points) + energy_point_of_user = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.multiplier = multiplier_value created_todo.save() - points_after_closing_todo = get_points('test@example.com') + points_after_closing_todo = get_points("test@example.com") # test max_points cap - self.assertNotEqual(points_after_closing_todo, - energy_point_of_user + round(todo_point_rule.points * multiplier_value)) + self.assertNotEqual( + points_after_closing_todo, + energy_point_of_user + round(todo_point_rule.points * multiplier_value), + ) - self.assertEqual(points_after_closing_todo, - energy_point_of_user + max_points) + self.assertEqual(points_after_closing_todo, energy_point_of_user + max_points) - clear_custom_fields('ToDo') + clear_custom_fields("ToDo") def test_disabled_energy_points(self): - settings = frappe.get_single('Energy Point Settings') + settings = frappe.get_single("Energy Point Settings") settings.enabled = 0 settings.save() - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") create_energy_point_rule_for_todo() - energy_point_of_user = get_points('test@example.com') + energy_point_of_user = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - points_after_closing_todo = get_points('test@example.com') + points_after_closing_todo = get_points("test@example.com") # no change in points self.assertEqual(points_after_closing_todo, energy_point_of_user) @@ -123,60 +128,60 @@ class TestEnergyPointLog(FrappeTestCase): def test_review(self): created_todo = create_a_todo() review_points = 20 - create_review_points_log('test2@example.com', review_points) + create_review_points_log("test2@example.com", review_points) # reviewer - frappe.set_user('test2@example.com') + frappe.set_user("test2@example.com") - review_points_before_review = get_points('test2@example.com', 'review_points') + review_points_before_review = get_points("test2@example.com", "review_points") self.assertEqual(review_points_before_review, review_points) # for appreciation appreciation_points = 5 - energy_points_before_review = get_points('test@example.com') - review(created_todo, appreciation_points, 'test@example.com', 'good job') - energy_points_after_review = get_points('test@example.com') - review_points_after_review = get_points('test2@example.com', 'review_points') + energy_points_before_review = get_points("test@example.com") + review(created_todo, appreciation_points, "test@example.com", "good job") + energy_points_after_review = get_points("test@example.com") + review_points_after_review = get_points("test2@example.com", "review_points") self.assertEqual(energy_points_after_review, energy_points_before_review + appreciation_points) self.assertEqual(review_points_after_review, review_points_before_review - appreciation_points) # for criticism criticism_points = 2 - todo = create_a_todo(description='Bad patch') + todo = create_a_todo(description="Bad patch") energy_points_before_review = energy_points_after_review review_points_before_review = review_points_after_review - review(todo, criticism_points, 'test@example.com', 'You could have done better.', 'Criticism') - energy_points_after_review = get_points('test@example.com') - review_points_after_review = get_points('test2@example.com', 'review_points') + review(todo, criticism_points, "test@example.com", "You could have done better.", "Criticism") + energy_points_after_review = get_points("test@example.com") + review_points_after_review = get_points("test2@example.com", "review_points") self.assertEqual(energy_points_after_review, energy_points_before_review - criticism_points) self.assertEqual(review_points_after_review, review_points_before_review - criticism_points) def test_user_energy_point_as_admin(self): - frappe.set_user('Administrator') + frappe.set_user("Administrator") create_energy_point_rule_for_todo() created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - points_after_closing_todo = get_points('Administrator') + points_after_closing_todo = get_points("Administrator") # no points for admin self.assertEqual(points_after_closing_todo, 0) def test_revert_points_on_cancelled_doc(self): - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") create_energy_point_rule_for_todo() created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - energy_point_logs = frappe.get_all('Energy Point Log') + energy_point_logs = frappe.get_all("Energy Point Log") self.assertEqual(len(energy_point_logs), 1) # for submit and cancel permission - frappe.set_user('Administrator') + frappe.set_user("Administrator") # submit created_todo.docstatus = 1 created_todo.save() @@ -185,103 +190,110 @@ class TestEnergyPointLog(FrappeTestCase): created_todo.docstatus = 2 created_todo.save() - energy_point_logs = frappe.get_all('Energy Point Log', fields=['reference_name', 'type', 'reverted']) + energy_point_logs = frappe.get_all( + "Energy Point Log", fields=["reference_name", "type", "reverted"] + ) - self.assertListEqual(energy_point_logs, [ - {'reference_name': created_todo.name, 'type': 'Revert', 'reverted': 0}, - {'reference_name': created_todo.name, 'type': 'Auto', 'reverted': 1} - ]) + self.assertListEqual( + energy_point_logs, + [ + {"reference_name": created_todo.name, "type": "Revert", "reverted": 0}, + {"reference_name": created_todo.name, "type": "Auto", "reverted": 1}, + ], + ) def test_energy_point_for_new_document_creation(self): - frappe.set_user('test@example.com') - todo_point_rule = create_energy_point_rule_for_todo(for_doc_event='New') + frappe.set_user("test@example.com") + todo_point_rule = create_energy_point_rule_for_todo(for_doc_event="New") - points_before_todo_creation = get_points('test@example.com') + points_before_todo_creation = get_points("test@example.com") create_a_todo() - points_after_todo_creation = get_points('test@example.com') + points_after_todo_creation = get_points("test@example.com") - self.assertEqual(points_after_todo_creation, - points_before_todo_creation + todo_point_rule.points) + self.assertEqual( + points_after_todo_creation, points_before_todo_creation + todo_point_rule.points + ) def test_point_allocation_for_assigned_users(self): todo = create_a_todo() - assign_users_to_todo(todo.name, ['test@example.com', 'test2@example.com']) + assign_users_to_todo(todo.name, ["test@example.com", "test2@example.com"]) - test_user_before_points = get_points('test@example.com') - test2_user_before_points = get_points('test2@example.com') + test_user_before_points = get_points("test@example.com") + test2_user_before_points = get_points("test2@example.com") rule = create_energy_point_rule_for_todo(for_assigned_users=1) - todo.status = 'Closed' + todo.status = "Closed" todo.save() - test_user_after_points = get_points('test@example.com') - test2_user_after_points = get_points('test2@example.com') + test_user_after_points = get_points("test@example.com") + test2_user_after_points = get_points("test2@example.com") - self.assertEqual(test_user_after_points, - test_user_before_points + rule.points) + self.assertEqual(test_user_after_points, test_user_before_points + rule.points) - self.assertEqual(test2_user_after_points, - test2_user_before_points + rule.points) + self.assertEqual(test2_user_after_points, test2_user_before_points + rule.points) def test_points_on_field_value_change(self): - rule = create_energy_point_rule_for_todo(for_doc_event='Value Change', - field_to_check='description') + rule = create_energy_point_rule_for_todo( + for_doc_event="Value Change", field_to_check="description" + ) - frappe.set_user('test@example.com') - points_before_todo_creation = get_points('test@example.com') + frappe.set_user("test@example.com") + points_before_todo_creation = get_points("test@example.com") todo = create_a_todo() - todo.status = 'Closed' + todo.status = "Closed" todo.save() - points_after_closing_todo = get_points('test@example.com') - self.assertEqual(points_after_closing_todo, - points_before_todo_creation) + points_after_closing_todo = get_points("test@example.com") + self.assertEqual(points_after_closing_todo, points_before_todo_creation) - todo.description = 'This is new todo' + todo.description = "This is new todo" todo.save() - points_after_changing_todo_description = get_points('test@example.com') - self.assertEqual(points_after_changing_todo_description, - points_before_todo_creation + rule.points) + points_after_changing_todo_description = get_points("test@example.com") + self.assertEqual( + points_after_changing_todo_description, points_before_todo_creation + rule.points + ) def test_apply_only_once(self): - frappe.set_user('test@example.com') - todo_point_rule = create_energy_point_rule_for_todo(apply_once=True, user_field='modified_by') - first_user_points = get_points('test@example.com') + frappe.set_user("test@example.com") + todo_point_rule = create_energy_point_rule_for_todo(apply_once=True, user_field="modified_by") + first_user_points = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - first_user_points_after_closing_todo = get_points('test@example.com') + first_user_points_after_closing_todo = get_points("test@example.com") - self.assertEqual(first_user_points_after_closing_todo, first_user_points + todo_point_rule.points) + self.assertEqual( + first_user_points_after_closing_todo, first_user_points + todo_point_rule.points + ) - frappe.set_user('test2@example.com') - second_user_points = get_points('test2@example.com') + frappe.set_user("test2@example.com") + second_user_points = get_points("test2@example.com") created_todo.save(ignore_permissions=True) - second_user_points_after_closing_todo = get_points('test2@example.com') + second_user_points_after_closing_todo = get_points("test2@example.com") # point should not be awarded more than once for same doc (irrespective of user) self.assertEqual(second_user_points_after_closing_todo, second_user_points) def test_allow_creation_of_new_log_if_the_previous_log_was_reverted(self): - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") todo_point_rule = create_energy_point_rule_for_todo() - energy_point_of_user = get_points('test@example.com') + energy_point_of_user = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - points_after_closing_todo = get_points('test@example.com') + points_after_closing_todo = get_points("test@example.com") - log_name = frappe.db.exists('Energy Point Log', {'reference_name': created_todo.name}) - frappe.get_doc('Energy Point Log', log_name).revert('Just for test') - points_after_reverting_todo = get_points('test@example.com') + log_name = frappe.db.exists("Energy Point Log", {"reference_name": created_todo.name}) + frappe.get_doc("Energy Point Log", log_name).revert("Just for test") + points_after_reverting_todo = get_points("test@example.com") created_todo.save() - points_after_saving_todo_again = get_points('test@example.com') + points_after_saving_todo_again = get_points("test@example.com") rule_points = todo_point_rule.points self.assertEqual(points_after_closing_todo, energy_point_of_user + rule_points) @@ -289,18 +301,18 @@ class TestEnergyPointLog(FrappeTestCase): self.assertEqual(points_after_saving_todo_again, points_after_reverting_todo + rule_points) def test_energy_points_disabled_user(self): - frappe.set_user('test@example.com') - user = frappe.get_doc('User', 'test@example.com') + frappe.set_user("test@example.com") + user = frappe.get_doc("User", "test@example.com") user.enabled = 0 user.save() todo_point_rule = create_energy_point_rule_for_todo() - energy_point_of_user = get_points('test@example.com') + energy_point_of_user = get_points("test@example.com") created_todo = create_a_todo() - created_todo.status = 'Closed' + created_todo.status = "Closed" created_todo.save() - points_after_closing_todo = get_points('test@example.com') + points_after_closing_todo = get_points("test@example.com") # do not update energy points for disabled user self.assertEqual(points_after_closing_todo, energy_point_of_user) @@ -309,49 +321,58 @@ class TestEnergyPointLog(FrappeTestCase): user.save() created_todo.save() - points_after_re_saving_todo = get_points('test@example.com') + points_after_re_saving_todo = get_points("test@example.com") self.assertEqual(points_after_re_saving_todo, energy_point_of_user + todo_point_rule.points) -def create_energy_point_rule_for_todo(multiplier_field=None, for_doc_event='Custom', max_points=None, - for_assigned_users=0, field_to_check=None, apply_once=False, user_field='owner'): - name = 'ToDo Closed' - point_rule_exists = frappe.db.exists('Energy Point Rule', name) - - if point_rule_exists: return frappe.get_doc('Energy Point Rule', name) - - return frappe.get_doc({ - 'doctype': 'Energy Point Rule', - 'rule_name': name, - 'points': 5, - 'reference_doctype': 'ToDo', - 'condition': 'doc.status == "Closed"', - 'for_doc_event': for_doc_event, - 'user_field': user_field, - 'for_assigned_users': for_assigned_users, - 'multiplier_field': multiplier_field, - 'max_points': max_points, - 'field_to_check': field_to_check, - 'apply_only_once': apply_once - }).insert(ignore_permissions=1) +def create_energy_point_rule_for_todo( + multiplier_field=None, + for_doc_event="Custom", + max_points=None, + for_assigned_users=0, + field_to_check=None, + apply_once=False, + user_field="owner", +): + name = "ToDo Closed" + point_rule_exists = frappe.db.exists("Energy Point Rule", name) + + if point_rule_exists: + return frappe.get_doc("Energy Point Rule", name) + + return frappe.get_doc( + { + "doctype": "Energy Point Rule", + "rule_name": name, + "points": 5, + "reference_doctype": "ToDo", + "condition": 'doc.status == "Closed"', + "for_doc_event": for_doc_event, + "user_field": user_field, + "for_assigned_users": for_assigned_users, + "multiplier_field": multiplier_field, + "max_points": max_points, + "field_to_check": field_to_check, + "apply_only_once": apply_once, + } + ).insert(ignore_permissions=1) def create_a_todo(description=None): if not description: - description = 'Fix a bug' - return frappe.get_doc({ - 'doctype': 'ToDo', - 'description': description, - }).insert(ignore_permissions=True) + description = "Fix a bug" + return frappe.get_doc( + { + "doctype": "ToDo", + "description": description, + } + ).insert(ignore_permissions=True) -def get_points(user, point_type='energy_points'): +def get_points(user, point_type="energy_points"): return _get_energy_points(user).get(point_type) or 0 + def assign_users_to_todo(todo_name, users): for user in users: - assign_to({ - 'assign_to': [user], - 'doctype': 'ToDo', - 'name': todo_name - }) + assign_to({"assign_to": [user], "doctype": "ToDo", "name": todo_name}) diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py index 55bf55a3b0..a36581ec4c 100644 --- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py +++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py @@ -3,21 +3,23 @@ # License: MIT. See LICENSE import frappe -from frappe import _ import frappe.cache_manager +from frappe import _ from frappe.core.doctype.user.user import get_enabled_users from frappe.model import log_types from frappe.model.document import Document -from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled -from frappe.social.doctype.energy_point_log.energy_point_log import \ - create_energy_points_log +from frappe.social.doctype.energy_point_log.energy_point_log import create_energy_points_log +from frappe.social.doctype.energy_point_settings.energy_point_settings import ( + is_energy_point_enabled, +) + class EnergyPointRule(Document): def on_update(self): - frappe.cache_manager.clear_doctype_map('Energy Point Rule', self.reference_doctype) + frappe.cache_manager.clear_doctype_map("Energy Point Rule", self.reference_doctype) def on_trash(self): - frappe.cache_manager.clear_doctype_map('Energy Point Rule', self.reference_doctype) + frappe.cache_manager.clear_doctype_map("Energy Point Rule", self.reference_doctype) def apply(self, doc): if self.rule_condition_satisfied(doc): @@ -41,53 +43,60 @@ class EnergyPointRule(Document): rule = self.name # incase of zero as result after roundoff - if not points: return + if not points: + return try: for user in users: - if not is_eligible_user(user): continue - create_energy_points_log(reference_doctype, reference_name, { - 'points': points, - 'user': user, - 'rule': rule - }, self.apply_only_once) + if not is_eligible_user(user): + continue + create_energy_points_log( + reference_doctype, + reference_name, + {"points": points, "user": user, "rule": rule}, + self.apply_only_once, + ) except Exception as e: - frappe.log_error(frappe.get_traceback(), 'apply_energy_point') + frappe.log_error(frappe.get_traceback(), "apply_energy_point") def rule_condition_satisfied(self, doc): - if self.for_doc_event == 'New': + if self.for_doc_event == "New": # indicates that this was a new doc return doc.get_doc_before_save() is None - if self.for_doc_event == 'Submit': + if self.for_doc_event == "Submit": return doc.docstatus.is_submitted() - if self.for_doc_event == 'Cancel': + if self.for_doc_event == "Cancel": return doc.docstatus.is_cancelled() - if self.for_doc_event == 'Value Change': + if self.for_doc_event == "Value Change": field_to_check = self.field_to_check - if not field_to_check: return False + if not field_to_check: + return False doc_before_save = doc.get_doc_before_save() # check if the field has been changed # if condition is set check if it is satisfied - return doc_before_save \ - and doc_before_save.get(field_to_check) != doc.get(field_to_check) \ + return ( + doc_before_save + and doc_before_save.get(field_to_check) != doc.get(field_to_check) and (not self.condition or self.eval_condition(doc)) + ) - if self.for_doc_event == 'Custom' and self.condition: + if self.for_doc_event == "Custom" and self.condition: return self.eval_condition(doc) return False def eval_condition(self, doc): - return self.condition and frappe.safe_eval(self.condition, None, { - 'doc': doc.as_dict() - }) + return self.condition and frappe.safe_eval(self.condition, None, {"doc": doc.as_dict()}) + def process_energy_points(doc, state): - if (frappe.flags.in_patch + if ( + frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_migrate or frappe.flags.in_import or frappe.flags.in_setup_wizard - or doc.doctype in log_types): + or doc.doctype in log_types + ): return if not is_energy_point_enabled(): @@ -99,29 +108,30 @@ def process_energy_points(doc, state): if old_doc and old_doc.docstatus.is_submitted() and doc.docstatus.is_cancelled(): return revert_points_for_cancelled_doc(doc) - for d in frappe.cache_manager.get_doctype_map('Energy Point Rule', doc.doctype, - dict(reference_doctype = doc.doctype, enabled=1)): - frappe.get_doc('Energy Point Rule', d.get('name')).apply(doc) + for d in frappe.cache_manager.get_doctype_map( + "Energy Point Rule", doc.doctype, dict(reference_doctype=doc.doctype, enabled=1) + ): + frappe.get_doc("Energy Point Rule", d.get("name")).apply(doc) def revert_points_for_cancelled_doc(doc): - energy_point_logs = frappe.get_all('Energy Point Log', { - 'reference_doctype': doc.doctype, - 'reference_name': doc.name, - 'type': 'Auto' - }) + energy_point_logs = frappe.get_all( + "Energy Point Log", + {"reference_doctype": doc.doctype, "reference_name": doc.name, "type": "Auto"}, + ) for log in energy_point_logs: - reference_log = frappe.get_doc('Energy Point Log', log.name) - reference_log.revert(_('Reference document has been cancelled'), ignore_permissions=True) + reference_log = frappe.get_doc("Energy Point Log", log.name) + reference_log.revert(_("Reference document has been cancelled"), ignore_permissions=True) def get_energy_point_doctypes(): return [ - d.reference_doctype for d in frappe.get_all('Energy Point Rule', - ['reference_doctype'], {'enabled': 1}) + d.reference_doctype + for d in frappe.get_all("Energy Point Rule", ["reference_doctype"], {"enabled": 1}) ] + def is_eligible_user(user): - '''Checks if user is eligible to get energy points''' + """Checks if user is eligible to get energy points""" enabled_users = get_enabled_users() - return user and user in enabled_users and user != 'Administrator' + return user and user in enabled_users and user != "Administrator" diff --git a/frappe/social/doctype/energy_point_settings/energy_point_settings.py b/frappe/social/doctype/energy_point_settings/energy_point_settings.py index cac0e0b4a5..c1ff2707af 100644 --- a/frappe/social/doctype/energy_point_settings/energy_point_settings.py +++ b/frappe/social/doctype/energy_point_settings/energy_point_settings.py @@ -5,19 +5,23 @@ import frappe from frappe.model.document import Document from frappe.social.doctype.energy_point_log.energy_point_log import create_review_points_log -from frappe.utils import add_to_date, today, getdate +from frappe.utils import add_to_date, getdate, today + class EnergyPointSettings(Document): pass + def is_energy_point_enabled(): - return frappe.db.get_single_value('Energy Point Settings', 'enabled', True) + return frappe.db.get_single_value("Energy Point Settings", "enabled", True) + def allocate_review_points(): - settings = frappe.get_single('Energy Point Settings') + settings = frappe.get_single("Energy Point Settings") - if not can_allocate_today(settings.last_point_allocation_date, - settings.point_allocation_periodicity): + if not can_allocate_today( + settings.last_point_allocation_date, settings.point_allocation_periodicity + ): return user_point_map = {} @@ -35,15 +39,12 @@ def allocate_review_points(): settings.last_point_allocation_date = today() settings.save(ignore_permissions=True) + def can_allocate_today(last_date, periodicity): if not last_date: return True - days_to_add = { - 'Daily': 1, - 'Weekly': 7, - 'Monthly': 30 - }.get(periodicity, 1) + days_to_add = {"Daily": 1, "Weekly": 7, "Monthly": 30}.get(periodicity, 1) next_allocation_date = add_to_date(last_date, days=days_to_add) @@ -51,9 +52,15 @@ def can_allocate_today(last_date, periodicity): def get_users_with_role(role): - return [p[0] for p in frappe.db.sql("""SELECT DISTINCT `tabUser`.`name` + return [ + p[0] + for p in frappe.db.sql( + """SELECT DISTINCT `tabUser`.`name` FROM `tabHas Role`, `tabUser` WHERE `tabHas Role`.`role`=%s AND `tabUser`.`name`!='Administrator' AND `tabHas Role`.`parent`=`tabUser`.`name` - AND `tabUser`.`enabled`=1""", role)] \ No newline at end of file + AND `tabUser`.`enabled`=1""", + role, + ) + ] diff --git a/frappe/social/doctype/energy_point_settings/test_energy_point_settings.py b/frappe/social/doctype/energy_point_settings/test_energy_point_settings.py index 3b0a756878..10416a127b 100644 --- a/frappe/social/doctype/energy_point_settings/test_energy_point_settings.py +++ b/frappe/social/doctype/energy_point_settings/test_energy_point_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestEnergyPointSettings(unittest.TestCase): pass diff --git a/frappe/social/doctype/review_level/review_level.py b/frappe/social/doctype/review_level/review_level.py index 1951233552..b3418e913e 100644 --- a/frappe/social/doctype/review_level/review_level.py +++ b/frappe/social/doctype/review_level/review_level.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class ReviewLevel(Document): pass diff --git a/frappe/templates/includes/comments/comments.py b/frappe/templates/includes/comments/comments.py index 8d485423bf..dd94cfa989 100644 --- a/frappe/templates/includes/comments/comments.py +++ b/frappe/templates/includes/comments/comments.py @@ -1,60 +1,64 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import re -from frappe.website.utils import clear_cache + +import frappe +from frappe import _ from frappe.rate_limiter import rate_limit from frappe.utils import add_to_date, now -from frappe.website.doctype.blog_settings.blog_settings import get_comment_limit from frappe.utils.html_utils import clean_html +from frappe.website.doctype.blog_settings.blog_settings import get_comment_limit +from frappe.website.utils import clear_cache -from frappe import _ @frappe.whitelist(allow_guest=True) -@rate_limit(key='reference_name', limit=get_comment_limit, seconds=60*60) +@rate_limit(key="reference_name", limit=get_comment_limit, seconds=60 * 60) def add_comment(comment, comment_email, comment_by, reference_doctype, reference_name, route): doc = frappe.get_doc(reference_doctype, reference_name) - if frappe.session.user == 'Guest' and doc.doctype not in ['Blog Post', 'Web Page']: + if frappe.session.user == "Guest" and doc.doctype not in ["Blog Post", "Web Page"]: return if not comment.strip(): - frappe.msgprint(_('The comment cannot be empty')) + frappe.msgprint(_("The comment cannot be empty")) return False - url_regex = re.compile(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", re.IGNORECASE) + url_regex = re.compile( + r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", re.IGNORECASE + ) email_regex = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", re.IGNORECASE) if url_regex.search(comment) or email_regex.search(comment): - frappe.msgprint(_('Comments cannot have links or email addresses')) + frappe.msgprint(_("Comments cannot have links or email addresses")) return False comment = doc.add_comment( - text=clean_html(comment), - comment_email=comment_email, - comment_by=comment_by) + text=clean_html(comment), comment_email=comment_email, comment_by=comment_by + ) - comment.db_set('published', 1) + comment.db_set("published", 1) # since comments are embedded in the page, clear the web cache if route: clear_cache(route) - content = (comment.content - + "

{2}

".format(frappe.utils.get_request_site_address(), - comment.name, - _("View Comment"))) + content = ( + comment.content + + "

{2}

".format( + frappe.utils.get_request_site_address(), comment.name, _("View Comment") + ) + ) if doc.doctype == "Blog Post" and not doc.enable_email_notification: pass else: # notify creator frappe.sendmail( - recipients=frappe.db.get_value('User', doc.owner, 'email') or doc.owner, - subject=_('New Comment on {0}: {1}').format(doc.doctype, doc.name), + recipients=frappe.db.get_value("User", doc.owner, "email") or doc.owner, + subject=_("New Comment on {0}: {1}").format(doc.doctype, doc.name), message=content, reference_doctype=doc.doctype, - reference_name=doc.name + reference_name=doc.name, ) # revert with template if all clear (no backlinks) diff --git a/frappe/templates/includes/feedback/feedback.py b/frappe/templates/includes/feedback/feedback.py index 279ff05e6d..4c6a0588f5 100644 --- a/frappe/templates/includes/feedback/feedback.py +++ b/frappe/templates/includes/feedback/feedback.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals import frappe - from frappe import _ from frappe.rate_limiter import rate_limit from frappe.website.doctype.blog_settings.blog_settings import get_feedback_limit + @frappe.whitelist(allow_guest=True) -@rate_limit(key='reference_name', limit=get_feedback_limit, seconds=60*60) +@rate_limit(key="reference_name", limit=get_feedback_limit, seconds=60 * 60) def give_feedback(reference_doctype, reference_name, like): like = frappe.parse_json(like) ref_doc = frappe.get_doc(reference_doctype, reference_name) @@ -19,35 +19,38 @@ def give_feedback(reference_doctype, reference_name, like): filters = { "owner": frappe.session.user, "reference_doctype": reference_doctype, - "reference_name": reference_name + "reference_name": reference_name, } - d = frappe.get_all('Feedback', filters=filters, limit=1) + d = frappe.get_all("Feedback", filters=filters, limit=1) if d: - doc = frappe.get_doc('Feedback', d[0].name) + doc = frappe.get_doc("Feedback", d[0].name) else: - doc = doc = frappe.new_doc('Feedback') + doc = doc = frappe.new_doc("Feedback") doc.reference_doctype = reference_doctype doc.reference_name = reference_name doc.ip_address = frappe.local.request_ip doc.like = like doc.save(ignore_permissions=True) - subject = _('Feedback on {0}: {1}').format(reference_doctype, reference_name) + subject = _("Feedback on {0}: {1}").format(reference_doctype, reference_name) ref_doc.enable_email_notification and send_mail(doc, subject) return doc + def send_mail(feedback, subject): doc = frappe.get_doc(feedback.reference_doctype, feedback.reference_name) if feedback.like: - message = "

Hey,

You have received a ❤️ heart on your blog post {0}

".format(feedback.reference_name) + message = "

Hey,

You have received a ❤️ heart on your blog post {0}

".format( + feedback.reference_name + ) else: return # notify creator frappe.sendmail( - recipients=frappe.db.get_value('User', doc.owner, 'email') or doc.owner, + recipients=frappe.db.get_value("User", doc.owner, "email") or doc.owner, subject=subject, message=message, reference_doctype=doc.doctype, - reference_name=doc.name + reference_name=doc.name, ) diff --git a/frappe/templates/pages/integrations/braintree_checkout.py b/frappe/templates/pages/integrations/braintree_checkout.py index 26c1d0842a..c4c79ea74f 100644 --- a/frappe/templates/pages/integrations/braintree_checkout.py +++ b/frappe/templates/pages/integrations/braintree_checkout.py @@ -1,16 +1,30 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ +from frappe.integrations.doctype.braintree_settings.braintree_settings import ( + get_client_token, + get_gateway_controller, +) from frappe.utils import flt -import json -from frappe.integrations.doctype.braintree_settings.braintree_settings import get_client_token, get_gateway_controller no_cache = 1 -expected_keys = ('amount', 'title', 'description', 'reference_doctype', 'reference_docname', - 'payer_name', 'payer_email', 'order_id', 'currency') +expected_keys = ( + "amount", + "title", + "description", + "reference_doctype", + "reference_docname", + "payer_name", + "payer_email", + "order_id", + "currency", +) + def get_context(context): context.no_cache = 1 @@ -22,26 +36,29 @@ def get_context(context): context.client_token = get_client_token(context.reference_docname) - context['amount'] = flt(context['amount']) + context["amount"] = flt(context["amount"]) gateway_controller = get_gateway_controller(context.reference_docname) - context['header_img'] = frappe.db.get_value("Braintree Settings", gateway_controller, "header_img") + context["header_img"] = frappe.db.get_value( + "Braintree Settings", gateway_controller, "header_img" + ) else: - frappe.redirect_to_message(_('Some information is missing'), - _('Looks like someone sent you to an incomplete URL. Please ask them to look into it.')) + frappe.redirect_to_message( + _("Some information is missing"), + _("Looks like someone sent you to an incomplete URL. Please ask them to look into it."), + ) frappe.local.flags.redirect_location = frappe.local.response.location raise frappe.Redirect + @frappe.whitelist(allow_guest=True) def make_payment(payload_nonce, data, reference_doctype, reference_docname): data = json.loads(data) - data.update({ - "payload_nonce": payload_nonce - }) + data.update({"payload_nonce": payload_nonce}) gateway_controller = get_gateway_controller(reference_docname) - data = frappe.get_doc("Braintree Settings", gateway_controller).create_payment_request(data) + data = frappe.get_doc("Braintree Settings", gateway_controller).create_payment_request(data) frappe.db.commit() return data diff --git a/frappe/templates/pages/integrations/payment_cancel.py b/frappe/templates/pages/integrations/payment_cancel.py index 9c0b972cb2..cf2a10f8c7 100644 --- a/frappe/templates/pages/integrations/payment_cancel.py +++ b/frappe/templates/pages/integrations/payment_cancel.py @@ -3,6 +3,7 @@ import frappe + def get_context(context): token = frappe.local.form_dict.token diff --git a/frappe/templates/pages/integrations/payment_success.py b/frappe/templates/pages/integrations/payment_success.py index bb584ee577..8985850a81 100644 --- a/frappe/templates/pages/integrations/payment_success.py +++ b/frappe/templates/pages/integrations/payment_success.py @@ -2,13 +2,14 @@ # License: MIT. See LICENSE import frappe + no_cache = True + def get_context(context): - token = frappe.local.form_dict.token - doc = frappe.get_doc(frappe.local.form_dict.doctype, frappe.local.form_dict.docname) + token = frappe.local.form_dict.token + doc = frappe.get_doc(frappe.local.form_dict.doctype, frappe.local.form_dict.docname) - context.payment_message = '' - if hasattr(doc, 'get_payment_success_message'): + context.payment_message = "" + if hasattr(doc, "get_payment_success_message"): context.payment_message = doc.get_payment_success_message() - diff --git a/frappe/templates/pages/integrations/paytm_checkout.py b/frappe/templates/pages/integrations/paytm_checkout.py index c003844157..93097e038b 100644 --- a/frappe/templates/pages/integrations/paytm_checkout.py +++ b/frappe/templates/pages/integrations/paytm_checkout.py @@ -1,16 +1,21 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -import json -from frappe.integrations.doctype.paytm_settings.paytm_settings import get_paytm_params, get_paytm_config +from frappe.integrations.doctype.paytm_settings.paytm_settings import ( + get_paytm_config, + get_paytm_params, +) + def get_context(context): context.no_cache = 1 paytm_config = get_paytm_config() try: - doc = frappe.get_doc("Integration Request", frappe.form_dict['order_id']) + doc = frappe.get_doc("Integration Request", frappe.form_dict["order_id"]) context.payment_details = get_paytm_params(json.loads(doc.data), doc.name, paytm_config) @@ -18,9 +23,12 @@ def get_context(context): except Exception: frappe.log_error() - frappe.redirect_to_message(_('Invalid Token'), - _('Seems token you are using is invalid!'), - http_status_code=400, indicator_color='red') + frappe.redirect_to_message( + _("Invalid Token"), + _("Seems token you are using is invalid!"), + http_status_code=400, + indicator_color="red", + ) frappe.local.flags.redirect_location = frappe.local.response.location - raise frappe.Redirect \ No newline at end of file + raise frappe.Redirect diff --git a/frappe/templates/pages/integrations/razorpay_checkout.py b/frappe/templates/pages/integrations/razorpay_checkout.py index 51904fca46..aed832119b 100644 --- a/frappe/templates/pages/integrations/razorpay_checkout.py +++ b/frappe/templates/pages/integrations/razorpay_checkout.py @@ -1,39 +1,54 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -from frappe.utils import flt, cint -import json +from frappe.utils import cint, flt no_cache = 1 -expected_keys = ('amount', 'title', 'description', 'reference_doctype', 'reference_docname', - 'payer_name', 'payer_email', 'order_id') +expected_keys = ( + "amount", + "title", + "description", + "reference_doctype", + "reference_docname", + "payer_name", + "payer_email", + "order_id", +) + def get_context(context): context.no_cache = 1 context.api_key = get_api_key() try: - doc = frappe.get_doc("Integration Request", frappe.form_dict['token']) + doc = frappe.get_doc("Integration Request", frappe.form_dict["token"]) payment_details = json.loads(doc.data) for key in expected_keys: context[key] = payment_details[key] - context['token'] = frappe.form_dict['token'] - context['amount'] = flt(context['amount']) - context['subscription_id'] = payment_details['subscription_id'] \ - if payment_details.get('subscription_id') else '' + context["token"] = frappe.form_dict["token"] + context["amount"] = flt(context["amount"]) + context["subscription_id"] = ( + payment_details["subscription_id"] if payment_details.get("subscription_id") else "" + ) except Exception as e: - frappe.redirect_to_message(_('Invalid Token'), - _('Seems token you are using is invalid!'), - http_status_code=400, indicator_color='red') + frappe.redirect_to_message( + _("Invalid Token"), + _("Seems token you are using is invalid!"), + http_status_code=400, + indicator_color="red", + ) frappe.local.flags.redirect_location = frappe.local.response.location raise frappe.Redirect + def get_api_key(): api_key = frappe.db.get_value("Razorpay Settings", None, "api_key") if cint(frappe.form_dict.get("use_sandbox")): @@ -41,6 +56,7 @@ def get_api_key(): return api_key + @frappe.whitelist(allow_guest=True) def make_payment(razorpay_payment_id, options, reference_doctype, reference_docname, token): data = {} @@ -48,13 +64,15 @@ def make_payment(razorpay_payment_id, options, reference_doctype, reference_docn if isinstance(options, str): data = json.loads(options) - data.update({ - "razorpay_payment_id": razorpay_payment_id, - "reference_docname": reference_docname, - "reference_doctype": reference_doctype, - "token": token - }) + data.update( + { + "razorpay_payment_id": razorpay_payment_id, + "reference_docname": reference_docname, + "reference_doctype": reference_doctype, + "token": token, + } + ) - data = frappe.get_doc("Razorpay Settings").create_request(data) + data = frappe.get_doc("Razorpay Settings").create_request(data) frappe.db.commit() return data diff --git a/frappe/templates/pages/integrations/stripe_checkout.py b/frappe/templates/pages/integrations/stripe_checkout.py index eb9c23ad22..1c0e20c631 100644 --- a/frappe/templates/pages/integrations/stripe_checkout.py +++ b/frappe/templates/pages/integrations/stripe_checkout.py @@ -1,15 +1,26 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe from frappe import _ -from frappe.utils import cint, fmt_money -import json from frappe.integrations.doctype.stripe_settings.stripe_settings import get_gateway_controller +from frappe.utils import cint, fmt_money no_cache = 1 -expected_keys = ('amount', 'title', 'description', 'reference_doctype', 'reference_docname', - 'payer_name', 'payer_email', 'order_id', 'currency') +expected_keys = ( + "amount", + "title", + "description", + "reference_doctype", + "reference_docname", + "payer_name", + "payer_email", + "order_id", + "currency", +) + def get_context(context): context.no_cache = 1 @@ -23,20 +34,25 @@ def get_context(context): context.publishable_key = get_api_key(context.reference_docname, gateway_controller) context.image = get_header_image(context.reference_docname, gateway_controller) - context['amount'] = fmt_money(amount=context['amount'], currency=context['currency']) + context["amount"] = fmt_money(amount=context["amount"], currency=context["currency"]) if is_a_subscription(context.reference_doctype, context.reference_docname): - payment_plan = frappe.db.get_value(context.reference_doctype, context.reference_docname, "payment_plan") + payment_plan = frappe.db.get_value( + context.reference_doctype, context.reference_docname, "payment_plan" + ) recurrence = frappe.db.get_value("Payment Plan", payment_plan, "recurrence") - context['amount'] = context['amount'] + " " + _(recurrence) + context["amount"] = context["amount"] + " " + _(recurrence) else: - frappe.redirect_to_message(_('Some information is missing'), - _('Looks like someone sent you to an incomplete URL. Please ask them to look into it.')) + frappe.redirect_to_message( + _("Some information is missing"), + _("Looks like someone sent you to an incomplete URL. Please ask them to look into it."), + ) frappe.local.flags.redirect_location = frappe.local.response.location raise frappe.Redirect + def get_api_key(doc, gateway_controller): publishable_key = frappe.db.get_value("Stripe Settings", gateway_controller, "publishable_key") if cint(frappe.form_dict.get("use_sandbox")): @@ -44,31 +60,32 @@ def get_api_key(doc, gateway_controller): return publishable_key + def get_header_image(doc, gateway_controller): header_image = frappe.db.get_value("Stripe Settings", gateway_controller, "header_img") return header_image + @frappe.whitelist(allow_guest=True) def make_payment(stripe_token_id, data, reference_doctype=None, reference_docname=None): data = json.loads(data) - data.update({ - "stripe_token_id": stripe_token_id - }) + data.update({"stripe_token_id": stripe_token_id}) - gateway_controller = get_gateway_controller(reference_doctype,reference_docname) + gateway_controller = get_gateway_controller(reference_doctype, reference_docname) if is_a_subscription(reference_doctype, reference_docname): reference = frappe.get_doc(reference_doctype, reference_docname) - data = reference.create_subscription("stripe", gateway_controller, data) + data = reference.create_subscription("stripe", gateway_controller, data) else: - data = frappe.get_doc("Stripe Settings", gateway_controller).create_request(data) + data = frappe.get_doc("Stripe Settings", gateway_controller).create_request(data) frappe.db.commit() return data + def is_a_subscription(reference_doctype, reference_docname): - if not frappe.get_meta(reference_doctype).has_field('is_a_subscription'): + if not frappe.get_meta(reference_doctype).has_field("is_a_subscription"): return False - return frappe.db.get_value(reference_doctype, reference_docname, "is_a_subscription") \ No newline at end of file + return frappe.db.get_value(reference_doctype, reference_docname, "is_a_subscription") diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 6c1aae8907..5fa9f60197 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -1,19 +1,25 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import unittest, json, sys, os -import time +import cProfile import importlib -from frappe.modules import load_doctype_module, get_module_name -import frappe.utils.scheduler -import cProfile, pstats -from io import StringIO +import json +import os +import pstats +import sys +import time +import unittest from importlib import reload +from io import StringIO + +import frappe +import frappe.utils.scheduler from frappe.model.naming import revert_series_if_last +from frappe.modules import get_module_name, load_doctype_module unittest_runner = unittest.TextTestRunner SLOW_TEST_THRESHOLD = 2 + def xmlrunner_wrapper(output): """Convenience wrapper to keep method signature unchanged for XMLTestRunner and TextTestRunner""" try: @@ -24,26 +30,42 @@ def xmlrunner_wrapper(output): raise def _runner(*args, **kwargs): - kwargs['output'] = output + kwargs["output"] = output return xmlrunner.XMLTestRunner(*args, **kwargs) + return _runner -def main(app=None, module=None, doctype=None, verbose=False, tests=(), - force=False, profile=False, junit_xml_output=None, ui_tests=False, - doctype_list_path=None, skip_test_records=False, failfast=False, case=None): + +def main( + app=None, + module=None, + doctype=None, + verbose=False, + tests=(), + force=False, + profile=False, + junit_xml_output=None, + ui_tests=False, + doctype_list_path=None, + skip_test_records=False, + failfast=False, + case=None, +): global unittest_runner if doctype_list_path: app, doctype_list_path = doctype_list_path.split(os.path.sep, 1) - with open(frappe.get_app_path(app, doctype_list_path), 'r') as f: + with open(frappe.get_app_path(app, doctype_list_path), "r") as f: doctype = f.read().strip().splitlines() if ui_tests: - print("Selenium testing has been deprecated\nUse bench --site {site_name} run-ui-tests for Cypress tests") + print( + "Selenium testing has been deprecated\nUse bench --site {site_name} run-ui-tests for Cypress tests" + ) xmloutput_fh = None if junit_xml_output: - xmloutput_fh = open(junit_xml_output, 'wb') + xmloutput_fh = open(junit_xml_output, "wb") unittest_runner = xmlrunner_wrapper(xmloutput_fh) else: unittest_runner = unittest.TextTestRunner @@ -62,7 +84,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), frappe.utils.scheduler.disable_scheduler() set_test_email_config() - frappe.conf.update({'bench_id': 'test_bench', 'use_rq_auth': False}) + frappe.conf.update({"bench_id": "test_bench", "use_rq_auth": False}) if not frappe.flags.skip_before_tests: if verbose: @@ -71,16 +93,29 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), frappe.get_attr(fn)() if doctype: - ret = run_tests_for_doctype(doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output) + ret = run_tests_for_doctype( + doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output + ) elif module: - ret = run_tests_for_module(module, verbose, tests, profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case) + ret = run_tests_for_module( + module, + verbose, + tests, + profile, + failfast=failfast, + junit_xml_output=junit_xml_output, + case=case, + ) else: - ret = run_all_tests(app, verbose, profile, ui_tests, failfast=failfast, junit_xml_output=junit_xml_output) + ret = run_all_tests( + app, verbose, profile, ui_tests, failfast=failfast, junit_xml_output=junit_xml_output + ) if not scheduler_disabled_by_user: frappe.utils.scheduler.enable_scheduler() - if frappe.db: frappe.db.commit() + if frappe.db: + frappe.db.commit() # workaround! since there is no separate test db frappe.clear_cache() @@ -93,13 +128,15 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), def set_test_email_config(): - frappe.conf.update({ - "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", - "mail_login": "test@example.com", - "mail_password": "test", - "admin_password": "admin" - }) + frappe.conf.update( + { + "auto_email_id": "test@example.com", + "mail_server": "smtp.example.com", + "mail_login": "test@example.com", + "mail_password": "test", + "admin_password": "admin", + } + ) class TimeLoggingTestResult(unittest.TextTestResult): @@ -115,7 +152,9 @@ class TimeLoggingTestResult(unittest.TextTestResult): super(TimeLoggingTestResult, self).addSuccess(test) -def run_all_tests(app=None, verbose=False, profile=False, ui_tests=False, failfast=False, junit_xml_output=False): +def run_all_tests( + app=None, verbose=False, profile=False, ui_tests=False, failfast=False, junit_xml_output=False +): import os apps = [app] if app else frappe.get_installed_apps() @@ -123,7 +162,7 @@ def run_all_tests(app=None, verbose=False, profile=False, ui_tests=False, failfa test_suite = unittest.TestSuite() for app in apps: for path, folders, files in os.walk(frappe.get_pymodule_path(app)): - for dontwalk in ('locals', '.git', 'public', '__pycache__'): + for dontwalk in ("locals", ".git", "public", "__pycache__"): if dontwalk in folders: folders.remove(dontwalk) @@ -133,16 +172,16 @@ def run_all_tests(app=None, verbose=False, profile=False, ui_tests=False, failfa # print path for filename in files: - if filename.startswith("test_") and filename.endswith(".py")\ - and filename != 'test_runner.py': + if filename.startswith("test_") and filename.endswith(".py") and filename != "test_runner.py": # print filename[:-3] - _add_test(app, path, filename, verbose, - test_suite, ui_tests) + _add_test(app, path, filename, verbose, test_suite, ui_tests) if junit_xml_output: - runner = unittest_runner(verbosity=1+(verbose and 1 or 0), failfast=failfast) + runner = unittest_runner(verbosity=1 + (verbose and 1 or 0), failfast=failfast) else: - runner = unittest_runner(resultclass=TimeLoggingTestResult, verbosity=1+(verbose and 1 or 0), failfast=failfast) + runner = unittest_runner( + resultclass=TimeLoggingTestResult, verbosity=1 + (verbose and 1 or 0), failfast=failfast + ) if profile: pr = cProfile.Profile() @@ -153,13 +192,22 @@ def run_all_tests(app=None, verbose=False, profile=False, ui_tests=False, failfa if profile: pr.disable() s = StringIO() - ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps = pstats.Stats(pr, stream=s).sort_stats("cumulative") ps.print_stats() print(s.getvalue()) return out -def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profile=False, failfast=False, junit_xml_output=False): + +def run_tests_for_doctype( + doctypes, + verbose=False, + tests=(), + force=False, + profile=False, + failfast=False, + junit_xml_output=False, +): modules = [] if not isinstance(doctypes, (list, tuple)): doctypes = [doctypes] @@ -167,7 +215,7 @@ def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profil for doctype in doctypes: module = frappe.db.get_value("DocType", doctype, "module") if not module: - print('Invalid doctype {0}'.format(doctype)) + print("Invalid doctype {0}".format(doctype)) sys.exit(1) test_module = get_module_name(doctype, module, "test_") @@ -177,18 +225,39 @@ def run_tests_for_doctype(doctypes, verbose=False, tests=(), force=False, profil make_test_records(doctype, verbose=verbose, force=force) modules.append(importlib.import_module(test_module)) - return _run_unittest(modules, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output) + return _run_unittest( + modules, + verbose=verbose, + tests=tests, + profile=profile, + failfast=failfast, + junit_xml_output=junit_xml_output, + ) -def run_tests_for_module(module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None): + +def run_tests_for_module( + module, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None +): module = importlib.import_module(module) if hasattr(module, "test_dependencies"): for doctype in module.test_dependencies: make_test_records(doctype, verbose=verbose) frappe.db.commit() - return _run_unittest(module, verbose=verbose, tests=tests, profile=profile, failfast=failfast, junit_xml_output=junit_xml_output, case=case) - -def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None): + return _run_unittest( + module, + verbose=verbose, + tests=tests, + profile=profile, + failfast=failfast, + junit_xml_output=junit_xml_output, + case=case, + ) + + +def _run_unittest( + modules, verbose=False, tests=(), profile=False, failfast=False, junit_xml_output=False, case=None +): frappe.db.begin() test_suite = unittest.TestSuite() @@ -210,9 +279,11 @@ def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=Fals test_suite.addTest(module_test_cases) if junit_xml_output: - runner = unittest_runner(verbosity=1+(verbose and 1 or 0), failfast=failfast) + runner = unittest_runner(verbosity=1 + (verbose and 1 or 0), failfast=failfast) else: - runner = unittest_runner(resultclass=TimeLoggingTestResult, verbosity=1+(verbose and 1 or 0), failfast=failfast) + runner = unittest_runner( + resultclass=TimeLoggingTestResult, verbosity=1 + (verbose and 1 or 0), failfast=failfast + ) if profile: pr = cProfile.Profile() @@ -222,16 +293,16 @@ def _run_unittest(modules, verbose=False, tests=(), profile=False, failfast=Fals out = runner.run(test_suite) - if profile: pr.disable() s = StringIO() - ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps = pstats.Stats(pr, stream=s).sort_stats("cumulative") ps.print_stats() print(s.getvalue()) return out + def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): import os @@ -241,11 +312,12 @@ def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): app_path = frappe.get_pymodule_path(app) relative_path = os.path.relpath(path, app_path) - if relative_path=='.': + if relative_path == ".": module_name = app else: - module_name = '{app}.{relative_path}.{module_name}'.format(app=app, - relative_path=relative_path.replace('/', '.'), module_name=filename[:-3]) + module_name = "{app}.{relative_path}.{module_name}".format( + app=app, relative_path=relative_path.replace("/", "."), module_name=filename[:-3] + ) module = importlib.import_module(module_name) @@ -253,7 +325,7 @@ def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): for doctype in module.test_dependencies: make_test_records(doctype, verbose=verbose) - is_ui_test = True if hasattr(module, 'TestDriver') else False + is_ui_test = True if hasattr(module, "TestDriver") else False if is_ui_test != ui_tests: return @@ -261,16 +333,17 @@ def _add_test(app, path, filename, verbose, test_suite=None, ui_tests=False): if not test_suite: test_suite = unittest.TestSuite() - if os.path.basename(os.path.dirname(path))=="doctype": + if os.path.basename(os.path.dirname(path)) == "doctype": txt_file = os.path.join(path, filename[5:].replace(".py", ".json")) if os.path.exists(txt_file): - with open(txt_file, 'r') as f: + with open(txt_file, "r") as f: doc = json.loads(f.read()) doctype = doc["name"] make_test_records(doctype, verbose) test_suite.addTest(unittest.TestLoader().loadTestsFromModule(module)) + def make_test_records(doctype, verbose=0, force=False): if not frappe.db: frappe.connect() @@ -287,6 +360,7 @@ def make_test_records(doctype, verbose=0, force=False): make_test_records(options, verbose, force) make_test_records_for_doctype(options, verbose, force) + def get_modules(doctype): module = frappe.db.get_value("DocType", doctype, "module") try: @@ -298,6 +372,7 @@ def get_modules(doctype): return module, test_module + def get_dependencies(doctype): module, test_module = get_modules(doctype) meta = frappe.get_meta(doctype) @@ -322,6 +397,7 @@ def get_dependencies(doctype): return options_list + def make_test_records_for_doctype(doctype, verbose=0, force=False): if not force and doctype in get_test_record_log(): return @@ -336,9 +412,13 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False): elif hasattr(test_module, "test_records"): if doctype in frappe.local.test_objects: - frappe.local.test_objects[doctype] += make_test_objects(doctype, test_module.test_records, verbose, force) + frappe.local.test_objects[doctype] += make_test_objects( + doctype, test_module.test_records, verbose, force + ) else: - frappe.local.test_objects[doctype] = make_test_objects(doctype, test_module.test_records, verbose, force) + frappe.local.test_objects[doctype] = make_test_objects( + doctype, test_module.test_records, verbose, force + ) else: test_records = frappe.get_test_records(doctype) @@ -350,15 +430,15 @@ def make_test_records_for_doctype(doctype, verbose=0, force=False): add_to_test_record_log(doctype) + def make_test_objects(doctype, test_records=None, verbose=None, reset=False): - '''Make test objects from given list of `test_records` or from `test_records.json`''' + """Make test objects from given list of `test_records` or from `test_records.json`""" records = [] def revert_naming(d): - if getattr(d, 'naming_series', None): + if getattr(d, "naming_series", None): revert_series_if_last(d.naming_series, d.name) - if test_records is None: test_records = frappe.get_test_records(doctype) @@ -372,8 +452,8 @@ def make_test_objects(doctype, test_records=None, verbose=None, reset=False): if not d.naming_series: d.naming_series = "_T-" + d.doctype + "-" - if doc.get('name'): - d.name = doc.get('name') + if doc.get("name"): + d.name = doc.get("name") else: d.set_new_name() @@ -398,7 +478,10 @@ def make_test_objects(doctype, test_records=None, verbose=None, reset=False): revert_naming(d) except Exception as e: - if d.flags.ignore_these_exceptions_in_test and e.__class__ in d.flags.ignore_these_exceptions_in_test: + if ( + d.flags.ignore_these_exceptions_in_test + and e.__class__ in d.flags.ignore_these_exceptions_in_test + ): revert_naming(d) else: raise @@ -408,30 +491,33 @@ def make_test_objects(doctype, test_records=None, verbose=None, reset=False): frappe.db.commit() return records + def print_mandatory_fields(doctype): print("Please setup make_test_records for: " + doctype) print("-" * 60) meta = frappe.get_meta(doctype) print("Autoname: " + (meta.autoname or "")) print("Mandatory Fields: ") - for d in meta.get("fields", {"reqd":1}): + for d in meta.get("fields", {"reqd": 1}): print(d.parent + ":" + d.fieldname + " | " + d.fieldtype + " | " + (d.options or "")) print() + def add_to_test_record_log(doctype): - '''Add `doctype` to site/.test_log - `.test_log` is a cache of all doctypes for which test records are created''' + """Add `doctype` to site/.test_log + `.test_log` is a cache of all doctypes for which test records are created""" test_record_log = get_test_record_log() if doctype not in test_record_log: frappe.flags.test_record_log.append(doctype) - with open(frappe.get_site_path('.test_log'), 'w') as f: - f.write('\n'.join(filter(None, frappe.flags.test_record_log))) + with open(frappe.get_site_path(".test_log"), "w") as f: + f.write("\n".join(filter(None, frappe.flags.test_record_log))) + def get_test_record_log(): - '''Return the list of doctypes for which test records have been created''' - if 'test_record_log' not in frappe.flags: - if os.path.exists(frappe.get_site_path('.test_log')): - with open(frappe.get_site_path('.test_log'), 'r') as f: + """Return the list of doctypes for which test records have been created""" + if "test_record_log" not in frappe.flags: + if os.path.exists(frappe.get_site_path(".test_log")): + with open(frappe.get_site_path(".test_log"), "r") as f: frappe.flags.test_record_log = f.read().splitlines() else: frappe.flags.test_record_log = [] diff --git a/frappe/tests/__init__.py b/frappe/tests/__init__.py index a310864d83..5a44cae5f1 100644 --- a/frappe/tests/__init__.py +++ b/frappe/tests/__init__.py @@ -1,12 +1,15 @@ import frappe + def update_system_settings(args): - doc = frappe.get_doc('System Settings') + doc = frappe.get_doc("System Settings") doc.update(args) doc.flags.ignore_mandatory = 1 doc.save() + def get_system_setting(key): return frappe.db.get_single_value("System Settings", key) -global_test_dependencies = ['User'] + +global_test_dependencies = ["User"] diff --git a/frappe/tests/test_api.py b/frappe/tests/test_api.py index 8d41545c4c..bbb2280578 100644 --- a/frappe/tests/test_api.py +++ b/frappe/tests/test_api.py @@ -20,6 +20,7 @@ except Exception: authorization_token = None + @contextmanager def suppress_stdout(): """Supress stdout for tests which expectedly make noise @@ -31,7 +32,9 @@ def suppress_stdout(): sys.stdout = sys.__stdout__ -def make_request(target: str, args: Optional[Tuple] = None, kwargs: Optional[Dict] = None) -> TestResponse: +def make_request( + target: str, args: Optional[Tuple] = None, kwargs: Optional[Dict] = None +) -> TestResponse: t = ThreadWithReturnValue(target=target, args=args, kwargs=kwargs) t.start() t.join() @@ -78,22 +81,22 @@ class FrappeAPITestCase(unittest.TestCase): set_request(path="/") frappe.local.cookie_manager = CookieManager() frappe.local.login_manager = LoginManager() - frappe.local.login_manager.login_as('Administrator') + frappe.local.login_manager.login_as("Administrator") self._sid = frappe.session.sid return self._sid def get(self, path: str, params: Optional[Dict] = None, **kwargs) -> TestResponse: - return make_request(target=self.TEST_CLIENT.get, args=(path, ), kwargs={"data": params, **kwargs}) + return make_request(target=self.TEST_CLIENT.get, args=(path,), kwargs={"data": params, **kwargs}) def post(self, path, data, **kwargs) -> TestResponse: - return make_request(target=self.TEST_CLIENT.post, args=(path, ), kwargs={"data": data, **kwargs}) + return make_request(target=self.TEST_CLIENT.post, args=(path,), kwargs={"data": data, **kwargs}) def put(self, path, data, **kwargs) -> TestResponse: - return make_request(target=self.TEST_CLIENT.put, args=(path, ), kwargs={"data": data, **kwargs}) + return make_request(target=self.TEST_CLIENT.put, args=(path,), kwargs={"data": data, **kwargs}) def delete(self, path, **kwargs) -> TestResponse: - return make_request(target=self.TEST_CLIENT.delete, args=(path, ), kwargs=kwargs) + return make_request(target=self.TEST_CLIENT.delete, args=(path,), kwargs=kwargs) class TestResourceAPI(FrappeAPITestCase): @@ -103,9 +106,7 @@ class TestResourceAPI(FrappeAPITestCase): @classmethod def setUpClass(cls): for _ in range(10): - doc = frappe.get_doc( - {"doctype": "ToDo", "description": frappe.mock("paragraph")} - ).insert() + doc = frappe.get_doc({"doctype": "ToDo", "description": frappe.mock("paragraph")}).insert() cls.GENERATED_DOCUMENTS.append(doc.name) frappe.db.commit() @@ -157,7 +158,9 @@ class TestResourceAPI(FrappeAPITestCase): def test_get_list_fields(self): # test 6: fetch response with fields - response = self.get(f"/api/resource/{self.DOCTYPE}", {"sid": self.sid, "fields": '["description"]'}) + response = self.get( + f"/api/resource/{self.DOCTYPE}", {"sid": self.sid, "fields": '["description"]'} + ) self.assertEqual(response.status_code, 200) json = frappe._dict(response.json) self.assertIn("description", json.data[0]) @@ -205,10 +208,14 @@ class TestResourceAPI(FrappeAPITestCase): self.assertIn(response.status_code, (403, 200)) if response.status_code == 403: - self.assertTrue(set(response.json.keys()) == {'exc_type', 'exception', 'exc', '_server_messages'}) - self.assertEqual(response.json.get('exc_type'), 'PermissionError') - self.assertEqual(response.json.get('exception'), 'frappe.exceptions.PermissionError: Not permitted') - self.assertIsInstance(response.json.get('exc'), str) + self.assertTrue( + set(response.json.keys()) == {"exc_type", "exception", "exc", "_server_messages"} + ) + self.assertEqual(response.json.get("exc_type"), "PermissionError") + self.assertEqual( + response.json.get("exception"), "frappe.exceptions.PermissionError: Not permitted" + ) + self.assertIsInstance(response.json.get("exc"), str) elif response.status_code == 200: data = response.json.get("data") @@ -222,6 +229,7 @@ class TestMethodAPI(FrappeAPITestCase): def setUp(self): if self._testMethodName == "test_auth_cycle": from frappe.core.doctype.user.user import generate_keys + generate_keys("Administrator") frappe.db.commit() diff --git a/frappe/tests/test_assign.py b/frappe/tests/test_assign.py index 48fe4e04e5..aa646d2827 100644 --- a/frappe/tests/test_assign.py +++ b/frappe/tests/test_assign.py @@ -1,16 +1,19 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + +import frappe import frappe.desk.form.assign_to -from frappe.desk.listview import get_group_by_count from frappe.automation.doctype.assignment_rule.test_assignment_rule import make_note from frappe.desk.form.load import get_assignments +from frappe.desk.listview import get_group_by_count + class TestAssign(unittest.TestCase): def test_assign(self): - todo = frappe.get_doc({"doctype":"ToDo", "description": "test"}).insert() + todo = frappe.get_doc({"doctype": "ToDo", "description": "test"}).insert() if not frappe.db.exists("User", "test@example.com"): - frappe.get_doc({"doctype":"User", "email":"test@example.com", "first_name":"Test"}).insert() + frappe.get_doc({"doctype": "User", "email": "test@example.com", "first_name": "Test"}).insert() added = assign(todo, "test@example.com") @@ -19,17 +22,31 @@ class TestAssign(unittest.TestCase): removed = frappe.desk.form.assign_to.remove(todo.doctype, todo.name, "test@example.com") # assignment is cleared - assignments = frappe.desk.form.assign_to.get(dict(doctype = todo.doctype, name=todo.name)) + assignments = frappe.desk.form.assign_to.get(dict(doctype=todo.doctype, name=todo.name)) self.assertEqual(len(assignments), 0) def test_assignment_count(self): frappe.db.delete("ToDo") if not frappe.db.exists("User", "test_assign1@example.com"): - frappe.get_doc({"doctype":"User", "email":"test_assign1@example.com", "first_name":"Test", "roles": [{"role": "System Manager"}]}).insert() + frappe.get_doc( + { + "doctype": "User", + "email": "test_assign1@example.com", + "first_name": "Test", + "roles": [{"role": "System Manager"}], + } + ).insert() if not frappe.db.exists("User", "test_assign2@example.com"): - frappe.get_doc({"doctype":"User", "email":"test_assign2@example.com", "first_name":"Test", "roles": [{"role": "System Manager"}]}).insert() + frappe.get_doc( + { + "doctype": "User", + "email": "test_assign2@example.com", + "first_name": "Test", + "roles": [{"role": "System Manager"}], + } + ).insert() note = make_note() assign(note, "test_assign1@example.com") @@ -43,23 +60,23 @@ class TestAssign(unittest.TestCase): note = make_note() assign(note, "test_assign2@example.com") - data = {d.name: d.count for d in get_group_by_count('Note', '[]', 'assigned_to')} + data = {d.name: d.count for d in get_group_by_count("Note", "[]", "assigned_to")} - self.assertTrue('test_assign1@example.com' in data) - self.assertEqual(data['test_assign1@example.com'], 1) - self.assertEqual(data['test_assign2@example.com'], 3) + self.assertTrue("test_assign1@example.com" in data) + self.assertEqual(data["test_assign1@example.com"], 1) + self.assertEqual(data["test_assign2@example.com"], 3) - data = {d.name: d.count for d in get_group_by_count('Note', '[{"public": 1}]', 'assigned_to')} + data = {d.name: d.count for d in get_group_by_count("Note", '[{"public": 1}]', "assigned_to")} - self.assertFalse('test_assign1@example.com' in data) - self.assertEqual(data['test_assign2@example.com'], 2) + self.assertFalse("test_assign1@example.com" in data) + self.assertEqual(data["test_assign2@example.com"], 2) frappe.db.rollback() def test_assignment_removal(self): - todo = frappe.get_doc({"doctype":"ToDo", "description": "test"}).insert() + todo = frappe.get_doc({"doctype": "ToDo", "description": "test"}).insert() if not frappe.db.exists("User", "test@example.com"): - frappe.get_doc({"doctype":"User", "email":"test@example.com", "first_name":"Test"}).insert() + frappe.get_doc({"doctype": "User", "email": "test@example.com", "first_name": "Test"}).insert() new_todo = assign(todo, "test@example.com") @@ -68,10 +85,13 @@ class TestAssign(unittest.TestCase): self.assertFalse(get_assignments("ToDo", todo.name)) + def assign(doc, user): - return frappe.desk.form.assign_to.add({ - "assign_to": [user], - "doctype": doc.doctype, - "name": doc.name, - "description": 'test', - }) + return frappe.desk.form.assign_to.add( + { + "assign_to": [user], + "doctype": doc.doctype, + "name": doc.name, + "description": "test", + } + ) diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py index ae4f78735b..b9c492715c 100644 --- a/frappe/tests/test_auth.py +++ b/frappe/tests/test_auth.py @@ -6,14 +6,14 @@ import unittest import frappe import frappe.utils from frappe.auth import LoginAttemptTracker -from frappe.frappeclient import FrappeClient, AuthError +from frappe.frappeclient import AuthError, FrappeClient def add_user(email, password, username=None, mobile_no=None): - first_name = email.split('@', 1)[0] + first_name = email.split("@", 1)[0] user = frappe.get_doc( - dict(doctype='User', email=email, first_name=first_name, username=username, mobile_no=mobile_no) - ).insert() + dict(doctype="User", email=email, first_name=first_name, username=username, mobile_no=mobile_no) + ).insert() user.new_password = password user.add_roles("System Manager") frappe.db.commit() @@ -22,22 +22,25 @@ def add_user(email, password, username=None, mobile_no=None): class TestAuth(unittest.TestCase): @classmethod def setUpClass(cls): - cls.HOST_NAME = ( - frappe.get_site_config().host_name - or frappe.utils.get_site_url(frappe.local.site) + cls.HOST_NAME = frappe.get_site_config().host_name or frappe.utils.get_site_url( + frappe.local.site ) - cls.test_user_email = 'test_auth@test.com' - cls.test_user_name = 'test_auth_user' - cls.test_user_mobile = '+911234567890' - cls.test_user_password = 'pwd_012' + cls.test_user_email = "test_auth@test.com" + cls.test_user_name = "test_auth_user" + cls.test_user_mobile = "+911234567890" + cls.test_user_password = "pwd_012" cls.tearDownClass() - add_user(email=cls.test_user_email, password=cls.test_user_password, - username=cls.test_user_name, mobile_no=cls.test_user_mobile) + add_user( + email=cls.test_user_email, + password=cls.test_user_password, + username=cls.test_user_name, + mobile_no=cls.test_user_mobile, + ) @classmethod def tearDownClass(cls): - frappe.delete_doc('User', cls.test_user_email, force=True) + frappe.delete_doc("User", cls.test_user_email, force=True) def set_system_settings(self, k, v): frappe.db.set_value("System Settings", "System Settings", k, v) @@ -45,8 +48,8 @@ class TestAuth(unittest.TestCase): frappe.db.commit() def test_allow_login_using_mobile(self): - self.set_system_settings('allow_login_using_mobile_number', 1) - self.set_system_settings('allow_login_using_user_name', 0) + self.set_system_settings("allow_login_using_mobile_number", 1) + self.set_system_settings("allow_login_using_user_name", 0) # Login by both email and mobile should work FrappeClient(self.HOST_NAME, self.test_user_mobile, self.test_user_password) @@ -57,8 +60,8 @@ class TestAuth(unittest.TestCase): FrappeClient(self.HOST_NAME, self.test_user_name, self.test_user_password) def test_allow_login_using_only_email(self): - self.set_system_settings('allow_login_using_mobile_number', 0) - self.set_system_settings('allow_login_using_user_name', 0) + self.set_system_settings("allow_login_using_mobile_number", 0) + self.set_system_settings("allow_login_using_user_name", 0) # Login by mobile number should fail with self.assertRaises(AuthError): @@ -72,8 +75,8 @@ class TestAuth(unittest.TestCase): FrappeClient(self.HOST_NAME, self.test_user_email, self.test_user_password) def test_allow_login_using_username(self): - self.set_system_settings('allow_login_using_mobile_number', 0) - self.set_system_settings('allow_login_using_user_name', 1) + self.set_system_settings("allow_login_using_mobile_number", 0) + self.set_system_settings("allow_login_using_user_name", 1) # Mobile login should fail with self.assertRaises(AuthError): @@ -84,8 +87,8 @@ class TestAuth(unittest.TestCase): FrappeClient(self.HOST_NAME, self.test_user_name, self.test_user_password) def test_allow_login_using_username_and_mobile(self): - self.set_system_settings('allow_login_using_mobile_number', 1) - self.set_system_settings('allow_login_using_user_name', 1) + self.set_system_settings("allow_login_using_mobile_number", 1) + self.set_system_settings("allow_login_using_user_name", 1) # Both email and username and mobile logins should work FrappeClient(self.HOST_NAME, self.test_user_mobile, self.test_user_password) @@ -93,7 +96,7 @@ class TestAuth(unittest.TestCase): FrappeClient(self.HOST_NAME, self.test_user_name, self.test_user_password) def test_deny_multiple_login(self): - self.set_system_settings('deny_multiple_sessions', 1) + self.set_system_settings("deny_multiple_sessions", 1) first_login = FrappeClient(self.HOST_NAME, self.test_user_email, self.test_user_password) first_login.get_list("ToDo") @@ -113,9 +116,10 @@ class TestAuth(unittest.TestCase): class TestLoginAttemptTracker(unittest.TestCase): def test_account_lock(self): - """Make sure that account locks after `n consecutive failures - """ - tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=3, lock_interval=60) + """Make sure that account locks after `n consecutive failures""" + tracker = LoginAttemptTracker( + user_name="tester", max_consecutive_login_attempts=3, lock_interval=60 + ) # Clear the cache by setting attempt as success tracker.add_success_attempt() @@ -132,10 +136,11 @@ class TestLoginAttemptTracker(unittest.TestCase): self.assertFalse(tracker.is_user_allowed()) def test_account_unlock(self): - """Make sure that locked account gets unlocked after lock_interval of time. - """ - lock_interval = 2 # In sec - tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=1, lock_interval=lock_interval) + """Make sure that locked account gets unlocked after lock_interval of time.""" + lock_interval = 2 # In sec + tracker = LoginAttemptTracker( + user_name="tester", max_consecutive_login_attempts=1, lock_interval=lock_interval + ) # Clear the cache by setting attempt as success tracker.add_success_attempt() diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 75f6cc8fe3..6c7dda51f1 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -1,11 +1,11 @@ +import time import unittest from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn, generate_qname -import time +from frappe.utils.background_jobs import generate_qname, get_redis_conn class TestBackgroundJobs(unittest.TestCase): diff --git a/frappe/tests/test_boilerplate.py b/frappe/tests/test_boilerplate.py index 6a9544b2e9..4dfb6e615b 100644 --- a/frappe/tests/test_boilerplate.py +++ b/frappe/tests/test_boilerplate.py @@ -42,7 +42,7 @@ class TestBoilerPlate(unittest.TestCase): "setup.py", "license.txt", cls.git_folder, - cls.gitignore_file + cls.gitignore_file, ] cls.paths_inside_app = [ "__init__.py", @@ -52,7 +52,7 @@ class TestBoilerPlate(unittest.TestCase): "www", "config", "modules.txt", - "public" + "public", ] @classmethod @@ -70,10 +70,7 @@ class TestBoilerPlate(unittest.TestCase): paths = self.get_paths(new_app_dir, self.app_names[0]) for path in paths: - self.assertTrue( - os.path.exists(path), - msg=f"{path} should exist in {self.app_names[0]} app" - ) + self.assertTrue(os.path.exists(path), msg=f"{path} should exist in {self.app_names[0]} app") self.check_parsable_python_files(new_app_dir) @@ -87,14 +84,10 @@ class TestBoilerPlate(unittest.TestCase): for path in paths: if os.path.basename(path) in (self.git_folder, self.gitignore_file): self.assertFalse( - os.path.exists(path), - msg=f"{path} shouldn't exist in {self.app_names[1]} app" + os.path.exists(path), msg=f"{path} shouldn't exist in {self.app_names[1]} app" ) else: - self.assertTrue( - os.path.exists(path), - msg=f"{path} should exist in {self.app_names[1]} app" - ) + self.assertTrue(os.path.exists(path), msg=f"{path} should exist in {self.app_names[1]} app") self.check_parsable_python_files(new_app_dir) diff --git a/frappe/tests/test_bot.py b/frappe/tests/test_bot.py index 9d20895338..222f35b99b 100644 --- a/frappe/tests/test_bot.py +++ b/frappe/tests/test_bot.py @@ -5,5 +5,6 @@ import unittest + class TestBot(unittest.TestCase): pass diff --git a/frappe/tests/test_child_table.py b/frappe/tests/test_child_table.py index 8cdfd08599..eb632eac90 100644 --- a/frappe/tests/test_child_table.py +++ b/frappe/tests/test_child_table.py @@ -1,9 +1,9 @@ -import frappe -from frappe.model import child_table_fields - import unittest from typing import Callable +import frappe +from frappe.model import child_table_fields + class TestChildTable(unittest.TestCase): def tearDown(self) -> None: @@ -13,31 +13,29 @@ class TestChildTable(unittest.TestCase): pass def test_child_table_doctype_creation_and_transitioning(self) -> None: - ''' + """ This method tests the creation of child table doctype as well as it's transitioning from child table to normal and normal to child table doctype - ''' + """ self.doctype_name = "Test Newy Child Table" try: - doc = frappe.get_doc({ - "doctype": "DocType", - "name": self.doctype_name, - "istable": 1, - "custom": 1, - "module": "Integrations", - "fields": [{ - "label": "Some Field", - "fieldname": "some_fieldname", - "fieldtype": "Data", - "reqd": 1 - }] - }).insert(ignore_permissions=True) + doc = frappe.get_doc( + { + "doctype": "DocType", + "name": self.doctype_name, + "istable": 1, + "custom": 1, + "module": "Integrations", + "fields": [ + {"label": "Some Field", "fieldname": "some_fieldname", "fieldtype": "Data", "reqd": 1} + ], + } + ).insert(ignore_permissions=True) except Exception: self.fail("Not able to create Child Table Doctype") - for column in child_table_fields: self.assertTrue(frappe.db.has_column(self.doctype_name, column)) @@ -59,7 +57,6 @@ class TestChildTable(unittest.TestCase): self.check_valid_columns(self.assertTrue) - def check_valid_columns(self, assertion_method: Callable) -> None: valid_columns = frappe.get_meta(self.doctype_name).get_valid_columns() for column in child_table_fields: diff --git a/frappe/tests/test_client.py b/frappe/tests/test_client.py index 40639e4f98..1bb2867665 100644 --- a/frappe/tests/test_client.py +++ b/frappe/tests/test_client.py @@ -1,22 +1,23 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors import unittest + import frappe class TestClient(unittest.TestCase): def test_set_value(self): - todo = frappe.get_doc(dict(doctype='ToDo', description='test')).insert() - frappe.set_value('ToDo', todo.name, 'description', 'test 1') - self.assertEqual(frappe.get_value('ToDo', todo.name, 'description'), 'test 1') + todo = frappe.get_doc(dict(doctype="ToDo", description="test")).insert() + frappe.set_value("ToDo", todo.name, "description", "test 1") + self.assertEqual(frappe.get_value("ToDo", todo.name, "description"), "test 1") - frappe.set_value('ToDo', todo.name, {'description': 'test 2'}) - self.assertEqual(frappe.get_value('ToDo', todo.name, 'description'), 'test 2') + frappe.set_value("ToDo", todo.name, {"description": "test 2"}) + self.assertEqual(frappe.get_value("ToDo", todo.name, "description"), "test 2") def test_delete(self): from frappe.client import delete - todo = frappe.get_doc(dict(doctype='ToDo', description='description')).insert() + todo = frappe.get_doc(dict(doctype="ToDo", description="description")).insert() delete("ToDo", todo.name) self.assertFalse(frappe.db.exists("ToDo", todo.name)) @@ -29,15 +30,14 @@ class TestClient(unittest.TestCase): frappe.set_user("Administrator") frappe.local.request = frappe._dict() - frappe.local.request.method = 'POST' + frappe.local.request.method = "POST" - frappe.local.form_dict = frappe._dict({ - 'doc': dict(doctype='ToDo', description='Valid http method'), - 'cmd': 'frappe.client.save' - }) - todo = execute_cmd('frappe.client.save') + frappe.local.form_dict = frappe._dict( + {"doc": dict(doctype="ToDo", description="Valid http method"), "cmd": "frappe.client.save"} + ) + todo = execute_cmd("frappe.client.save") - self.assertEqual(todo.get('description'), 'Valid http method') + self.assertEqual(todo.get("description"), "Valid http method") delete("ToDo", todo.name) @@ -47,87 +47,85 @@ class TestClient(unittest.TestCase): frappe.set_user("Administrator") frappe.local.request = frappe._dict() - frappe.local.request.method = 'GET' + frappe.local.request.method = "GET" - frappe.local.form_dict = frappe._dict({ - 'doc': dict(doctype='ToDo', description='Invalid http method'), - 'cmd': 'frappe.client.save' - }) + frappe.local.form_dict = frappe._dict( + {"doc": dict(doctype="ToDo", description="Invalid http method"), "cmd": "frappe.client.save"} + ) - self.assertRaises(frappe.PermissionError, execute_cmd, 'frappe.client.save') + self.assertRaises(frappe.PermissionError, execute_cmd, "frappe.client.save") def test_run_doc_method(self): from frappe.handler import execute_cmd - if not frappe.db.exists('Report', 'Test Run Doc Method'): - report = frappe.get_doc({ - 'doctype': 'Report', - 'ref_doctype': 'User', - 'report_name': 'Test Run Doc Method', - 'report_type': 'Query Report', - 'is_standard': 'No', - 'roles': [ - {'role': 'System Manager'} - ] - }).insert() + if not frappe.db.exists("Report", "Test Run Doc Method"): + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": "Test Run Doc Method", + "report_type": "Query Report", + "is_standard": "No", + "roles": [{"role": "System Manager"}], + } + ).insert() else: - report = frappe.get_doc('Report', 'Test Run Doc Method') + report = frappe.get_doc("Report", "Test Run Doc Method") frappe.local.request = frappe._dict() - frappe.local.request.method = 'GET' + frappe.local.request.method = "GET" # Whitelisted, works as expected - frappe.local.form_dict = frappe._dict({ - 'dt': report.doctype, - 'dn': report.name, - 'method': 'toggle_disable', - 'cmd': 'run_doc_method', - 'args': 0 - }) + frappe.local.form_dict = frappe._dict( + { + "dt": report.doctype, + "dn": report.name, + "method": "toggle_disable", + "cmd": "run_doc_method", + "args": 0, + } + ) execute_cmd(frappe.local.form_dict.cmd) # Not whitelisted, throws permission error - frappe.local.form_dict = frappe._dict({ - 'dt': report.doctype, - 'dn': report.name, - 'method': 'create_report_py', - 'cmd': 'run_doc_method', - 'args': 0 - }) - - self.assertRaises( - frappe.PermissionError, - execute_cmd, - frappe.local.form_dict.cmd + frappe.local.form_dict = frappe._dict( + { + "dt": report.doctype, + "dn": report.name, + "method": "create_report_py", + "cmd": "run_doc_method", + "args": 0, + } ) + self.assertRaises(frappe.PermissionError, execute_cmd, frappe.local.form_dict.cmd) + def test_array_values_in_request_args(self): import requests + from frappe.auth import CookieManager, LoginManager frappe.utils.set_request(path="/") frappe.local.cookie_manager = CookieManager() frappe.local.login_manager = LoginManager() - frappe.local.login_manager.login_as('Administrator') + frappe.local.login_manager.login_as("Administrator") params = { - 'doctype': 'DocType', - 'fields': ['name', 'modified'], - 'sid': frappe.session.sid, + "doctype": "DocType", + "fields": ["name", "modified"], + "sid": frappe.session.sid, } headers = { - 'accept': 'application/json', - 'content-type': 'application/json', + "accept": "application/json", + "content-type": "application/json", } - url = f'http://{frappe.local.site}:{frappe.conf.webserver_port}/api/method/frappe.client.get_list' - res = requests.post( - url, - json=params, - headers=headers + url = ( + f"http://{frappe.local.site}:{frappe.conf.webserver_port}/api/method/frappe.client.get_list" ) + res = requests.post(url, json=params, headers=headers) self.assertEqual(res.status_code, 200) data = res.json() - first_item = data['message'][0] - self.assertTrue('name' in first_item) - self.assertTrue('modified' in first_item) + first_item = data["message"][0] + self.assertTrue("name" in first_item) + self.assertTrue("modified" in first_item) frappe.local.login_manager.logout() diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 68605444f1..67de6c6e41 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -19,8 +19,8 @@ from unittest.mock import patch # imports - third party imports import click -from click.testing import CliRunner, Result from click import Command +from click.testing import CliRunner, Result # imports - module imports import frappe @@ -32,7 +32,7 @@ from frappe.utils import add_to_date, get_bench_path, get_bench_relative_path, n from frappe.utils.backups import fetch_latest_backups _result: Optional[Result] = None -TEST_SITE = "commands-site-O4PN2QKA.test" # added random string tag to avoid collisions +TEST_SITE = "commands-site-O4PN2QKA.test" # added random string tag to avoid collisions CLI_CONTEXT = frappe._dict(sites=[TEST_SITE]) @@ -40,10 +40,10 @@ def clean(value) -> str: """Strips and converts bytes to str Args: - value ([type]): [description] + value ([type]): [description] Returns: - [type]: [description] + [type]: [description] """ if isinstance(value, bytes): value = value.decode() @@ -56,33 +56,28 @@ def missing_in_backup(doctypes: List, file: os.PathLike) -> List: """Returns list of missing doctypes in the backup. Args: - doctypes (list): List of DocTypes to be checked - file (str): Path of the database file + doctypes (list): List of DocTypes to be checked + file (str): Path of the database file Returns: - doctypes(list): doctypes that are missing in backup + doctypes(list): doctypes that are missing in backup """ - predicate = ( - 'COPY public."tab{}"' - if frappe.conf.db_type == "postgres" - else "CREATE TABLE `tab{}`" - ) + predicate = 'COPY public."tab{}"' if frappe.conf.db_type == "postgres" else "CREATE TABLE `tab{}`" with gzip.open(file, "rb") as f: content = f.read().decode("utf8").lower() - return [doctype for doctype in doctypes - if predicate.format(doctype).lower() not in content] + return [doctype for doctype in doctypes if predicate.format(doctype).lower() not in content] def exists_in_backup(doctypes: List, file: os.PathLike) -> bool: """Checks if the list of doctypes exist in the database.sql.gz file supplied Args: - doctypes (list): List of DocTypes to be checked - file (str): Path of the database file + doctypes (list): List of DocTypes to be checked + file (str): Path of the database file Returns: - bool: True if all tables exist + bool: True if all tables exist """ missing_doctypes = missing_in_backup(doctypes, file) return len(missing_doctypes) == 0 @@ -108,6 +103,7 @@ def pass_test_context(f): @wraps(f) def decorated_function(*args, **kwargs): return f(CLI_CONTEXT, *args, **kwargs) + return decorated_function @@ -150,9 +146,7 @@ class BaseTestCommands(unittest.TestCase): cmd_input = kwargs.get("cmd_input", None) if cmd_input: if not isinstance(cmd_input, bytes): - raise Exception( - f"The input should be of type bytes, not {type(cmd_input).__name__}" - ) + raise Exception(f"The input should be of type bytes, not {type(cmd_input).__name__}") del kwargs["cmd_input"] kwargs.update(site) @@ -163,7 +157,9 @@ class BaseTestCommands(unittest.TestCase): click.secho(self.command, fg="bright_black") command = shlex.split(self.command) - self._proc = subprocess.run(command, input=cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self._proc = subprocess.run( + command, input=cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) self.stdout = clean(self._proc.stdout) self.stderr = clean(self._proc.stderr) self.returncode = clean(self._proc.returncode) @@ -178,12 +174,9 @@ class BaseTestCommands(unittest.TestCase): "db_type": frappe.conf.db_type, } - if not os.path.exists( - os.path.join(TEST_SITE, "site_config.json") - ): + if not os.path.exists(os.path.join(TEST_SITE, "site_config.json")): cls.execute( - "bench new-site {test_site} --admin-password {admin_password} --db-type" - " {db_type}", + "bench new-site {test_site} --admin-password {admin_password} --db-type" " {db_type}", cmd_config, ) @@ -201,14 +194,16 @@ class BaseTestCommands(unittest.TestCase): stderr = self.stderr returncode = self.returncode - cmd_execution_summary = "\n".join([ - "-" * 70, - "Last Command Execution Summary:", - "Command: {}".format(command) if command else "", - "Standard Output: {}".format(stdout) if stdout else "", - "Standard Error: {}".format(stderr) if stderr else "", - "Return Code: {}".format(returncode) if returncode else "", - ]).strip() + cmd_execution_summary = "\n".join( + [ + "-" * 70, + "Last Command Execution Summary:", + "Command: {}".format(command) if command else "", + "Standard Output: {}".format(stdout) if stdout else "", + "Standard Error: {}".format(stderr) if stderr else "", + "Return Code: {}".format(returncode) if returncode else "", + ] + ).strip() return "{}\n\n{}".format(output, cmd_execution_summary) @@ -260,8 +255,7 @@ class TestCommands(BaseTestCommands): self.execute("bench --site {test_site} backup --exclude 'ToDo'", site_data) site_data.update({"kw": "\"{'partial':True}\""}) self.execute( - "bench --site {test_site} execute" - " frappe.utils.backups.fetch_latest_backups --kwargs {kw}", + "bench --site {test_site} execute" " frappe.utils.backups.fetch_latest_backups --kwargs {kw}", site_data, ) site_data.update({"database": json.loads(self.stdout)["database"]}) @@ -271,11 +265,13 @@ class TestCommands(BaseTestCommands): def test_partial_restore(self): _now = now() for num in range(10): - frappe.get_doc({ - "doctype": "ToDo", - "date": add_to_date(_now, days=num), - "description": frappe.mock("paragraph") - }).insert() + frappe.get_doc( + { + "doctype": "ToDo", + "date": add_to_date(_now, days=num), + "description": frappe.mock("paragraph"), + } + ).insert() frappe.db.commit() todo_count = frappe.db.count("ToDo") @@ -326,9 +322,7 @@ class TestCommands(BaseTestCommands): # test 2: bare functionality for single site self.execute("bench --site {site} list-apps") self.assertEqual(self.returncode, 0) - list_apps = set( - _x.split()[0] for _x in self.stdout.split("\n") - ) + list_apps = set(_x.split()[0] for _x in self.stdout.split("\n")) doctype = frappe.get_single("Installed Applications").installed_applications if doctype: installed_apps = set(x.app_name for x in doctype) @@ -390,7 +384,7 @@ class TestCommands(BaseTestCommands): os.remove(test2_path) def test_frappe_site_env(self): - os.putenv('FRAPPE_SITE', frappe.local.site) + os.putenv("FRAPPE_SITE", frappe.local.site) self.execute("bench execute frappe.ping") self.assertEqual(self.returncode, 0) self.assertIn("pong", self.stdout) @@ -411,43 +405,39 @@ class TestCommands(BaseTestCommands): self.execute("bench --site {site} set-password Administrator test1") self.assertEqual(self.returncode, 0) - self.assertEqual(check_password('Administrator', 'test1'), 'Administrator') + self.assertEqual(check_password("Administrator", "test1"), "Administrator") # to release the lock taken by check_password frappe.db.commit() self.execute("bench --site {site} set-admin-password test2") self.assertEqual(self.returncode, 0) - self.assertEqual(check_password('Administrator', 'test2'), 'Administrator') + self.assertEqual(check_password("Administrator", "test2"), "Administrator") def test_make_app(self): user_input = [ - b"Test App", # title - b"This app's description contains 'single quotes' and \"double quotes\".", # description - b"Test Publisher", # publisher - b"example@example.org", # email - b"", # icon - b"", # color - b"MIT" # app_license + b"Test App", # title + b"This app's description contains 'single quotes' and \"double quotes\".", # description + b"Test Publisher", # publisher + b"example@example.org", # email + b"", # icon + b"", # color + b"MIT", # app_license ] app_name = "testapp0" apps_path = os.path.join(get_bench_path(), "apps") test_app_path = os.path.join(apps_path, app_name) - self.execute(f"bench make-app {apps_path} {app_name}", {"cmd_input": b'\n'.join(user_input)}) + self.execute(f"bench make-app {apps_path} {app_name}", {"cmd_input": b"\n".join(user_input)}) self.assertEqual(self.returncode, 0) - self.assertTrue( - os.path.exists(test_app_path) - ) + self.assertTrue(os.path.exists(test_app_path)) # cleanup shutil.rmtree(test_app_path) @skipIf( not ( - frappe.conf.root_password - and frappe.conf.admin_password - and frappe.conf.db_type == "mariadb" + frappe.conf.root_password and frappe.conf.admin_password and frappe.conf.db_type == "mariadb" ), - "DB Root password and Admin password not set in config" + "DB Root password and Admin password not set in config", ) def test_bench_drop_site_should_archive_site(self): # TODO: Make this test postgres compatible @@ -465,9 +455,9 @@ class TestCommands(BaseTestCommands): self.assertEqual(self.returncode, 0) bench_path = get_bench_path() - site_directory = os.path.join(bench_path, f'sites/{site}') + site_directory = os.path.join(bench_path, f"sites/{site}") self.assertFalse(os.path.exists(site_directory)) - archive_directory = os.path.join(bench_path, f'archived/sites/{site}') + archive_directory = os.path.join(bench_path, f"archived/sites/{site}") self.assertTrue(os.path.exists(archive_directory)) @@ -479,13 +469,7 @@ class TestBackups(BaseTestCommands): "Note", ] }, - "excludes": { - "excludes": [ - "Activity Log", - "Access Log", - "Error Log" - ] - } + "excludes": {"excludes": ["Activity Log", "Access Log", "Error Log"]}, } home = os.path.expanduser("~") site_backup_path = frappe.utils.get_site_path("private", "backups") @@ -503,8 +487,7 @@ class TestBackups(BaseTestCommands): pass def test_backup_no_options(self): - """Take a backup without any options - """ + """Take a backup without any options""" before_backup = fetch_latest_backups(partial=True) self.execute("bench --site {site} backup") after_backup = fetch_latest_backups(partial=True) @@ -514,8 +497,7 @@ class TestBackups(BaseTestCommands): self.assertNotEqual(before_backup["database"], after_backup["database"]) def test_backup_with_files(self): - """Take a backup with files (--with-files) - """ + """Take a backup with files (--with-files)""" before_backup = fetch_latest_backups() self.execute("bench --site {site} backup --with-files") after_backup = fetch_latest_backups() @@ -528,18 +510,18 @@ class TestBackups(BaseTestCommands): self.assertIsNotNone(after_backup["private"]) def test_backup_with_custom_path(self): - """Backup to a custom path (--backup-path) - """ + """Backup to a custom path (--backup-path)""" backup_path = os.path.join(self.home, "backups") - self.execute("bench --site {site} backup --backup-path {backup_path}", {"backup_path": backup_path}) + self.execute( + "bench --site {site} backup --backup-path {backup_path}", {"backup_path": backup_path} + ) self.assertEqual(self.returncode, 0) self.assertTrue(os.path.exists(backup_path)) self.assertGreaterEqual(len(os.listdir(backup_path)), 2) def test_backup_with_different_file_paths(self): - """Backup with different file paths (--backup-path-db, --backup-path-files, --backup-path-private-files, --backup-path-conf) - """ + """Backup with different file paths (--backup-path-db, --backup-path-files, --backup-path-private-files, --backup-path-conf)""" kwargs = { key: os.path.join(self.home, key, value) for key, value in { @@ -565,22 +547,19 @@ class TestBackups(BaseTestCommands): self.assertTrue(os.path.exists(path)) def test_backup_compress_files(self): - """Take a compressed backup (--compress) - """ + """Take a compressed backup (--compress)""" self.execute("bench --site {site} backup --with-files --compress") self.assertEqual(self.returncode, 0) compressed_files = glob(f"{self.site_backup_path}/*.tgz") self.assertGreater(len(compressed_files), 0) def test_backup_verbose(self): - """Take a verbose backup (--verbose) - """ + """Take a verbose backup (--verbose)""" self.execute("bench --site {site} backup --verbose") self.assertEqual(self.returncode, 0) def test_backup_only_specific_doctypes(self): - """Take a backup with (include) backup options set in the site config `frappe.conf.backup.includes` - """ + """Take a backup with (include) backup options set in the site config `frappe.conf.backup.includes`""" self.execute( "bench --site {site} set-config backup '{includes}' --parse", {"includes": json.dumps(self.backup_map["includes"])}, @@ -591,8 +570,7 @@ class TestBackups(BaseTestCommands): self.assertEqual([], missing_in_backup(self.backup_map["includes"]["includes"], database)) def test_backup_excluding_specific_doctypes(self): - """Take a backup with (exclude) backup options set (`frappe.conf.backup.excludes`, `--exclude`) - """ + """Take a backup with (exclude) backup options set (`frappe.conf.backup.excludes`, `--exclude`)""" # test 1: take a backup with frappe.conf.backup.excludes self.execute( "bench --site {site} set-config backup '{excludes}' --parse", @@ -614,8 +592,7 @@ class TestBackups(BaseTestCommands): self.assertFalse(exists_in_backup(self.backup_map["excludes"]["excludes"], database)) def test_selective_backup_priority_resolution(self): - """Take a backup with conflicting backup options set (`frappe.conf.excludes`, `--include`) - """ + """Take a backup with conflicting backup options set (`frappe.conf.excludes`, `--include`)""" self.execute( "bench --site {site} backup --include '{include}'", {"include": ",".join(self.backup_map["includes"]["includes"])}, @@ -625,8 +602,7 @@ class TestBackups(BaseTestCommands): self.assertEqual([], missing_in_backup(self.backup_map["includes"]["includes"], database)) def test_dont_backup_conf(self): - """Take a backup ignoring frappe.conf.backup settings (with --ignore-backup-conf option) - """ + """Take a backup ignoring frappe.conf.backup settings (with --ignore-backup-conf option)""" self.execute("bench --site {site} backup --ignore-backup-conf") self.assertEqual(self.returncode, 0) database = fetch_latest_backups()["database"] @@ -636,9 +612,9 @@ class TestBackups(BaseTestCommands): class TestRemoveApp(unittest.TestCase): def test_delete_modules(self): from frappe.installer import ( - _delete_doctypes, - _delete_modules, - _get_module_linked_doctype_field_map, + _delete_doctypes, + _delete_modules, + _get_module_linked_doctype_field_map, ) test_module = frappe.new_doc("Module Def") @@ -646,18 +622,17 @@ class TestRemoveApp(unittest.TestCase): test_module.update({"module_name": "RemoveThis", "app_name": "frappe"}) test_module.save() - module_def_linked_doctype = frappe.get_doc({ - "doctype": "DocType", - "name": "Doctype linked with module def", - "module": "RemoveThis", - "custom": 1, - "fields": [{ - "label": "Modulen't", - "fieldname": "notmodule", - "fieldtype": "Link", - "options": "Module Def" - }] - }).insert() + module_def_linked_doctype = frappe.get_doc( + { + "doctype": "DocType", + "name": "Doctype linked with module def", + "module": "RemoveThis", + "custom": 1, + "fields": [ + {"label": "Modulen't", "fieldname": "notmodule", "fieldtype": "Link", "options": "Module Def"} + ], + } + ).insert() doctype_to_link_field_map = _get_module_linked_doctype_field_map() diff --git a/frappe/tests/test_cors.py b/frappe/tests/test_cors.py index 0376531712..723cb7e804 100644 --- a/frappe/tests/test_cors.py +++ b/frappe/tests/test_cors.py @@ -1,55 +1,63 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + from werkzeug.wrappers import Response + +import frappe from frappe.app import process_response -HEADERS = ('Access-Control-Allow-Origin', 'Access-Control-Allow-Credentials', - 'Access-Control-Allow-Methods', 'Access-Control-Allow-Headers') +HEADERS = ( + "Access-Control-Allow-Origin", + "Access-Control-Allow-Credentials", + "Access-Control-Allow-Methods", + "Access-Control-Allow-Headers", +) + class TestCORS(unittest.TestCase): - def make_request_and_test(self, origin='http://example.com', absent=False): - self.origin = origin + def make_request_and_test(self, origin="http://example.com", absent=False): + self.origin = origin - headers = {} - if origin: - headers = {'Origin': origin} + headers = {} + if origin: + headers = {"Origin": origin} - frappe.utils.set_request(headers=headers) + frappe.utils.set_request(headers=headers) - self.response = Response() - process_response(self.response) + self.response = Response() + process_response(self.response) - for header in HEADERS: - if absent: - self.assertNotIn(header, self.response.headers) - else: - if header == 'Access-Control-Allow-Origin': - self.assertEqual(self.response.headers.get(header), self.origin) - else: - self.assertIn(header, self.response.headers) + for header in HEADERS: + if absent: + self.assertNotIn(header, self.response.headers) + else: + if header == "Access-Control-Allow-Origin": + self.assertEqual(self.response.headers.get(header), self.origin) + else: + self.assertIn(header, self.response.headers) - def test_cors_disabled(self): - frappe.conf.allow_cors = None - self.make_request_and_test('http://example.com', True) + def test_cors_disabled(self): + frappe.conf.allow_cors = None + self.make_request_and_test("http://example.com", True) - def test_request_without_origin(self): - frappe.conf.allow_cors = 'http://example.com' - self.make_request_and_test(None, True) + def test_request_without_origin(self): + frappe.conf.allow_cors = "http://example.com" + self.make_request_and_test(None, True) - def test_valid_origin(self): - frappe.conf.allow_cors = 'http://example.com' - self.make_request_and_test() + def test_valid_origin(self): + frappe.conf.allow_cors = "http://example.com" + self.make_request_and_test() - frappe.conf.allow_cors = "*" - self.make_request_and_test() + frappe.conf.allow_cors = "*" + self.make_request_and_test() - frappe.conf.allow_cors = ['http://example.com', 'https://example.com'] - self.make_request_and_test() + frappe.conf.allow_cors = ["http://example.com", "https://example.com"] + self.make_request_and_test() - def test_invalid_origin(self): - frappe.conf.allow_cors = 'http://example1.com' - self.make_request_and_test(absent=True) + def test_invalid_origin(self): + frappe.conf.allow_cors = "http://example1.com" + self.make_request_and_test(absent=True) - frappe.conf.allow_cors = ['http://example1.com', 'https://example.com'] - self.make_request_and_test(absent=True) + frappe.conf.allow_cors = ["http://example1.com", "https://example.com"] + self.make_request_and_test(absent=True) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 33a5006161..624f346716 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -14,7 +14,7 @@ from frappe.database.database import Database from frappe.query_builder import Field from frappe.query_builder.functions import Concat_ws from frappe.tests.test_query_builder import db_type_is, run_only_if -from frappe.utils import add_days, now, random_string, cint +from frappe.utils import add_days, cint, now, random_string from frappe.utils.testutils import clear_custom_fields @@ -49,11 +49,15 @@ class TestDB(unittest.TestCase): order_by=None, ), ) - self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], - frappe.db.get_value("User", {"name": [">", "s"]})) + self.assertEqual( + frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], + frappe.db.get_value("User", {"name": [">", "s"]}), + ) - self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0], - frappe.db.get_value("User", {"name": [">=", "t"]})) + self.assertEqual( + frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0], + frappe.db.get_value("User", {"name": [">=", "t"]}), + ) self.assertEqual( frappe.db.get_values( "User", @@ -79,9 +83,7 @@ class TestDB(unittest.TestCase): ) self.assertEqual( frappe.db.sql("select email from tabUser where name='Administrator' order by modified DESC"), - frappe.db.get_values( - "User", filters=[["name", "=", "Administrator"]], fieldname="email" - ), + frappe.db.get_values("User", filters=[["name", "=", "Administrator"]], fieldname="email"), ) def test_get_value_limits(self): @@ -96,8 +98,9 @@ class TestDB(unittest.TestCase): self.assertGreaterEqual(2, cint(frappe.db._cursor.rowcount)) # without limits length == count - self.assertEqual(len(frappe.db.get_values("User", filters=filter)), - frappe.db.count("User", filter)) + self.assertEqual( + len(frappe.db.get_values("User", filters=filter)), frappe.db.count("User", filter) + ) frappe.db.get_value("User", filters=filter) self.assertGreaterEqual(1, cint(frappe.db._cursor.rowcount)) @@ -109,7 +112,7 @@ class TestDB(unittest.TestCase): frappe.db.escape("香港濟生堂製藥有限公司 - IT".encode("utf-8")) def test_get_single_value(self): - #setup + # setup values_dict = { "Float": 1.5, "Int": 1, @@ -118,76 +121,150 @@ class TestDB(unittest.TestCase): "Data": "Test", "Date": datetime.datetime.now().date(), "Datetime": datetime.datetime.now(), - "Time": datetime.timedelta(hours=9, minutes=45, seconds=10) + "Time": datetime.timedelta(hours=9, minutes=45, seconds=10), } - test_inputs = [{ - "fieldtype": fieldtype, - "value": value} for fieldtype, value in values_dict.items()] + test_inputs = [ + {"fieldtype": fieldtype, "value": value} for fieldtype, value in values_dict.items() + ] for fieldtype in values_dict.keys(): - create_custom_field("Print Settings", { - "fieldname": f"test_{fieldtype.lower()}", - "label": f"Test {fieldtype}", - "fieldtype": fieldtype, - }) + create_custom_field( + "Print Settings", + { + "fieldname": f"test_{fieldtype.lower()}", + "label": f"Test {fieldtype}", + "fieldtype": fieldtype, + }, + ) - #test + # test for inp in test_inputs: fieldname = f"test_{inp['fieldtype'].lower()}" frappe.db.set_value("Print Settings", "Print Settings", fieldname, inp["value"]) self.assertEqual(frappe.db.get_single_value("Print Settings", fieldname), inp["value"]) - #teardown + # teardown clear_custom_fields("Print Settings") def test_log_touched_tables(self): frappe.flags.in_migrate = True frappe.flags.touched_tables = set() - frappe.db.set_value('System Settings', 'System Settings', 'backup_limit', 5) - self.assertIn('tabSingles', frappe.flags.touched_tables) + frappe.db.set_value("System Settings", "System Settings", "backup_limit", 5) + self.assertIn("tabSingles", frappe.flags.touched_tables) frappe.flags.touched_tables = set() - todo = frappe.get_doc({'doctype': 'ToDo', 'description': 'Random Description'}) + todo = frappe.get_doc({"doctype": "ToDo", "description": "Random Description"}) todo.save() - self.assertIn('tabToDo', frappe.flags.touched_tables) + self.assertIn("tabToDo", frappe.flags.touched_tables) frappe.flags.touched_tables = set() todo.description = "Another Description" todo.save() - self.assertIn('tabToDo', frappe.flags.touched_tables) + self.assertIn("tabToDo", frappe.flags.touched_tables) if frappe.db.db_type != "postgres": frappe.flags.touched_tables = set() frappe.db.sql("UPDATE tabToDo SET description = 'Updated Description'") - self.assertNotIn('tabToDo SET', frappe.flags.touched_tables) - self.assertIn('tabToDo', frappe.flags.touched_tables) + self.assertNotIn("tabToDo SET", frappe.flags.touched_tables) + self.assertIn("tabToDo", frappe.flags.touched_tables) frappe.flags.touched_tables = set() todo.delete() - self.assertIn('tabToDo', frappe.flags.touched_tables) + self.assertIn("tabToDo", frappe.flags.touched_tables) frappe.flags.touched_tables = set() - create_custom_field('ToDo', {'label': 'ToDo Custom Field'}) + create_custom_field("ToDo", {"label": "ToDo Custom Field"}) - self.assertIn('tabToDo', frappe.flags.touched_tables) - self.assertIn('tabCustom Field', frappe.flags.touched_tables) + self.assertIn("tabToDo", frappe.flags.touched_tables) + self.assertIn("tabCustom Field", frappe.flags.touched_tables) frappe.flags.in_migrate = False frappe.flags.touched_tables.clear() - def test_db_keywords_as_fields(self): """Tests if DB keywords work as docfield names. If they're wrapped with grave accents.""" # Using random.choices, picked out a list of 40 keywords for testing all_keywords = { - "mariadb": ["CHARACTER", "DELAYED", "LINES", "EXISTS", "YEAR_MONTH", "LOCALTIME", "BOTH", "MEDIUMINT", - "LEFT", "BINARY", "DEFAULT", "KILL", "WRITE", "SQL_SMALL_RESULT", "CURRENT_TIME", "CROSS", "INHERITS", - "SELECT", "TABLE", "ALTER", "CURRENT_TIMESTAMP", "XOR", "CASE", "ALL", "WHERE", "INT", "TO", "SOME", - "DAY_MINUTE", "ERRORS", "OPTIMIZE", "REPLACE", "HIGH_PRIORITY", "VARBINARY", "HELP", "IS", - "CHAR", "DESCRIBE", "KEY"], - "postgres": ["WORK", "LANCOMPILER", "REAL", "HAVING", "REPEATABLE", "DATA", "USING", "BIT", "DEALLOCATE", - "SERIALIZABLE", "CURSOR", "INHERITS", "ARRAY", "TRUE", "IGNORE", "PARAMETER_MODE", "ROW", "CHECKPOINT", - "SHOW", "BY", "SIZE", "SCALE", "UNENCRYPTED", "WITH", "AND", "CONVERT", "FIRST", "SCOPE", "WRITE", "INTERVAL", - "CHARACTER_SET_SCHEMA", "ADD", "SCROLL", "NULL", "WHEN", "TRANSACTION_ACTIVE", - "INT", "FORTRAN", "STABLE"] + "mariadb": [ + "CHARACTER", + "DELAYED", + "LINES", + "EXISTS", + "YEAR_MONTH", + "LOCALTIME", + "BOTH", + "MEDIUMINT", + "LEFT", + "BINARY", + "DEFAULT", + "KILL", + "WRITE", + "SQL_SMALL_RESULT", + "CURRENT_TIME", + "CROSS", + "INHERITS", + "SELECT", + "TABLE", + "ALTER", + "CURRENT_TIMESTAMP", + "XOR", + "CASE", + "ALL", + "WHERE", + "INT", + "TO", + "SOME", + "DAY_MINUTE", + "ERRORS", + "OPTIMIZE", + "REPLACE", + "HIGH_PRIORITY", + "VARBINARY", + "HELP", + "IS", + "CHAR", + "DESCRIBE", + "KEY", + ], + "postgres": [ + "WORK", + "LANCOMPILER", + "REAL", + "HAVING", + "REPEATABLE", + "DATA", + "USING", + "BIT", + "DEALLOCATE", + "SERIALIZABLE", + "CURSOR", + "INHERITS", + "ARRAY", + "TRUE", + "IGNORE", + "PARAMETER_MODE", + "ROW", + "CHECKPOINT", + "SHOW", + "BY", + "SIZE", + "SCALE", + "UNENCRYPTED", + "WITH", + "AND", + "CONVERT", + "FIRST", + "SCOPE", + "WRITE", + "INTERVAL", + "CHARACTER_SET_SCHEMA", + "ADD", + "SCROLL", + "NULL", + "WHEN", + "TRANSACTION_ACTIVE", + "INT", + "FORTRAN", + "STABLE", + ], } created_docs = [] @@ -197,11 +274,14 @@ class TestDB(unittest.TestCase): test_doctype = "ToDo" def add_custom_field(field): - create_custom_field(test_doctype, { - "fieldname": field.lower(), - "label": field.title(), - "fieldtype": 'Data', - }) + create_custom_field( + test_doctype, + { + "fieldname": field.lower(), + "label": field.title(), + "fieldtype": "Data", + }, + ) # Create custom fields for test_doctype for field in fields: @@ -219,30 +299,40 @@ class TestDB(unittest.TestCase): random_value = random_string(20) # Testing read - self.assertEqual(list(frappe.get_all("ToDo", fields=[random_field], limit=1)[0])[0], random_field) - self.assertEqual(list(frappe.get_all("ToDo", fields=[f"`{random_field}` as total"], limit=1)[0])[0], "total") + self.assertEqual( + list(frappe.get_all("ToDo", fields=[random_field], limit=1)[0])[0], random_field + ) + self.assertEqual( + list(frappe.get_all("ToDo", fields=[f"`{random_field}` as total"], limit=1)[0])[0], "total" + ) # Testing read for distinct and sql functions - self.assertEqual(list( - frappe.get_all("ToDo", - fields=[f"`{random_field}` as total"], - distinct=True, - limit=1, - )[0] - )[0], "total") - self.assertEqual(list( - frappe.get_all("ToDo", - fields=[f"`{random_field}`"], - distinct=True, - limit=1, - )[0] - )[0], random_field) - self.assertEqual(list( - frappe.get_all("ToDo", - fields=[f"count(`{random_field}`)"], - limit=1 - )[0] - )[0], "count" if frappe.conf.db_type == "postgres" else f"count(`{random_field}`)") + self.assertEqual( + list( + frappe.get_all( + "ToDo", + fields=[f"`{random_field}` as total"], + distinct=True, + limit=1, + )[0] + )[0], + "total", + ) + self.assertEqual( + list( + frappe.get_all( + "ToDo", + fields=[f"`{random_field}`"], + distinct=True, + limit=1, + )[0] + )[0], + random_field, + ) + self.assertEqual( + list(frappe.get_all("ToDo", fields=[f"count(`{random_field}`)"], limit=1)[0])[0], + "count" if frappe.conf.db_type == "postgres" else f"count(`{random_field}`)", + ) # Testing update frappe.db.set_value(test_doctype, random_doc, random_field, random_value) @@ -302,6 +392,7 @@ class TestDB(unittest.TestCase): def test_transaction_writes_error(self): from frappe.database.database import Database + frappe.db.rollback() frappe.db.MAX_WRITES_PER_TRANSACTION = 1 @@ -320,11 +411,14 @@ class TestDB(unittest.TestCase): self.assertEqual(1, frappe.db.transaction_writes - writes) writes = frappe.db.transaction_writes - frappe.db.sql(""" + frappe.db.sql( + """ update `tabNote` set content = 'abc' where name = %s - """, note.name) + """, + note.name, + ) self.assertEqual(1, frappe.db.transaction_writes - writes) def test_pk_collision_ignoring(self): @@ -333,7 +427,9 @@ class TestDB(unittest.TestCase): frappe.get_doc(doctype="Note", title="duplicate name").insert(ignore_if_duplicate=True) with savepoint(): - self.assertRaises(frappe.DuplicateEntryError, frappe.get_doc(doctype="Note", title="duplicate name").insert) + self.assertRaises( + frappe.DuplicateEntryError, frappe.get_doc(doctype="Note", title="duplicate name").insert + ) # recover transaction to continue other tests raise Exception @@ -345,9 +441,7 @@ class TestDB(unittest.TestCase): filters = {"doctype": dt, "name": ("like", "Admin%")} self.assertEqual(frappe.db.exists(filters), dn) - self.assertEqual( - filters["doctype"], dt - ) # make sure that doctype was not removed from filters + self.assertEqual(filters["doctype"], dt) # make sure that doctype was not removed from filters self.assertEqual(frappe.db.exists(dt, [["name", "=", dn]]), dn) @@ -420,7 +514,9 @@ class TestDBSetValue(unittest.TestCase): value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") changed_value = not value - frappe.db.set_value("System Settings", "System Settings", "deny_multiple_sessions", changed_value) + frappe.db.set_value( + "System Settings", "System Settings", "deny_multiple_sessions", changed_value + ) current_value = frappe.db.get_single_value("System Settings", "deny_multiple_sessions") self.assertEqual(current_value, changed_value) @@ -442,40 +538,47 @@ class TestDBSetValue(unittest.TestCase): def test_update_single_row_multiple_columns(self): description, status = "Upated by test_update_single_row_multiple_columns", "Closed" - frappe.db.set_value("ToDo", self.todo1.name, { - "description": description, - "status": status, - }, update_modified=False) + frappe.db.set_value( + "ToDo", + self.todo1.name, + { + "description": description, + "status": status, + }, + update_modified=False, + ) - updated_desciption, updated_status = frappe.db.get_value("ToDo", - filters={"name": self.todo1.name}, - fieldname=["description", "status"] + updated_desciption, updated_status = frappe.db.get_value( + "ToDo", filters={"name": self.todo1.name}, fieldname=["description", "status"] ) self.assertEqual(description, updated_desciption) self.assertEqual(status, updated_status) def test_update_multiple_rows_single_column(self): - frappe.db.set_value("ToDo", {"description": ("like", "%test_set_value%")}, "description", "change 2") + frappe.db.set_value( + "ToDo", {"description": ("like", "%test_set_value%")}, "description", "change 2" + ) self.assertEqual(frappe.db.get_value("ToDo", self.todo1.name, "description"), "change 2") self.assertEqual(frappe.db.get_value("ToDo", self.todo2.name, "description"), "change 2") def test_update_multiple_rows_multiple_columns(self): - todos_to_update = frappe.get_all("ToDo", filters={ - "description": ("like", "%test_set_value%"), - "status": ("!=", "Closed") - }, pluck="name") + todos_to_update = frappe.get_all( + "ToDo", + filters={"description": ("like", "%test_set_value%"), "status": ("!=", "Closed")}, + pluck="name", + ) - frappe.db.set_value("ToDo", { - "description": ("like", "%test_set_value%"), - "status": ("!=", "Closed") - }, { - "status": "Closed", - "priority": "High" - }) + frappe.db.set_value( + "ToDo", + {"description": ("like", "%test_set_value%"), "status": ("!=", "Closed")}, + {"status": "Closed", "priority": "High"}, + ) - test_result = frappe.get_all("ToDo", filters={"name": ("in", todos_to_update)}, fields=["status", "priority"]) + test_result = frappe.get_all( + "ToDo", filters={"name": ("in", todos_to_update)}, fields=["status", "priority"] + ) self.assertTrue(all(x for x in test_result if x["status"] == "Closed")) self.assertTrue(all(x for x in test_result if x["priority"] == "High")) @@ -492,10 +595,17 @@ class TestDBSetValue(unittest.TestCase): self.assertEqual(updated_description, frappe.db.get_value("ToDo", todo.name, "description")) self.assertEqual(todo.modified, frappe.db.get_value("ToDo", todo.name, "modified")) - frappe.db.set_value("ToDo", todo.name, "description", "test_set_value change 1", modified=custom_modified, modified_by=custom_modified_by) + frappe.db.set_value( + "ToDo", + todo.name, + "description", + "test_set_value change 1", + modified=custom_modified, + modified_by=custom_modified_by, + ) self.assertTupleEqual( (custom_modified, custom_modified_by), - frappe.db.get_value("ToDo", todo.name, ["modified", "modified_by"]) + frappe.db.get_value("ToDo", todo.name, ["modified", "modified_by"]), ) def test_for_update(self): @@ -506,7 +616,7 @@ class TestDBSetValue(unittest.TestCase): self.todo1.doctype, self.todo1.name, "description", - f"{self.todo1.description}-edit by `test_for_update`" + f"{self.todo1.description}-edit by `test_for_update`", ) first_query = sql_called.call_args_list[0].args[0] second_query = sql_called.call_args_list[1].args[0] @@ -515,6 +625,7 @@ class TestDBSetValue(unittest.TestCase): self.assertTrue("FOR UPDATE" in first_query) if frappe.conf.db_type == "postgres": from frappe.database.postgres.database import modify_query + self.assertTrue(modify_query("UPDATE `tabToDo` SET") in second_query) if frappe.conf.db_type == "mariadb": self.assertTrue("UPDATE `tabToDo` SET" in second_query) @@ -527,13 +638,19 @@ class TestDBSetValue(unittest.TestCase): self.todo2.doctype, self.todo2.name, "description", - f"{self.todo2.description}-edit by `test_cleared_cache`" + f"{self.todo2.description}-edit by `test_cleared_cache`", ) clear_cache.assert_called() def test_update_alias(self): args = (self.todo1.doctype, self.todo1.name, "description", "Updated by `test_update_alias`") - kwargs = {"for_update": False, "modified": None, "modified_by": None, "update_modified": True, "debug": False} + kwargs = { + "for_update": False, + "modified": None, + "modified_by": None, + "update_modified": True, + "debug": False, + } self.assertTrue("return self.set_value(" in inspect.getsource(frappe.db.update)) @@ -579,9 +696,7 @@ class TestDDLCommandsPost(unittest.TestCase): self.test_table_name = new_table_name def test_describe(self) -> None: - self.assertEqual( - [("id",), ("content",)], frappe.db.describe(self.test_table_name) - ) + self.assertEqual([("id",), ("content",)], frappe.db.describe(self.test_table_name)) def test_change_type(self) -> None: frappe.db.change_column_type(self.test_table_name, "id", "varchar(255)") @@ -619,14 +734,16 @@ class TestDDLCommandsPost(unittest.TestCase): query = "select * from `tabtree b` where lft > 13 and rgt <= 16 and name =1.0 and parent = 4134qrsdc and isgroup = 1.00045" self.assertEqual( - "select * from \"tabtree b\" where lft > \'13\' and rgt <= '16' and name = '1' and parent = 4134qrsdc and isgroup = 1.00045", - modify_query(query) + "select * from \"tabtree b\" where lft > '13' and rgt <= '16' and name = '1' and parent = 4134qrsdc and isgroup = 1.00045", + modify_query(query), ) - query = "select locate(\".io\", \"frappe.io\"), locate(\"3\", cast(3 as varchar)), locate(\"3\", 3::varchar)" + query = ( + 'select locate(".io", "frappe.io"), locate("3", cast(3 as varchar)), locate("3", 3::varchar)' + ) self.assertEqual( - "select strpos( \"frappe.io\", \".io\"), strpos( cast(3 as varchar), \"3\"), strpos( 3::varchar, \"3\")", - modify_query(query) + 'select strpos( "frappe.io", ".io"), strpos( cast(3 as varchar), "3"), strpos( 3::varchar, "3")', + modify_query(query), ) @run_only_if(db_type_is.POSTGRES) @@ -635,12 +752,9 @@ class TestDDLCommandsPost(unittest.TestCase): self.assertEqual( {"abcd": "23", "efgh": "23", "ijkl": 23.0345, "mnop": "wow"}, - modify_values({"abcd": 23, "efgh": 23.0, "ijkl": 23.0345, "mnop": "wow"}) - ) - self.assertEqual( - ["23", "23", 23.00004345, "wow"], - modify_values((23, 23.0, 23.00004345, "wow")) + modify_values({"abcd": 23, "efgh": 23.0, "ijkl": 23.0345, "mnop": "wow"}), ) + self.assertEqual(["23", "23", 23.00004345, "wow"], modify_values((23, 23.0, 23.00004345, "wow"))) def test_sequence_table_creation(self): from frappe.core.doctype.doctype.test_doctype import new_doctype @@ -649,13 +763,17 @@ class TestDDLCommandsPost(unittest.TestCase): if frappe.db.db_type == "postgres": self.assertTrue( - frappe.db.sql("""select sequence_name FROM information_schema.sequences - where sequence_name ilike 'autoinc_dt_seq_test%'""")[0][0] + frappe.db.sql( + """select sequence_name FROM information_schema.sequences + where sequence_name ilike 'autoinc_dt_seq_test%'""" + )[0][0] ) else: self.assertTrue( - frappe.db.sql("""select data_type FROM information_schema.tables - where table_type = 'SEQUENCE' and table_name like 'autoinc_dt_seq_test%'""")[0][0] + frappe.db.sql( + """select data_type FROM information_schema.tables + where table_type = 'SEQUENCE' and table_name like 'autoinc_dt_seq_test%'""" + )[0][0] ) dt.delete(ignore_permissions=True) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 47a5e775ee..90b047b3cd 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -1,32 +1,31 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import datetime import unittest -from frappe.model.db_query import DatabaseQuery -from frappe.desk.reportview import get_filters_cond -from frappe.query_builder import Column - -from frappe.core.page.permission_manager.permission_manager import update, reset, add -from frappe.permissions import add_user_permission, clear_user_permissions_for_doctype +import frappe +from frappe.core.page.permission_manager.permission_manager import add, reset, update from frappe.custom.doctype.property_setter.property_setter import make_property_setter +from frappe.desk.reportview import get_filters_cond from frappe.handler import execute_cmd - +from frappe.model.db_query import DatabaseQuery +from frappe.permissions import add_user_permission, clear_user_permissions_for_doctype +from frappe.query_builder import Column from frappe.utils.testutils import add_custom_field, clear_custom_fields -test_dependencies = ['User', 'Blog Post', 'Blog Category', 'Blogger'] +test_dependencies = ["User", "Blog Post", "Blog Category", "Blogger"] + class TestReportview(unittest.TestCase): def setUp(self): frappe.set_user("Administrator") def test_basic(self): - self.assertTrue({"name":"DocType"} in DatabaseQuery("DocType").execute(limit_page_length=None)) + self.assertTrue({"name": "DocType"} in DatabaseQuery("DocType").execute(limit_page_length=None)) def test_extract_tables(self): db_query = DatabaseQuery("DocType") - add_custom_field("DocType", 'test_tab_field', 'Data') + add_custom_field("DocType", "test_tab_field", "Data") db_query.fields = ["tabNote.creation", "test_tab_field", "tabDocType.test_tab_field"] db_query.extract_tables() @@ -37,14 +36,14 @@ class TestReportview(unittest.TestCase): clear_custom_fields("DocType") def test_build_match_conditions(self): - clear_user_permissions_for_doctype('Blog Post', 'test2@example.com') + clear_user_permissions_for_doctype("Blog Post", "test2@example.com") - test2user = frappe.get_doc('User', 'test2@example.com') - test2user.add_roles('Blogger') - frappe.set_user('test2@example.com') + test2user = frappe.get_doc("User", "test2@example.com") + test2user.add_roles("Blogger") + frappe.set_user("test2@example.com") # this will get match conditions for Blog Post - build_match_conditions = DatabaseQuery('Blog Post').build_match_conditions + build_match_conditions = DatabaseQuery("Blog Post").build_match_conditions # Before any user permission is applied # get as filters @@ -52,68 +51,85 @@ class TestReportview(unittest.TestCase): # get as conditions self.assertEqual(build_match_conditions(as_condition=True), "") - add_user_permission('Blog Post', '-test-blog-post', 'test2@example.com', True) - add_user_permission('Blog Post', '-test-blog-post-1', 'test2@example.com', True) + add_user_permission("Blog Post", "-test-blog-post", "test2@example.com", True) + add_user_permission("Blog Post", "-test-blog-post-1", "test2@example.com", True) # After applying user permission # get as filters - self.assertTrue({'Blog Post': ['-test-blog-post-1', '-test-blog-post']} in build_match_conditions(as_condition=False)) + self.assertTrue( + {"Blog Post": ["-test-blog-post-1", "-test-blog-post"]} + in build_match_conditions(as_condition=False) + ) # get as conditions - self.assertEqual(build_match_conditions(as_condition=True), - """(((ifnull(`tabBlog Post`.`name`, '')='' or `tabBlog Post`.`name` in ('-test-blog-post-1', '-test-blog-post'))))""") + self.assertEqual( + build_match_conditions(as_condition=True), + """(((ifnull(`tabBlog Post`.`name`, '')='' or `tabBlog Post`.`name` in ('-test-blog-post-1', '-test-blog-post'))))""", + ) - frappe.set_user('Administrator') + frappe.set_user("Administrator") def test_fields(self): - self.assertTrue({"name":"DocType", "issingle":0} \ - in DatabaseQuery("DocType").execute(fields=["name", "issingle"], limit_page_length=None)) + self.assertTrue( + {"name": "DocType", "issingle": 0} + in DatabaseQuery("DocType").execute(fields=["name", "issingle"], limit_page_length=None) + ) def test_filters_1(self): - self.assertFalse({"name":"DocType"} \ - in DatabaseQuery("DocType").execute(filters=[["DocType", "name", "like", "J%"]])) + self.assertFalse( + {"name": "DocType"} + in DatabaseQuery("DocType").execute(filters=[["DocType", "name", "like", "J%"]]) + ) def test_filters_2(self): - self.assertFalse({"name":"DocType"} \ - in DatabaseQuery("DocType").execute(filters=[{"name": ["like", "J%"]}])) + self.assertFalse( + {"name": "DocType"} in DatabaseQuery("DocType").execute(filters=[{"name": ["like", "J%"]}]) + ) def test_filters_3(self): - self.assertFalse({"name":"DocType"} \ - in DatabaseQuery("DocType").execute(filters={"name": ["like", "J%"]})) + self.assertFalse( + {"name": "DocType"} in DatabaseQuery("DocType").execute(filters={"name": ["like", "J%"]}) + ) def test_filters_4(self): - self.assertTrue({"name":"DocField"} \ - in DatabaseQuery("DocType").execute(filters={"name": "DocField"})) + self.assertTrue( + {"name": "DocField"} in DatabaseQuery("DocType").execute(filters={"name": "DocField"}) + ) def test_in_not_in_filters(self): self.assertFalse(DatabaseQuery("DocType").execute(filters={"name": ["in", None]})) - self.assertTrue({"name":"DocType"} \ - in DatabaseQuery("DocType").execute(filters={"name": ["not in", None]})) + self.assertTrue( + {"name": "DocType"} in DatabaseQuery("DocType").execute(filters={"name": ["not in", None]}) + ) - for result in [{"name":"DocType"}, {"name":"DocField"}]: - self.assertTrue(result - in DatabaseQuery("DocType").execute(filters={"name": ["in", 'DocType,DocField']})) + for result in [{"name": "DocType"}, {"name": "DocField"}]: + self.assertTrue( + result in DatabaseQuery("DocType").execute(filters={"name": ["in", "DocType,DocField"]}) + ) - for result in [{"name":"DocType"}, {"name":"DocField"}]: - self.assertFalse(result - in DatabaseQuery("DocType").execute(filters={"name": ["not in", 'DocType,DocField']})) + for result in [{"name": "DocType"}, {"name": "DocField"}]: + self.assertFalse( + result in DatabaseQuery("DocType").execute(filters={"name": ["not in", "DocType,DocField"]}) + ) def test_none_filter(self): query = frappe.db.query.get_sql("DocType", fields="name", filters={"restrict_to_domain": None}) - sql = str(query).replace('`', '').replace('"', '') - condition = 'restrict_to_domain IS NULL' + sql = str(query).replace("`", "").replace('"', "") + condition = "restrict_to_domain IS NULL" self.assertIn(condition, sql) def test_or_filters(self): data = DatabaseQuery("DocField").execute( - filters={"parent": "DocType"}, fields=["fieldname", "fieldtype"], - or_filters=[{"fieldtype":"Table"}, {"fieldtype":"Select"}]) + filters={"parent": "DocType"}, + fields=["fieldname", "fieldtype"], + or_filters=[{"fieldtype": "Table"}, {"fieldtype": "Select"}], + ) - self.assertTrue({"fieldtype":"Table", "fieldname":"fields"} in data) - self.assertTrue({"fieldtype":"Select", "fieldname":"document_type"} in data) - self.assertFalse({"fieldtype":"Check", "fieldname":"issingle"} in data) + self.assertTrue({"fieldtype": "Table", "fieldname": "fields"} in data) + self.assertTrue({"fieldtype": "Select", "fieldname": "document_type"} in data) + self.assertFalse({"fieldtype": "Check", "fieldname": "issingle"} in data) def test_between_filters(self): - """ test case to check between filter for date fields """ + """test case to check between filter for date fields""" frappe.db.delete("Event") # create events to test the between operator filter @@ -124,177 +140,269 @@ class TestReportview(unittest.TestCase): event4 = create_event(starts_on="2016-07-08 00:00:01") # if the values are not passed in filters then event should be filter as current datetime - data = DatabaseQuery("Event").execute( - filters={"starts_on": ["between", None]}, fields=["name"]) + data = DatabaseQuery("Event").execute(filters={"starts_on": ["between", None]}, fields=["name"]) - self.assertTrue({ "name": event1.name } not in data) + self.assertTrue({"name": event1.name} not in data) # if both from and to_date values are passed data = DatabaseQuery("Event").execute( - filters={"starts_on": ["between", ["2016-07-06", "2016-07-07"]]}, - fields=["name"]) + filters={"starts_on": ["between", ["2016-07-06", "2016-07-07"]]}, fields=["name"] + ) - self.assertTrue({ "name": event2.name } in data) - self.assertTrue({ "name": event3.name } in data) - self.assertTrue({ "name": event1.name } not in data) - self.assertTrue({ "name": event4.name } not in data) + self.assertTrue({"name": event2.name} in data) + self.assertTrue({"name": event3.name} in data) + self.assertTrue({"name": event1.name} not in data) + self.assertTrue({"name": event4.name} not in data) # if only one value is passed in the filter data = DatabaseQuery("Event").execute( - filters={"starts_on": ["between", ["2016-07-07"]]}, - fields=["name"]) + filters={"starts_on": ["between", ["2016-07-07"]]}, fields=["name"] + ) - self.assertTrue({ "name": event3.name } in data) - self.assertTrue({ "name": event4.name } in data) - self.assertTrue({ "name": todays_event.name } in data) - self.assertTrue({ "name": event1.name } not in data) - self.assertTrue({ "name": event2.name } not in data) + self.assertTrue({"name": event3.name} in data) + self.assertTrue({"name": event4.name} in data) + self.assertTrue({"name": todays_event.name} in data) + self.assertTrue({"name": event1.name} not in data) + self.assertTrue({"name": event2.name} not in data) # test between is formatted for creation column data = DatabaseQuery("Event").execute( - filters={"creation": ["between", ["2016-07-06", "2016-07-07"]]}, - fields=["name"]) + filters={"creation": ["between", ["2016-07-06", "2016-07-07"]]}, fields=["name"] + ) def test_ignore_permissions_for_get_filters_cond(self): - frappe.set_user('test2@example.com') - self.assertRaises(frappe.PermissionError, get_filters_cond, 'DocType', dict(istable=1), []) - self.assertTrue(get_filters_cond('DocType', dict(istable=1), [], ignore_permissions=True)) - frappe.set_user('Administrator') + frappe.set_user("test2@example.com") + self.assertRaises(frappe.PermissionError, get_filters_cond, "DocType", dict(istable=1), []) + self.assertTrue(get_filters_cond("DocType", dict(istable=1), [], ignore_permissions=True)) + frappe.set_user("Administrator") def test_query_fields_sanitizer(self): - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "issingle, version()"], limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "issingle, version()"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, fields=["name", "issingle, IF(issingle=1, (select name from tabUser), count(name))"], - limit_start=0, limit_page_length=1) + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, fields=["name", "issingle, (select count(*) from tabSessions)"], - limit_start=0, limit_page_length=1) + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, fields=["name", "issingle, SELECT LOCATE('', `tabUser`.`user`) AS user;"], - limit_start=0, limit_page_length=1) + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, fields=["name", "issingle, IF(issingle=1, (SELECT name from tabUser), count(*))"], - limit_start=0, limit_page_length=1) + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "issingle ''"],limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "issingle ''"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "issingle,'"],limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "issingle,'"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "select * from tabSessions"],limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "select * from tabSessions"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "issingle from --"],limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "issingle from --"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "issingle from tabDocType order by 2 --"],limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "issingle from tabDocType order by 2 --"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name", "1' UNION SELECT * FROM __Auth --"],limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name", "1' UNION SELECT * FROM __Auth --"], + limit_start=0, + limit_page_length=1, + ) - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["@@version"], limit_start=0, limit_page_length=1) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["@@version"], + limit_start=0, + limit_page_length=1, + ) - data = DatabaseQuery("DocType").execute(fields=["count(`name`) as count"], - limit_start=0, limit_page_length=1) - self.assertTrue('count' in data[0]) + data = DatabaseQuery("DocType").execute( + fields=["count(`name`) as count"], limit_start=0, limit_page_length=1 + ) + self.assertTrue("count" in data[0]) - data = DatabaseQuery("DocType").execute(fields=["name", "issingle", "locate('', name) as _relevance"], - limit_start=0, limit_page_length=1) - self.assertTrue('_relevance' in data[0]) + data = DatabaseQuery("DocType").execute( + fields=["name", "issingle", "locate('', name) as _relevance"], + limit_start=0, + limit_page_length=1, + ) + self.assertTrue("_relevance" in data[0]) - data = DatabaseQuery("DocType").execute(fields=["name", "issingle", "date(creation) as creation"], - limit_start=0, limit_page_length=1) - self.assertTrue('creation' in data[0]) + data = DatabaseQuery("DocType").execute( + fields=["name", "issingle", "date(creation) as creation"], limit_start=0, limit_page_length=1 + ) + self.assertTrue("creation" in data[0]) - if frappe.db.db_type != 'postgres': + if frappe.db.db_type != "postgres": # datediff function does not exist in postgres - data = DatabaseQuery("DocType").execute(fields=["name", "issingle", - "datediff(modified, creation) as date_diff"], limit_start=0, limit_page_length=1) - self.assertTrue('date_diff' in data[0]) + data = DatabaseQuery("DocType").execute( + fields=["name", "issingle", "datediff(modified, creation) as date_diff"], + limit_start=0, + limit_page_length=1, + ) + self.assertTrue("date_diff" in data[0]) def test_nested_permission(self): - frappe.set_user('Administrator') + frappe.set_user("Administrator") create_nested_doctype() create_nested_doctype_records() - clear_user_permissions_for_doctype('Nested DocType') + clear_user_permissions_for_doctype("Nested DocType") # user permission for only one root folder - add_user_permission('Nested DocType', 'Level 1 A', 'test2@example.com') + add_user_permission("Nested DocType", "Level 1 A", "test2@example.com") from frappe.core.page.permission_manager.permission_manager import update + # to avoid if_owner filter - update('Nested DocType', 'All', 0, 'if_owner', 0) + update("Nested DocType", "All", 0, "if_owner", 0) - frappe.set_user('test2@example.com') - data = DatabaseQuery('Nested DocType').execute() + frappe.set_user("test2@example.com") + data = DatabaseQuery("Nested DocType").execute() # children of root folder (for which we added user permission) should be accessible - self.assertTrue({'name': 'Level 2 A'} in data) - self.assertTrue({'name': 'Level 2 A'} in data) + self.assertTrue({"name": "Level 2 A"} in data) + self.assertTrue({"name": "Level 2 A"} in data) # other folders should not be accessible - self.assertFalse({'name': 'Level 1 B'} in data) - self.assertFalse({'name': 'Level 2 B'} in data) - update('Nested DocType', 'All', 0, 'if_owner', 1) - frappe.set_user('Administrator') + self.assertFalse({"name": "Level 1 B"} in data) + self.assertFalse({"name": "Level 2 B"} in data) + update("Nested DocType", "All", 0, "if_owner", 1) + frappe.set_user("Administrator") def test_filter_sanitizer(self): - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name"], filters={'istable,': 1}, limit_start=0, limit_page_length=1) - - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name"], filters={'editable_grid,': 1}, or_filters={'istable,': 1}, - limit_start=0, limit_page_length=1) - - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name"], filters={'editable_grid,': 1}, - or_filters=[['DocType', 'istable,', '=', 1]], - limit_start=0, limit_page_length=1) - - self.assertRaises(frappe.DataError, DatabaseQuery("DocType").execute, - fields=["name"], filters={'editable_grid,': 1}, - or_filters=[['DocType', 'istable', '=', 1], ['DocType', 'beta and 1=1', '=', 0]], - limit_start=0, limit_page_length=1) - - out = DatabaseQuery("DocType").execute(fields=["name"], - filters={'editable_grid': 1, 'module': 'Core'}, - or_filters=[['DocType', 'istable', '=', 1]], order_by='creation') - self.assertTrue('DocField' in [d['name'] for d in out]) - - out = DatabaseQuery("DocType").execute(fields=["name"], - filters={'issingle': 1}, or_filters=[['DocType', 'module', '=', 'Core']], - order_by='creation') - self.assertTrue('Role Permission for Page and Report' in [d['name'] for d in out]) - - out = DatabaseQuery("DocType").execute(fields=["name"], - filters={'track_changes': 1, 'module': 'Core'}, - order_by='creation') - self.assertTrue('File' in [d['name'] for d in out]) - - out = DatabaseQuery("DocType").execute(fields=["name"], - filters=[ - ['DocType', 'ifnull(track_changes, 0)', '=', 0], - ['DocType', 'module', '=', 'Core'] - ], order_by='creation') - self.assertTrue('DefaultValue' in [d['name'] for d in out]) + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name"], + filters={"istable,": 1}, + limit_start=0, + limit_page_length=1, + ) + + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name"], + filters={"editable_grid,": 1}, + or_filters={"istable,": 1}, + limit_start=0, + limit_page_length=1, + ) + + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name"], + filters={"editable_grid,": 1}, + or_filters=[["DocType", "istable,", "=", 1]], + limit_start=0, + limit_page_length=1, + ) + + self.assertRaises( + frappe.DataError, + DatabaseQuery("DocType").execute, + fields=["name"], + filters={"editable_grid,": 1}, + or_filters=[["DocType", "istable", "=", 1], ["DocType", "beta and 1=1", "=", 0]], + limit_start=0, + limit_page_length=1, + ) + + out = DatabaseQuery("DocType").execute( + fields=["name"], + filters={"editable_grid": 1, "module": "Core"}, + or_filters=[["DocType", "istable", "=", 1]], + order_by="creation", + ) + self.assertTrue("DocField" in [d["name"] for d in out]) + + out = DatabaseQuery("DocType").execute( + fields=["name"], + filters={"issingle": 1}, + or_filters=[["DocType", "module", "=", "Core"]], + order_by="creation", + ) + self.assertTrue("Role Permission for Page and Report" in [d["name"] for d in out]) + + out = DatabaseQuery("DocType").execute( + fields=["name"], filters={"track_changes": 1, "module": "Core"}, order_by="creation" + ) + self.assertTrue("File" in [d["name"] for d in out]) + + out = DatabaseQuery("DocType").execute( + fields=["name"], + filters=[["DocType", "ifnull(track_changes, 0)", "=", 0], ["DocType", "module", "=", "Core"]], + order_by="creation", + ) + self.assertTrue("DefaultValue" in [d["name"] for d in out]) def test_of_not_of_descendant_ancestors(self): - frappe.set_user('Administrator') - clear_user_permissions_for_doctype('Nested DocType') + frappe.set_user("Administrator") + clear_user_permissions_for_doctype("Nested DocType") # in descendants filter - data = frappe.get_all('Nested DocType', {'name': ('descendants of', 'Level 2 A')}) + data = frappe.get_all("Nested DocType", {"name": ("descendants of", "Level 2 A")}) self.assertTrue({"name": "Level 3 A"} in data) - data = frappe.get_all('Nested DocType', {'name': ('descendants of', 'Level 1 A')}) + data = frappe.get_all("Nested DocType", {"name": ("descendants of", "Level 1 A")}) self.assertTrue({"name": "Level 3 A"} in data) self.assertTrue({"name": "Level 2 A"} in data) self.assertFalse({"name": "Level 2 B"} in data) @@ -303,7 +411,7 @@ class TestReportview(unittest.TestCase): self.assertFalse({"name": "Root"} in data) # in ancestors of filter - data = frappe.get_all('Nested DocType', {'name': ('ancestors of', 'Level 2 A')}) + data = frappe.get_all("Nested DocType", {"name": ("ancestors of", "Level 2 A")}) self.assertFalse({"name": "Level 3 A"} in data) self.assertFalse({"name": "Level 2 A"} in data) self.assertFalse({"name": "Level 2 B"} in data) @@ -311,7 +419,7 @@ class TestReportview(unittest.TestCase): self.assertTrue({"name": "Level 1 A"} in data) self.assertTrue({"name": "Root"} in data) - data = frappe.get_all('Nested DocType', {'name': ('ancestors of', 'Level 1 A')}) + data = frappe.get_all("Nested DocType", {"name": ("ancestors of", "Level 1 A")}) self.assertFalse({"name": "Level 3 A"} in data) self.assertFalse({"name": "Level 2 A"} in data) self.assertFalse({"name": "Level 2 B"} in data) @@ -320,14 +428,14 @@ class TestReportview(unittest.TestCase): self.assertTrue({"name": "Root"} in data) # not descendants filter - data = frappe.get_all('Nested DocType', {'name': ('not descendants of', 'Level 2 A')}) + data = frappe.get_all("Nested DocType", {"name": ("not descendants of", "Level 2 A")}) self.assertFalse({"name": "Level 3 A"} in data) self.assertTrue({"name": "Level 2 A"} in data) self.assertTrue({"name": "Level 2 B"} in data) self.assertTrue({"name": "Level 1 A"} in data) self.assertTrue({"name": "Root"} in data) - data = frappe.get_all('Nested DocType', {'name': ('not descendants of', 'Level 1 A')}) + data = frappe.get_all("Nested DocType", {"name": ("not descendants of", "Level 1 A")}) self.assertFalse({"name": "Level 3 A"} in data) self.assertFalse({"name": "Level 2 A"} in data) self.assertTrue({"name": "Level 2 B"} in data) @@ -336,7 +444,7 @@ class TestReportview(unittest.TestCase): self.assertTrue({"name": "Root"} in data) # not ancestors of filter - data = frappe.get_all('Nested DocType', {'name': ('not ancestors of', 'Level 2 A')}) + data = frappe.get_all("Nested DocType", {"name": ("not ancestors of", "Level 2 A")}) self.assertTrue({"name": "Level 3 A"} in data) self.assertTrue({"name": "Level 2 A"} in data) self.assertTrue({"name": "Level 2 B"} in data) @@ -344,7 +452,7 @@ class TestReportview(unittest.TestCase): self.assertTrue({"name": "Level 1 A"} not in data) self.assertTrue({"name": "Root"} not in data) - data = frappe.get_all('Nested DocType', {'name': ('not ancestors of', 'Level 1 A')}) + data = frappe.get_all("Nested DocType", {"name": ("not ancestors of", "Level 1 A")}) self.assertTrue({"name": "Level 3 A"} in data) self.assertTrue({"name": "Level 2 A"} in data) self.assertTrue({"name": "Level 2 B"} in data) @@ -352,29 +460,33 @@ class TestReportview(unittest.TestCase): self.assertTrue({"name": "Level 1 A"} in data) self.assertFalse({"name": "Root"} in data) - data = frappe.get_all('Nested DocType', {'name': ('ancestors of', 'Root')}) + data = frappe.get_all("Nested DocType", {"name": ("ancestors of", "Root")}) self.assertTrue(len(data) == 0) - self.assertTrue(len(frappe.get_all('Nested DocType', {'name': ('not ancestors of', 'Root')})) == len(frappe.get_all('Nested DocType'))) + self.assertTrue( + len(frappe.get_all("Nested DocType", {"name": ("not ancestors of", "Root")})) + == len(frappe.get_all("Nested DocType")) + ) def test_is_set_is_not_set(self): - res = DatabaseQuery('DocType').execute(filters={'autoname': ['is', 'not set']}) - self.assertTrue({'name': 'Integration Request'} in res) - self.assertTrue({'name': 'User'} in res) - self.assertFalse({'name': 'Blogger'} in res) + res = DatabaseQuery("DocType").execute(filters={"autoname": ["is", "not set"]}) + self.assertTrue({"name": "Integration Request"} in res) + self.assertTrue({"name": "User"} in res) + self.assertFalse({"name": "Blogger"} in res) - res = DatabaseQuery('DocType').execute(filters={'autoname': ['is', 'set']}) - self.assertTrue({'name': 'DocField'} in res) - self.assertTrue({'name': 'Prepared Report'} in res) - self.assertFalse({'name': 'Property Setter'} in res) + res = DatabaseQuery("DocType").execute(filters={"autoname": ["is", "set"]}) + self.assertTrue({"name": "DocField"} in res) + self.assertTrue({"name": "Prepared Report"} in res) + self.assertFalse({"name": "Property Setter"} in res) def test_set_field_tables(self): # Tests _in_standard_sql_methods method in test_set_field_tables # The following query will break if the above method is broken - data = frappe.db.get_list("Web Form", - filters=[['Web Form Field', 'reqd', '=', 1]], - group_by='amount_field', - fields=['count(*) as count', '`amount_field` as name'], - order_by='count desc', + data = frappe.db.get_list( + "Web Form", + filters=[["Web Form Field", "reqd", "=", 1]], + group_by="amount_field", + fields=["count(*) as count", "`amount_field` as name"], + order_by="count desc", limit=50, ) @@ -389,7 +501,8 @@ class TestReportview(unittest.TestCase): def test_prepare_select_args(self): # frappe.get_all inserts modified field into order_by clause # test to make sure this is inserted into select field when postgres - doctypes = frappe.get_all("DocType", + doctypes = frappe.get_all( + "DocType", filters={"docstatus": 0, "document_type": ("!=", "")}, group_by="document_type", fields=["document_type", "sum(is_submittable) as is_submittable"], @@ -403,8 +516,7 @@ class TestReportview(unittest.TestCase): self.assertTrue(isinstance(doctypes[0][2], datetime.datetime)) def test_column_comparison(self): - """Test DatabaseQuery.execute to test column comparison - """ + """Test DatabaseQuery.execute to test column comparison""" users_unedited = frappe.get_all( "User", filters={"creation": Column("modified")}, @@ -439,18 +551,22 @@ class TestReportview(unittest.TestCase): frappe.local.request = frappe._dict() frappe.local.request.method = "POST" - frappe.local.form_dict = frappe._dict({ - "doctype": "Blog Post", - "fields": ["published", "title", "`tabTest Child`.`test_field`"], - }) + frappe.local.form_dict = frappe._dict( + { + "doctype": "Blog Post", + "fields": ["published", "title", "`tabTest Child`.`test_field`"], + } + ) # even if * is passed, fields which are not accessible should be filtered out response = execute_cmd("frappe.desk.reportview.get") self.assertListEqual(response["keys"], ["title"]) - frappe.local.form_dict = frappe._dict({ - "doctype": "Blog Post", - "fields": ["*"], - }) + frappe.local.form_dict = frappe._dict( + { + "doctype": "Blog Post", + "fields": ["*"], + } + ) response = execute_cmd("frappe.desk.reportview.get") self.assertNotIn("published", response["keys"]) @@ -462,13 +578,15 @@ class TestReportview(unittest.TestCase): frappe.set_user("Administrator") # Admin should be able to see access all fields - frappe.local.form_dict = frappe._dict({ - "doctype": "Blog Post", - "fields": ["published", "title", "`tabTest Child`.`test_field`"], - }) + frappe.local.form_dict = frappe._dict( + { + "doctype": "Blog Post", + "fields": ["published", "title", "`tabTest Child`.`test_field`"], + } + ) response = execute_cmd("frappe.desk.reportview.get") - self.assertListEqual(response["keys"], ['published', 'title', 'test_field']) + self.assertListEqual(response["keys"], ["published", "title", "test_field"]) # reset user roles user.remove_roles("Blogger", "Website Manager") @@ -476,23 +594,27 @@ class TestReportview(unittest.TestCase): def test_reportview_get_aggregation(self): # test aggregation based on child table field - frappe.local.form_dict = frappe._dict({ - "doctype": "DocType", - "fields": """["`tabDocField`.`label` as field_label","`tabDocField`.`name` as field_name"]""", - "filters": "[]", - "order_by": "_aggregate_column desc", - "start": 0, - "page_length": 20, - "view": "Report", - "with_comment_count": 0, - "group_by": "field_label, field_name", - "aggregate_on_field": "columns", - "aggregate_on_doctype": "DocField", - "aggregate_function": "sum" - }) + frappe.local.form_dict = frappe._dict( + { + "doctype": "DocType", + "fields": """["`tabDocField`.`label` as field_label","`tabDocField`.`name` as field_name"]""", + "filters": "[]", + "order_by": "_aggregate_column desc", + "start": 0, + "page_length": 20, + "view": "Report", + "with_comment_count": 0, + "group_by": "field_label, field_name", + "aggregate_on_field": "columns", + "aggregate_on_doctype": "DocField", + "aggregate_function": "sum", + } + ) response = execute_cmd("frappe.desk.reportview.get") - self.assertListEqual(response["keys"], ["field_label", "field_name", "_aggregate_column", 'columns']) + self.assertListEqual( + response["keys"], ["field_label", "field_name", "_aggregate_column", "columns"] + ) def test_cast_name(self): from frappe.core.doctype.doctype.test_doctype import new_doctype @@ -502,14 +624,14 @@ class TestReportview(unittest.TestCase): query = DatabaseQuery("autoinc_dt_test").execute( fields=["locate('1', `tabautoinc_dt_test`.`name`)", "`tabautoinc_dt_test`.`name`"], filters={"name": 1}, - run=False + run=False, ) if frappe.db.db_type == "postgres": - self.assertTrue("strpos( cast( \"tabautoinc_dt_test\".\"name\" as varchar), \'1\')" in query) - self.assertTrue("where cast(\"tabautoinc_dt_test\".\"name\" as varchar) = \'1\'" in query) + self.assertTrue('strpos( cast( "tabautoinc_dt_test"."name" as varchar), \'1\')' in query) + self.assertTrue('where cast("tabautoinc_dt_test"."name" as varchar) = \'1\'' in query) else: - self.assertTrue("locate(\'1\', `tabautoinc_dt_test`.`name`)" in query) + self.assertTrue("locate('1', `tabautoinc_dt_test`.`name`)" in query) self.assertTrue("where `tabautoinc_dt_test`.`name` = 1" in query) dt.delete(ignore_permissions=True) @@ -519,18 +641,12 @@ class TestReportview(unittest.TestCase): dt = new_doctype( "dt_with_int_named_fieldname", - fields=[{ - "label": "1field", - "fieldname": "1field", - "fieldtype": "Int" - }] + fields=[{"label": "1field", "fieldname": "1field", "fieldtype": "Int"}], ).insert(ignore_permissions=True) - frappe.get_doc({ - "doctype": "dt_with_int_named_fieldname", - "1field": 10 - }).insert(ignore_permissions=True) - + frappe.get_doc({"doctype": "dt_with_int_named_fieldname", "1field": 10}).insert( + ignore_permissions=True + ) query = DatabaseQuery("dt_with_int_named_fieldname") self.assertTrue(query.execute(filters={"1field": 10})) @@ -543,77 +659,78 @@ class TestReportview(unittest.TestCase): def add_child_table_to_blog_post(): - child_table = frappe.get_doc({ - 'doctype': 'DocType', - 'istable': 1, - 'custom': 1, - 'name': 'Test Child', - 'module': 'Custom', - 'autoname': 'Prompt', - 'fields': [{ - 'fieldname': 'test_field', - 'fieldtype': 'Data', - 'permlevel': 1 - }], - }) + child_table = frappe.get_doc( + { + "doctype": "DocType", + "istable": 1, + "custom": 1, + "name": "Test Child", + "module": "Custom", + "autoname": "Prompt", + "fields": [{"fieldname": "test_field", "fieldtype": "Data", "permlevel": 1}], + } + ) child_table.insert(ignore_permissions=True, ignore_if_duplicate=True) - clear_custom_fields('Blog Post') - add_custom_field('Blog Post', 'child_table', 'Table', child_table.name) + clear_custom_fields("Blog Post") + add_custom_field("Blog Post", "child_table", "Table", child_table.name) + def create_event(subject="_Test Event", starts_on=None): - """ create a test event """ + """create a test event""" from frappe.utils import get_datetime - event = frappe.get_doc({ - "doctype": "Event", - "subject": subject, - "event_type": "Public", - "starts_on": get_datetime(starts_on), - }).insert(ignore_permissions=True) + event = frappe.get_doc( + { + "doctype": "Event", + "subject": subject, + "event_type": "Public", + "starts_on": get_datetime(starts_on), + } + ).insert(ignore_permissions=True) return event + def create_nested_doctype(): - if frappe.db.exists('DocType', 'Nested DocType'): + if frappe.db.exists("DocType", "Nested DocType"): return - frappe.get_doc({ - 'doctype': 'DocType', - 'name': 'Nested DocType', - 'module': 'Custom', - 'is_tree': 1, - 'custom': 1, - 'autoname': 'Prompt', - 'fields': [ - {'label': 'Description', 'fieldname': 'description'} - ], - 'permissions': [ - {'role': 'Blogger'} - ] - }).insert() + frappe.get_doc( + { + "doctype": "DocType", + "name": "Nested DocType", + "module": "Custom", + "is_tree": 1, + "custom": 1, + "autoname": "Prompt", + "fields": [{"label": "Description", "fieldname": "description"}], + "permissions": [{"role": "Blogger"}], + } + ).insert() + def create_nested_doctype_records(): - ''' + """ Create a structure like: - Root - - Level 1 A - - Level 2 A - - Level 3 A - - Level 1 B - - Level 2 B - ''' + - Level 1 A + - Level 2 A + - Level 3 A + - Level 1 B + - Level 2 B + """ records = [ - {'name': 'Root', 'is_group': 1}, - {'name': 'Level 1 A', 'parent_nested_doctype': 'Root', 'is_group': 1}, - {'name': 'Level 2 A', 'parent_nested_doctype': 'Level 1 A', 'is_group': 1}, - {'name': 'Level 3 A', 'parent_nested_doctype': 'Level 2 A'}, - {'name': 'Level 1 B', 'parent_nested_doctype': 'Root', 'is_group': 1}, - {'name': 'Level 2 B', 'parent_nested_doctype': 'Level 1 B'}, + {"name": "Root", "is_group": 1}, + {"name": "Level 1 A", "parent_nested_doctype": "Root", "is_group": 1}, + {"name": "Level 2 A", "parent_nested_doctype": "Level 1 A", "is_group": 1}, + {"name": "Level 3 A", "parent_nested_doctype": "Level 2 A"}, + {"name": "Level 1 B", "parent_nested_doctype": "Root", "is_group": 1}, + {"name": "Level 2 B", "parent_nested_doctype": "Level 1 B"}, ] for r in records: - d = frappe.new_doc('Nested DocType') + d = frappe.new_doc("Nested DocType") d.update(r) d.insert(ignore_permissions=True, ignore_if_duplicate=True) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 66eb05391a..7c615f2df5 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -1,80 +1,82 @@ import unittest -import frappe -from frappe.utils import cstr +import frappe from frappe.core.utils import find from frappe.custom.doctype.property_setter.property_setter import make_property_setter +from frappe.utils import cstr class TestDBUpdate(unittest.TestCase): def test_db_update(self): - doctype = 'User' - frappe.reload_doctype('User', force=True) - frappe.model.meta.trim_tables('User') - make_property_setter(doctype, 'bio', 'fieldtype', 'Text', 'Data') - make_property_setter(doctype, 'middle_name', 'fieldtype', 'Data', 'Text') - make_property_setter(doctype, 'enabled', 'default', '1', 'Int') + doctype = "User" + frappe.reload_doctype("User", force=True) + frappe.model.meta.trim_tables("User") + make_property_setter(doctype, "bio", "fieldtype", "Text", "Data") + make_property_setter(doctype, "middle_name", "fieldtype", "Data", "Text") + make_property_setter(doctype, "enabled", "default", "1", "Int") frappe.db.updatedb(doctype) field_defs = get_field_defs(doctype) - table_columns = frappe.db.get_table_columns_description('tab{}'.format(doctype)) + table_columns = frappe.db.get_table_columns_description("tab{}".format(doctype)) self.assertEqual(len(field_defs), len(table_columns)) for field_def in field_defs: - fieldname = field_def.get('fieldname') - table_column = find(table_columns, lambda d: d.get('name') == fieldname) + fieldname = field_def.get("fieldname") + table_column = find(table_columns, lambda d: d.get("name") == fieldname) fieldtype = get_fieldtype_from_def(field_def) - fallback_default = '0' if field_def.get('fieldtype') in frappe.model.numeric_fieldtypes else 'NULL' + fallback_default = ( + "0" if field_def.get("fieldtype") in frappe.model.numeric_fieldtypes else "NULL" + ) default = field_def.default if field_def.default is not None else fallback_default self.assertEqual(fieldtype, table_column.type) - self.assertIn(cstr(table_column.default) or 'NULL', [cstr(default), "'{}'".format(default)]) + self.assertIn(cstr(table_column.default) or "NULL", [cstr(default), "'{}'".format(default)]) def test_index_and_unique_constraints(self): doctype = "User" - frappe.reload_doctype('User', force=True) - frappe.model.meta.trim_tables('User') + frappe.reload_doctype("User", force=True) + frappe.model.meta.trim_tables("User") - make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') + make_property_setter(doctype, "restrict_ip", "unique", "1", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertTrue(restrict_ip_in_table.unique) - make_property_setter(doctype, 'restrict_ip', 'unique', '0', 'Int') + make_property_setter(doctype, "restrict_ip", "unique", "0", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertFalse(restrict_ip_in_table.unique) - make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') + make_property_setter(doctype, "restrict_ip", "search_index", "1", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertTrue(restrict_ip_in_table.index) - make_property_setter(doctype, 'restrict_ip', 'search_index', '0', 'Int') + make_property_setter(doctype, "restrict_ip", "search_index", "0", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertFalse(restrict_ip_in_table.index) - make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') - make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') + make_property_setter(doctype, "restrict_ip", "search_index", "1", "Int") + make_property_setter(doctype, "restrict_ip", "unique", "1", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertTrue(restrict_ip_in_table.index) self.assertTrue(restrict_ip_in_table.unique) - make_property_setter(doctype, 'restrict_ip', 'search_index', '1', 'Int') - make_property_setter(doctype, 'restrict_ip', 'unique', '0', 'Int') + make_property_setter(doctype, "restrict_ip", "search_index", "1", "Int") + make_property_setter(doctype, "restrict_ip", "unique", "0", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertTrue(restrict_ip_in_table.index) self.assertFalse(restrict_ip_in_table.unique) - make_property_setter(doctype, 'restrict_ip', 'search_index', '0', 'Int') - make_property_setter(doctype, 'restrict_ip', 'unique', '1', 'Int') + make_property_setter(doctype, "restrict_ip", "search_index", "0", "Int") + make_property_setter(doctype, "restrict_ip", "unique", "1", "Int") frappe.db.updatedb(doctype) restrict_ip_in_table = get_table_column("User", "restrict_ip") self.assertFalse(restrict_ip_in_table.index) @@ -86,44 +88,51 @@ class TestDBUpdate(unittest.TestCase): email_sig_column = get_table_column("User", "email_signature") self.assertEqual(email_sig_column.index, 1) + def get_fieldtype_from_def(field_def): - fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) + fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ("", 0)) fieldtype = fieldtuple[0] - if fieldtype in ('varchar', 'datetime', 'int'): - fieldtype += '({})'.format(field_def.length or fieldtuple[1]) + if fieldtype in ("varchar", "datetime", "int"): + fieldtype += "({})".format(field_def.length or fieldtuple[1]) return fieldtype + def get_field_defs(doctype): meta = frappe.get_meta(doctype, cached=False) field_defs = meta.get_fieldnames_with_value(True) field_defs += get_other_fields_meta(meta) return field_defs + def get_other_fields_meta(meta): default_fields_map = { - 'name': ('Data', 0), - 'owner': ('Data', 0), - 'modified_by': ('Data', 0), - 'creation': ('Datetime', 0), - 'modified': ('Datetime', 0), - 'idx': ('Int', 8), - 'docstatus': ('Check', 0) + "name": ("Data", 0), + "owner": ("Data", 0), + "modified_by": ("Data", 0), + "creation": ("Datetime", 0), + "modified": ("Datetime", 0), + "idx": ("Int", 8), + "docstatus": ("Check", 0), } optional_fields = frappe.db.OPTIONAL_COLUMNS if meta.track_seen: - optional_fields.append('_seen') + optional_fields.append("_seen") child_table_fields_map = {} if meta.istable: - child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS}) + child_table_fields_map.update({field: ("Data", 0) for field in frappe.db.CHILD_TABLE_COLUMNS}) - optional_fields_map = {field: ('Text', 0) for field in optional_fields} + optional_fields_map = {field: ("Text", 0) for field in optional_fields} fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map) - field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()] + field_map = [ + frappe._dict({"fieldname": field, "fieldtype": _type, "length": _length}) + for field, (_type, _length) in fields.items() + ] return field_map + def get_table_column(doctype, fieldname): - table_columns = frappe.db.get_table_columns_description('tab{}'.format(doctype)) - return find(table_columns, lambda d: d.get('name') == fieldname) + table_columns = frappe.db.get_table_columns_description("tab{}".format(doctype)) + return find(table_columns, lambda d: d.get("name") == fieldname) diff --git a/frappe/tests/test_defaults.py b/frappe/tests/test_defaults.py index 84d482d8ca..3841bde34e 100644 --- a/frappe/tests/test_defaults.py +++ b/frappe/tests/test_defaults.py @@ -1,9 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest +import frappe from frappe.defaults import * + class TestDefaults(unittest.TestCase): def test_global(self): clear_user_default("key1") @@ -52,19 +54,21 @@ class TestDefaults(unittest.TestCase): self.assertEqual(get_user_default_as_list("language"), ["en"]) old_user = frappe.session.user - user = 'test@example.com' + user = "test@example.com" frappe.set_user(user) - perm_doc = frappe.get_doc(dict( - doctype='User Permission', - user=frappe.session.user, - allow="Language", - for_value="en-GB", - )).insert(ignore_permissions = True) + perm_doc = frappe.get_doc( + dict( + doctype="User Permission", + user=frappe.session.user, + allow="Language", + for_value="en-GB", + ) + ).insert(ignore_permissions=True) self.assertEqual(get_global_default("language"), None) self.assertEqual(get_user_default("language"), None) self.assertEqual(get_user_default_as_list("language"), []) - frappe.delete_doc('User Permission', perm_doc.name) + frappe.delete_doc("User Permission", perm_doc.name) frappe.set_user(old_user) diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 5caccb167e..40bc8e2d45 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -19,7 +19,7 @@ class CustomTestNote(Note): class TestDocument(unittest.TestCase): def test_get_return_empty_list_for_table_field_if_none(self): - d = frappe.get_doc({"doctype":"User"}) + d = frappe.get_doc({"doctype": "User"}) self.assertEqual(d.get("roles"), []) def test_load(self): @@ -29,7 +29,7 @@ class TestDocument(unittest.TestCase): self.assertEqual(d.allow_rename, 1) self.assertTrue(isinstance(d.fields, list)) self.assertTrue(isinstance(d.permissions, list)) - self.assertTrue(filter(lambda d: d.fieldname=="email", d.fields)) + self.assertTrue(filter(lambda d: d.fieldname == "email", d.fields)) def test_load_single(self): d = frappe.get_doc("Website Settings", "Website Settings") @@ -38,32 +38,34 @@ class TestDocument(unittest.TestCase): self.assertTrue(d.disable_signup in (0, 1)) def test_insert(self): - d = frappe.get_doc({ - "doctype":"Event", - "subject":"test-doc-test-event 1", - "starts_on": "2014-01-01", - "event_type": "Public" - }) + d = frappe.get_doc( + { + "doctype": "Event", + "subject": "test-doc-test-event 1", + "starts_on": "2014-01-01", + "event_type": "Public", + } + ) d.insert() self.assertTrue(d.name.startswith("EV")) - self.assertEqual(frappe.db.get_value("Event", d.name, "subject"), - "test-doc-test-event 1") + self.assertEqual(frappe.db.get_value("Event", d.name, "subject"), "test-doc-test-event 1") # test if default values are added self.assertEqual(d.send_reminder, 1) return d def test_insert_with_child(self): - d = frappe.get_doc({ - "doctype":"Event", - "subject":"test-doc-test-event 2", - "starts_on": "2014-01-01", - "event_type": "Public" - }) + d = frappe.get_doc( + { + "doctype": "Event", + "subject": "test-doc-test-event 2", + "starts_on": "2014-01-01", + "event_type": "Public", + } + ) d.insert() self.assertTrue(d.name.startswith("EV")) - self.assertEqual(frappe.db.get_value("Event", d.name, "subject"), - "test-doc-test-event 2") + self.assertEqual(frappe.db.get_value("Event", d.name, "subject"), "test-doc-test-event 2") def test_update(self): d = self.test_insert() @@ -76,17 +78,19 @@ class TestDocument(unittest.TestCase): d = self.test_insert() d.subject = "subject changed again" d.save() - self.assertTrue(d.has_value_changed('subject')) - self.assertFalse(d.has_value_changed('event_type')) + self.assertTrue(d.has_value_changed("subject")) + self.assertFalse(d.has_value_changed("event_type")) def test_mandatory(self): # TODO: recheck if it is OK to force delete frappe.delete_doc_if_exists("User", "test_mandatory@example.com", 1) - d = frappe.get_doc({ - "doctype": "User", - "email": "test_mandatory@example.com", - }) + d = frappe.get_doc( + { + "doctype": "User", + "email": "test_mandatory@example.com", + } + ) self.assertRaises(frappe.MandatoryError, d.insert) d.set("first_name", "Test Mandatory") @@ -123,22 +127,18 @@ class TestDocument(unittest.TestCase): def test_link_validation(self): frappe.delete_doc_if_exists("User", "test_link_validation@example.com", 1) - d = frappe.get_doc({ - "doctype": "User", - "email": "test_link_validation@example.com", - "first_name": "Link Validation", - "roles": [ - { - "role": "ABC" - } - ] - }) + d = frappe.get_doc( + { + "doctype": "User", + "email": "test_link_validation@example.com", + "first_name": "Link Validation", + "roles": [{"role": "ABC"}], + } + ) self.assertRaises(frappe.LinkValidationError, d.insert) d.roles = [] - d.append("roles", { - "role": "System Manager" - }) + d.append("roles", {"role": "System Manager"}) d.insert() self.assertEqual(frappe.db.get_value("User", d.name), d.name) @@ -166,7 +166,7 @@ class TestDocument(unittest.TestCase): def test_varchar_length(self): d = self.test_insert() - d.sender = "abcde"*100 + "@user.com" + d.sender = "abcde" * 100 + "@user.com" self.assertRaises(frappe.CharacterLengthExceededError, d.save) def test_xss_filter(self): @@ -174,7 +174,7 @@ class TestDocument(unittest.TestCase): # script xss = '' - escaped_xss = xss.replace('<', '<').replace('>', '>') + escaped_xss = xss.replace("<", "<").replace(">", ">") d.subject += xss d.save() d.reload() @@ -184,7 +184,7 @@ class TestDocument(unittest.TestCase): # onload xss = '
Test
' - escaped_xss = '
Test
' + escaped_xss = "
Test
" d.subject += xss d.save() d.reload() @@ -210,57 +210,56 @@ class TestDocument(unittest.TestCase): prefix = series if ".#" in series: - prefix = series.rsplit('.',1)[0] + prefix = series.rsplit(".", 1)[0] prefix = parse_naming_series(prefix) - old_current = frappe.db.get_value('Series', prefix, "current", order_by="name") + old_current = frappe.db.get_value("Series", prefix, "current", order_by="name") revert_series_if_last(series, name) - new_current = cint(frappe.db.get_value('Series', prefix, "current", order_by="name")) + new_current = cint(frappe.db.get_value("Series", prefix, "current", order_by="name")) self.assertEqual(cint(old_current) - 1, new_current) def test_non_negative_check(self): frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1) - d = frappe.get_doc({ - 'doctype': 'Currency', - 'currency_name': 'Frappe Coin', - 'smallest_currency_fraction_value': -1 - }) + d = frappe.get_doc( + {"doctype": "Currency", "currency_name": "Frappe Coin", "smallest_currency_fraction_value": -1} + ) self.assertRaises(frappe.NonNegativeError, d.insert) - d.set('smallest_currency_fraction_value', 1) + d.set("smallest_currency_fraction_value", 1) d.insert() self.assertEqual(frappe.db.get_value("Currency", d.name), d.name) frappe.delete_doc_if_exists("Currency", "Frappe Coin", 1) def test_get_formatted(self): - frappe.get_doc({ - 'doctype': 'DocType', - 'name': 'Test Formatted', - 'module': 'Custom', - 'custom': 1, - 'fields': [ - {'label': 'Currency', 'fieldname': 'currency', 'reqd': 1, 'fieldtype': 'Currency'}, - ] - }).insert(ignore_if_duplicate=True) + frappe.get_doc( + { + "doctype": "DocType", + "name": "Test Formatted", + "module": "Custom", + "custom": 1, + "fields": [ + {"label": "Currency", "fieldname": "currency", "reqd": 1, "fieldtype": "Currency"}, + ], + } + ).insert(ignore_if_duplicate=True) frappe.delete_doc_if_exists("Currency", "INR", 1) - d = frappe.get_doc({ - 'doctype': 'Currency', - 'currency_name': 'INR', - 'symbol': '₹', - }).insert() + d = frappe.get_doc( + { + "doctype": "Currency", + "currency_name": "INR", + "symbol": "₹", + } + ).insert() - d = frappe.get_doc({ - 'doctype': 'Test Formatted', - 'currency': 100000 - }) - self.assertEqual(d.get_formatted('currency', currency='INR', format="#,###.##"), '₹ 100,000.00') + d = frappe.get_doc({"doctype": "Test Formatted", "currency": 100000}) + self.assertEqual(d.get_formatted("currency", currency="INR", format="#,###.##"), "₹ 100,000.00") # should work even if options aren't set in df # and currency param is not passed @@ -275,29 +274,30 @@ class TestDocument(unittest.TestCase): self.assertEqual(len(doc.get("fields", filters={"fieldtype": "Data"}, limit=3)), 3) def test_virtual_fields(self): - """Virtual fields are accessible via API and Form views, whenever .as_dict is invoked - """ - frappe.db.delete("Custom Field", {"dt": "Note", "fieldname":"age"}) + """Virtual fields are accessible via API and Form views, whenever .as_dict is invoked""" + frappe.db.delete("Custom Field", {"dt": "Note", "fieldname": "age"}) note = frappe.new_doc("Note") note.content = "some content" note.title = frappe.generate_hash(length=20) note.insert() def patch_note(): - return patch("frappe.controllers", new={frappe.local.site: {'Note': CustomTestNote}}) + return patch("frappe.controllers", new={frappe.local.site: {"Note": CustomTestNote}}) @contextmanager def customize_note(with_options=False): options = "frappe.utils.now_datetime() - doc.creation" if with_options else "" - custom_field = frappe.get_doc({ - "doctype": "Custom Field", - "dt": "Note", - "fieldname": "age", - "fieldtype": "Data", - "read_only": True, - "is_virtual": True, - "options": options, - }) + custom_field = frappe.get_doc( + { + "doctype": "Custom Field", + "dt": "Note", + "fieldname": "age", + "fieldtype": "Data", + "read_only": True, + "is_virtual": True, + "options": options, + } + ) try: yield custom_field.insert(ignore_if_duplicate=True) diff --git a/frappe/tests/test_document_locks.py b/frappe/tests/test_document_locks.py index 4173410024..36e07ca2ca 100644 --- a/frappe/tests/test_document_locks.py +++ b/frappe/tests/test_document_locks.py @@ -1,11 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + +import frappe + class TestDocumentLocks(unittest.TestCase): def test_locking(self): - todo = frappe.get_doc(dict(doctype='ToDo', description='test')).insert() - todo_1 = frappe.get_doc('ToDo', todo.name) + todo = frappe.get_doc(dict(doctype="ToDo", description="test")).insert() + todo_1 = frappe.get_doc("ToDo", todo.name) todo.lock() self.assertRaises(frappe.DocumentLockedError, todo_1.lock) diff --git a/frappe/tests/test_domainification.py b/frappe/tests/test_domainification.py index 054a9031c3..575445e34c 100644 --- a/frappe/tests/test_domainification.py +++ b/frappe/tests/test_domainification.py @@ -1,11 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest, frappe -from frappe.core.page.permission_manager.permission_manager import get_roles_and_doctypes -from frappe.desk.doctype.desktop_icon.desktop_icon import (get_desktop_icons, add_user_icon, - clear_desktop_icons_cache) +import unittest +import frappe from frappe.core.doctype.domain_settings.domain_settings import get_active_modules +from frappe.core.page.permission_manager.permission_manager import get_roles_and_doctypes +from frappe.desk.doctype.desktop_icon.desktop_icon import ( + add_user_icon, + clear_desktop_icons_cache, + get_desktop_icons, +) + class TestDomainification(unittest.TestCase): def setUp(self): @@ -20,21 +25,21 @@ class TestDomainification(unittest.TestCase): frappe.db.delete("Role", {"name": "_Test Role"}) frappe.db.delete("Has Role", {"role": "_Test Role"}) frappe.db.delete("Domain", {"name": ("in", ("_Test Domain 1", "_Test Domain 2"))}) - frappe.delete_doc('DocType', 'Test Domainification') + frappe.delete_doc("DocType", "Test Domainification") self.remove_from_active_domains(remove_all=True) def add_active_domain(self, domain): - """ add domain in active domain """ + """add domain in active domain""" if not domain: return domain_settings = frappe.get_doc("Domain Settings", "Domain Settings") - domain_settings.append("active_domains", { "domain": domain }) + domain_settings.append("active_domains", {"domain": domain}) domain_settings.save() def remove_from_active_domains(self, domain=None, remove_all=False): - """ remove domain from domain settings """ + """remove domain from domain settings""" if not (domain or remove_all): return @@ -44,27 +49,26 @@ class TestDomainification(unittest.TestCase): domain_settings.set("active_domains", []) else: to_remove = [] - [ to_remove.append(row) for row in domain_settings.active_domains if row.domain == domain ] - [ domain_settings.remove(row) for row in to_remove ] + [to_remove.append(row) for row in domain_settings.active_domains if row.domain == domain] + [domain_settings.remove(row) for row in to_remove] domain_settings.save() def new_domain(self, domain): # create new domain - frappe.get_doc({ - "doctype": "Domain", - "domain": domain - }).insert() + frappe.get_doc({"doctype": "Domain", "domain": domain}).insert() def new_doctype(self, name): - return frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "fields": [{"label": "Some Field", "fieldname": "some_fieldname", "fieldtype": "Data"}], - "permissions": [{"role": "System Manager", "read": 1}], - "name": name - }) + return frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "custom": 1, + "fields": [{"label": "Some Field", "fieldname": "some_fieldname", "fieldtype": "Data"}], + "permissions": [{"role": "System Manager", "read": 1}], + "name": name, + } + ) def test_active_domains(self): self.assertTrue("_Test Domain 1" in frappe.get_active_domains()) @@ -78,17 +82,14 @@ class TestDomainification(unittest.TestCase): def test_doctype_and_role_domainification(self): """ - test if doctype is hidden if the doctype's restrict to domain is not included - in active domains + test if doctype is hidden if the doctype's restrict to domain is not included + in active domains """ test_doctype = self.new_doctype("Test Domainification") test_doctype.insert() - test_role = frappe.get_doc({ - "doctype": "Role", - "role_name": "_Test Role" - }).insert() + test_role = frappe.get_doc({"doctype": "Role", "role_name": "_Test Role"}).insert() # doctype should be hidden in desktop icon, role permissions results = get_roles_and_doctypes() @@ -113,32 +114,38 @@ class TestDomainification(unittest.TestCase): self.assertTrue("_Test Role" not in [d.get("value") for d in results.get("roles")]) def test_desktop_icon_for_domainification(self): - """ desktop icon should be hidden if doctype's restrict to domain is not in active domains """ + """desktop icon should be hidden if doctype's restrict to domain is not in active domains""" test_doctype = self.new_doctype("Test Domainification") test_doctype.restrict_to_domain = "_Test Domain 2" test_doctype.insert() self.add_active_domain("_Test Domain 2") - add_user_icon('Test Domainification') + add_user_icon("Test Domainification") icons = get_desktop_icons() - doctypes = [icon.get("_doctype") for icon in icons if icon.get("_doctype") == "Test Domainification" \ - and icon.get("blocked") == 0] + doctypes = [ + icon.get("_doctype") + for icon in icons + if icon.get("_doctype") == "Test Domainification" and icon.get("blocked") == 0 + ] self.assertTrue("Test Domainification" in doctypes) # doctype should be hidden from the desk self.remove_from_active_domains("_Test Domain 2") - clear_desktop_icons_cache() # clear cache to fetch the desktop icon according to new active domains + clear_desktop_icons_cache() # clear cache to fetch the desktop icon according to new active domains icons = get_desktop_icons() - doctypes = [icon.get("_doctype") for icon in icons if icon.get("_doctype") == "Test Domainification" \ - and icon.get("blocked") == 0] + doctypes = [ + icon.get("_doctype") + for icon in icons + if icon.get("_doctype") == "Test Domainification" and icon.get("blocked") == 0 + ] self.assertFalse("Test Domainification" in doctypes) def test_module_def_for_domainification(self): - """ modules should be hidden if module def's restrict to domain is not in active domains""" + """modules should be hidden if module def's restrict to domain is not in active domains""" test_module_def = frappe.get_doc("Module Def", "Contacts") test_module_def.restrict_to_domain = "_Test Domain 2" diff --git a/frappe/tests/test_dynamic_links.py b/frappe/tests/test_dynamic_links.py index b74c5867ca..ebbacae1f2 100644 --- a/frappe/tests/test_dynamic_links.py +++ b/frappe/tests/test_dynamic_links.py @@ -1,64 +1,76 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest +import unittest + +import frappe + class TestDynamicLinks(unittest.TestCase): def setUp(self): frappe.db.delete("Email Unsubscribe") def test_delete_normal(self): - event = frappe.get_doc({ - 'doctype': 'Event', - 'subject':'test-for-delete', - 'starts_on': '2014-01-01', - 'event_type': 'Public' - }).insert() - - unsub = frappe.get_doc({ - 'doctype': 'Email Unsubscribe', - 'email': 'test@example.com', - 'reference_doctype': event.doctype, - 'reference_name': event.name - }).insert() + event = frappe.get_doc( + { + "doctype": "Event", + "subject": "test-for-delete", + "starts_on": "2014-01-01", + "event_type": "Public", + } + ).insert() + + unsub = frappe.get_doc( + { + "doctype": "Email Unsubscribe", + "email": "test@example.com", + "reference_doctype": event.doctype, + "reference_name": event.name, + } + ).insert() event.delete() - self.assertFalse(frappe.db.exists('Email Unsubscribe', unsub.name)) + self.assertFalse(frappe.db.exists("Email Unsubscribe", unsub.name)) def test_delete_with_comment(self): - event = frappe.get_doc({ - 'doctype': 'Event', - 'subject':'test-for-delete-1', - 'starts_on': '2014-01-01', - 'event_type': 'Public' - }).insert() - event.add_comment('Comment', 'test') - - self.assertTrue(frappe.get_all('Comment', - filters={'reference_doctype':'Event', 'reference_name':event.name})) + event = frappe.get_doc( + { + "doctype": "Event", + "subject": "test-for-delete-1", + "starts_on": "2014-01-01", + "event_type": "Public", + } + ).insert() + event.add_comment("Comment", "test") + + self.assertTrue( + frappe.get_all("Comment", filters={"reference_doctype": "Event", "reference_name": event.name}) + ) event.delete() - self.assertFalse(frappe.get_all('Comment', - filters={'reference_doctype':'Event', 'reference_name':event.name})) + self.assertFalse( + frappe.get_all("Comment", filters={"reference_doctype": "Event", "reference_name": event.name}) + ) def test_custom_fields(self): from frappe.utils.testutils import add_custom_field, clear_custom_fields - add_custom_field('Event', 'test_ref_doc', 'Link', 'DocType') - add_custom_field('Event', 'test_ref_name', 'Dynamic Link', 'test_ref_doc') - - unsub = frappe.get_doc({ - 'doctype': 'Email Unsubscribe', - 'email': 'test@example.com', - 'global_unsubscribe': 1 - }).insert() - - event = frappe.get_doc({ - 'doctype': 'Event', - 'subject':'test-for-delete-2', - 'starts_on': '2014-01-01', - 'event_type': 'Public', - 'test_ref_doc': unsub.doctype, - 'test_ref_name': unsub.name - }).insert() + + add_custom_field("Event", "test_ref_doc", "Link", "DocType") + add_custom_field("Event", "test_ref_name", "Dynamic Link", "test_ref_doc") + + unsub = frappe.get_doc( + {"doctype": "Email Unsubscribe", "email": "test@example.com", "global_unsubscribe": 1} + ).insert() + + event = frappe.get_doc( + { + "doctype": "Event", + "subject": "test-for-delete-2", + "starts_on": "2014-01-01", + "event_type": "Public", + "test_ref_doc": unsub.doctype, + "test_ref_name": unsub.name, + } + ).insert() self.assertRaises(frappe.LinkExistsError, unsub.delete) @@ -68,4 +80,4 @@ class TestDynamicLinks(unittest.TestCase): unsub.delete() - clear_custom_fields('Event') + clear_custom_fields("Event") diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index e4a18c06e5..3d6773044f 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -1,11 +1,15 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest, frappe, re, email +import email +import re +import unittest +import frappe from frappe.email.doctype.email_account.test_email_account import TestEmailAccount -test_dependencies = ['Email Account'] +test_dependencies = ["Email Account"] + class TestEmail(unittest.TestCase): def setUp(self): @@ -14,103 +18,177 @@ class TestEmail(unittest.TestCase): frappe.db.delete("Email Queue Recipient") def test_email_queue(self, send_after=None): - frappe.sendmail(recipients=['test@example.com', 'test1@example.com'], - sender="admin@example.com", - reference_doctype='User', reference_name='Administrator', - subject='Testing Queue', message='This mail is queued!', - unsubscribe_message="Unsubscribe", send_after=send_after) - - email_queue = frappe.db.sql("""select name,message from `tabEmail Queue` where status='Not Sent'""", as_dict=1) + frappe.sendmail( + recipients=["test@example.com", "test1@example.com"], + sender="admin@example.com", + reference_doctype="User", + reference_name="Administrator", + subject="Testing Queue", + message="This mail is queued!", + unsubscribe_message="Unsubscribe", + send_after=send_after, + ) + + email_queue = frappe.db.sql( + """select name,message from `tabEmail Queue` where status='Not Sent'""", as_dict=1 + ) self.assertEqual(len(email_queue), 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""SELECT recipient FROM `tabEmail Queue Recipient` - WHERE status='Not Sent'""", as_dict=1)] - self.assertTrue('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """SELECT recipient FROM `tabEmail Queue Recipient` + WHERE status='Not Sent'""", + as_dict=1, + ) + ] + self.assertTrue("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) self.assertEqual(len(queue_recipients), 2) - self.assertTrue('' in email_queue[0]['message']) + self.assertTrue("" in email_queue[0]["message"]) def test_send_after(self): self.test_email_queue(send_after=1) from frappe.email.queue import flush + flush(from_test=True) - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Sent'""", as_dict=1 + ) self.assertEqual(len(email_queue), 0) def test_flush(self): self.test_email_queue() from frappe.email.queue import flush + flush(from_test=True) - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Sent'""", as_dict=1 + ) self.assertEqual(len(email_queue), 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` - where status='Sent'""", as_dict=1)] - self.assertTrue('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """select recipient from `tabEmail Queue Recipient` + where status='Sent'""", + as_dict=1, + ) + ] + self.assertTrue("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) self.assertEqual(len(queue_recipients), 2) - self.assertTrue('Unsubscribe' in frappe.safe_decode(frappe.flags.sent_mail)) + self.assertTrue("Unsubscribe" in frappe.safe_decode(frappe.flags.sent_mail)) def test_cc_header(self): # test if sending with cc's makes it into header - frappe.sendmail(recipients=['test@example.com'], - cc=['test1@example.com'], - sender="admin@example.com", - reference_doctype='User', reference_name="Administrator", - subject='Testing Email Queue', message='This is mail is queued!', - unsubscribe_message="Unsubscribe", expose_recipients="header") - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""", as_dict=1) + frappe.sendmail( + recipients=["test@example.com"], + cc=["test1@example.com"], + sender="admin@example.com", + reference_doctype="User", + reference_name="Administrator", + subject="Testing Email Queue", + message="This is mail is queued!", + unsubscribe_message="Unsubscribe", + expose_recipients="header", + ) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Not Sent'""", as_dict=1 + ) self.assertEqual(len(email_queue), 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` - where status='Not Sent'""", as_dict=1)] - self.assertTrue('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) - - message = frappe.db.sql("""select message from `tabEmail Queue` - where status='Not Sent'""", as_dict=1)[0].message - self.assertTrue('To: test@example.com' in message) - self.assertTrue('CC: test1@example.com' in message) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """select recipient from `tabEmail Queue Recipient` + where status='Not Sent'""", + as_dict=1, + ) + ] + self.assertTrue("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) + + message = frappe.db.sql( + """select message from `tabEmail Queue` + where status='Not Sent'""", + as_dict=1, + )[0].message + self.assertTrue("To: test@example.com" in message) + self.assertTrue("CC: test1@example.com" in message) def test_cc_footer(self): frappe.conf.use_ssl = True # test if sending with cc's makes it into header - frappe.sendmail(recipients=['test@example.com'], - cc=['test1@example.com'], - sender="admin@example.com", - reference_doctype='User', reference_name="Administrator", - subject='Testing Email Queue', message='This is mail is queued!', - unsubscribe_message="Unsubscribe", expose_recipients="footer", now=True) - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1) + frappe.sendmail( + recipients=["test@example.com"], + cc=["test1@example.com"], + sender="admin@example.com", + reference_doctype="User", + reference_name="Administrator", + subject="Testing Email Queue", + message="This is mail is queued!", + unsubscribe_message="Unsubscribe", + expose_recipients="footer", + now=True, + ) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Sent'""", as_dict=1 + ) self.assertEqual(len(email_queue), 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` - where status='Sent'""", as_dict=1)] - self.assertTrue('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) - - self.assertTrue('This email was sent to test@example.com and copied to test1@example.com' in frappe.safe_decode( - frappe.flags.sent_mail)) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """select recipient from `tabEmail Queue Recipient` + where status='Sent'""", + as_dict=1, + ) + ] + self.assertTrue("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) + + self.assertTrue( + "This email was sent to test@example.com and copied to test1@example.com" + in frappe.safe_decode(frappe.flags.sent_mail) + ) # check for email tracker - self.assertTrue('mark_email_as_seen' in frappe.safe_decode(frappe.flags.sent_mail)) + self.assertTrue("mark_email_as_seen" in frappe.safe_decode(frappe.flags.sent_mail)) frappe.conf.use_ssl = False def test_expose(self): from frappe.utils.verified_command import verify_request - frappe.sendmail(recipients=['test@example.com'], - cc=['test1@example.com'], - sender="admin@example.com", - reference_doctype='User', reference_name="Administrator", - subject='Testing Email Queue', message='This is mail is queued!', - unsubscribe_message="Unsubscribe", now=True) - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Sent'""", as_dict=1) - self.assertEqual(len(email_queue), 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` - where status='Sent'""", as_dict=1)] - self.assertTrue('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) - message = frappe.db.sql("""select message from `tabEmail Queue` - where status='Sent'""", as_dict=1)[0].message - self.assertTrue('' in message) + frappe.sendmail( + recipients=["test@example.com"], + cc=["test1@example.com"], + sender="admin@example.com", + reference_doctype="User", + reference_name="Administrator", + subject="Testing Email Queue", + message="This is mail is queued!", + unsubscribe_message="Unsubscribe", + now=True, + ) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Sent'""", as_dict=1 + ) + self.assertEqual(len(email_queue), 1) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """select recipient from `tabEmail Queue Recipient` + where status='Sent'""", + as_dict=1, + ) + ] + self.assertTrue("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) + + message = frappe.db.sql( + """select message from `tabEmail Queue` + where status='Sent'""", + as_dict=1, + )[0].message + self.assertTrue("" in message) email_obj = email.message_from_string(frappe.safe_decode(frappe.flags.sent_mail)) for part in email_obj.walk(): @@ -119,9 +197,9 @@ class TestEmail(unittest.TestCase): if content: eol = "\r\n" - frappe.local.flags.signed_query_string = \ - re.search(r'(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=' + eol + ')', - content.decode()).group(0) + frappe.local.flags.signed_query_string = re.search( + r"(?<=/api/method/frappe.email.queue.unsubscribe\?).*(?=" + eol + ")", content.decode() + ).group(0) self.assertTrue(verify_request()) break @@ -130,61 +208,82 @@ class TestEmail(unittest.TestCase): frappe.db.sql("UPDATE `tabEmail Queue` SET `modified`=(NOW() - INTERVAL '8' day)") from frappe.email.queue import set_expiry_for_email_queue + set_expiry_for_email_queue() - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Expired'""", as_dict=1) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Expired'""", as_dict=1 + ) self.assertEqual(len(email_queue), 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` - where parent = %s""", email_queue[0].name, as_dict=1)] - self.assertTrue('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """select recipient from `tabEmail Queue Recipient` + where parent = %s""", + email_queue[0].name, + as_dict=1, + ) + ] + self.assertTrue("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) self.assertEqual(len(queue_recipients), 2) def test_unsubscribe(self): - from frappe.email.queue import unsubscribe from frappe.email.doctype.email_queue.email_queue import QueueBuilder + from frappe.email.queue import unsubscribe + unsubscribe(doctype="User", name="Administrator", email="test@example.com") - self.assertTrue(frappe.db.get_value("Email Unsubscribe", - {"reference_doctype": "User", "reference_name": "Administrator", - "email": "test@example.com"})) + self.assertTrue( + frappe.db.get_value( + "Email Unsubscribe", + {"reference_doctype": "User", "reference_name": "Administrator", "email": "test@example.com"}, + ) + ) - before = frappe.db.sql("""select count(name) from `tabEmail Queue` where status='Not Sent'""")[0][0] + before = frappe.db.sql("""select count(name) from `tabEmail Queue` where status='Not Sent'""")[ + 0 + ][0] - builder = QueueBuilder(recipients=['test@example.com', 'test1@example.com'], + builder = QueueBuilder( + recipients=["test@example.com", "test1@example.com"], sender="admin@example.com", - reference_doctype='User', reference_name="Administrator", - subject='Testing Email Queue', message='This is mail is queued!', unsubscribe_message="Unsubscribe") + reference_doctype="User", + reference_name="Administrator", + subject="Testing Email Queue", + message="This is mail is queued!", + unsubscribe_message="Unsubscribe", + ) builder.process() # this is sent async (?) - email_queue = frappe.db.sql("""select name from `tabEmail Queue` where status='Not Sent'""", - as_dict=1) + email_queue = frappe.db.sql( + """select name from `tabEmail Queue` where status='Not Sent'""", as_dict=1 + ) self.assertEqual(len(email_queue), before + 1) - queue_recipients = [r.recipient for r in frappe.db.sql("""select recipient from `tabEmail Queue Recipient` - where status='Not Sent'""", as_dict=1)] - self.assertFalse('test@example.com' in queue_recipients) - self.assertTrue('test1@example.com' in queue_recipients) + queue_recipients = [ + r.recipient + for r in frappe.db.sql( + """select recipient from `tabEmail Queue Recipient` + where status='Not Sent'""", + as_dict=1, + ) + ] + self.assertFalse("test@example.com" in queue_recipients) + self.assertTrue("test1@example.com" in queue_recipients) self.assertEqual(len(queue_recipients), 1) - self.assertTrue('Unsubscribe' in frappe.safe_decode(frappe.flags.sent_mail)) + self.assertTrue("Unsubscribe" in frappe.safe_decode(frappe.flags.sent_mail)) def test_image_parsing(self): import re - email_account = frappe.get_doc('Email Account', '_Test Email Account 1') + + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") frappe.db.delete("Communication", {"sender": "sukh@yyy.com"}) - with open(frappe.get_app_path('frappe', 'tests', 'data', 'email_with_image.txt'), 'r') as raw: + with open(frappe.get_app_path("frappe", "tests", "data", "email_with_image.txt"), "r") as raw: messages = { - '"INBOX"': { - 'latest_messages': [ - raw.read() - ], - 'seen_status': { - 2: 'UNSEEN' - }, - 'uid_list': [2] - } + '"INBOX"': {"latest_messages": [raw.read()], "seen_status": {2: "UNSEEN"}, "uid_list": [2]} } email_account = frappe.get_doc("Email Account", "_Test Email Account 1") @@ -200,13 +299,17 @@ class TestEmail(unittest.TestCase): communication = mails[0].process() - self.assertTrue(re.search(''']*src=["']/private/files/rtco1.png[^>]*>''', communication.content)) - self.assertTrue(re.search(''']*src=["']/private/files/rtco2.png[^>]*>''', communication.content)) + self.assertTrue( + re.search("""]*src=["']/private/files/rtco1.png[^>]*>""", communication.content) + ) + self.assertTrue( + re.search("""]*src=["']/private/files/rtco2.png[^>]*>""", communication.content) + ) if changed_flag: email_account.enable_incoming = False -if __name__ == '__main__': +if __name__ == "__main__": frappe.connect() unittest.main() diff --git a/frappe/tests/test_exporter_fixtures.py b/frappe/tests/test_exporter_fixtures.py index f1b9e8fa5a..0f82954dc6 100644 --- a/frappe/tests/test_exporter_fixtures.py +++ b/frappe/tests/test_exporter_fixtures.py @@ -1,16 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import os +import unittest + import frappe import frappe.defaults from frappe.core.doctype.data_import.data_import import export_csv -import unittest -import os + class TestDataImportFixtures(unittest.TestCase): def setUp(self): pass - #start test for Client Script + # start test for Client Script def test_Custom_Script_fixture_simple(self): fixture = "Client Script" path = frappe.scrub(fixture) + "_original_style.csv" @@ -20,7 +22,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Script_fixture_simple_name_equal_default(self): - fixture = ["Client Script", {"name":["Item"]}] + fixture = ["Client Script", {"name": ["Item"]}] path = frappe.scrub(fixture[0]) + "_simple_name_equal_default.csv" export_csv(fixture, path) @@ -28,7 +30,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Script_fixture_simple_name_equal(self): - fixture = ["Client Script", {"name":["Item"],"op":"="}] + fixture = ["Client Script", {"name": ["Item"], "op": "="}] path = frappe.scrub(fixture[0]) + "_simple_name_equal.csv" export_csv(fixture, path) @@ -36,16 +38,16 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Script_fixture_simple_name_not_equal(self): - fixture = ["Client Script", {"name":["Item"],"op":"!="}] + fixture = ["Client Script", {"name": ["Item"], "op": "!="}] path = frappe.scrub(fixture[0]) + "_simple_name_not_equal.csv" export_csv(fixture, path) self.assertTrue(True) os.remove(path) - #without [] around the name... + # without [] around the name... def test_Custom_Script_fixture_simple_name_at_least_equal(self): - fixture = ["Client Script", {"name":"Item-Cli"}] + fixture = ["Client Script", {"name": "Item-Cli"}] path = frappe.scrub(fixture[0]) + "_simple_name_at_least_equal.csv" export_csv(fixture, path) @@ -53,7 +55,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Script_fixture_multi_name_equal(self): - fixture = ["Client Script", {"name":["Item", "Customer"],"op":"="}] + fixture = ["Client Script", {"name": ["Item", "Customer"], "op": "="}] path = frappe.scrub(fixture[0]) + "_multi_name_equal.csv" export_csv(fixture, path) @@ -61,7 +63,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Script_fixture_multi_name_not_equal(self): - fixture = ["Client Script", {"name":["Item", "Customer"],"op":"!="}] + fixture = ["Client Script", {"name": ["Item", "Customer"], "op": "!="}] path = frappe.scrub(fixture[0]) + "_multi_name_not_equal.csv" export_csv(fixture, path) @@ -86,7 +88,7 @@ class TestDataImportFixtures(unittest.TestCase): # Client Script regular expression def test_Custom_Script_fixture_rex_no_flags(self): - fixture = ["Client Script", {"name":r"^[i|A]"}] + fixture = ["Client Script", {"name": r"^[i|A]"}] path = frappe.scrub(fixture[0]) + "_rex_no_flags.csv" export_csv(fixture, path) @@ -94,14 +96,14 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Script_fixture_rex_with_flags(self): - fixture = ["Client Script", {"name":r"^[i|A]", "flags":"L,M"}] + fixture = ["Client Script", {"name": r"^[i|A]", "flags": "L,M"}] path = frappe.scrub(fixture[0]) + "_rex_with_flags.csv" export_csv(fixture, path) self.assertTrue(True) os.remove(path) - #start test for Custom Field + # start test for Custom Field def test_Custom_Field_fixture_simple(self): fixture = "Custom Field" path = frappe.scrub(fixture) + "_original_style.csv" @@ -111,7 +113,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Field_fixture_simple_name_equal_default(self): - fixture = ["Custom Field", {"name":["Item-vat"]}] + fixture = ["Custom Field", {"name": ["Item-vat"]}] path = frappe.scrub(fixture[0]) + "_simple_name_equal_default.csv" export_csv(fixture, path) @@ -119,7 +121,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Field_fixture_simple_name_equal(self): - fixture = ["Custom Field", {"name":["Item-vat"],"op":"="}] + fixture = ["Custom Field", {"name": ["Item-vat"], "op": "="}] path = frappe.scrub(fixture[0]) + "_simple_name_equal.csv" export_csv(fixture, path) @@ -127,16 +129,16 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Field_fixture_simple_name_not_equal(self): - fixture = ["Custom Field", {"name":["Item-vat"],"op":"!="}] + fixture = ["Custom Field", {"name": ["Item-vat"], "op": "!="}] path = frappe.scrub(fixture[0]) + "_simple_name_not_equal.csv" export_csv(fixture, path) self.assertTrue(True) os.remove(path) - #without [] around the name... + # without [] around the name... def test_Custom_Field_fixture_simple_name_at_least_equal(self): - fixture = ["Custom Field", {"name":"Item-va"}] + fixture = ["Custom Field", {"name": "Item-va"}] path = frappe.scrub(fixture[0]) + "_simple_name_at_least_equal.csv" export_csv(fixture, path) @@ -144,7 +146,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Field_fixture_multi_name_equal(self): - fixture = ["Custom Field", {"name":["Item-vat", "Bin-vat"],"op":"="}] + fixture = ["Custom Field", {"name": ["Item-vat", "Bin-vat"], "op": "="}] path = frappe.scrub(fixture[0]) + "_multi_name_equal.csv" export_csv(fixture, path) @@ -152,7 +154,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Field_fixture_multi_name_not_equal(self): - fixture = ["Custom Field", {"name":["Item-vat", "Bin-vat"],"op":"!="}] + fixture = ["Custom Field", {"name": ["Item-vat", "Bin-vat"], "op": "!="}] path = frappe.scrub(fixture[0]) + "_multi_name_not_equal.csv" export_csv(fixture, path) @@ -177,7 +179,7 @@ class TestDataImportFixtures(unittest.TestCase): # Custom Field regular expression def test_Custom_Field_fixture_rex_no_flags(self): - fixture = ["Custom Field", {"name":r"^[r|L]"}] + fixture = ["Custom Field", {"name": r"^[r|L]"}] path = frappe.scrub(fixture[0]) + "_rex_no_flags.csv" export_csv(fixture, path) @@ -185,15 +187,14 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Custom_Field_fixture_rex_with_flags(self): - fixture = ["Custom Field", {"name":r"^[i|A]", "flags":"L,M"}] + fixture = ["Custom Field", {"name": r"^[i|A]", "flags": "L,M"}] path = frappe.scrub(fixture[0]) + "_rex_with_flags.csv" export_csv(fixture, path) self.assertTrue(True) os.remove(path) - - #start test for Doctype + # start test for Doctype def test_Doctype_fixture_simple(self): fixture = "ToDo" path = "Doctype_" + frappe.scrub(fixture) + "_original_style_should_be_all.csv" @@ -203,7 +204,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Doctype_fixture_simple_name_equal_default(self): - fixture = ["ToDo", {"name":["TDI00000008"]}] + fixture = ["ToDo", {"name": ["TDI00000008"]}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_simple_name_equal_default.csv" export_csv(fixture, path) @@ -211,7 +212,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Doctype_fixture_simple_name_equal(self): - fixture = ["ToDo", {"name":["TDI00000002"],"op":"="}] + fixture = ["ToDo", {"name": ["TDI00000002"], "op": "="}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_simple_name_equal.csv" export_csv(fixture, path) @@ -219,16 +220,16 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Doctype_simple_name_not_equal(self): - fixture = ["ToDo", {"name":["TDI00000002"],"op":"!="}] + fixture = ["ToDo", {"name": ["TDI00000002"], "op": "!="}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_simple_name_not_equal.csv" export_csv(fixture, path) self.assertTrue(True) os.remove(path) - #without [] around the name... + # without [] around the name... def test_Doctype_fixture_simple_name_at_least_equal(self): - fixture = ["ToDo", {"name":"TDI"}] + fixture = ["ToDo", {"name": "TDI"}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_simple_name_at_least_equal.csv" export_csv(fixture, path) @@ -236,7 +237,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Doctype_multi_name_equal(self): - fixture = ["ToDo", {"name":["TDI00000002", "TDI00000008"],"op":"="}] + fixture = ["ToDo", {"name": ["TDI00000002", "TDI00000008"], "op": "="}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_multi_name_equal.csv" export_csv(fixture, path) @@ -244,7 +245,7 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Doctype_multi_name_not_equal(self): - fixture = ["ToDo", {"name":["TDI00000002", "TDI00000008"],"op":"!="}] + fixture = ["ToDo", {"name": ["TDI00000002", "TDI00000008"], "op": "!="}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_multi_name_not_equal.csv" export_csv(fixture, path) @@ -269,7 +270,7 @@ class TestDataImportFixtures(unittest.TestCase): # Doctype regular expression def test_Doctype_fixture_rex_no_flags(self): - fixture = ["ToDo", {"name":r"^TDi"}] + fixture = ["ToDo", {"name": r"^TDi"}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_rex_no_flags_should_be_all.csv" export_csv(fixture, path) @@ -277,10 +278,9 @@ class TestDataImportFixtures(unittest.TestCase): os.remove(path) def test_Doctype_fixture_rex_with_flags(self): - fixture = ["ToDo", {"name":r"^TDi", "flags":"L,M"}] + fixture = ["ToDo", {"name": r"^TDi", "flags": "L,M"}] path = "Doctype_" + frappe.scrub(fixture[0]) + "_rex_with_flags_should_be_none.csv" export_csv(fixture, path) self.assertTrue(True) os.remove(path) - diff --git a/frappe/tests/test_fixture_import.py b/frappe/tests/test_fixture_import.py index 2fe7e40d0d..5b776c0392 100644 --- a/frappe/tests/test_fixture_import.py +++ b/frappe/tests/test_fixture_import.py @@ -64,9 +64,7 @@ class TestFixtureImport(unittest.TestCase): singles_doctype = frappe.qb.DocType("Singles") truncate_query = ( - frappe.qb.from_(singles_doctype) - .delete() - .where(singles_doctype.doctype == "temp_singles") + frappe.qb.from_(singles_doctype).delete().where(singles_doctype.doctype == "temp_singles") ) truncate_query.run() diff --git a/frappe/tests/test_fmt_datetime.py b/frappe/tests/test_fmt_datetime.py index ea75747157..031b8c323c 100644 --- a/frappe/tests/test_fmt_datetime.py +++ b/frappe/tests/test_fmt_datetime.py @@ -1,28 +1,36 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import datetime +import unittest import frappe from frappe.utils import ( - getdate, get_datetime, get_time, - get_user_date_format, get_user_time_format, - formatdate, format_datetime, format_time) -import unittest + format_datetime, + format_time, + formatdate, + get_datetime, + get_time, + get_user_date_format, + get_user_time_format, + getdate, +) test_date_obj = datetime.datetime.now() -test_date = test_date_obj.strftime('%Y-%m-%d') -test_time = test_date_obj.strftime('%H:%M:%S.%f') -test_datetime = test_date_obj.strftime('%Y-%m-%d %H:%M:%S.%f') +test_date = test_date_obj.strftime("%Y-%m-%d") +test_time = test_date_obj.strftime("%H:%M:%S.%f") +test_datetime = test_date_obj.strftime("%Y-%m-%d %H:%M:%S.%f") test_date_formats = { - 'yyyy-mm-dd': test_date_obj.strftime('%Y-%m-%d'), - 'dd-mm-yyyy': test_date_obj.strftime('%d-%m-%Y'), - 'dd/mm/yyyy': test_date_obj.strftime('%d/%m/%Y'), - 'dd.mm.yyyy': test_date_obj.strftime('%d.%m.%Y'), - 'mm/dd/yyyy': test_date_obj.strftime('%m/%d/%Y'), - 'mm-dd-yyyy': test_date_obj.strftime('%m-%d-%Y')} + "yyyy-mm-dd": test_date_obj.strftime("%Y-%m-%d"), + "dd-mm-yyyy": test_date_obj.strftime("%d-%m-%Y"), + "dd/mm/yyyy": test_date_obj.strftime("%d/%m/%Y"), + "dd.mm.yyyy": test_date_obj.strftime("%d.%m.%Y"), + "mm/dd/yyyy": test_date_obj.strftime("%m/%d/%Y"), + "mm-dd-yyyy": test_date_obj.strftime("%m-%d-%Y"), +} test_time_formats = { - 'HH:mm:ss': test_date_obj.strftime('%H:%M:%S'), - 'HH:mm': test_date_obj.strftime('%H:%M')} + "HH:mm:ss": test_date_obj.strftime("%H:%M:%S"), + "HH:mm": test_date_obj.strftime("%H:%M"), +} class TestFmtDatetime(unittest.TestCase): @@ -65,25 +73,17 @@ class TestFmtDatetime(unittest.TestCase): def test_formatdate_forced(self): # Test with forced date formats - self.assertEqual( - formatdate(test_date, 'dd-yyyy-mm'), - test_date_obj.strftime('%d-%Y-%m')) - self.assertEqual( - formatdate(test_date, 'dd-yyyy-MM'), - test_date_obj.strftime('%d-%Y-%m')) + self.assertEqual(formatdate(test_date, "dd-yyyy-mm"), test_date_obj.strftime("%d-%Y-%m")) + self.assertEqual(formatdate(test_date, "dd-yyyy-MM"), test_date_obj.strftime("%d-%Y-%m")) def test_formatdate_forced_broken_locale(self): # Test with forced date formats lang = frappe.local.lang # Force fallback from Babel try: - frappe.local.lang = 'FAKE' - self.assertEqual( - formatdate(test_date, 'dd-yyyy-mm'), - test_date_obj.strftime('%d-%Y-%m')) - self.assertEqual( - formatdate(test_date, 'dd-yyyy-MM'), - test_date_obj.strftime('%d-%Y-%m')) + frappe.local.lang = "FAKE" + self.assertEqual(formatdate(test_date, "dd-yyyy-mm"), test_date_obj.strftime("%d-%Y-%m")) + self.assertEqual(formatdate(test_date, "dd-yyyy-MM"), test_date_obj.strftime("%d-%Y-%m")) finally: frappe.local.lang = lang @@ -98,9 +98,7 @@ class TestFmtDatetime(unittest.TestCase): # Test time formatters def test_format_time_forced(self): # Test with forced time formats - self.assertEqual( - format_time(test_time, 'ss:mm:HH'), - test_date_obj.strftime('%S:%M:%H')) + self.assertEqual(format_time(test_time, "ss:mm:HH"), test_date_obj.strftime("%S:%M:%H")) def test_format_time(self): # Test format_time with various default time formats set @@ -115,8 +113,9 @@ class TestFmtDatetime(unittest.TestCase): def test_format_datetime_forced(self): # Test with forced date formats self.assertEqual( - format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'), - test_date_obj.strftime('%d-%Y-%m %S:%M:%H')) + format_datetime(test_datetime, "dd-yyyy-MM ss:mm:HH"), + test_date_obj.strftime("%d-%Y-%m %S:%M:%H"), + ) def test_format_datetime(self): # Test formatdate with various default date formats set @@ -126,6 +125,5 @@ class TestFmtDatetime(unittest.TestCase): for time_fmt, valid_time in test_time_formats.items(): frappe.db.set_default("time_format", time_fmt) frappe.local.user_time_format = None - valid_fmt = valid_date + ' ' + valid_time - self.assertEqual( - format_datetime(test_datetime), valid_fmt) + valid_fmt = valid_date + " " + valid_time + self.assertEqual(format_datetime(test_datetime), valid_fmt) diff --git a/frappe/tests/test_fmt_money.py b/frappe/tests/test_fmt_money.py index 61a2e8c69e..c27c13ad3b 100644 --- a/frappe/tests/test_fmt_money.py +++ b/frappe/tests/test_fmt_money.py @@ -1,8 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import unittest + import frappe from frappe.utils import fmt_money -import unittest + class TestFmtMoney(unittest.TestCase): def test_standard(self): @@ -38,7 +40,6 @@ class TestFmtMoney(unittest.TestCase): self.assertEqual(fmt_money(-100000000), "-100.000.000,00") self.assertEqual(fmt_money(-1000000000), "-1.000.000.000,00") - def test_lacs(self): frappe.db.set_default("number_format", "#,##,###.##") self.assertEqual(fmt_money(100), "100.00") @@ -93,8 +94,9 @@ class TestFmtMoney(unittest.TestCase): frappe.db.set_default("currency_precision", "") def test_custom_fmt_money_format(self): - self.assertEqual(fmt_money(100000, format="#,###.##"), '100,000.00') + self.assertEqual(fmt_money(100000, format="#,###.##"), "100,000.00") + -if __name__=="__main__": +if __name__ == "__main__": frappe.connect() - unittest.main() \ No newline at end of file + unittest.main() diff --git a/frappe/tests/test_form_load.py b/frappe/tests/test_form_load.py index 92694cf022..e92b8c3ff2 100644 --- a/frappe/tests/test_form_load.py +++ b/frappe/tests/test_form_load.py @@ -1,46 +1,51 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest -from frappe.desk.form.load import getdoctype, getdoc, get_docinfo -from frappe.core.page.permission_manager.permission_manager import update, reset, add +import unittest + +import frappe +from frappe.core.page.permission_manager.permission_manager import add, reset, update from frappe.custom.doctype.property_setter.property_setter import make_property_setter +from frappe.desk.form.load import get_docinfo, getdoc, getdoctype from frappe.utils.file_manager import save_file -test_dependencies = ['Blog Category', 'Blogger'] +test_dependencies = ["Blog Category", "Blogger"] + class TestFormLoad(unittest.TestCase): def test_load(self): getdoctype("DocType") - meta = list(filter(lambda d: d.name=="DocType", frappe.response.docs))[0] + meta = list(filter(lambda d: d.name == "DocType", frappe.response.docs))[0] self.assertEqual(meta.name, "DocType") self.assertTrue(meta.get("__js")) frappe.response.docs = [] getdoctype("Event") - meta = list(filter(lambda d: d.name=="Event", frappe.response.docs))[0] + meta = list(filter(lambda d: d.name == "Event", frappe.response.docs))[0] self.assertTrue(meta.get("__calendar_js")) def test_fieldlevel_permissions_in_load(self): - blog = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category-1", - "blog_intro": "Test Blog Intro", - "blogger": "_Test Blogger 1", - "content": "Test Blog Content", - "title": "_Test Blog Post {}".format(frappe.utils.now()), - "published": 0 - }) + blog = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category-1", + "blog_intro": "Test Blog Intro", + "blogger": "_Test Blogger 1", + "content": "Test Blog Content", + "title": "_Test Blog Post {}".format(frappe.utils.now()), + "published": 0, + } + ) blog.insert() - user = frappe.get_doc('User', 'test@example.com') + user = frappe.get_doc("User", "test@example.com") user_roles = frappe.get_roles() user.remove_roles(*user_roles) - user.add_roles('Blogger') + user.add_roles("Blogger") - blog_post_property_setter = make_property_setter('Blog Post', 'published', 'permlevel', 1, 'Int') - reset('Blog Post') + blog_post_property_setter = make_property_setter("Blog Post", "published", "permlevel", 1, "Int") + reset("Blog Post") # test field level permission before role level permissions are defined frappe.set_user(user.name) @@ -57,9 +62,9 @@ class TestFormLoad(unittest.TestCase): self.assertEqual(blog_doc.published, 0) # test field level permission after role level permissions are defined - frappe.set_user('Administrator') - add('Blog Post', 'Website Manager', 1) - update('Blog Post', 'Website Manager', 1, 'write', 1) + frappe.set_user("Administrator") + add("Blog Post", "Website Manager", 1) + update("Blog Post", "Website Manager", 1, "write", 1) frappe.set_user(user.name) blog_doc = get_blog(blog.name) @@ -76,11 +81,11 @@ class TestFormLoad(unittest.TestCase): # since published field has higher permlevel self.assertEqual(blog_doc.published, 0) - frappe.set_user('Administrator') - user.add_roles('Website Manager') + frappe.set_user("Administrator") + user.add_roles("Website Manager") frappe.set_user(user.name) - doc = frappe.get_doc('Blog Post', blog.name) + doc = frappe.get_doc("Blog Post", blog.name) doc.published = 1 doc.save() @@ -89,55 +94,55 @@ class TestFormLoad(unittest.TestCase): # (after adding Website Manager role) self.assertEqual(blog_doc.published, 1) - frappe.set_user('Administrator') + frappe.set_user("Administrator") # reset user roles - user.remove_roles('Blogger', 'Website Manager') + user.remove_roles("Blogger", "Website Manager") user.add_roles(*user_roles) blog_doc.delete() frappe.delete_doc(blog_post_property_setter.doctype, blog_post_property_setter.name) def test_fieldlevel_permissions_in_load_for_child_table(self): - contact = frappe.new_doc('Contact') - contact.first_name = '_Test Contact 1' - contact.append('phone_nos', {'phone': '123456'}) + contact = frappe.new_doc("Contact") + contact.first_name = "_Test Contact 1" + contact.append("phone_nos", {"phone": "123456"}) contact.insert() - user = frappe.get_doc('User', 'test@example.com') + user = frappe.get_doc("User", "test@example.com") user_roles = frappe.get_roles() user.remove_roles(*user_roles) - user.add_roles('Accounts User') + user.add_roles("Accounts User") - make_property_setter('Contact Phone', 'phone', 'permlevel', 1, 'Int') - reset('Contact Phone') - add('Contact', 'Sales User', 1) - update('Contact', 'Sales User', 1, 'write', 1) + make_property_setter("Contact Phone", "phone", "permlevel", 1, "Int") + reset("Contact Phone") + add("Contact", "Sales User", 1) + update("Contact", "Sales User", 1, "write", 1) frappe.set_user(user.name) - contact = frappe.get_doc('Contact', '_Test Contact 1') + contact = frappe.get_doc("Contact", "_Test Contact 1") - contact.phone_nos[0].phone = '654321' + contact.phone_nos[0].phone = "654321" contact.save() - self.assertEqual(contact.phone_nos[0].phone, '123456') + self.assertEqual(contact.phone_nos[0].phone, "123456") - frappe.set_user('Administrator') - user.add_roles('Sales User') + frappe.set_user("Administrator") + user.add_roles("Sales User") frappe.set_user(user.name) - contact.phone_nos[0].phone = '654321' + contact.phone_nos[0].phone = "654321" contact.save() - contact = frappe.get_doc('Contact', '_Test Contact 1') - self.assertEqual(contact.phone_nos[0].phone, '654321') + contact = frappe.get_doc("Contact", "_Test Contact 1") + self.assertEqual(contact.phone_nos[0].phone, "654321") - frappe.set_user('Administrator') + frappe.set_user("Administrator") # reset user roles - user.remove_roles('Accounts User', 'Sales User') + user.remove_roles("Accounts User", "Sales User") user.add_roles(*user_roles) contact.delete() @@ -160,13 +165,15 @@ class TestFormLoad(unittest.TestCase): # empty attachment save_file("test_file", b"", note.doctype, note.name, decode=True) - frappe.get_doc({ - "doctype": "Communication", - "communication_type": "Communication", - "content": "test email", - "reference_doctype": note.doctype, - "reference_name": note.name, - }).insert() + frappe.get_doc( + { + "doctype": "Communication", + "communication_type": "Communication", + "content": "test email", + "reference_doctype": note.doctype, + "reference_name": note.name, + } + ).insert() get_docinfo(note) docinfo = frappe.response["docinfo"] @@ -188,6 +195,6 @@ class TestFormLoad(unittest.TestCase): def get_blog(blog_name): frappe.response.docs = [] - getdoc('Blog Post', blog_name) + getdoc("Blog Post", blog_name) doc = frappe.response.docs[0] return doc diff --git a/frappe/tests/test_formatter.py b/frappe/tests/test_formatter.py index 5454c2b1cd..1a195f9218 100644 --- a/frappe/tests/test_formatter.py +++ b/frappe/tests/test_formatter.py @@ -1,25 +1,21 @@ # -*- coding: utf-8 -*- +import unittest + import frappe from frappe import format -import unittest + class TestFormatter(unittest.TestCase): def test_currency_formatting(self): - df = frappe._dict({ - 'fieldname': 'amount', - 'fieldtype': 'Currency', - 'options': 'currency' - }) + df = frappe._dict({"fieldname": "amount", "fieldtype": "Currency", "options": "currency"}) - doc = frappe._dict({ - 'amount': 5 - }) - frappe.db.set_default("currency", 'INR') + doc = frappe._dict({"amount": 5}) + frappe.db.set_default("currency", "INR") # if currency field is not passed then default currency should be used. - self.assertEqual(format(100000, df, doc, format="#,###.##"), '₹ 100,000.00') + self.assertEqual(format(100000, df, doc, format="#,###.##"), "₹ 100,000.00") - doc.currency = 'USD' + doc.currency = "USD" self.assertEqual(format(100000, df, doc, format="#,###.##"), "$ 100,000.00") - frappe.db.set_default("currency", None) \ No newline at end of file + frappe.db.set_default("currency", None) diff --git a/frappe/tests/test_frappe_client.py b/frappe/tests/test_frappe_client.py index e4588a16f1..2cca251718 100644 --- a/frappe/tests/test_frappe_client.py +++ b/frappe/tests/test_frappe_client.py @@ -21,28 +21,32 @@ class TestFrappeClient(unittest.TestCase): try: FrappeClient(site_url, "Administrator", cls.PASSWORD, verify=False) except AuthError: - raise unittest.SkipTest(f"AuthError raised for {site_url} [usr=Administrator, pwd={cls.PASSWORD}]") + raise unittest.SkipTest( + f"AuthError raised for {site_url} [usr=Administrator, pwd={cls.PASSWORD}]" + ) return super().setUpClass() def test_insert_many(self): server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) - frappe.db.delete("Note", {"title": ("in", ('Sing','a','song','of','sixpence'))}) + frappe.db.delete("Note", {"title": ("in", ("Sing", "a", "song", "of", "sixpence"))}) frappe.db.commit() - server.insert_many([ - {"doctype": "Note", "public": True, "title": "Sing"}, - {"doctype": "Note", "public": True, "title": "a"}, - {"doctype": "Note", "public": True, "title": "song"}, - {"doctype": "Note", "public": True, "title": "of"}, - {"doctype": "Note", "public": True, "title": "sixpence"}, - ]) + server.insert_many( + [ + {"doctype": "Note", "public": True, "title": "Sing"}, + {"doctype": "Note", "public": True, "title": "a"}, + {"doctype": "Note", "public": True, "title": "song"}, + {"doctype": "Note", "public": True, "title": "of"}, + {"doctype": "Note", "public": True, "title": "sixpence"}, + ] + ) - self.assertTrue(frappe.db.get_value('Note', {'title': 'Sing'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'a'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'song'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'of'})) - self.assertTrue(frappe.db.get_value('Note', {'title': 'sixpence'})) + self.assertTrue(frappe.db.get_value("Note", {"title": "Sing"})) + self.assertTrue(frappe.db.get_value("Note", {"title": "a"})) + self.assertTrue(frappe.db.get_value("Note", {"title": "song"})) + self.assertTrue(frappe.db.get_value("Note", {"title": "of"})) + self.assertTrue(frappe.db.get_value("Note", {"title": "sixpence"})) def test_create_doc(self): server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) @@ -51,7 +55,7 @@ class TestFrappeClient(unittest.TestCase): server.insert({"doctype": "Note", "public": True, "title": "test_create"}) - self.assertTrue(frappe.db.get_value('Note', {'title': 'test_create'})) + self.assertTrue(frappe.db.get_value("Note", {"title": "test_create"})) def test_list_docs(self): server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) @@ -64,9 +68,11 @@ class TestFrappeClient(unittest.TestCase): frappe.db.delete("Note", {"title": "get_this"}) frappe.db.commit() - server.insert_many([ - {"doctype": "Note", "public": True, "title": "get_this"}, - ]) + server.insert_many( + [ + {"doctype": "Note", "public": True, "title": "get_this"}, + ] + ) doc = server.get_doc("Note", "get_this") self.assertTrue(doc) @@ -77,31 +83,46 @@ class TestFrappeClient(unittest.TestCase): test_content = "test get value" - server.insert_many([ - {"doctype": "Note", "public": True, "title": "get_value", "content": test_content}, - ]) - self.assertEqual(server.get_value("Note", "content", {"title": "get_value"}).get('content'), test_content) - name = server.get_value("Note", "name", {"title": "get_value"}).get('name') + server.insert_many( + [ + {"doctype": "Note", "public": True, "title": "get_value", "content": test_content}, + ] + ) + self.assertEqual( + server.get_value("Note", "content", {"title": "get_value"}).get("content"), test_content + ) + name = server.get_value("Note", "name", {"title": "get_value"}).get("name") # test by name - self.assertEqual(server.get_value("Note", "content", name).get('content'), test_content) - - self.assertRaises(FrappeException, server.get_value, "Note", "(select (password) from(__Auth) order by name desc limit 1)", {"title": "get_value"}) + self.assertEqual(server.get_value("Note", "content", name).get("content"), test_content) + + self.assertRaises( + FrappeException, + server.get_value, + "Note", + "(select (password) from(__Auth) order by name desc limit 1)", + {"title": "get_value"}, + ) def test_get_single(self): server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) - server.set_value('Website Settings', 'Website Settings', 'title_prefix', 'test-prefix') - self.assertEqual(server.get_value('Website Settings', 'title_prefix', 'Website Settings').get('title_prefix'), 'test-prefix') - self.assertEqual(server.get_value('Website Settings', 'title_prefix').get('title_prefix'), 'test-prefix') - frappe.db.set_value('Website Settings', None, 'title_prefix', '') + server.set_value("Website Settings", "Website Settings", "title_prefix", "test-prefix") + self.assertEqual( + server.get_value("Website Settings", "title_prefix", "Website Settings").get("title_prefix"), + "test-prefix", + ) + self.assertEqual( + server.get_value("Website Settings", "title_prefix").get("title_prefix"), "test-prefix" + ) + frappe.db.set_value("Website Settings", None, "title_prefix", "") def test_update_doc(self): server = FrappeClient(get_url(), "Administrator", self.PASSWORD, verify=False) frappe.db.delete("Note", {"title": ("in", ("Sing", "sing"))}) frappe.db.commit() - server.insert({"doctype":"Note", "public": True, "title": "Sing"}) - doc = server.get_doc("Note", 'Sing') + server.insert({"doctype": "Note", "public": True, "title": "Sing"}) + doc = server.get_doc("Note", "Sing") changed_title = "sing" doc["title"] = changed_title doc = server.update(doc) @@ -118,27 +139,32 @@ class TestFrappeClient(unittest.TestCase): frappe.db.commit() # create multiple contacts - server.insert_many([ - {"doctype": "Contact", "first_name": "George", "last_name": "Steevens"}, - {"doctype": "Contact", "first_name": "William", "last_name": "Shakespeare"} - ]) + server.insert_many( + [ + {"doctype": "Contact", "first_name": "George", "last_name": "Steevens"}, + {"doctype": "Contact", "first_name": "William", "last_name": "Shakespeare"}, + ] + ) # create an event with one of the created contacts - event = server.insert({ - "doctype": "Event", - "subject": "Sing a song of sixpence", - "event_participants": [{ - "reference_doctype": "Contact", - "reference_docname": "George Steevens" - }] - }) + event = server.insert( + { + "doctype": "Event", + "subject": "Sing a song of sixpence", + "event_participants": [ + {"reference_doctype": "Contact", "reference_docname": "George Steevens"} + ], + } + ) # update the event's contact to the second contact - server.update({ - "doctype": "Event Participants", - "name": event.get("event_participants")[0].get("name"), - "reference_docname": "William Shakespeare" - }) + server.update( + { + "doctype": "Event Participants", + "name": event.get("event_participants")[0].get("name"), + "reference_docname": "William Shakespeare", + } + ) # the change should run the parent document's validations and # create a Communication record with the new contact @@ -149,19 +175,21 @@ class TestFrappeClient(unittest.TestCase): frappe.db.delete("Note", {"title": "delete"}) frappe.db.commit() - server.insert_many([ - {"doctype": "Note", "public": True, "title": "delete"}, - ]) + server.insert_many( + [ + {"doctype": "Note", "public": True, "title": "delete"}, + ] + ) server.delete("Note", "delete") - self.assertFalse(frappe.db.get_value('Note', {'title': 'delete'})) + self.assertFalse(frappe.db.get_value("Note", {"title": "delete"})) def test_auth_via_api_key_secret(self): # generate API key and API secret for administrator keys = generate_keys("Administrator") frappe.db.commit() generated_secret = frappe.utils.password.get_decrypted_password( - "User", "Administrator", fieldname='api_secret' + "User", "Administrator", fieldname="api_secret" ) api_key = frappe.db.get_value("User", "Administrator", "api_key") @@ -170,9 +198,13 @@ class TestFrappeClient(unittest.TestCase): self.assertEqual(res.status_code, 200) self.assertEqual("Administrator", res.json()["message"]) - self.assertEqual(keys['api_secret'], generated_secret) + self.assertEqual(keys["api_secret"], generated_secret) - header = {"Authorization": "Basic {}".format(base64.b64encode(frappe.safe_encode("{}:{}".format(api_key, generated_secret))).decode())} + header = { + "Authorization": "Basic {}".format( + base64.b64encode(frappe.safe_encode("{}:{}".format(api_key, generated_secret))).decode() + ) + } res = requests.post(get_url() + "/api/method/frappe.auth.get_logged_user", headers=header) self.assertEqual(res.status_code, 200) self.assertEqual("Administrator", res.json()["message"]) diff --git a/frappe/tests/test_geo_ip.py b/frappe/tests/test_geo_ip.py index b1811917e5..5e98339ffa 100644 --- a/frappe/tests/test_geo_ip.py +++ b/frappe/tests/test_geo_ip.py @@ -2,10 +2,12 @@ # License: MIT. See LICENSE import unittest + class TestGeoIP(unittest.TestCase): def test_geo_ip(self): return from frappe.sessions import get_geo_ip_country + self.assertEqual(get_geo_ip_country("223.29.223.255"), "India") self.assertEqual(get_geo_ip_country("4.18.32.80"), "United States") - self.assertEqual(get_geo_ip_country("217.194.147.25"), "United States") \ No newline at end of file + self.assertEqual(get_geo_ip_country("217.194.147.25"), "United States") diff --git a/frappe/tests/test_global_search.py b/frappe/tests/test_global_search.py index 0faff55e7f..1b9662219b 100644 --- a/frappe/tests/test_global_search.py +++ b/frappe/tests/test_global_search.py @@ -4,18 +4,17 @@ import unittest import frappe - from frappe.custom.doctype.property_setter.property_setter import make_property_setter from frappe.desk.page.setup_wizard.install_fixtures import update_global_search_doctypes -from frappe.utils import global_search, now_datetime from frappe.test_runner import make_test_objects +from frappe.utils import global_search, now_datetime class TestGlobalSearch(unittest.TestCase): def setUp(self): update_global_search_doctypes() global_search.setup_global_search_table() - self.assertTrue('__global_search' in frappe.db.get_tables()) + self.assertTrue("__global_search" in frappe.db.get_tables()) doctype = "Event" global_search.reset() make_property_setter(doctype, "subject", "in_global_search", 1, "Int") @@ -25,93 +24,105 @@ class TestGlobalSearch(unittest.TestCase): def tearDown(self): frappe.db.delete("Property Setter", {"doc_type": "Event"}) - frappe.clear_cache(doctype='Event') + frappe.clear_cache(doctype="Event") frappe.db.delete("Event") frappe.db.delete("__global_search") - make_test_objects('Event') + make_test_objects("Event") frappe.db.commit() def insert_test_events(self): frappe.db.delete("Event") - phrases = ['"The Sixth Extinction II: Amor Fati" is the second episode of the seventh season of the American science fiction.', - 'After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ', - 'Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy.'] + phrases = [ + '"The Sixth Extinction II: Amor Fati" is the second episode of the seventh season of the American science fiction.', + "After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ", + "Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy.", + ] for text in phrases: - frappe.get_doc(dict( - doctype='Event', - subject=text, - repeat_on='Monthly', - starts_on=now_datetime())).insert() + frappe.get_doc( + dict(doctype="Event", subject=text, repeat_on="Monthly", starts_on=now_datetime()) + ).insert() global_search.sync_global_search() frappe.db.commit() def test_search(self): self.insert_test_events() - results = global_search.search('awakens') - self.assertTrue('After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ' in results[0].content) - - results = global_search.search('extraterrestrial') - self.assertTrue('Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy.' in results[0].content) - results = global_search.search('awakens & duty & alien') - self.assertTrue('After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. ' in results[0].content) + results = global_search.search("awakens") + self.assertTrue( + "After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. " + in results[0].content + ) + + results = global_search.search("extraterrestrial") + self.assertTrue( + "Carter explored themes of extraterrestrial involvement in ancient mass extinctions in this episode, the third in a trilogy." + in results[0].content + ) + results = global_search.search("awakens & duty & alien") + self.assertTrue( + "After Mulder awakens from his coma, he realizes his duty to prevent alien colonization. " + in results[0].content + ) def test_update_doc(self): self.insert_test_events() - test_subject = 'testing global search' - event = frappe.get_doc('Event', frappe.get_all('Event')[0].name) + test_subject = "testing global search" + event = frappe.get_doc("Event", frappe.get_all("Event")[0].name) event.subject = test_subject event.save() frappe.db.commit() global_search.sync_global_search() - results = global_search.search('testing global search') + results = global_search.search("testing global search") - self.assertTrue('testing global search' in results[0].content) + self.assertTrue("testing global search" in results[0].content) def test_update_fields(self): self.insert_test_events() - results = global_search.search('Monthly') + results = global_search.search("Monthly") self.assertEqual(len(results), 0) doctype = "Event" make_property_setter(doctype, "repeat_on", "in_global_search", 1, "Int") global_search.rebuild_for_doctype(doctype) - results = global_search.search('Monthly') + results = global_search.search("Monthly") self.assertEqual(len(results), 3) def test_delete_doc(self): self.insert_test_events() - event_name = frappe.get_all('Event')[0].name - event = frappe.get_doc('Event', event_name) + event_name = frappe.get_all("Event")[0].name + event = frappe.get_doc("Event", event_name) test_subject = event.subject results = global_search.search(test_subject) - self.assertTrue(any(r["name"] == event_name for r in results), msg="Failed to search document by exact name") + self.assertTrue( + any(r["name"] == event_name for r in results), msg="Failed to search document by exact name" + ) - frappe.delete_doc('Event', event_name) + frappe.delete_doc("Event", event_name) global_search.sync_global_search() frappe.db.commit() results = global_search.search(test_subject) - self.assertTrue(all(r["name"] != event_name for r in results), msg="Deleted documents appearing in global search.") + self.assertTrue( + all(r["name"] != event_name for r in results), + msg="Deleted documents appearing in global search.", + ) def test_insert_child_table(self): frappe.db.delete("Event") - phrases = ['Hydrus is a small constellation in the deep southern sky. ', - 'It was first depicted on a celestial atlas by Johann Bayer in his 1603 Uranometria. ', - 'The French explorer and astronomer Nicolas Louis de Lacaille charted the brighter stars and gave their Bayer designations in 1756. ', - 'Its name means "male water snake", as opposed to Hydra, a much larger constellation that represents a female water snake. ', - 'It remains below the horizon for most Northern Hemisphere observers.', - 'The brightest star is the 2.8-magnitude Beta Hydri, also the closest reasonably bright star to the south celestial pole. ', - 'Pulsating between magnitude 3.26 and 3.33, Gamma Hydri is a variable red giant some 60 times the diameter of our Sun. ', - 'Lying near it is VW Hydri, one of the brightest dwarf novae in the heavens. ', - 'Four star systems have been found to have exoplanets to date, most notably HD 10180, which could bear up to nine planetary companions.'] + phrases = [ + "Hydrus is a small constellation in the deep southern sky. ", + "It was first depicted on a celestial atlas by Johann Bayer in his 1603 Uranometria. ", + "The French explorer and astronomer Nicolas Louis de Lacaille charted the brighter stars and gave their Bayer designations in 1756. ", + 'Its name means "male water snake", as opposed to Hydra, a much larger constellation that represents a female water snake. ', + "It remains below the horizon for most Northern Hemisphere observers.", + "The brightest star is the 2.8-magnitude Beta Hydri, also the closest reasonably bright star to the south celestial pole. ", + "Pulsating between magnitude 3.26 and 3.33, Gamma Hydri is a variable red giant some 60 times the diameter of our Sun. ", + "Lying near it is VW Hydri, one of the brightest dwarf novae in the heavens. ", + "Four star systems have been found to have exoplanets to date, most notably HD 10180, which could bear up to nine planetary companions.", + ] for text in phrases: - doc = frappe.get_doc({ - 'doctype':'Event', - 'subject': text, - 'starts_on': now_datetime() - }) + doc = frappe.get_doc({"doctype": "Event", "subject": text, "starts_on": now_datetime()}) doc.insert() global_search.sync_global_search() @@ -121,7 +132,7 @@ class TestGlobalSearch(unittest.TestCase): cases = [ { "case_type": "generic", - "data": ''' + "data": """ Lorem Ipsum Dolor Sit Amet - ''', - "result": "Description : Lorem Ipsum Dolor Sit Amet" - } + """, + "result": "Description : Lorem Ipsum Dolor Sit Amet", + }, ] for case in cases: - doc = frappe.get_doc({ - 'doctype':'Event', - 'subject': 'Lorem Ipsum', - 'starts_on': now_datetime(), - 'description': case["data"] - }) - - field_as_text = '' + doc = frappe.get_doc( + { + "doctype": "Event", + "subject": "Lorem Ipsum", + "starts_on": now_datetime(), + "description": case["data"], + } + ) + + field_as_text = "" for field in doc.meta.fields: - if field.fieldname == 'description': + if field.fieldname == "description": field_as_text = global_search.get_formatted_value(doc.description, field) self.assertEqual(case["result"], field_as_text) @@ -186,8 +200,9 @@ class TestGlobalSearch(unittest.TestCase): global_search.update_global_search_for_all_web_pages() global_search.sync_global_search() frappe.db.commit() - results = global_search.web_search('unsubscribe') - self.assertTrue('Unsubscribe' in results[0].content) - results = global_search.web_search(text='unsubscribe', - scope="manufacturing\" UNION ALL SELECT 1,2,3,4,doctype from __global_search") + results = global_search.web_search("unsubscribe") + self.assertTrue("Unsubscribe" in results[0].content) + results = global_search.web_search( + text="unsubscribe", scope='manufacturing" UNION ALL SELECT 1,2,3,4,doctype from __global_search' + ) self.assertTrue(results == []) diff --git a/frappe/tests/test_goal.py b/frappe/tests/test_goal.py index 35028d939f..b624717c64 100644 --- a/frappe/tests/test_goal.py +++ b/frappe/tests/test_goal.py @@ -3,9 +3,9 @@ import frappe from frappe.test_runner import make_test_objects +from frappe.tests.utils import FrappeTestCase from frappe.utils import format_date, today from frappe.utils.goal import get_monthly_goal_graph_data, get_monthly_results -from frappe.tests.utils import FrappeTestCase class TestGoal(FrappeTestCase): @@ -29,9 +29,7 @@ class TestGoal(FrappeTestCase): def test_get_monthly_goal_graph_data(self): """Test for accurate values in graph data (based on test_get_monthly_results)""" - docname = frappe.get_list("Event", filters={"subject": ["=", "_Test Event 1"]})[ - 0 - ]["name"] + docname = frappe.get_list("Event", filters={"subject": ["=", "_Test Event 1"]})[0]["name"] frappe.db.set_value("Event", docname, "description", 1) data = get_monthly_goal_graph_data( "Test", diff --git a/frappe/tests/test_hooks.py b/frappe/tests/test_hooks.py index e172a52828..1311d59341 100644 --- a/frappe/tests/test_hooks.py +++ b/frappe/tests/test_hooks.py @@ -2,9 +2,11 @@ # License: MIT. See LICENSE import unittest + import frappe -from frappe.desk.doctype.todo.todo import ToDo from frappe.cache_manager import clear_controller_cache +from frappe.desk.doctype.todo.todo import ToDo + class TestHooks(unittest.TestCase): def test_hooks(self): @@ -13,40 +15,38 @@ class TestHooks(unittest.TestCase): self.assertTrue(isinstance(hooks.get("doc_events"), dict)) self.assertTrue(isinstance(hooks.get("doc_events").get("*"), dict)) self.assertTrue(isinstance(hooks.get("doc_events").get("*"), dict)) - self.assertTrue("frappe.desk.notifications.clear_doctype_notifications" in - hooks.get("doc_events").get("*").get("on_update")) + self.assertTrue( + "frappe.desk.notifications.clear_doctype_notifications" + in hooks.get("doc_events").get("*").get("on_update") + ) def test_override_doctype_class(self): from frappe import hooks # Set hook - hooks.override_doctype_class = { - 'ToDo': ['frappe.tests.test_hooks.CustomToDo'] - } + hooks.override_doctype_class = {"ToDo": ["frappe.tests.test_hooks.CustomToDo"]} # Clear cache - frappe.cache().delete_value('app_hooks') - clear_controller_cache('ToDo') + frappe.cache().delete_value("app_hooks") + clear_controller_cache("ToDo") - todo = frappe.get_doc(doctype='ToDo', description='asdf') + todo = frappe.get_doc(doctype="ToDo", description="asdf") self.assertTrue(isinstance(todo, CustomToDo)) def test_has_permission(self): from frappe import hooks # Set hook - address_has_permission_hook = hooks.has_permission.get('Address', []) + address_has_permission_hook = hooks.has_permission.get("Address", []) if isinstance(address_has_permission_hook, str): address_has_permission_hook = [address_has_permission_hook] - address_has_permission_hook.append( - 'frappe.tests.test_hooks.custom_has_permission' - ) + address_has_permission_hook.append("frappe.tests.test_hooks.custom_has_permission") - hooks.has_permission['Address'] = address_has_permission_hook + hooks.has_permission["Address"] = address_has_permission_hook # Clear cache - frappe.cache().delete_value('app_hooks') + frappe.cache().delete_value("app_hooks") # Init User and Address username = "test@example.com" @@ -55,14 +55,10 @@ class TestHooks(unittest.TestCase): address = frappe.new_doc("Address") # Test! - self.assertTrue( - frappe.has_permission("Address", doc=address, user=username) - ) + self.assertTrue(frappe.has_permission("Address", doc=address, user=username)) address.flags.dont_touch_me = True - self.assertFalse( - frappe.has_permission("Address", doc=address, user=username) - ) + self.assertFalse(frappe.has_permission("Address", doc=address, user=username)) def custom_has_permission(doc, ptype, user): diff --git a/frappe/tests/test_linked_with.py b/frappe/tests/test_linked_with.py index ec461c7d5f..d1be3cb775 100644 --- a/frappe/tests/test_linked_with.py +++ b/frappe/tests/test_linked_with.py @@ -11,105 +11,118 @@ class TestLinkedWith(unittest.TestCase): parent_doc.is_submittable = 1 parent_doc.insert() - child_doc1 = new_doctype("Child Doc1", + child_doc1 = new_doctype( + "Child Doc1", fields=[ { "label": "Parent Doc", "fieldname": "parent_doc", "fieldtype": "Link", - "options": "Parent Doc" + "options": "Parent Doc", }, { "label": "Reference field", "fieldname": "reference_name", "fieldtype": "Dynamic Link", - "options": "reference_doctype" + "options": "reference_doctype", }, { "label": "Reference Doctype", "fieldname": "reference_doctype", "fieldtype": "Link", - "options": "DocType" - } - - ], unique=0) + "options": "DocType", + }, + ], + unique=0, + ) child_doc1.is_submittable = 1 child_doc1.insert() - child_doc2 = new_doctype("Child Doc2", + child_doc2 = new_doctype( + "Child Doc2", fields=[ { "label": "Parent Doc", "fieldname": "parent_doc", "fieldtype": "Link", - "options": "Parent Doc" + "options": "Parent Doc", }, { "label": "Child Doc1", "fieldname": "child_doc1", "fieldtype": "Link", - "options": "Child Doc1" - } - - ], unique=0) + "options": "Child Doc1", + }, + ], + unique=0, + ) child_doc2.is_submittable = 1 child_doc2.insert() def tearDown(self): - for doctype in ['Parent Doc', 'Child Doc1', 'Child Doc2']: + for doctype in ["Parent Doc", "Child Doc1", "Child Doc2"]: frappe.delete_doc("DocType", doctype) def test_get_doctype_references_by_link_field(self): - references = linked_with.get_references_across_doctypes_by_link_field(to_doctypes = ['Parent Doc']) - self.assertEqual(len(references['Parent Doc']), 3) - self.assertIn({'doctype': 'Child Doc1', 'fieldname': 'parent_doc'}, references['Parent Doc']) - self.assertIn({'doctype': 'Child Doc2', 'fieldname': 'parent_doc'}, references['Parent Doc']) + references = linked_with.get_references_across_doctypes_by_link_field(to_doctypes=["Parent Doc"]) + self.assertEqual(len(references["Parent Doc"]), 3) + self.assertIn({"doctype": "Child Doc1", "fieldname": "parent_doc"}, references["Parent Doc"]) + self.assertIn({"doctype": "Child Doc2", "fieldname": "parent_doc"}, references["Parent Doc"]) - references = linked_with.get_references_across_doctypes_by_link_field(to_doctypes = ['Child Doc1']) - self.assertEqual(len(references['Child Doc1']), 2) - self.assertIn({'doctype': 'Child Doc2', 'fieldname': 'child_doc1'}, references['Child Doc1']) + references = linked_with.get_references_across_doctypes_by_link_field(to_doctypes=["Child Doc1"]) + self.assertEqual(len(references["Child Doc1"]), 2) + self.assertIn({"doctype": "Child Doc2", "fieldname": "child_doc1"}, references["Child Doc1"]) references = linked_with.get_references_across_doctypes_by_link_field( - to_doctypes = ['Child Doc1', 'Parent Doc'], limit_link_doctypes=['Child Doc1']) - self.assertEqual(len(references['Child Doc1']), 1) - self.assertEqual(len(references['Parent Doc']), 1) - self.assertIn({'doctype': 'Child Doc1', 'fieldname': 'parent_doc'}, references['Parent Doc']) + to_doctypes=["Child Doc1", "Parent Doc"], limit_link_doctypes=["Child Doc1"] + ) + self.assertEqual(len(references["Child Doc1"]), 1) + self.assertEqual(len(references["Parent Doc"]), 1) + self.assertIn({"doctype": "Child Doc1", "fieldname": "parent_doc"}, references["Parent Doc"]) def test_get_doctype_references_by_dlink_field(self): references = linked_with.get_references_across_doctypes_by_dynamic_link_field( - to_doctypes = ['Parent Doc'], limit_link_doctypes = ['Parent Doc', 'Child Doc1', 'Child Doc2']) + to_doctypes=["Parent Doc"], limit_link_doctypes=["Parent Doc", "Child Doc1", "Child Doc2"] + ) self.assertFalse(references) - parent_record = frappe.get_doc({'doctype': 'Parent Doc'}).insert() + parent_record = frappe.get_doc({"doctype": "Parent Doc"}).insert() - child_record = frappe.get_doc({ - 'doctype': 'Child Doc1', - 'reference_doctype': 'Parent Doc', - 'reference_name': parent_record.name - }).insert() + child_record = frappe.get_doc( + { + "doctype": "Child Doc1", + "reference_doctype": "Parent Doc", + "reference_name": parent_record.name, + } + ).insert() references = linked_with.get_references_across_doctypes_by_dynamic_link_field( - to_doctypes = ['Parent Doc'], limit_link_doctypes = ['Parent Doc', 'Child Doc1', 'Child Doc2']) + to_doctypes=["Parent Doc"], limit_link_doctypes=["Parent Doc", "Child Doc1", "Child Doc2"] + ) - self.assertEqual(len(references['Parent Doc']), 1) - self.assertEqual(references['Parent Doc'][0]['doctype'], 'Child Doc1') - self.assertEqual(references['Parent Doc'][0]['doctype_fieldname'], 'reference_doctype') + self.assertEqual(len(references["Parent Doc"]), 1) + self.assertEqual(references["Parent Doc"][0]["doctype"], "Child Doc1") + self.assertEqual(references["Parent Doc"][0]["doctype_fieldname"], "reference_doctype") child_record.delete() parent_record.delete() def test_get_submitted_linked_docs(self): - parent_record = frappe.get_doc({'doctype': 'Parent Doc'}).insert() - - child_record = frappe.get_doc({ - 'doctype': 'Child Doc1', - 'reference_doctype': 'Parent Doc', - 'reference_name': parent_record.name, - 'docstatus': 1 - }).insert() - - linked_docs = linked_with.get_submitted_linked_docs(parent_record.doctype, parent_record.name)["docs"] - self.assertIn(child_record.name,linked_docs[0]['name']) + parent_record = frappe.get_doc({"doctype": "Parent Doc"}).insert() + + child_record = frappe.get_doc( + { + "doctype": "Child Doc1", + "reference_doctype": "Parent Doc", + "reference_name": parent_record.name, + "docstatus": 1, + } + ).insert() + + linked_docs = linked_with.get_submitted_linked_docs(parent_record.doctype, parent_record.name)[ + "docs" + ] + self.assertIn(child_record.name, linked_docs[0]["name"]) child_record.cancel() child_record.delete() parent_record.delete() diff --git a/frappe/tests/test_listview.py b/frappe/tests/test_listview.py index 3dbe2f3fa5..7d75e80f95 100644 --- a/frappe/tests/test_listview.py +++ b/frappe/tests/test_listview.py @@ -1,10 +1,11 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json import unittest + import frappe -import json +from frappe.desk.listview import get_group_by_count, get_list_settings, set_list_settings -from frappe.desk.listview import get_list_settings, set_list_settings, get_group_by_count class TestListView(unittest.TestCase): def setUp(self): @@ -34,7 +35,7 @@ class TestListView(unittest.TestCase): def test_set_list_settings_without_settings(self): set_list_settings("DocType", json.dumps({})) - settings = frappe.get_doc("List View Settings","DocType") + settings = frappe.get_doc("List View Settings", "DocType") self.assertEqual(settings.disable_auto_refresh, 0) self.assertEqual(settings.disable_count, 0) @@ -43,7 +44,7 @@ class TestListView(unittest.TestCase): def test_set_list_settings_with_existing_settings(self): frappe.get_doc({"doctype": "List View Settings", "name": "DocType", "disable_count": 1}).insert() set_list_settings("DocType", json.dumps({"disable_count": 0, "disable_auto_refresh": 1})) - settings = frappe.get_doc("List View Settings","DocType") + settings = frappe.get_doc("List View Settings", "DocType") self.assertEqual(settings.disable_auto_refresh, 1) self.assertEqual(settings.disable_count, 0) @@ -53,9 +54,14 @@ class TestListView(unittest.TestCase): if frappe.db.exists("Note", "Test created by filter with child table filter"): frappe.delete_doc("Note", "Test created by filter with child table filter") - doc = frappe.get_doc({"doctype": "Note", "title": "Test created by filter with child table filter", "public": 1}) + doc = frappe.get_doc( + {"doctype": "Note", "title": "Test created by filter with child table filter", "public": 1} + ) doc.append("seen_by", {"user": "Administrator"}) doc.insert() - data = {d.name: d.count for d in get_group_by_count('Note', '[["Note Seen By","user","=","Administrator"]]', 'owner')} - self.assertEqual(data['Administrator'], 1) \ No newline at end of file + data = { + d.name: d.count + for d in get_group_by_count("Note", '[["Note Seen By","user","=","Administrator"]]', "owner") + } + self.assertEqual(data["Administrator"], 1) diff --git a/frappe/tests/test_monitor.py b/frappe/tests/test_monitor.py index 7b3a6f7147..0c26ec0e28 100644 --- a/frappe/tests/test_monitor.py +++ b/frappe/tests/test_monitor.py @@ -3,11 +3,12 @@ # License: MIT. See LICENSE import unittest + import frappe import frappe.monitor +from frappe.monitor import MONITOR_REDIS_KEY from frappe.utils import set_request from frappe.utils.response import build_response -from frappe.monitor import MONITOR_REDIS_KEY class TestMonitor(unittest.TestCase): diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 5572f9c233..e57d2ae4cd 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -2,38 +2,43 @@ # License: MIT. See LICENSE import unittest + import frappe +from frappe.model.naming import ( + append_number_if_name_exists, + determine_consecutive_week_number, + getseries, + parse_naming_series, + revert_series_if_last, +) from frappe.utils import now_datetime -from frappe.model.naming import getseries -from frappe.model.naming import append_number_if_name_exists, revert_series_if_last -from frappe.model.naming import determine_consecutive_week_number, parse_naming_series class TestNaming(unittest.TestCase): def setUp(self): - frappe.db.delete('Note') + frappe.db.delete("Note") def tearDown(self): frappe.db.rollback() def test_append_number_if_name_exists(self): - ''' + """ Append number to name based on existing values if Bottle exists - Bottle -> Bottle-1 + Bottle -> Bottle-1 if Bottle-1 exists - Bottle -> Bottle-2 - ''' + Bottle -> Bottle-2 + """ - note = frappe.new_doc('Note') - note.title = 'Test' + note = frappe.new_doc("Note") + note.title = "Test" note.insert() - title2 = append_number_if_name_exists('Note', 'Test') - self.assertEqual(title2, 'Test-1') + title2 = append_number_if_name_exists("Note", "Test") + self.assertEqual(title2, "Test-1") - title2 = append_number_if_name_exists('Note', 'Test', 'title', '_') - self.assertEqual(title2, 'Test_1') + title2 = append_number_if_name_exists("Note", "Test", "title", "_") + self.assertEqual(title2, "Test_1") def test_field_autoname_name_sync(self): @@ -47,131 +52,144 @@ class TestNaming(unittest.TestCase): self.assertEqual(country.name, country.country_name) def test_format_autoname(self): - ''' + """ Test if braced params are replaced in format autoname - ''' - doctype = 'ToDo' + """ + doctype = "ToDo" - todo_doctype = frappe.get_doc('DocType', doctype) - todo_doctype.autoname = 'format:TODO-{MM}-{status}-{##}' + todo_doctype = frappe.get_doc("DocType", doctype) + todo_doctype.autoname = "format:TODO-{MM}-{status}-{##}" todo_doctype.save() - description = 'Format' + description = "Format" todo = frappe.new_doc(doctype) todo.description = description todo.insert() - series = getseries('', 2) + series = getseries("", 2) - series = str(int(series)-1) + series = str(int(series) - 1) if len(series) < 2: - series = '0' + series + series = "0" + series - self.assertEqual(todo.name, 'TODO-{month}-{status}-{series}'.format( - month=now_datetime().strftime('%m'), status=todo.status, series=series)) + self.assertEqual( + todo.name, + "TODO-{month}-{status}-{series}".format( + month=now_datetime().strftime("%m"), status=todo.status, series=series + ), + ) def test_format_autoname_for_consecutive_week_number(self): - ''' + """ Test if braced params are replaced for consecutive week number in format autoname - ''' - doctype = 'ToDo' + """ + doctype = "ToDo" - todo_doctype = frappe.get_doc('DocType', doctype) - todo_doctype.autoname = 'format:TODO-{WW}-{##}' + todo_doctype = frappe.get_doc("DocType", doctype) + todo_doctype.autoname = "format:TODO-{WW}-{##}" todo_doctype.save() - description = 'Format' + description = "Format" todo = frappe.new_doc(doctype) todo.description = description todo.insert() - series = getseries('', 2) + series = getseries("", 2) - series = str(int(series)-1) + series = str(int(series) - 1) if len(series) < 2: - series = '0' + series + series = "0" + series week = determine_consecutive_week_number(now_datetime()) - self.assertEqual(todo.name, 'TODO-{week}-{series}'.format( - week=week, series=series)) + self.assertEqual(todo.name, "TODO-{week}-{series}".format(week=week, series=series)) def test_revert_series(self): from datetime import datetime + year = datetime.now().year - series = 'TEST-{}-'.format(year) - key = 'TEST-.YYYY.-' - name = 'TEST-{}-00001'.format(year) + series = "TEST-{}-".format(year) + key = "TEST-.YYYY.-" + name = "TEST-{}-00001".format(year) frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 1)""", (series,)) revert_series_if_last(key, name) - current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] + current_index = frappe.db.sql( + """SELECT current from `tabSeries` where name = %s""", series, as_dict=True + )[0] - self.assertEqual(current_index.get('current'), 0) + self.assertEqual(current_index.get("current"), 0) frappe.db.delete("Series", {"name": series}) - series = 'TEST-{}-'.format(year) - key = 'TEST-.YYYY.-.#####' - name = 'TEST-{}-00002'.format(year) + series = "TEST-{}-".format(year) + key = "TEST-.YYYY.-.#####" + name = "TEST-{}-00002".format(year) frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 2)""", (series,)) revert_series_if_last(key, name) - current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] + current_index = frappe.db.sql( + """SELECT current from `tabSeries` where name = %s""", series, as_dict=True + )[0] - self.assertEqual(current_index.get('current'), 1) + self.assertEqual(current_index.get("current"), 1) frappe.db.delete("Series", {"name": series}) - series = 'TEST-' - key = 'TEST-' - name = 'TEST-00003' + series = "TEST-" + key = "TEST-" + name = "TEST-00003" frappe.db.delete("Series", {"name": series}) frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,)) revert_series_if_last(key, name) - current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] + current_index = frappe.db.sql( + """SELECT current from `tabSeries` where name = %s""", series, as_dict=True + )[0] - self.assertEqual(current_index.get('current'), 2) + self.assertEqual(current_index.get("current"), 2) frappe.db.delete("Series", {"name": series}) - series = 'TEST1-' - key = 'TEST1-.#####.-2021-22' - name = 'TEST1-00003-2021-22' + series = "TEST1-" + key = "TEST1-.#####.-2021-22" + name = "TEST1-00003-2021-22" frappe.db.delete("Series", {"name": series}) frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,)) revert_series_if_last(key, name) - current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] + current_index = frappe.db.sql( + """SELECT current from `tabSeries` where name = %s""", series, as_dict=True + )[0] - self.assertEqual(current_index.get('current'), 2) + self.assertEqual(current_index.get("current"), 2) frappe.db.delete("Series", {"name": series}) - series = '' - key = '.#####.-2021-22' - name = '00003-2021-22' + series = "" + key = ".#####.-2021-22" + name = "00003-2021-22" frappe.db.delete("Series", {"name": series}) frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,)) revert_series_if_last(key, name) - current_index = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] + current_index = frappe.db.sql( + """SELECT current from `tabSeries` where name = %s""", series, as_dict=True + )[0] - self.assertEqual(current_index.get('current'), 2) + self.assertEqual(current_index.get("current"), 2) frappe.db.delete("Series", {"name": series}) def test_naming_for_cancelled_and_amended_doc(self): - submittable_doctype = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "is_submittable": 1, - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": 'Submittable Doctype' - }).insert(ignore_if_duplicate=True) - - doc = frappe.new_doc('Submittable Doctype') + submittable_doctype = frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "custom": 1, + "is_submittable": 1, + "permissions": [{"role": "System Manager", "read": 1}], + "name": "Submittable Doctype", + } + ).insert(ignore_if_duplicate=True) + + doc = frappe.new_doc("Submittable Doctype") doc.save() original_name = doc.name @@ -192,7 +210,6 @@ class TestNaming(unittest.TestCase): submittable_doctype.delete() - def test_determine_consecutive_week_number(self): from datetime import datetime @@ -219,41 +236,26 @@ class TestNaming(unittest.TestCase): def test_naming_validations(self): # case 1: check same name as doctype # set name via prompt - tag = frappe.get_doc({ - 'doctype': 'Tag', - '__newname': 'Tag' - }) + tag = frappe.get_doc({"doctype": "Tag", "__newname": "Tag"}) self.assertRaises(frappe.NameError, tag.insert) # set by passing set_name as ToDo self.assertRaises(frappe.NameError, make_invalid_todo) # set new name - Note - note = frappe.get_doc({ - 'doctype': 'Note', - 'title': 'Note' - }) + note = frappe.get_doc({"doctype": "Note", "title": "Note"}) self.assertRaises(frappe.NameError, note.insert) # case 2: set name with "New ---" - tag = frappe.get_doc({ - 'doctype': 'Tag', - '__newname': 'New Tag' - }) + tag = frappe.get_doc({"doctype": "Tag", "__newname": "New Tag"}) self.assertRaises(frappe.NameError, tag.insert) # case 3: set name with special characters - tag = frappe.get_doc({ - 'doctype': 'Tag', - '__newname': 'Tag<>' - }) + tag = frappe.get_doc({"doctype": "Tag", "__newname": "Tag<>"}) self.assertRaises(frappe.NameError, tag.insert) # case 4: no name specified - tag = frappe.get_doc({ - 'doctype': 'Tag', - '__newname': '' - }) + tag = frappe.get_doc({"doctype": "Tag", "__newname": ""}) self.assertRaises(frappe.ValidationError, tag.insert) def test_autoincremented_naming(self): @@ -269,7 +271,4 @@ class TestNaming(unittest.TestCase): def make_invalid_todo(): - frappe.get_doc({ - 'doctype': 'ToDo', - 'description': 'Test' - }).insert(set_name='ToDo') + frappe.get_doc({"doctype": "ToDo", "description": "Test"}).insert(set_name="ToDo") diff --git a/frappe/tests/test_oauth20.py b/frappe/tests/test_oauth20.py index 93daed2bf7..b809630204 100644 --- a/frappe/tests/test_oauth20.py +++ b/frappe/tests/test_oauth20.py @@ -13,7 +13,6 @@ from frappe.test_runner import make_test_records class TestOAuth20(unittest.TestCase): - def setUp(self): make_test_records("OAuth Client") make_test_records("User") @@ -49,12 +48,14 @@ class TestOAuth20(unittest.TestCase): try: session.get( get_full_url("/api/method/frappe.integrations.oauth2.authorize"), - params=encode_params({ - "client_id": self.client_id, - "scope": self.scope, - "response_type": "code", - "redirect_uri": self.redirect_uri - }) + params=encode_params( + { + "client_id": self.client_id, + "scope": self.scope, + "response_type": "code", + "redirect_uri": self.redirect_uri, + } + ), ) except requests.exceptions.ConnectionError as ex: redirect_destination = ex.request.url @@ -67,13 +68,15 @@ class TestOAuth20(unittest.TestCase): token_response = requests.post( get_full_url("/api/method/frappe.integrations.oauth2.get_token"), headers=self.form_header, - data=encode_params({ - "grant_type": "authorization_code", - "code": auth_code, - "redirect_uri": self.redirect_uri, - "client_id": self.client_id, - "scope": self.scope, - }) + data=encode_params( + { + "grant_type": "authorization_code", + "code": auth_code, + "redirect_uri": self.redirect_uri, + "client_id": self.client_id, + "scope": self.scope, + } + ), ) # Parse bearer token json @@ -99,14 +102,16 @@ class TestOAuth20(unittest.TestCase): try: session.get( get_full_url("/api/method/frappe.integrations.oauth2.authorize"), - params=encode_params({ - "client_id": self.client_id, - "scope": self.scope, - "response_type": "code", - "redirect_uri": self.redirect_uri, - "code_challenge_method": 'S256', - "code_challenge": '21XaP8MJjpxCMRxgEzBP82sZ73PRLqkyBUta1R309J0' , - }) + params=encode_params( + { + "client_id": self.client_id, + "scope": self.scope, + "response_type": "code", + "redirect_uri": self.redirect_uri, + "code_challenge_method": "S256", + "code_challenge": "21XaP8MJjpxCMRxgEzBP82sZ73PRLqkyBUta1R309J0", + } + ), ) except requests.exceptions.ConnectionError as ex: redirect_destination = ex.request.url @@ -119,14 +124,16 @@ class TestOAuth20(unittest.TestCase): token_response = requests.post( get_full_url("/api/method/frappe.integrations.oauth2.get_token"), headers=self.form_header, - data=encode_params({ - "grant_type": "authorization_code", - "code": auth_code, - "redirect_uri": self.redirect_uri, - "client_id": self.client_id, - "scope": self.scope, - "code_verifier": "420", - }) + data=encode_params( + { + "grant_type": "authorization_code", + "code": auth_code, + "redirect_uri": self.redirect_uri, + "client_id": self.client_id, + "scope": self.scope, + "code_verifier": "420", + } + ), ) # Parse bearer token json @@ -151,12 +158,14 @@ class TestOAuth20(unittest.TestCase): try: session.get( get_full_url("/api/method/frappe.integrations.oauth2.authorize"), - params=encode_params({ - "client_id": self.client_id, - "scope": self.scope, - "response_type": "code", - "redirect_uri": self.redirect_uri - }) + params=encode_params( + { + "client_id": self.client_id, + "scope": self.scope, + "response_type": "code", + "redirect_uri": self.redirect_uri, + } + ), ) except requests.exceptions.ConnectionError as ex: redirect_destination = ex.request.url @@ -169,12 +178,14 @@ class TestOAuth20(unittest.TestCase): token_response = requests.post( get_full_url("/api/method/frappe.integrations.oauth2.get_token"), headers=self.form_header, - data=encode_params({ - "grant_type": "authorization_code", - "code": auth_code, - "redirect_uri": self.redirect_uri, - "client_id": self.client_id - }) + data=encode_params( + { + "grant_type": "authorization_code", + "code": auth_code, + "redirect_uri": self.redirect_uri, + "client_id": self.client_id, + } + ), ) # Parse bearer token json @@ -184,7 +195,7 @@ class TestOAuth20(unittest.TestCase): revoke_token_response = requests.post( get_full_url("/api/method/frappe.integrations.oauth2.revoke_token"), headers=self.form_header, - data={"token": bearer_token.get("access_token")} + data={"token": bearer_token.get("access_token")}, ) self.assertTrue(revoke_token_response.status_code == 200) @@ -203,13 +214,15 @@ class TestOAuth20(unittest.TestCase): token_response = requests.post( get_full_url("/api/method/frappe.integrations.oauth2.get_token"), headers=self.form_header, - data=encode_params({ - "grant_type": "password", - "username": "test@example.com", - "password": "Eastern_43A1W", - "client_id": self.client_id, - "scope": self.scope - }) + data=encode_params( + { + "grant_type": "password", + "username": "test@example.com", + "password": "Eastern_43A1W", + "client_id": self.client_id, + "scope": self.scope, + } + ), ) # Parse bearer token json @@ -234,12 +247,14 @@ class TestOAuth20(unittest.TestCase): try: session.get( get_full_url("/api/method/frappe.integrations.oauth2.authorize"), - params=encode_params({ - "client_id": self.client_id, - "scope": self.scope, - "response_type": "token", - "redirect_uri": self.redirect_uri - }) + params=encode_params( + { + "client_id": self.client_id, + "scope": self.scope, + "response_type": "token", + "redirect_uri": self.redirect_uri, + } + ), ) except requests.exceptions.ConnectionError as ex: redirect_destination = ex.request.url @@ -266,13 +281,15 @@ class TestOAuth20(unittest.TestCase): try: session.get( get_full_url("/api/method/frappe.integrations.oauth2.authorize"), - params=encode_params({ - "client_id": self.client_id, - "scope": self.scope, - "response_type": "code", - "redirect_uri": self.redirect_uri, - "nonce": nonce, - }) + params=encode_params( + { + "client_id": self.client_id, + "scope": self.scope, + "response_type": "code", + "redirect_uri": self.redirect_uri, + "nonce": nonce, + } + ), ) except requests.exceptions.ConnectionError as ex: redirect_destination = ex.request.url @@ -285,13 +302,15 @@ class TestOAuth20(unittest.TestCase): token_response = requests.post( get_full_url("/api/method/frappe.integrations.oauth2.get_token"), headers=self.form_header, - data=encode_params({ - "grant_type": "authorization_code", - "code": auth_code, - "redirect_uri": self.redirect_uri, - "client_id": self.client_id, - "scope": self.scope, - }) + data=encode_params( + { + "grant_type": "authorization_code", + "code": auth_code, + "redirect_uri": self.redirect_uri, + "client_id": self.client_id, + "scope": self.scope, + } + ), ) # Parse bearer token json @@ -317,8 +336,7 @@ def check_valid_openid_response(access_token=None): # check openid for email test@example.com openid_response = requests.get( - get_full_url("/api/method/frappe.integrations.oauth2.openid_profile"), - headers=headers + get_full_url("/api/method/frappe.integrations.oauth2.openid_profile"), headers=headers ) return openid_response.status_code == 200 @@ -326,11 +344,7 @@ def check_valid_openid_response(access_token=None): def login(session): session.post( - get_full_url("/api/method/login"), - data={ - "usr": "test@example.com", - "pwd": "Eastern_43A1W" - } + get_full_url("/api/method/login"), data={"usr": "test@example.com", "pwd": "Eastern_43A1W"} ) diff --git a/frappe/tests/test_password.py b/frappe/tests/test_password.py index 6062a960e6..a6864f360c 100644 --- a/frappe/tests/test_password.py +++ b/frappe/tests/test_password.py @@ -1,26 +1,30 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest -from frappe.utils.password import update_password, check_password, passlibctx, encrypt, decrypt + from cryptography.fernet import Fernet + +import frappe +from frappe.utils.password import check_password, decrypt, encrypt, passlibctx, update_password + + class TestPassword(unittest.TestCase): def setUp(self): - frappe.delete_doc('Email Account', 'Test Email Account Password') - frappe.delete_doc('Email Account', 'Test Email Account Password-new') + frappe.delete_doc("Email Account", "Test Email Account Password") + frappe.delete_doc("Email Account", "Test Email Account Password-new") def test_encrypted_password(self): doc = self.make_email_account() - new_password = 'test-password' + new_password = "test-password" doc.password = new_password doc.save() - self.assertEqual(doc.password, '*' * len(new_password)) + self.assertEqual(doc.password, "*" * len(new_password)) password_list = get_password_list(doc) - auth_password = password_list[0].get('password', '') + auth_password = password_list[0].get("password", "") # encrypted self.assertTrue(auth_password != new_password) @@ -30,29 +34,31 @@ class TestPassword(unittest.TestCase): return doc, new_password - def make_email_account(self, name='Test Email Account Password'): - if not frappe.db.exists('Email Account', name): - return frappe.get_doc({ - 'doctype': 'Email Account', - 'domain': 'example.com', - 'email_account_name': name, - 'append_to': 'Communication', - 'smtp_server': 'test.example.com', - 'pop3_server': 'pop.test.example.com', - 'email_id': 'test-password@example.com', - 'password': 'password', - }).insert() + def make_email_account(self, name="Test Email Account Password"): + if not frappe.db.exists("Email Account", name): + return frappe.get_doc( + { + "doctype": "Email Account", + "domain": "example.com", + "email_account_name": name, + "append_to": "Communication", + "smtp_server": "test.example.com", + "pop3_server": "pop.test.example.com", + "email_id": "test-password@example.com", + "password": "password", + } + ).insert() else: - return frappe.get_doc('Email Account', name) + return frappe.get_doc("Email Account", name) - def test_hashed_password(self, user='test@example.com'): - old_password = 'Eastern_43A1W' - new_password = 'Eastern_43A1W-new' + def test_hashed_password(self, user="test@example.com"): + old_password = "Eastern_43A1W" + new_password = "Eastern_43A1W-new" update_password(user, new_password) - auth = get_password_list(dict(doctype='User', name=user))[0] + auth = get_password_list(dict(doctype="User", name=user))[0] # is not plain text self.assertTrue(auth.password != new_password) @@ -70,14 +76,14 @@ class TestPassword(unittest.TestCase): self.assertRaises(frappe.AuthenticationError, check_password, user, new_password) def test_password_on_rename_user(self): - password = 'test-rename-password' + password = "test-rename-password" doc = self.make_email_account() doc.password = password doc.save() old_name = doc.name - new_name = old_name + '-new' + new_name = old_name + "-new" frappe.rename_doc(doc.doctype, old_name, new_name) new_doc = frappe.get_doc(doc.doctype, new_name) @@ -96,16 +102,16 @@ class TestPassword(unittest.TestCase): def test_password_unset(self): doc = self.make_email_account() - doc.password = 'asdf' + doc.password = "asdf" doc.save() - self.assertEqual(doc.get_password(raise_exception=False), 'asdf') + self.assertEqual(doc.get_password(raise_exception=False), "asdf") - doc.password = '' + doc.password = "" doc.save() self.assertEqual(doc.get_password(raise_exception=False), None) def test_custom_encryption_key(self): - text = 'Frappe Framework' + text = "Frappe Framework" custom_encryption_key = Fernet.generate_key().decode() encrypted_text = encrypt(text, encryption_key=custom_encryption_key) @@ -115,9 +121,14 @@ class TestPassword(unittest.TestCase): pass + def get_password_list(doc): - return frappe.db.sql("""SELECT `password` + return frappe.db.sql( + """SELECT `password` FROM `__Auth` WHERE `doctype`=%s AND `name`=%s - AND `fieldname`='password' LIMIT 1""", (doc.get('doctype'), doc.get('name')), as_dict=1) + AND `fieldname`='password' LIMIT 1""", + (doc.get("doctype"), doc.get("name")), + as_dict=1, + ) diff --git a/frappe/tests/test_patches.py b/frappe/tests/test_patches.py index 32e7b7ff3a..6ddd8e01d3 100644 --- a/frappe/tests/test_patches.py +++ b/frappe/tests/test_patches.py @@ -1,10 +1,8 @@ - import unittest -import frappe -from frappe.modules import patch_handler - from unittest.mock import mock_open, patch +import frappe +from frappe.modules import patch_handler EMTPY_FILE = "" EMTPY_SECTION = """ @@ -48,6 +46,7 @@ app.module.patch3 app.module.patch4 """ + class TestPatches(unittest.TestCase): def test_patch_module_names(self): frappe.flags.final_patches = [] @@ -57,7 +56,7 @@ class TestPatches(unittest.TestCase): pass else: if patchmodule.startswith("finally:"): - patchmodule = patchmodule.split('finally:')[-1] + patchmodule = patchmodule.split("finally:")[-1] self.assertTrue(frappe.get_attr(patchmodule.split()[0] + ".execute")) frappe.flags.in_install = False @@ -78,14 +77,12 @@ class TestPatches(unittest.TestCase): self.assertGreaterEqual(finished_patches, len(all_patches)) - class TestPatchReader(unittest.TestCase): - def get_patches(self): return ( patch_handler.get_patches_from_app("frappe"), patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.pre_model_sync), - patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync) + patch_handler.get_patches_from_app("frappe", patch_handler.PatchType.post_model_sync), ) @patch("builtins.open", new_callable=mock_open, read_data=EMTPY_FILE) @@ -95,7 +92,6 @@ class TestPatchReader(unittest.TestCase): self.assertEqual(pre, []) self.assertEqual(post, []) - @patch("builtins.open", new_callable=mock_open, read_data=EMTPY_SECTION) def test_empty_sections(self, _file): all, pre, post = self.get_patches() @@ -108,7 +104,12 @@ class TestPatchReader(unittest.TestCase): all, pre, post = self.get_patches() self.assertEqual(all, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) self.assertEqual(pre, ["app.module.patch1", "app.module.patch2"]) - self.assertEqual(post, ["app.module.patch3",]) + self.assertEqual( + post, + [ + "app.module.patch3", + ], + ) @patch("builtins.open", new_callable=mock_open, read_data=OLD_STYLE_PATCH_TXT) def test_old_style(self, _file): @@ -117,16 +118,18 @@ class TestPatchReader(unittest.TestCase): self.assertEqual(pre, ["app.module.patch1", "app.module.patch2", "app.module.patch3"]) self.assertEqual(post, []) - @patch("builtins.open", new_callable=mock_open, read_data=EDGE_CASES) def test_new_style_edge_cases(self, _file): all, pre, post = self.get_patches() - self.assertEqual(pre, [ - "App.module.patch1", - "app.module.patch2 # rerun", - 'execute:frappe.db.updatedb("Item")', - 'execute:frappe.function(arg="1")', - ]) + self.assertEqual( + pre, + [ + "App.module.patch1", + "app.module.patch2 # rerun", + 'execute:frappe.db.updatedb("Item")', + 'execute:frappe.function(arg="1")', + ], + ) @patch("builtins.open", new_callable=mock_open, read_data=COMMENTED_OUT) def test_ignore_comments(self, _file): diff --git a/frappe/tests/test_pdf.py b/frappe/tests/test_pdf.py index f23db32845..497546ebd5 100644 --- a/frappe/tests/test_pdf.py +++ b/frappe/tests/test_pdf.py @@ -35,9 +35,9 @@ class TestPdf(unittest.TestCase): def test_read_options_from_html(self): _, html_options = pdfgen.read_options_from_html(self.html) - self.assertTrue(html_options['margin-top'] == '0') - self.assertTrue(html_options['margin-left'] == '10') - self.assertTrue(html_options['margin-right'] == '0') + self.assertTrue(html_options["margin-top"] == "0") + self.assertTrue(html_options["margin-left"] == "10") + self.assertTrue(html_options["margin-right"] == "0") def test_pdf_encryption(self): password = "qwe" diff --git a/frappe/tests/test_permissions.py b/frappe/tests/test_permissions.py index f8ceb5f34c..70297a4f54 100644 --- a/frappe/tests/test_permissions.py +++ b/frappe/tests/test_permissions.py @@ -5,17 +5,22 @@ import frappe import frappe.defaults import frappe.model.meta -from frappe.permissions import (add_user_permission, remove_user_permission, - clear_user_permissions_for_doctype, get_doc_permissions, add_permission, update_permission_property) -from frappe.core.page.permission_manager.permission_manager import update, reset -from frappe.test_runner import make_test_records_for_doctype from frappe.core.doctype.user_permission.user_permission import clear_user_permissions +from frappe.core.page.permission_manager.permission_manager import reset, update from frappe.desk.form.load import getdoc -from frappe.utils.data import now_datetime - +from frappe.permissions import ( + add_permission, + add_user_permission, + clear_user_permissions_for_doctype, + get_doc_permissions, + remove_user_permission, + update_permission_property, +) +from frappe.test_runner import make_test_records_for_doctype from frappe.tests.utils import FrappeTestCase +from frappe.utils.data import now_datetime -test_dependencies = ['Blogger', 'Blog Post', "User", "Contact", "Salutation"] +test_dependencies = ["Blogger", "Blog Post", "User", "Contact", "Salutation"] class TestPermissions(FrappeTestCase): @@ -38,8 +43,8 @@ class TestPermissions(FrappeTestCase): frappe.flags.permission_user_setup_done = True - reset('Blogger') - reset('Blog Post') + reset("Blogger") + reset("Blog Post") frappe.db.delete("User Permission") @@ -66,10 +71,10 @@ class TestPermissions(FrappeTestCase): def test_select_permission(self): # grant only select perm to blog post - add_permission('Blog Post', 'Sales User', 0) - update_permission_property('Blog Post', 'Sales User', 0, 'select', 1) - update_permission_property('Blog Post', 'Sales User', 0, 'read', 0) - update_permission_property('Blog Post', 'Sales User', 0, 'write', 0) + add_permission("Blog Post", "Sales User", 0) + update_permission_property("Blog Post", "Sales User", 0, "select", 1) + update_permission_property("Blog Post", "Sales User", 0, "read", 0) + update_permission_property("Blog Post", "Sales User", 0, "write", 0) frappe.clear_cache(doctype="Blog Post") frappe.set_user("test3@example.com") @@ -83,8 +88,7 @@ class TestPermissions(FrappeTestCase): self.assertRaises(frappe.PermissionError, post.save) def test_user_permissions_in_doc(self): - add_user_permission("Blog Category", "-test-blog-category-1", - "test2@example.com") + add_user_permission("Blog Category", "-test-blog-category-1", "test2@example.com") frappe.set_user("test2@example.com") @@ -117,13 +121,21 @@ class TestPermissions(FrappeTestCase): self.assertEqual(doc.get("blog_category"), "-test-blog-category-1") # Don't fetch default if user permissions is more than 1 - add_user_permission("Blog Category", "-test-blog-category", "test2@example.com", ignore_permissions=True) + add_user_permission( + "Blog Category", "-test-blog-category", "test2@example.com", ignore_permissions=True + ) frappe.clear_cache() doc = frappe.new_doc("Blog Post") self.assertFalse(doc.get("blog_category")) # Fetch user permission set as default from multiple user permission - add_user_permission("Blog Category", "-test-blog-category-2", "test2@example.com", ignore_permissions=True, is_default=1) + add_user_permission( + "Blog Category", + "-test-blog-category-2", + "test2@example.com", + ignore_permissions=True, + is_default=1, + ) frappe.clear_cache() doc = frappe.new_doc("Blog Post") self.assertEqual(doc.get("blog_category"), "-test-blog-category-2") @@ -160,8 +172,9 @@ class TestPermissions(FrappeTestCase): frappe.set_user("test2@example.com") # this user can't add user permissions - self.assertRaises(frappe.PermissionError, add_user_permission, - "Blog Post", "-test-blog-post", "test2@example.com") + self.assertRaises( + frappe.PermissionError, add_user_permission, "Blog Post", "-test-blog-post", "test2@example.com" + ) def test_read_if_explicit_user_permissions_are_set(self): self.test_set_user_permissions() @@ -182,8 +195,13 @@ class TestPermissions(FrappeTestCase): frappe.set_user("test2@example.com") # user cannot remove their own user permissions - self.assertRaises(frappe.PermissionError, remove_user_permission, - "Blog Post", "-test-blog-post", "test2@example.com") + self.assertRaises( + frappe.PermissionError, + remove_user_permission, + "Blog Post", + "-test-blog-post", + "test2@example.com", + ) def test_user_permissions_if_applied_on_doc_being_evaluated(self): frappe.set_user("test2@example.com") @@ -229,7 +247,7 @@ class TestPermissions(FrappeTestCase): def test_set_only_once(self): blog_post = frappe.get_meta("Blog Post") doc = frappe.get_doc("Blog Post", "-test-blog-post-1") - doc.db_set('title', 'Old') + doc.db_set("title", "Old") blog_post.get_field("title").set_only_once = 1 doc.title = "New" self.assertRaises(frappe.CannotChangeConstantError, doc.save) @@ -243,7 +261,7 @@ class TestPermissions(FrappeTestCase): # remove last one doc.fields = doc.fields[:-1] self.assertRaises(frappe.CannotChangeConstantError, doc.save) - frappe.clear_cache(doctype='DocType') + frappe.clear_cache(doctype="DocType") def test_set_only_once_child_table_row_value(self): doctype_meta = frappe.get_meta("DocType") @@ -251,9 +269,9 @@ class TestPermissions(FrappeTestCase): doc = frappe.get_doc("DocType", "Blog Post") # change one property from the child table - doc.fields[-1].fieldtype = 'Check' + doc.fields[-1].fieldtype = "Check" self.assertRaises(frappe.CannotChangeConstantError, doc.save) - frappe.clear_cache(doctype='DocType') + frappe.clear_cache(doctype="DocType") def test_set_only_once_child_table_okay(self): doctype_meta = frappe.get_meta("DocType") @@ -262,13 +280,11 @@ class TestPermissions(FrappeTestCase): doc.load_doc_before_save() self.assertFalse(doc.validate_set_only_once()) - frappe.clear_cache(doctype='DocType') + frappe.clear_cache(doctype="DocType") def test_user_permission_doctypes(self): - add_user_permission("Blog Category", "-test-blog-category-1", - "test2@example.com") - add_user_permission("Blogger", "_Test Blogger 1", - "test2@example.com") + add_user_permission("Blog Category", "-test-blog-category-1", "test2@example.com") + add_user_permission("Blogger", "_Test Blogger 1", "test2@example.com") frappe.set_user("test2@example.com") @@ -283,12 +299,10 @@ class TestPermissions(FrappeTestCase): frappe.clear_cache(doctype="Blog Post") def if_owner_setup(self): - update('Blog Post', 'Blogger', 0, 'if_owner', 1) + update("Blog Post", "Blogger", 0, "if_owner", 1) - add_user_permission("Blog Category", "-test-blog-category-1", - "test2@example.com") - add_user_permission("Blogger", "_Test Blogger 1", - "test2@example.com") + add_user_permission("Blog Category", "-test-blog-category-1", "test2@example.com") + add_user_permission("Blogger", "_Test Blogger 1", "test2@example.com") frappe.clear_cache(doctype="Blog Post") @@ -296,32 +310,32 @@ class TestPermissions(FrappeTestCase): """If `If Owner` is checked for a Role, check if that document is allowed to be read, updated, submitted, etc. except be created, even if the document is restricted based on User Permissions.""" - frappe.delete_doc('Blog Post', '-test-blog-post-title') + frappe.delete_doc("Blog Post", "-test-blog-post-title") self.if_owner_setup() frappe.set_user("test2@example.com") - doc = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category", - "blogger": "_Test Blogger 1", - "title": "_Test Blog Post Title", - "content": "_Test Blog Post Content" - }) + doc = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category", + "blogger": "_Test Blogger 1", + "title": "_Test Blog Post Title", + "content": "_Test Blog Post Content", + } + ) self.assertRaises(frappe.PermissionError, doc.insert) - frappe.set_user('test1@example.com') - add_user_permission("Blog Category", "-test-blog-category", - "test2@example.com") + frappe.set_user("test1@example.com") + add_user_permission("Blog Category", "-test-blog-category", "test2@example.com") frappe.set_user("test2@example.com") doc.insert() frappe.set_user("Administrator") - remove_user_permission("Blog Category", "-test-blog-category", - "test2@example.com") + remove_user_permission("Blog Category", "-test-blog-category", "test2@example.com") frappe.set_user("test2@example.com") doc = frappe.get_doc(doc.doctype, doc.name) @@ -331,64 +345,64 @@ class TestPermissions(FrappeTestCase): # delete created record frappe.set_user("Administrator") - frappe.delete_doc('Blog Post', '-test-blog-post-title') + frappe.delete_doc("Blog Post", "-test-blog-post-title") def test_ignore_user_permissions_if_missing(self): """If there are no user permissions, then allow as per role""" - add_user_permission("Blog Category", "-test-blog-category", - "test2@example.com") + add_user_permission("Blog Category", "-test-blog-category", "test2@example.com") frappe.set_user("test2@example.com") - doc = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category-2", - "blogger": "_Test Blogger 1", - "title": "_Test Blog Post Title", - "content": "_Test Blog Post Content" - }) + doc = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category-2", + "blogger": "_Test Blogger 1", + "title": "_Test Blog Post Title", + "content": "_Test Blog Post Content", + } + ) self.assertFalse(doc.has_permission("write")) frappe.set_user("Administrator") - remove_user_permission("Blog Category", "-test-blog-category", - "test2@example.com") + remove_user_permission("Blog Category", "-test-blog-category", "test2@example.com") frappe.set_user("test2@example.com") - self.assertTrue(doc.has_permission('write')) + self.assertTrue(doc.has_permission("write")) def test_strict_user_permissions(self): """If `Strict User Permissions` is checked in System Settings, - show records even if User Permissions are missing for a linked - doctype""" + show records even if User Permissions are missing for a linked + doctype""" - frappe.set_user('Administrator') + frappe.set_user("Administrator") frappe.db.delete("Contact") frappe.db.delete("Contact Email") frappe.db.delete("Contact Phone") - reset('Salutation') - reset('Contact') + reset("Salutation") + reset("Contact") - make_test_records_for_doctype('Contact', force=True) + make_test_records_for_doctype("Contact", force=True) add_user_permission("Salutation", "Mr", "test3@example.com") self.set_strict_user_permissions(0) - allowed_contact = frappe.get_doc('Contact', '_Test Contact For _Test Customer') - other_contact = frappe.get_doc('Contact', '_Test Contact For _Test Supplier') + allowed_contact = frappe.get_doc("Contact", "_Test Contact For _Test Customer") + other_contact = frappe.get_doc("Contact", "_Test Contact For _Test Supplier") frappe.set_user("test3@example.com") - self.assertTrue(allowed_contact.has_permission('read')) - self.assertTrue(other_contact.has_permission('read')) + self.assertTrue(allowed_contact.has_permission("read")) + self.assertTrue(other_contact.has_permission("read")) self.assertEqual(len(frappe.get_list("Contact")), 2) frappe.set_user("Administrator") self.set_strict_user_permissions(1) frappe.set_user("test3@example.com") - self.assertTrue(allowed_contact.has_permission('read')) - self.assertFalse(other_contact.has_permission('read')) + self.assertTrue(allowed_contact.has_permission("read")) + self.assertFalse(other_contact.has_permission("read")) self.assertTrue(len(frappe.get_list("Contact")), 1) frappe.set_user("Administrator") @@ -398,14 +412,14 @@ class TestPermissions(FrappeTestCase): clear_user_permissions_for_doctype("Contact") def test_user_permissions_not_applied_if_user_can_edit_user_permissions(self): - add_user_permission('Blogger', '_Test Blogger 1', 'test1@example.com') + add_user_permission("Blogger", "_Test Blogger 1", "test1@example.com") # test1@example.com has rights to create user permissions # so it should not matter if explicit user permissions are not set - self.assertTrue(frappe.get_doc('Blogger', '_Test Blogger').has_permission('read')) + self.assertTrue(frappe.get_doc("Blogger", "_Test Blogger").has_permission("read")) def test_user_permission_is_not_applied_if_user_roles_does_not_have_permission(self): - add_user_permission('Blog Post', '-test-blog-post-1', 'test3@example.com') + add_user_permission("Blog Post", "-test-blog-post-1", "test3@example.com") frappe.set_user("test3@example.com") doc = frappe.get_doc("Blog Post", "-test-blog-post-1") self.assertFalse(doc.has_permission("read")) @@ -421,60 +435,68 @@ class TestPermissions(FrappeTestCase): def test_contextual_user_permission(self): # should be applicable for across all doctypes - add_user_permission('Blogger', '_Test Blogger', 'test2@example.com') + add_user_permission("Blogger", "_Test Blogger", "test2@example.com") # should be applicable only while accessing Blog Post - add_user_permission('Blogger', '_Test Blogger 1', 'test2@example.com', applicable_for='Blog Post') + add_user_permission( + "Blogger", "_Test Blogger 1", "test2@example.com", applicable_for="Blog Post" + ) # should be applicable only while accessing User - add_user_permission('Blogger', '_Test Blogger 2', 'test2@example.com', applicable_for='User') + add_user_permission("Blogger", "_Test Blogger 2", "test2@example.com", applicable_for="User") - posts = frappe.get_all('Blog Post', fields=['name', 'blogger']) + posts = frappe.get_all("Blog Post", fields=["name", "blogger"]) # Get all posts for admin self.assertEqual(len(posts), 4) - frappe.set_user('test2@example.com') + frappe.set_user("test2@example.com") - posts = frappe.get_list('Blog Post', fields=['name', 'blogger']) + posts = frappe.get_list("Blog Post", fields=["name", "blogger"]) # Should get only posts with allowed blogger via user permission # only '_Test Blogger', '_Test Blogger 1' are allowed in Blog Post self.assertEqual(len(posts), 3) for post in posts: - self.assertIn(post.blogger, ['_Test Blogger', '_Test Blogger 1'], 'A post from {} is not expected.'.format(post.blogger)) + self.assertIn( + post.blogger, + ["_Test Blogger", "_Test Blogger 1"], + "A post from {} is not expected.".format(post.blogger), + ) def test_if_owner_permission_overrides_properly(self): # check if user is not granted access if the user is not the owner of the doc # Blogger has only read access on the blog post unless he is the owner of the blog - update('Blog Post', 'Blogger', 0, 'if_owner', 1) - update('Blog Post', 'Blogger', 0, 'read', 1) - update('Blog Post', 'Blogger', 0, 'write', 1) - update('Blog Post', 'Blogger', 0, 'delete', 1) + update("Blog Post", "Blogger", 0, "if_owner", 1) + update("Blog Post", "Blogger", 0, "read", 1) + update("Blog Post", "Blogger", 0, "write", 1) + update("Blog Post", "Blogger", 0, "delete", 1) # currently test2 user has not created any document # still he should be able to do get_list query which should # not raise permission error but simply return empty list frappe.set_user("test2@example.com") - self.assertEqual(frappe.get_list('Blog Post'), []) + self.assertEqual(frappe.get_list("Blog Post"), []) frappe.set_user("Administrator") # creates a custom docperm with just read access # now any user can read any blog post (but other rights are limited to the blog post owner) - add_permission('Blog Post', 'Blogger') + add_permission("Blog Post", "Blogger") frappe.clear_cache(doctype="Blog Post") - frappe.delete_doc('Blog Post', '-test-blog-post-title') + frappe.delete_doc("Blog Post", "-test-blog-post-title") frappe.set_user("test1@example.com") - doc = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category", - "blogger": "_Test Blogger 1", - "title": "_Test Blog Post Title", - "content": "_Test Blog Post Content" - }) + doc = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category", + "blogger": "_Test Blogger 1", + "title": "_Test Blog Post Title", + "content": "_Test Blog Post Content", + } + ) doc.insert() @@ -494,47 +516,51 @@ class TestPermissions(FrappeTestCase): self.assertTrue(doc.has_permission("delete")) # delete the created doc - frappe.delete_doc('Blog Post', '-test-blog-post-title') + frappe.delete_doc("Blog Post", "-test-blog-post-title") def test_if_owner_permission_on_getdoc(self): - update('Blog Post', 'Blogger', 0, 'if_owner', 1) - update('Blog Post', 'Blogger', 0, 'read', 1) - update('Blog Post', 'Blogger', 0, 'write', 1) - update('Blog Post', 'Blogger', 0, 'delete', 1) + update("Blog Post", "Blogger", 0, "if_owner", 1) + update("Blog Post", "Blogger", 0, "read", 1) + update("Blog Post", "Blogger", 0, "write", 1) + update("Blog Post", "Blogger", 0, "delete", 1) frappe.clear_cache(doctype="Blog Post") frappe.set_user("test1@example.com") - doc = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category", - "blogger": "_Test Blogger 1", - "title": "_Test Blog Post Title New", - "content": "_Test Blog Post Content" - }) + doc = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category", + "blogger": "_Test Blogger 1", + "title": "_Test Blog Post Title New", + "content": "_Test Blog Post Content", + } + ) doc.insert() - getdoc('Blog Post', doc.name) + getdoc("Blog Post", doc.name) doclist = [d.name for d in frappe.response.docs] self.assertTrue(doc.name in doclist) frappe.set_user("test2@example.com") - self.assertRaises(frappe.PermissionError, getdoc, 'Blog Post', doc.name) + self.assertRaises(frappe.PermissionError, getdoc, "Blog Post", doc.name) def test_if_owner_permission_on_get_list(self): - doc = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category", - "blogger": "_Test Blogger 1", - "title": "_Test If Owner Permissions on Get List", - "content": "_Test Blog Post Content" - }) + doc = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category", + "blogger": "_Test Blogger 1", + "title": "_Test If Owner Permissions on Get List", + "content": "_Test Blog Post Content", + } + ) doc.insert(ignore_if_duplicate=True) - update('Blog Post', 'Blogger', 0, 'if_owner', 1) - update('Blog Post', 'Blogger', 0, 'read', 1) + update("Blog Post", "Blogger", 0, "if_owner", 1) + update("Blog Post", "Blogger", 0, "read", 1) user = frappe.get_doc("User", "test2@example.com") user.add_roles("Website Manager") frappe.clear_cache(doctype="Blog Post") @@ -551,71 +577,71 @@ class TestPermissions(FrappeTestCase): self.assertNotIn(doc.name, frappe.get_list("Blog Post", pluck="name")) def test_if_owner_permission_on_delete(self): - update('Blog Post', 'Blogger', 0, 'if_owner', 1) - update('Blog Post', 'Blogger', 0, 'read', 1) - update('Blog Post', 'Blogger', 0, 'write', 1) - update('Blog Post', 'Blogger', 0, 'delete', 1) + update("Blog Post", "Blogger", 0, "if_owner", 1) + update("Blog Post", "Blogger", 0, "read", 1) + update("Blog Post", "Blogger", 0, "write", 1) + update("Blog Post", "Blogger", 0, "delete", 1) # Remove delete perm - update('Blog Post', 'Website Manager', 0, 'delete', 0) + update("Blog Post", "Website Manager", 0, "delete", 0) frappe.clear_cache(doctype="Blog Post") frappe.set_user("test2@example.com") - doc = frappe.get_doc({ - "doctype": "Blog Post", - "blog_category": "-test-blog-category", - "blogger": "_Test Blogger 1", - "title": "_Test Blog Post Title New 1", - "content": "_Test Blog Post Content" - }) + doc = frappe.get_doc( + { + "doctype": "Blog Post", + "blog_category": "-test-blog-category", + "blogger": "_Test Blogger 1", + "title": "_Test Blog Post Title New 1", + "content": "_Test Blog Post Content", + } + ) doc.insert() - getdoc('Blog Post', doc.name) + getdoc("Blog Post", doc.name) doclist = [d.name for d in frappe.response.docs] self.assertTrue(doc.name in doclist) frappe.set_user("testperm@example.com") # Website Manager able to read - getdoc('Blog Post', doc.name) + getdoc("Blog Post", doc.name) doclist = [d.name for d in frappe.response.docs] self.assertTrue(doc.name in doclist) # Website Manager should not be able to delete - self.assertRaises(frappe.PermissionError, frappe.delete_doc, 'Blog Post', doc.name) + self.assertRaises(frappe.PermissionError, frappe.delete_doc, "Blog Post", doc.name) frappe.set_user("test2@example.com") - frappe.delete_doc('Blog Post', '-test-blog-post-title-new-1') - update('Blog Post', 'Website Manager', 0, 'delete', 1) + frappe.delete_doc("Blog Post", "-test-blog-post-title-new-1") + update("Blog Post", "Website Manager", 0, "delete", 1) def test_clear_user_permissions(self): current_user = frappe.session.user - frappe.set_user('Administrator') - clear_user_permissions_for_doctype('Blog Category', 'test2@example.com') - clear_user_permissions_for_doctype('Blog Post', 'test2@example.com') + frappe.set_user("Administrator") + clear_user_permissions_for_doctype("Blog Category", "test2@example.com") + clear_user_permissions_for_doctype("Blog Post", "test2@example.com") - add_user_permission('Blog Post', '-test-blog-post-1', 'test2@example.com') - add_user_permission('Blog Post', '-test-blog-post-2', 'test2@example.com') - add_user_permission("Blog Category", '-test-blog-category-1', 'test2@example.com') + add_user_permission("Blog Post", "-test-blog-post-1", "test2@example.com") + add_user_permission("Blog Post", "-test-blog-post-2", "test2@example.com") + add_user_permission("Blog Category", "-test-blog-category-1", "test2@example.com") - deleted_user_permission_count = clear_user_permissions('test2@example.com', 'Blog Post') + deleted_user_permission_count = clear_user_permissions("test2@example.com", "Blog Post") self.assertEqual(deleted_user_permission_count, 2) - blog_post_user_permission_count = frappe.db.count('User Permission', filters={ - 'user': 'test2@example.com', - 'allow': 'Blog Post' - }) + blog_post_user_permission_count = frappe.db.count( + "User Permission", filters={"user": "test2@example.com", "allow": "Blog Post"} + ) self.assertEqual(blog_post_user_permission_count, 0) - blog_category_user_permission_count = frappe.db.count('User Permission', filters={ - 'user': 'test2@example.com', - 'allow': 'Blog Category' - }) + blog_category_user_permission_count = frappe.db.count( + "User Permission", filters={"user": "test2@example.com", "allow": "Blog Category"} + ) self.assertEqual(blog_category_user_permission_count, 1) @@ -625,9 +651,23 @@ class TestPermissions(FrappeTestCase): def test_child_table_permissions(self): frappe.set_user("test@example.com") self.assertIsInstance(frappe.get_list("Has Role", parent_doctype="User", limit=1), list) - self.assertRaisesRegex(frappe.exceptions.ValidationError, - ".* is not a valid parent DocType for .*", frappe.get_list, doctype="Has Role", parent_doctype="ToDo") - self.assertRaisesRegex(frappe.exceptions.ValidationError, - "Please specify a valid parent DocType for .*", frappe.get_list, "Has Role") - self.assertRaisesRegex(frappe.exceptions.ValidationError, - ".* is not a valid parent DocType for .*", frappe.get_list, doctype="Has Role", parent_doctype="Has Role") + self.assertRaisesRegex( + frappe.exceptions.ValidationError, + ".* is not a valid parent DocType for .*", + frappe.get_list, + doctype="Has Role", + parent_doctype="ToDo", + ) + self.assertRaisesRegex( + frappe.exceptions.ValidationError, + "Please specify a valid parent DocType for .*", + frappe.get_list, + "Has Role", + ) + self.assertRaisesRegex( + frappe.exceptions.ValidationError, + ".* is not a valid parent DocType for .*", + frappe.get_list, + doctype="Has Role", + parent_doctype="Has Role", + ) diff --git a/frappe/tests/test_printview.py b/frappe/tests/test_printview.py index 0fc4c4869b..31afc4bb12 100644 --- a/frappe/tests/test_printview.py +++ b/frappe/tests/test_printview.py @@ -14,9 +14,8 @@ class PrintViewTest(unittest.TestCase): messages_after = frappe.get_message_log() if len(messages_after) > len(messages_before): - new_messages = messages_after[len(messages_before):] - self.fail("Print view showing error/warnings: \n" - + "\n".join(str(msg) for msg in new_messages)) + new_messages = messages_after[len(messages_before) :] + self.fail("Print view showing error/warnings: \n" + "\n".join(str(msg) for msg in new_messages)) # html should exist self.assertTrue(bool(ret["html"])) diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index d86e1b1d19..45b1922767 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -2,15 +2,14 @@ import unittest from typing import Callable import frappe +from frappe.query_builder import Case from frappe.query_builder.custom import ConstantColumn -from frappe.query_builder.functions import Coalesce, GroupConcat, Match, CombineDatetime, Cast_ +from frappe.query_builder.functions import Cast_, Coalesce, CombineDatetime, GroupConcat, Match from frappe.query_builder.utils import db_type_is -from frappe.query_builder import Case + def run_only_if(dbtype: db_type_is) -> Callable: - return unittest.skipIf( - db_type_is(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}" - ) + return unittest.skipIf(db_type_is(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}") @run_only_if(db_type_is.MARIADB) @@ -20,38 +19,55 @@ class TestCustomFunctionsMariaDB(unittest.TestCase): def test_match(self): query = Match("Notes").Against("text") - self.assertEqual( - " MATCH('Notes') AGAINST ('+text*' IN BOOLEAN MODE)", query.get_sql() - ) + self.assertEqual(" MATCH('Notes') AGAINST ('+text*' IN BOOLEAN MODE)", query.get_sql()) def test_constant_column(self): - query = frappe.qb.from_("DocType").select( - "name", ConstantColumn("John").as_("User") - ) - self.assertEqual( - query.get_sql(), "SELECT `name`,'John' `User` FROM `tabDocType`" - ) + query = frappe.qb.from_("DocType").select("name", ConstantColumn("John").as_("User")) + self.assertEqual(query.get_sql(), "SELECT `name`,'John' `User` FROM `tabDocType`") def test_timestamp(self): note = frappe.qb.DocType("Note") - self.assertEqual("TIMESTAMP(posting_date,posting_time)", CombineDatetime(note.posting_date, note.posting_time).get_sql()) - self.assertEqual("TIMESTAMP('2021-01-01','00:00:21')", CombineDatetime("2021-01-01", "00:00:21").get_sql()) + self.assertEqual( + "TIMESTAMP(posting_date,posting_time)", + CombineDatetime(note.posting_date, note.posting_time).get_sql(), + ) + self.assertEqual( + "TIMESTAMP('2021-01-01','00:00:21')", CombineDatetime("2021-01-01", "00:00:21").get_sql() + ) todo = frappe.qb.DocType("ToDo") - select_query = (frappe.qb - .from_(note) - .join(todo).on(todo.refernce_name == note.name) - .select(CombineDatetime(note.posting_date, note.posting_time))) - self.assertIn("select timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`)", str(select_query).lower()) + select_query = ( + frappe.qb.from_(note) + .join(todo) + .on(todo.refernce_name == note.name) + .select(CombineDatetime(note.posting_date, note.posting_time)) + ) + self.assertIn( + "select timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`)", str(select_query).lower() + ) select_query = select_query.orderby(CombineDatetime(note.posting_date, note.posting_time)) - self.assertIn("order by timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`)", str(select_query).lower()) + self.assertIn( + "order by timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`)", + str(select_query).lower(), + ) - select_query = select_query.where(CombineDatetime(note.posting_date, note.posting_time) >= CombineDatetime("2021-01-01", "00:00:01")) - self.assertIn("timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`)>=timestamp('2021-01-01','00:00:01')", str(select_query).lower()) + select_query = select_query.where( + CombineDatetime(note.posting_date, note.posting_time) + >= CombineDatetime("2021-01-01", "00:00:01") + ) + self.assertIn( + "timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`)>=timestamp('2021-01-01','00:00:01')", + str(select_query).lower(), + ) - select_query = select_query.select(CombineDatetime(note.posting_date, note.posting_time, alias="timestamp")) - self.assertIn("timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`) `timestamp`", str(select_query).lower()) + select_query = select_query.select( + CombineDatetime(note.posting_date, note.posting_time, alias="timestamp") + ) + self.assertIn( + "timestamp(`tabnote`.`posting_date`,`tabnote`.`posting_time`) `timestamp`", + str(select_query).lower(), + ) def test_cast(self): note = frappe.qb.DocType("Note") @@ -66,41 +82,53 @@ class TestCustomFunctionsPostgres(unittest.TestCase): def test_match(self): query = Match("Notes").Against("text") - self.assertEqual( - "TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql() - ) + self.assertEqual("TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql()) def test_constant_column(self): - query = frappe.qb.from_("DocType").select( - "name", ConstantColumn("John").as_("User") - ) - self.assertEqual( - query.get_sql(), 'SELECT "name",\'John\' "User" FROM "tabDocType"' - ) + query = frappe.qb.from_("DocType").select("name", ConstantColumn("John").as_("User")) + self.assertEqual(query.get_sql(), 'SELECT "name",\'John\' "User" FROM "tabDocType"') def test_timestamp(self): note = frappe.qb.DocType("Note") - self.assertEqual("posting_date+posting_time", CombineDatetime(note.posting_date, note.posting_time).get_sql()) - self.assertEqual("CAST('2021-01-01' AS DATE)+CAST('00:00:21' AS TIME)", CombineDatetime("2021-01-01", "00:00:21").get_sql()) + self.assertEqual( + "posting_date+posting_time", CombineDatetime(note.posting_date, note.posting_time).get_sql() + ) + self.assertEqual( + "CAST('2021-01-01' AS DATE)+CAST('00:00:21' AS TIME)", + CombineDatetime("2021-01-01", "00:00:21").get_sql(), + ) todo = frappe.qb.DocType("ToDo") - select_query = (frappe.qb - .from_(note) - .join(todo).on(todo.refernce_name == note.name) - .select(CombineDatetime(note.posting_date, note.posting_time))) - self.assertIn('select "tabnote"."posting_date"+"tabnote"."posting_time"', str(select_query).lower()) + select_query = ( + frappe.qb.from_(note) + .join(todo) + .on(todo.refernce_name == note.name) + .select(CombineDatetime(note.posting_date, note.posting_time)) + ) + self.assertIn( + 'select "tabnote"."posting_date"+"tabnote"."posting_time"', str(select_query).lower() + ) select_query = select_query.orderby(CombineDatetime(note.posting_date, note.posting_time)) - self.assertIn('order by "tabnote"."posting_date"+"tabnote"."posting_time"', str(select_query).lower()) + self.assertIn( + 'order by "tabnote"."posting_date"+"tabnote"."posting_time"', str(select_query).lower() + ) select_query = select_query.where( - CombineDatetime(note.posting_date, note.posting_time) >= CombineDatetime('2021-01-01', '00:00:01') - ) - self.assertIn("""where "tabnote"."posting_date"+"tabnote"."posting_time">=cast('2021-01-01' as date)+cast('00:00:01' as time)""", - str(select_query).lower()) + CombineDatetime(note.posting_date, note.posting_time) + >= CombineDatetime("2021-01-01", "00:00:01") + ) + self.assertIn( + """where "tabnote"."posting_date"+"tabnote"."posting_time">=cast('2021-01-01' as date)+cast('00:00:01' as time)""", + str(select_query).lower(), + ) - select_query = select_query.select(CombineDatetime(note.posting_date, note.posting_time, alias="timestamp")) - self.assertIn('"tabnote"."posting_date"+"tabnote"."posting_time" "timestamp"', str(select_query).lower()) + select_query = select_query.select( + CombineDatetime(note.posting_date, note.posting_time, alias="timestamp") + ) + self.assertIn( + '"tabnote"."posting_date"+"tabnote"."posting_time" "timestamp"', str(select_query).lower() + ) def test_cast(self): note = frappe.qb.DocType("Note") @@ -126,9 +154,7 @@ class TestParameterization(unittest.TestCase): def test_where_conditions(self): DocType = frappe.qb.DocType("DocType") query = ( - frappe.qb.from_(DocType) - .select(DocType.name) - .where((DocType.owner == "Administrator' --")) + frappe.qb.from_(DocType).select(DocType.name).where((DocType.owner == "Administrator' --")) ) self.assertTrue("walk" in dir(query)) query, params = query.walk() @@ -165,14 +191,11 @@ class TestParameterization(unittest.TestCase): def test_case(self): DocType = frappe.qb.DocType("DocType") - query = ( - frappe.qb.from_(DocType) - .select( - Case() - .when(DocType.search_fields == "value", "other_value") - .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") - .else_("Overdue") - ) + query = frappe.qb.from_(DocType).select( + Case() + .when(DocType.search_fields == "value", "other_value") + .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") + .else_("Overdue") ) self.assertTrue("walk" in dir(query)) @@ -188,15 +211,12 @@ class TestParameterization(unittest.TestCase): def test_case_in_update(self): DocType = frappe.qb.DocType("DocType") - query = ( - frappe.qb.update(DocType) - .set( - "parent", - Case() - .when(DocType.search_fields == "value", "other_value") - .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") - .else_("Overdue") - ) + query = frappe.qb.update(DocType).set( + "parent", + Case() + .when(DocType.search_fields == "value", "other_value") + .when(Coalesce(DocType.search_fields == "subject_in_function"), "true_value") + .else_("Overdue"), ) self.assertTrue("walk" in dir(query)) @@ -211,27 +231,18 @@ class TestParameterization(unittest.TestCase): self.assertEqual(params["param5"], "Overdue") - @run_only_if(db_type_is.MARIADB) class TestBuilderMaria(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): - self.assertEqual( - "SELECT * FROM `tabNotes`", frappe.qb.from_("Notes").select("*").get_sql() - ) - self.assertEqual( - "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() - ) + self.assertEqual("SELECT * FROM `tabNotes`", frappe.qb.from_("Notes").select("*").get_sql()) + self.assertEqual("SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql()) @run_only_if(db_type_is.POSTGRES) class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): - self.assertEqual( - 'SELECT * FROM "tabNotes"', frappe.qb.from_("Notes").select("*").get_sql() - ) - self.assertEqual( - 'SELECT * FROM "__Auth"', frappe.qb.from_("__Auth").select("*").get_sql() - ) + self.assertEqual('SELECT * FROM "tabNotes"', frappe.qb.from_("Notes").select("*").get_sql()) + self.assertEqual('SELECT * FROM "__Auth"', frappe.qb.from_("__Auth").select("*").get_sql()) def test_replace_tables(self): info_schema = frappe.qb.Schema("information_schema") diff --git a/frappe/tests/test_query_report.py b/frappe/tests/test_query_report.py index 2117bc830e..60c248fb29 100644 --- a/frappe/tests/test_query_report.py +++ b/frappe/tests/test_query_report.py @@ -4,8 +4,8 @@ import unittest import frappe -from frappe.desk.query_report import build_xlsx_data import frappe.utils +from frappe.desk.query_report import build_xlsx_data class TestQueryReport(unittest.TestCase): diff --git a/frappe/tests/test_rate_limiter.py b/frappe/tests/test_rate_limiter.py index e3a1831159..5cfe2694f6 100644 --- a/frappe/tests/test_rate_limiter.py +++ b/frappe/tests/test_rate_limiter.py @@ -3,9 +3,10 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import time import unittest + from werkzeug.wrappers import Response -import time import frappe import frappe.rate_limiter diff --git a/frappe/tests/test_recorder.py b/frappe/tests/test_recorder.py index 9e8b194d72..94b97cb3a6 100644 --- a/frappe/tests/test_recorder.py +++ b/frappe/tests/test_recorder.py @@ -4,12 +4,14 @@ # License: MIT. See LICENSE import unittest + +import sqlparse + import frappe import frappe.recorder from frappe.utils import set_request from frappe.website.serve import get_response_content -import sqlparse class TestRecorder(unittest.TestCase): def setUp(self): @@ -25,7 +27,7 @@ class TestRecorder(unittest.TestCase): self.assertEqual(len(requests), 1) def test_do_not_record(self): - frappe.recorder.do_not_record(frappe.get_all)('DocType') + frappe.recorder.do_not_record(frappe.get_all)("DocType") frappe.recorder.dump() requests = frappe.recorder.get() self.assertEqual(len(requests), 0) @@ -36,7 +38,7 @@ class TestRecorder(unittest.TestCase): requests = frappe.recorder.get() self.assertEqual(len(requests), 1) - request = frappe.recorder.get(requests[0]['uuid']) + request = frappe.recorder.get(requests[0]["uuid"]) self.assertTrue(request) def test_delete(self): @@ -54,60 +56,61 @@ class TestRecorder(unittest.TestCase): frappe.recorder.dump() requests = frappe.recorder.get() - request = frappe.recorder.get(requests[0]['uuid']) + request = frappe.recorder.get(requests[0]["uuid"]) - self.assertEqual(len(request['calls']), 0) + self.assertEqual(len(request["calls"]), 0) def test_record_with_sql_queries(self): - frappe.get_all('DocType') + frappe.get_all("DocType") frappe.recorder.dump() requests = frappe.recorder.get() - request = frappe.recorder.get(requests[0]['uuid']) + request = frappe.recorder.get(requests[0]["uuid"]) - self.assertNotEqual(len(request['calls']), 0) + self.assertNotEqual(len(request["calls"]), 0) def test_explain(self): - frappe.db.sql('SELECT * FROM tabDocType') - frappe.db.sql('COMMIT') + frappe.db.sql("SELECT * FROM tabDocType") + frappe.db.sql("COMMIT") frappe.recorder.dump() requests = frappe.recorder.get() - request = frappe.recorder.get(requests[0]['uuid']) - - self.assertEqual(len(request['calls'][0]['explain_result']), 1) - self.assertEqual(len(request['calls'][1]['explain_result']), 0) + request = frappe.recorder.get(requests[0]["uuid"]) + self.assertEqual(len(request["calls"][0]["explain_result"]), 1) + self.assertEqual(len(request["calls"][1]["explain_result"]), 0) def test_multiple_queries(self): queries = [ - {'mariadb': 'SELECT * FROM tabDocType', 'postgres': 'SELECT * FROM "tabDocType"'}, - {'mariadb': 'SELECT COUNT(*) FROM tabDocType', 'postgres': 'SELECT COUNT(*) FROM "tabDocType"'}, - {'mariadb': 'COMMIT', 'postgres': 'COMMIT'}, + {"mariadb": "SELECT * FROM tabDocType", "postgres": 'SELECT * FROM "tabDocType"'}, + {"mariadb": "SELECT COUNT(*) FROM tabDocType", "postgres": 'SELECT COUNT(*) FROM "tabDocType"'}, + {"mariadb": "COMMIT", "postgres": "COMMIT"}, ] - sql_dialect = frappe.db.db_type or 'mariadb' + sql_dialect = frappe.db.db_type or "mariadb" for query in queries: frappe.db.sql(query[sql_dialect]) frappe.recorder.dump() requests = frappe.recorder.get() - request = frappe.recorder.get(requests[0]['uuid']) + request = frappe.recorder.get(requests[0]["uuid"]) - self.assertEqual(len(request['calls']), len(queries)) + self.assertEqual(len(request["calls"]), len(queries)) - for query, call in zip(queries, request['calls']): - self.assertEqual(call['query'], sqlparse.format(query[sql_dialect].strip(), keyword_case='upper', reindent=True)) + for query, call in zip(queries, request["calls"]): + self.assertEqual( + call["query"], sqlparse.format(query[sql_dialect].strip(), keyword_case="upper", reindent=True) + ) def test_duplicate_queries(self): queries = [ - ('SELECT * FROM tabDocType', 2), - ('SELECT COUNT(*) FROM tabDocType', 1), - ('select * from tabDocType', 2), - ('COMMIT', 3), - ('COMMIT', 3), - ('COMMIT', 3), + ("SELECT * FROM tabDocType", 2), + ("SELECT COUNT(*) FROM tabDocType", 1), + ("select * from tabDocType", 2), + ("COMMIT", 3), + ("COMMIT", 3), + ("COMMIT", 3), ] for query in queries: frappe.db.sql(query[0]) @@ -115,10 +118,10 @@ class TestRecorder(unittest.TestCase): frappe.recorder.dump() requests = frappe.recorder.get() - request = frappe.recorder.get(requests[0]['uuid']) + request = frappe.recorder.get(requests[0]["uuid"]) - for query, call in zip(queries, request['calls']): - self.assertEqual(call['exact_copies'], query[1]) + for query, call in zip(queries, request["calls"]): + self.assertEqual(call["exact_copies"], query[1]) def test_error_page_rendering(self): content = get_response_content("error") diff --git a/frappe/tests/test_redis.py b/frappe/tests/test_redis.py index 8dd50f2373..e06f6b0aa0 100644 --- a/frappe/tests/test_redis.py +++ b/frappe/tests/test_redis.py @@ -1,40 +1,44 @@ -import unittest import functools +import unittest import redis import frappe from frappe.utils import get_bench_id -from frappe.utils.redis_queue import RedisQueue from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.redis_queue import RedisQueue + def version_tuple(version): return tuple(map(int, (version.split(".")))) + def skip_if_redis_version_lt(version): def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): conn = get_redis_conn() - redis_version = conn.execute_command('info')['redis_version'] + redis_version = conn.execute_command("info")["redis_version"] if version_tuple(redis_version) < version_tuple(version): return return func(*args, **kwargs) + return wrapper + return decorator + class TestRedisAuth(unittest.TestCase): - @skip_if_redis_version_lt('6.0') + @skip_if_redis_version_lt("6.0") def test_rq_gen_acllist(self): - """Make sure that ACL list is genrated - """ + """Make sure that ACL list is genrated""" acl_list = RedisQueue.gen_acl_list() - self.assertEqual(acl_list[1]['bench'][0], get_bench_id()) + self.assertEqual(acl_list[1]["bench"][0], get_bench_id()) - @skip_if_redis_version_lt('6.0') + @skip_if_redis_version_lt("6.0") def test_adding_redis_user(self): acl_list = RedisQueue.gen_acl_list() - username, password = acl_list[1]['bench'] + username, password = acl_list[1]["bench"] conn = get_redis_conn() conn.acl_deluser(username) @@ -42,29 +46,28 @@ class TestRedisAuth(unittest.TestCase): self.assertTrue(conn.acl_getuser(username)) conn.acl_deluser(username) - @skip_if_redis_version_lt('6.0') + @skip_if_redis_version_lt("6.0") def test_rq_namespace(self): - """Make sure that user can access only their respective namespace. - """ + """Make sure that user can access only their respective namespace.""" # Current bench ID - bench_id = frappe.conf.get('bench_id') + bench_id = frappe.conf.get("bench_id") conn = get_redis_conn() - conn.set('rq:queue:test_bench1:abc', 'value') - conn.set(f'rq:queue:{bench_id}:abc', 'value') + conn.set("rq:queue:test_bench1:abc", "value") + conn.set(f"rq:queue:{bench_id}:abc", "value") # Create new Redis Queue user - tmp_bench_id = 'test_bench1' - username, password = tmp_bench_id, 'password1' + tmp_bench_id = "test_bench1" + username, password = tmp_bench_id, "password1" conn.acl_deluser(username) - frappe.conf.update({'bench_id': tmp_bench_id}) + frappe.conf.update({"bench_id": tmp_bench_id}) _ = RedisQueue(conn).add_user(username, password) test_bench1_conn = RedisQueue.get_connection(username, password) - self.assertEqual(test_bench1_conn.get('rq:queue:test_bench1:abc'), b'value') + self.assertEqual(test_bench1_conn.get("rq:queue:test_bench1:abc"), b"value") # User should not be able to access queues apart from their bench queues with self.assertRaises(redis.exceptions.NoPermissionError): - test_bench1_conn.get(f'rq:queue:{bench_id}:abc') + test_bench1_conn.get(f"rq:queue:{bench_id}:abc") - frappe.conf.update({'bench_id': bench_id}) + frappe.conf.update({"bench_id": bench_id}) conn.acl_deluser(username) diff --git a/frappe/tests/test_rename_doc.py b/frappe/tests/test_rename_doc.py index 0c1cba31fb..38fc7b32bd 100644 --- a/frappe/tests/test_rename_doc.py +++ b/frappe/tests/test_rename_doc.py @@ -12,7 +12,12 @@ from unittest.mock import patch import frappe from frappe.exceptions import DoesNotExistError, ValidationError from frappe.model.base_document import get_controller -from frappe.model.rename_doc import bulk_rename, get_fetch_fields, update_document_title, update_linked_doctypes +from frappe.model.rename_doc import ( + bulk_rename, + get_fetch_fields, + update_document_title, + update_linked_doctypes, +) from frappe.modules.utils import get_doc_path from frappe.utils import add_to_date, now @@ -50,29 +55,33 @@ class TestRenameDoc(unittest.TestCase): self.test_doctype = "ToDo" for num in range(1, 5): - doc = frappe.get_doc({ - "doctype": self.test_doctype, - "date": add_to_date(now(), days=num), - "description": "this is todo #{}".format(num), - }).insert() + doc = frappe.get_doc( + { + "doctype": self.test_doctype, + "date": add_to_date(now(), days=num), + "description": "this is todo #{}".format(num), + } + ).insert() self.available_documents.append(doc.name) # data generation: for controllers tests - self.doctype = frappe._dict({ - "old": "Test Rename Document Old", - "new": "Test Rename Document New", - }) - - frappe.get_doc({ - "doctype": "DocType", - "module": "Custom", - "name": self.doctype.old, - "custom": 0, - "fields": [ - {"label": "Some Field", "fieldname": "some_fieldname", "fieldtype": "Data"} - ], - "permissions": [{"role": "System Manager", "read": 1}], - }).insert() + self.doctype = frappe._dict( + { + "old": "Test Rename Document Old", + "new": "Test Rename Document New", + } + ) + + frappe.get_doc( + { + "doctype": "DocType", + "module": "Custom", + "name": self.doctype.old, + "custom": 0, + "fields": [{"label": "Some Field", "fieldname": "some_fieldname", "fieldtype": "Data"}], + "permissions": [{"role": "System Manager", "read": 1}], + } + ).insert() @classmethod def tearDownClass(self): @@ -181,17 +190,17 @@ class TestRenameDoc(unittest.TestCase): # having the same name old_name = to_rename_record.name new_name = "ToDo" - self.assertEqual( - new_name, frappe.rename_doc("Renamed Doc", old_name, new_name, force=True) - ) + self.assertEqual(new_name, frappe.rename_doc("Renamed Doc", old_name, new_name, force=True)) def test_update_document_title_api(self): test_doctype = "Module Def" - test_doc = frappe.get_doc({ - "doctype": test_doctype, - "module_name": f"Test-test_update_document_title_api-{frappe.generate_hash()}", - "custom": True, - }) + test_doc = frappe.get_doc( + { + "doctype": test_doctype, + "module_name": f"Test-test_update_document_title_api-{frappe.generate_hash()}", + "custom": True, + } + ) test_doc.insert(ignore_mandatory=True) dt = test_doc.doctype @@ -225,7 +234,8 @@ class TestRenameDoc(unittest.TestCase): self.assertEqual(len(message_log), len(self.available_documents)) self.assertIsInstance(message_log, list) enqueue.assert_called_with( - 'frappe.utils.global_search.rebuild_for_doctype', doctype=self.test_doctype, + "frappe.utils.global_search.rebuild_for_doctype", + doctype=self.test_doctype, ) def test_deprecated_utils(self): diff --git a/frappe/tests/test_safe_exec.py b/frappe/tests/test_safe_exec.py index 7fec292c49..5813fab721 100644 --- a/frappe/tests/test_safe_exec.py +++ b/frappe/tests/test_safe_exec.py @@ -1,56 +1,61 @@ +import unittest + +import frappe +from frappe.utils.safe_exec import get_safe_globals, safe_exec -import unittest, frappe -from frappe.utils.safe_exec import safe_exec, get_safe_globals class TestSafeExec(unittest.TestCase): def test_import_fails(self): - self.assertRaises(ImportError, safe_exec, 'import os') + self.assertRaises(ImportError, safe_exec, "import os") def test_internal_attributes(self): - self.assertRaises(SyntaxError, safe_exec, '().__class__.__call__') + self.assertRaises(SyntaxError, safe_exec, "().__class__.__call__") def test_utils(self): _locals = dict(out=None) - safe_exec('''out = frappe.utils.cint("1")''', None, _locals) - self.assertEqual(_locals['out'], 1) + safe_exec("""out = frappe.utils.cint("1")""", None, _locals) + self.assertEqual(_locals["out"], 1) def test_safe_eval(self): - self.assertEqual(frappe.safe_eval('1+1'), 2) - self.assertRaises(AttributeError, frappe.safe_eval, 'frappe.utils.os.path', get_safe_globals()) + self.assertEqual(frappe.safe_eval("1+1"), 2) + self.assertRaises(AttributeError, frappe.safe_eval, "frappe.utils.os.path", get_safe_globals()) def test_sql(self): _locals = dict(out=None) - safe_exec('''out = frappe.db.sql("select name from tabDocType where name='DocType'")''', None, _locals) - self.assertEqual(_locals['out'][0][0], 'DocType') + safe_exec( + """out = frappe.db.sql("select name from tabDocType where name='DocType'")""", None, _locals + ) + self.assertEqual(_locals["out"][0][0], "DocType") - self.assertRaises(frappe.PermissionError, safe_exec, 'frappe.db.sql("update tabToDo set description=NULL")') + self.assertRaises( + frappe.PermissionError, safe_exec, 'frappe.db.sql("update tabToDo set description=NULL")' + ) def test_query_builder(self): _locals = dict(out=None) - safe_exec(script='''out = frappe.qb.from_("User").select(frappe.qb.terms.PseudoColumn("Max(name)")).run()''', _globals=None, _locals=_locals) + safe_exec( + script="""out = frappe.qb.from_("User").select(frappe.qb.terms.PseudoColumn("Max(name)")).run()""", + _globals=None, + _locals=_locals, + ) self.assertEqual(frappe.db.sql("SELECT Max(name) FROM tabUser"), _locals["out"]) def test_safe_query_builder(self): - self.assertRaises(frappe.PermissionError, safe_exec, '''frappe.qb.from_("User").delete().run()''') + self.assertRaises( + frappe.PermissionError, safe_exec, """frappe.qb.from_("User").delete().run()""" + ) def test_call(self): # call non whitelisted method - self.assertRaises( - frappe.PermissionError, - safe_exec, - """frappe.call("frappe.get_user")""" - ) + self.assertRaises(frappe.PermissionError, safe_exec, """frappe.call("frappe.get_user")""") # call whitelisted method safe_exec("""frappe.call("ping")""") - def test_enqueue(self): # enqueue non whitelisted method self.assertRaises( - frappe.PermissionError, - safe_exec, - """frappe.enqueue("frappe.get_user", now=True)""" + frappe.PermissionError, safe_exec, """frappe.enqueue("frappe.get_user", now=True)""" ) # enqueue whitelisted method diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index f13bcbe06f..82bea12720 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -1,44 +1,53 @@ +import time from unittest import TestCase + from dateutil.relativedelta import relativedelta + +import frappe from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs -from frappe.utils.background_jobs import enqueue, get_jobs -from frappe.utils.scheduler import enqueue_events, is_dormant, schedule_jobs_based_on_activity from frappe.utils import add_days, get_datetime +from frappe.utils.background_jobs import enqueue, get_jobs from frappe.utils.doctor import purge_pending_jobs +from frappe.utils.scheduler import enqueue_events, is_dormant, schedule_jobs_based_on_activity -import frappe -import time def test_timeout(): time.sleep(100) + def test_timeout_10(): time.sleep(10) + def test_method(): pass + + class TestScheduler(TestCase): def setUp(self): purge_pending_jobs() - if not frappe.get_all('Scheduled Job Type', limit=1): + if not frappe.get_all("Scheduled Job Type", limit=1): sync_jobs() def test_enqueue_jobs(self): frappe.db.sql("update `tabScheduled Job Type` set last_execution = '2010-01-01 00:00:00'") frappe.flags.execute_job = True - enqueue_events(site = frappe.local.site) + enqueue_events(site=frappe.local.site) frappe.flags.execute_job = False - self.assertTrue('frappe.email.queue.set_expiry_for_email_queue', frappe.flags.enqueued_jobs) - self.assertTrue('frappe.utils.change_log.check_for_update', frappe.flags.enqueued_jobs) - self.assertTrue('frappe.email.doctype.auto_email_report.auto_email_report.send_monthly', frappe.flags.enqueued_jobs) + self.assertTrue("frappe.email.queue.set_expiry_for_email_queue", frappe.flags.enqueued_jobs) + self.assertTrue("frappe.utils.change_log.check_for_update", frappe.flags.enqueued_jobs) + self.assertTrue( + "frappe.email.doctype.auto_email_report.auto_email_report.send_monthly", + frappe.flags.enqueued_jobs, + ) def test_queue_peeking(self): job = get_test_job() self.assertTrue(job.enqueue()) - job.db_set('last_execution', '2010-01-01 00:00:00') + job.db_set("last_execution", "2010-01-01 00:00:00") frappe.db.commit() time.sleep(0.5) @@ -48,26 +57,38 @@ class TestScheduler(TestCase): frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": job.name}) def test_is_dormant(self): - self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00'))) - self.assertTrue(is_dormant(check_time = add_days(frappe.db.get_last_created('Activity Log'), 5))) - self.assertFalse(is_dormant(check_time = frappe.db.get_last_created('Activity Log'))) + self.assertTrue(is_dormant(check_time=get_datetime("2100-01-01 00:00:00"))) + self.assertTrue(is_dormant(check_time=add_days(frappe.db.get_last_created("Activity Log"), 5))) + self.assertFalse(is_dormant(check_time=frappe.db.get_last_created("Activity Log"))) def test_once_a_day_for_dormant(self): - frappe.db.clear_table('Scheduled Job Log') - self.assertTrue(schedule_jobs_based_on_activity(check_time= get_datetime('2100-01-01 00:00:00'))) - self.assertTrue(schedule_jobs_based_on_activity(check_time = add_days(frappe.db.get_last_created('Activity Log'), 5))) + frappe.db.clear_table("Scheduled Job Log") + self.assertTrue(schedule_jobs_based_on_activity(check_time=get_datetime("2100-01-01 00:00:00"))) + self.assertTrue( + schedule_jobs_based_on_activity( + check_time=add_days(frappe.db.get_last_created("Activity Log"), 5) + ) + ) # create a fake job executed 5 days from now - job = get_test_job(method='frappe.tests.test_scheduler.test_method', frequency='Daily') + job = get_test_job(method="frappe.tests.test_scheduler.test_method", frequency="Daily") job.execute() - job_log = frappe.get_doc('Scheduled Job Log', dict(scheduled_job_type=job.name)) - job_log.db_set('creation', add_days(frappe.db.get_last_created('Activity Log'), 5)) + job_log = frappe.get_doc("Scheduled Job Log", dict(scheduled_job_type=job.name)) + job_log.db_set("creation", add_days(frappe.db.get_last_created("Activity Log"), 5)) # inactive site with recent job, don't run - self.assertFalse(schedule_jobs_based_on_activity(check_time = add_days(frappe.db.get_last_created('Activity Log'), 5))) + self.assertFalse( + schedule_jobs_based_on_activity( + check_time=add_days(frappe.db.get_last_created("Activity Log"), 5) + ) + ) # one more day has passed - self.assertTrue(schedule_jobs_based_on_activity(check_time = add_days(frappe.db.get_last_created('Activity Log'), 6))) + self.assertTrue( + schedule_jobs_based_on_activity( + check_time=add_days(frappe.db.get_last_created("Activity Log"), 6) + ) + ) frappe.db.rollback() @@ -78,24 +99,26 @@ class TestScheduler(TestCase): while count > 0: count -= 1 time.sleep(5) - if job.get_status()=='failed': + if job.get_status() == "failed": break self.assertTrue(job.is_failed) -def get_test_job(method='frappe.tests.test_scheduler.test_timeout_10', frequency='All'): - if not frappe.db.exists('Scheduled Job Type', dict(method=method)): - job = frappe.get_doc(dict( - doctype = 'Scheduled Job Type', - method = method, - last_execution = '2010-01-01 00:00:00', - frequency = frequency - )).insert() + +def get_test_job(method="frappe.tests.test_scheduler.test_timeout_10", frequency="All"): + if not frappe.db.exists("Scheduled Job Type", dict(method=method)): + job = frappe.get_doc( + dict( + doctype="Scheduled Job Type", + method=method, + last_execution="2010-01-01 00:00:00", + frequency=frequency, + ) + ).insert() else: - job = frappe.get_doc('Scheduled Job Type', dict(method=method)) - job.db_set('last_execution', '2010-01-01 00:00:00') - job.db_set('frequency', frequency) + job = frappe.get_doc("Scheduled Job Type", dict(method=method)) + job.db_set("last_execution", "2010-01-01 00:00:00") + job.db_set("frequency", frequency) frappe.db.commit() return job - diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py index 38a00f689a..eea5783ccb 100644 --- a/frappe/tests/test_search.py +++ b/frappe/tests/test_search.py @@ -2,8 +2,9 @@ # License: MIT. See LICENSE import unittest + import frappe -from frappe.desk.search import search_link, search_widget, get_names_for_mentions +from frappe.desk.search import get_names_for_mentions, search_link, search_widget class TestSearch(unittest.TestCase): @@ -17,147 +18,208 @@ class TestSearch(unittest.TestCase): def test_search_field_sanitizer(self): # pass - search_link('DocType', 'User', query=None, filters=None, page_length=20, searchfield='name') - result = frappe.response['results'][0] - self.assertTrue('User' in result['value']) - - #raise exception on injection - self.assertRaises(frappe.DataError, - search_link, 'DocType', 'Customer', query=None, filters=None, - page_length=20, searchfield='1=1') - - self.assertRaises(frappe.DataError, - search_link, 'DocType', 'Customer', query=None, filters=None, - page_length=20, searchfield='select * from tabSessions) --') - - self.assertRaises(frappe.DataError, - search_link, 'DocType', 'Customer', query=None, filters=None, - page_length=20, searchfield='name or (select * from tabSessions)') - - self.assertRaises(frappe.DataError, - search_link, 'DocType', 'Customer', query=None, filters=None, - page_length=20, searchfield='*') - - self.assertRaises(frappe.DataError, - search_link, 'DocType', 'Customer', query=None, filters=None, - page_length=20, searchfield=';') - - self.assertRaises(frappe.DataError, - search_link, 'DocType', 'Customer', query=None, filters=None, - page_length=20, searchfield=';') + search_link("DocType", "User", query=None, filters=None, page_length=20, searchfield="name") + result = frappe.response["results"][0] + self.assertTrue("User" in result["value"]) + + # raise exception on injection + self.assertRaises( + frappe.DataError, + search_link, + "DocType", + "Customer", + query=None, + filters=None, + page_length=20, + searchfield="1=1", + ) + + self.assertRaises( + frappe.DataError, + search_link, + "DocType", + "Customer", + query=None, + filters=None, + page_length=20, + searchfield="select * from tabSessions) --", + ) + + self.assertRaises( + frappe.DataError, + search_link, + "DocType", + "Customer", + query=None, + filters=None, + page_length=20, + searchfield="name or (select * from tabSessions)", + ) + + self.assertRaises( + frappe.DataError, + search_link, + "DocType", + "Customer", + query=None, + filters=None, + page_length=20, + searchfield="*", + ) + + self.assertRaises( + frappe.DataError, + search_link, + "DocType", + "Customer", + query=None, + filters=None, + page_length=20, + searchfield=";", + ) + + self.assertRaises( + frappe.DataError, + search_link, + "DocType", + "Customer", + query=None, + filters=None, + page_length=20, + searchfield=";", + ) def test_only_enabled_in_mention(self): - email = 'test_disabled_user_in_mentions@example.com' - frappe.delete_doc('User', email) - if not frappe.db.exists('User', email): - user = frappe.new_doc('User') - user.update({ - 'email' : email, - 'first_name' : email.split("@")[0], - 'enabled' : False, - 'allowed_in_mentions' : True, - }) + email = "test_disabled_user_in_mentions@example.com" + frappe.delete_doc("User", email) + if not frappe.db.exists("User", email): + user = frappe.new_doc("User") + user.update( + { + "email": email, + "first_name": email.split("@")[0], + "enabled": False, + "allowed_in_mentions": True, + } + ) # saved when roles are added - user.add_roles('System Manager',) + user.add_roles( + "System Manager", + ) - names_for_mention = [user.get('id') for user in get_names_for_mentions('')] + names_for_mention = [user.get("id") for user in get_names_for_mentions("")] self.assertNotIn(email, names_for_mention) def test_link_field_order(self): # Making a request to the search_link with the tree doctype - search_link(doctype=self.tree_doctype_name, txt='all', query=None, - filters=None, page_length=20, searchfield=None) - result = frappe.response['results'] + search_link( + doctype=self.tree_doctype_name, + txt="all", + query=None, + filters=None, + page_length=20, + searchfield=None, + ) + result = frappe.response["results"] # Check whether the result is sorted or not - self.assertEqual(self.parent_doctype_name, result[0]['value']) + self.assertEqual(self.parent_doctype_name, result[0]["value"]) # Check whether searching for parent also list out children self.assertEqual(len(result), len(self.child_doctypes_names) + 1) - #Search for the word "pay", part of the word "pays" (country) in french. + # Search for the word "pay", part of the word "pays" (country) in french. def test_link_search_in_foreign_language(self): try: - frappe.local.lang = 'fr' + frappe.local.lang = "fr" search_widget(doctype="DocType", txt="pay", page_length=20) output = frappe.response["values"] - result = [['found' for x in y if x=="Country"] for y in output] - self.assertTrue(['found'] in result) + result = [["found" for x in y if x == "Country"] for y in output] + self.assertTrue(["found"] in result) finally: - frappe.local.lang = 'en' + frappe.local.lang = "en" def test_validate_and_sanitize_search_inputs(self): # should raise error if searchfield is injectable - self.assertRaises(frappe.DataError, - get_data, *('User', 'Random', 'select * from tabSessions) --', '1', '10', dict())) + self.assertRaises( + frappe.DataError, + get_data, + *("User", "Random", "select * from tabSessions) --", "1", "10", dict()) + ) # page_len and start should be converted to int - self.assertListEqual(get_data('User', 'Random', 'email', 'name or (select * from tabSessions)', '10', dict()), - ['User', 'Random', 'email', 0, 10, {}]) - self.assertListEqual(get_data('User', 'Random', 'email', page_len='2', start='10', filters=dict()), - ['User', 'Random', 'email', 10, 2, {}]) + self.assertListEqual( + get_data("User", "Random", "email", "name or (select * from tabSessions)", "10", dict()), + ["User", "Random", "email", 0, 10, {}], + ) + self.assertListEqual( + get_data("User", "Random", "email", page_len="2", start="10", filters=dict()), + ["User", "Random", "email", 10, 2, {}], + ) # DocType can be passed as None which should be accepted - self.assertListEqual(get_data(None, 'Random', 'email', '2', '10', dict()), - [None, 'Random', 'email', 2, 10, {}]) + self.assertListEqual( + get_data(None, "Random", "email", "2", "10", dict()), [None, "Random", "email", 2, 10, {}] + ) # return empty string if passed doctype is invalid - self.assertListEqual(get_data("Random DocType", 'Random', 'email', '2', '10', dict()), []) + self.assertListEqual(get_data("Random DocType", "Random", "email", "2", "10", dict()), []) # should not fail if function is called via frappe.call with extra arguments - args = ("Random DocType", 'Random', 'email', '2', '10', dict()) - kwargs = {'as_dict': False} - self.assertListEqual(frappe.call('frappe.tests.test_search.get_data', *args, **kwargs), []) + args = ("Random DocType", "Random", "email", "2", "10", dict()) + kwargs = {"as_dict": False} + self.assertListEqual(frappe.call("frappe.tests.test_search.get_data", *args, **kwargs), []) # should not fail if query has @ symbol in it - search_link('User', 'user@random', searchfield='name') - self.assertListEqual(frappe.response['results'], []) + search_link("User", "user@random", searchfield="name") + self.assertListEqual(frappe.response["results"], []) + @frappe.validate_and_sanitize_search_inputs def get_data(doctype, txt, searchfield, start, page_len, filters): return [doctype, txt, searchfield, start, page_len, filters] + def setup_test_link_field_order(TestCase): - TestCase.tree_doctype_name = 'Test Tree Order' + TestCase.tree_doctype_name = "Test Tree Order" TestCase.child_doctype_list = [] - TestCase.child_doctypes_names = ['USA', 'India', 'Russia', 'China'] - TestCase.parent_doctype_name = 'All Territories' + TestCase.child_doctypes_names = ["USA", "India", "Russia", "China"] + TestCase.parent_doctype_name = "All Territories" # Create Tree doctype - TestCase.tree_doc = frappe.get_doc({ - 'doctype': 'DocType', - 'name': TestCase.tree_doctype_name, - 'module': 'Custom', - 'custom': 1, - 'is_tree': 1, - 'autoname': 'field:random', - 'fields': [{ - 'fieldname': 'random', - 'label': 'Random', - 'fieldtype': 'Data' - }] - }).insert() - TestCase.tree_doc.search_fields = 'parent_test_tree_order' + TestCase.tree_doc = frappe.get_doc( + { + "doctype": "DocType", + "name": TestCase.tree_doctype_name, + "module": "Custom", + "custom": 1, + "is_tree": 1, + "autoname": "field:random", + "fields": [{"fieldname": "random", "label": "Random", "fieldtype": "Data"}], + } + ).insert() + TestCase.tree_doc.search_fields = "parent_test_tree_order" TestCase.tree_doc.save() # Create root for the tree doctype - frappe.get_doc({ - "doctype": TestCase.tree_doctype_name, - "random": TestCase.parent_doctype_name, - "is_group": 1 - }).insert() + frappe.get_doc( + {"doctype": TestCase.tree_doctype_name, "random": TestCase.parent_doctype_name, "is_group": 1} + ).insert() # Create children for the root for child_name in TestCase.child_doctypes_names: - temp = frappe.get_doc({ - "doctype": TestCase.tree_doctype_name, - "random": child_name, - "parent_test_tree_order": TestCase.parent_doctype_name - }).insert() + temp = frappe.get_doc( + { + "doctype": TestCase.tree_doctype_name, + "random": child_name, + "parent_test_tree_order": TestCase.parent_doctype_name, + } + ).insert() TestCase.child_doctype_list.append(temp) + def teardown_test_link_field_order(TestCase): # Deleting all the created doctype for child_doctype in TestCase.child_doctype_list: diff --git a/frappe/tests/test_seen.py b/frappe/tests/test_seen.py index 48b6345005..8db9ec8ebe 100644 --- a/frappe/tests/test_seen.py +++ b/frappe/tests/test_seen.py @@ -1,45 +1,51 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, unittest, json +import json +import unittest + +import frappe + class TestSeen(unittest.TestCase): def tearDown(self): - frappe.set_user('Administrator') + frappe.set_user("Administrator") def test_if_user_is_added(self): - ev = frappe.get_doc({ - 'doctype': 'Event', - 'subject': 'test event for seen', - 'starts_on': '2016-01-01 10:10:00', - 'event_type': 'Public' - }).insert() + ev = frappe.get_doc( + { + "doctype": "Event", + "subject": "test event for seen", + "starts_on": "2016-01-01 10:10:00", + "event_type": "Public", + } + ).insert() - frappe.set_user('test@example.com') + frappe.set_user("test@example.com") from frappe.desk.form.load import getdoc # load the form - getdoc('Event', ev.name) + getdoc("Event", ev.name) # reload the event - ev = frappe.get_doc('Event', ev.name) + ev = frappe.get_doc("Event", ev.name) - self.assertTrue('test@example.com' in json.loads(ev._seen)) + self.assertTrue("test@example.com" in json.loads(ev._seen)) # test another user - frappe.set_user('test1@example.com') + frappe.set_user("test1@example.com") # load the form - getdoc('Event', ev.name) + getdoc("Event", ev.name) # reload the event - ev = frappe.get_doc('Event', ev.name) + ev = frappe.get_doc("Event", ev.name) - self.assertTrue('test@example.com' in json.loads(ev._seen)) - self.assertTrue('test1@example.com' in json.loads(ev._seen)) + self.assertTrue("test@example.com" in json.loads(ev._seen)) + self.assertTrue("test1@example.com" in json.loads(ev._seen)) ev.save() - ev = frappe.get_doc('Event', ev.name) + ev = frappe.get_doc("Event", ev.name) - self.assertFalse('test@example.com' in json.loads(ev._seen)) - self.assertTrue('test1@example.com' in json.loads(ev._seen)) + self.assertFalse("test@example.com" in json.loads(ev._seen)) + self.assertTrue("test1@example.com" in json.loads(ev._seen)) diff --git a/frappe/tests/test_sitemap.py b/frappe/tests/test_sitemap.py index e29a453a14..db91e0b1a6 100644 --- a/frappe/tests/test_sitemap.py +++ b/frappe/tests/test_sitemap.py @@ -1,12 +1,16 @@ -import frappe, unittest +import unittest + +import frappe from frappe.utils import get_html_for_route + class TestSitemap(unittest.TestCase): def test_sitemap(self): from frappe.test_runner import make_test_records - make_test_records('Blog Post') - blogs = frappe.db.get_all('Blog Post', {'published': 1}, ['route'], limit=1) - xml = get_html_for_route('sitemap.xml') - self.assertTrue('/about' in xml) - self.assertTrue('/contact' in xml) + + make_test_records("Blog Post") + blogs = frappe.db.get_all("Blog Post", {"published": 1}, ["route"], limit=1) + xml = get_html_for_route("sitemap.xml") + self.assertTrue("/about" in xml) + self.assertTrue("/contact" in xml) self.assertTrue(blogs[0].route in xml) diff --git a/frappe/tests/test_translate.py b/frappe/tests/test_translate.py index f5386914f7..d372baae04 100644 --- a/frappe/tests/test_translate.py +++ b/frappe/tests/test_translate.py @@ -12,17 +12,18 @@ from frappe.translate import get_language, get_parent_language from frappe.utils import set_request dirname = os.path.dirname(__file__) -translation_string_file = os.path.join(dirname, 'translation_test_file.txt') +translation_string_file = os.path.join(dirname, "translation_test_file.txt") first_lang, second_lang, third_lang, fourth_lang, fifth_lang = choices( # skip "en*" since it is a default language frappe.get_all("Language", pluck="name", filters=[["name", "not like", "en%"]]), - k=5 + k=5, ) + class TestTranslate(unittest.TestCase): guest_sessions_required = [ "test_guest_request_language_resolution_with_cookie", - "test_guest_request_language_resolution_with_request_header" + "test_guest_request_language_resolution_with_request_header", ] def setUp(self): @@ -38,12 +39,15 @@ class TestTranslate(unittest.TestCase): data = frappe.translate.get_messages_from_file(translation_string_file) exp_filename = "apps/frappe/frappe/tests/translation_test_file.txt" - self.assertEqual(len(data), len(expected_output), - msg=f"Mismatched output:\nExpected: {expected_output}\nFound: {data}") + self.assertEqual( + len(data), + len(expected_output), + msg=f"Mismatched output:\nExpected: {expected_output}\nFound: {data}", + ) for extracted, expected in zip(data, expected_output): - ext_filename, ext_message, ext_context, ext_line = extracted - exp_message, exp_context, exp_line = expected + ext_filename, ext_message, ext_context, ext_line = extracted + exp_message, exp_context, exp_line = expected self.assertEqual(ext_filename, exp_filename) self.assertEqual(ext_message, exp_message) self.assertEqual(ext_context, exp_context) @@ -51,11 +55,11 @@ class TestTranslate(unittest.TestCase): def test_translation_with_context(self): try: - frappe.local.lang = 'fr' - self.assertEqual(_('Change'), 'Changement') - self.assertEqual(_('Change', context='Coins'), 'la monnaie') + frappe.local.lang = "fr" + self.assertEqual(_("Change"), "Changement") + self.assertEqual(_("Change", context="Coins"), "la monnaie") finally: - frappe.local.lang = 'en' + frappe.local.lang = "en" def test_request_language_resolution_with_form_dict(self): """Test for frappe.translate.get_language @@ -76,11 +80,11 @@ class TestTranslate(unittest.TestCase): Case 2: frappe.form_dict._lang is not set, but preferred_language cookie is """ - with patch.object(frappe.translate, "get_preferred_language_cookie", return_value='fr'): - set_request(method="POST", path="/", headers=[("Accept-Language", 'hr')]) + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value="fr"): + set_request(method="POST", path="/", headers=[("Accept-Language", "hr")]) return_val = get_language() # system default language - self.assertEqual(return_val, 'en') + self.assertEqual(return_val, "en") self.assertNotIn(return_val, [second_lang, get_parent_language(second_lang)]) def test_guest_request_language_resolution_with_cookie(self): @@ -95,7 +99,6 @@ class TestTranslate(unittest.TestCase): self.assertIn(return_val, [second_lang, get_parent_language(second_lang)]) - def test_guest_request_language_resolution_with_request_header(self): """Test for frappe.translate.get_language @@ -118,16 +121,15 @@ class TestTranslate(unittest.TestCase): expected_output = [ - ('Warning: Unable to find {0} in any table related to {1}', 'This is some context', 2), - ('Warning: Unable to find {0} in any table related to {1}', None, 4), + ("Warning: Unable to find {0} in any table related to {1}", "This is some context", 2), + ("Warning: Unable to find {0} in any table related to {1}", None, 4), ("You don't have any messages yet.", None, 6), - ('Submit', 'Some DocType', 8), - ('Warning: Unable to find {0} in any table related to {1}', 'This is some context', 15), - ('Submit', 'Some DocType', 17), + ("Submit", "Some DocType", 8), + ("Warning: Unable to find {0} in any table related to {1}", "This is some context", 15), + ("Submit", "Some DocType", 17), ("You don't have any messages yet.", None, 19), ("You don't have any messages yet.", None, 21), ("Long string that needs its own line because of black formatting.", None, 24), ("Long string with", "context", 28), ("Long string with", "context on newline", 32), ] - diff --git a/frappe/tests/test_twofactor.py b/frappe/tests/test_twofactor.py index fadc61a551..82d39f40bb 100644 --- a/frappe/tests/test_twofactor.py +++ b/frappe/tests/test_twofactor.py @@ -1,101 +1,107 @@ # Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest, frappe, pyotp -from frappe.auth import HTTPRequest -from frappe.utils import cint -from frappe.utils import set_request -from frappe.auth import validate_ip_address, get_login_attempt_tracker -from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass, - two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, ExpiredLoginException) -from . import update_system_settings, get_system_setting - import time +import unittest + +import pyotp + +import frappe +from frappe.auth import HTTPRequest, get_login_attempt_tracker, validate_ip_address +from frappe.twofactor import ( + ExpiredLoginException, + authenticate_for_2factor, + confirm_otp_token, + get_cached_user_pass, + get_otpsecret_for_, + get_verification_obj, + should_run_2fa, + two_factor_is_enabled_for_, +) +from frappe.utils import cint, set_request + +from . import get_system_setting, update_system_settings + class TestTwoFactor(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestTwoFactor, self).__init__(*args, **kwargs) - self.default_allowed_login_attempts = get_system_setting('allow_consecutive_login_attempts') + self.default_allowed_login_attempts = get_system_setting("allow_consecutive_login_attempts") def setUp(self): self.http_requests = create_http_request() self.login_manager = frappe.local.login_manager self.user = self.login_manager.user - update_system_settings({ - 'allow_consecutive_login_attempts': 2 - }) + update_system_settings({"allow_consecutive_login_attempts": 2}) def tearDown(self): - frappe.local.response['verification'] = None - frappe.local.response['tmp_id'] = None + frappe.local.response["verification"] = None + frappe.local.response["tmp_id"] = None disable_2fa() frappe.clear_cache(user=self.user) - update_system_settings({ - 'allow_consecutive_login_attempts': self.default_allowed_login_attempts - }) + update_system_settings({"allow_consecutive_login_attempts": self.default_allowed_login_attempts}) def test_should_run_2fa(self): - '''Should return true if enabled.''' + """Should return true if enabled.""" toggle_2fa_all_role(state=True) self.assertTrue(should_run_2fa(self.user)) toggle_2fa_all_role(state=False) self.assertFalse(should_run_2fa(self.user)) def test_get_cached_user_pass(self): - '''Cached data should not contain user and pass before 2fa.''' - user,pwd = get_cached_user_pass() + """Cached data should not contain user and pass before 2fa.""" + user, pwd = get_cached_user_pass() self.assertTrue(all([not user, not pwd])) def test_authenticate_for_2factor(self): - '''Verification obj and tmp_id should be set in frappe.local.''' + """Verification obj and tmp_id should be set in frappe.local.""" authenticate_for_2factor(self.user) - verification_obj = frappe.local.response['verification'] - tmp_id = frappe.local.response['tmp_id'] + verification_obj = frappe.local.response["verification"] + tmp_id = frappe.local.response["tmp_id"] self.assertTrue(verification_obj) self.assertTrue(tmp_id) - for k in ['_usr','_pwd','_otp_secret']: - self.assertTrue(frappe.cache().get('{0}{1}'.format(tmp_id,k)), - '{} not available'.format(k)) + for k in ["_usr", "_pwd", "_otp_secret"]: + self.assertTrue(frappe.cache().get("{0}{1}".format(tmp_id, k)), "{} not available".format(k)) def test_two_factor_is_enabled(self): - ''' + """ 1. Should return true, if enabled and not bypass_2fa_for_retricted_ip_users 2. Should return false, if not enabled 3. Should return true, if enabled and not bypass_2fa_for_retricted_ip_users and ip in restrict_ip 4. Should return true, if enabled and bypass_2fa_for_retricted_ip_users and not restrict_ip 5. Should return false, if enabled and bypass_2fa_for_retricted_ip_users and ip in restrict_ip - ''' + """ - #Scenario 1 + # Scenario 1 enable_2fa() self.assertTrue(should_run_2fa(self.user)) - #Scenario 2 + # Scenario 2 disable_2fa() self.assertFalse(should_run_2fa(self.user)) - #Scenario 3 + # Scenario 3 enable_2fa() - user = frappe.get_doc('User', self.user) + user = frappe.get_doc("User", self.user) user.restrict_ip = frappe.local.request_ip user.save() self.assertTrue(should_run_2fa(self.user)) - #Scenario 4 - user = frappe.get_doc('User', self.user) + # Scenario 4 + user = frappe.get_doc("User", self.user) user.restrict_ip = "" user.save() enable_2fa(1) self.assertTrue(should_run_2fa(self.user)) - #Scenario 5 - user = frappe.get_doc('User', self.user) + # Scenario 5 + user = frappe.get_doc("User", self.user) user.restrict_ip = frappe.local.request_ip user.save() enable_2fa(1) self.assertFalse(should_run_2fa(self.user)) def test_two_factor_is_enabled_for_user(self): - '''Should return true if enabled for user.''' + """Should return true if enabled for user.""" toggle_2fa_all_role(state=True) self.assertTrue(two_factor_is_enabled_for_(self.user)) self.assertFalse(two_factor_is_enabled_for_("Administrator")) @@ -103,125 +109,128 @@ class TestTwoFactor(unittest.TestCase): self.assertFalse(two_factor_is_enabled_for_(self.user)) def test_get_otpsecret_for_user(self): - '''OTP secret should be set for user.''' + """OTP secret should be set for user.""" self.assertTrue(get_otpsecret_for_(self.user)) - self.assertTrue(frappe.db.get_default(self.user + '_otpsecret')) + self.assertTrue(frappe.db.get_default(self.user + "_otpsecret")) def test_confirm_otp_token(self): - '''Ensure otp is confirmed''' + """Ensure otp is confirmed""" frappe.flags.otp_expiry = 2 authenticate_for_2factor(self.user) - tmp_id = frappe.local.response['tmp_id'] - otp = 'wrongotp' + tmp_id = frappe.local.response["tmp_id"] + otp = "wrongotp" with self.assertRaises(frappe.AuthenticationError): - confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) + confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id) otp = get_otp(self.user) - self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)) + self.assertTrue(confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id)) frappe.flags.otp_expiry = None if frappe.flags.tests_verbose: - print('Sleeping for 2 secs to confirm token expires..') + print("Sleeping for 2 secs to confirm token expires..") time.sleep(2) with self.assertRaises(ExpiredLoginException): - confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) + confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id) def test_get_verification_obj(self): - '''Confirm verification object is returned.''' + """Confirm verification object is returned.""" otp_secret = get_otpsecret_for_(self.user) token = int(pyotp.TOTP(otp_secret).now()) - self.assertTrue(get_verification_obj(self.user,token,otp_secret)) + self.assertTrue(get_verification_obj(self.user, token, otp_secret)) def test_render_string_template(self): - '''String template renders as expected with variables.''' - args = {'issuer_name':'Frappe Technologies'} - _str = 'Verification Code from {{issuer_name}}' - _str = frappe.render_template(_str,args) - self.assertEqual(_str,'Verification Code from Frappe Technologies') + """String template renders as expected with variables.""" + args = {"issuer_name": "Frappe Technologies"} + _str = "Verification Code from {{issuer_name}}" + _str = frappe.render_template(_str, args) + self.assertEqual(_str, "Verification Code from Frappe Technologies") def test_bypass_restict_ip(self): - ''' + """ 1. Raise error if user not login from one of the restrict_ip, Bypass restrict ip check disabled by default 2. Bypass restrict ip check enabled in System Settings 3. Bypass restrict ip check enabled for User - ''' + """ - #1 - user = frappe.get_doc('User', self.user) - user.restrict_ip = "192.168.255.254" #Dummy IP + # 1 + user = frappe.get_doc("User", self.user) + user.restrict_ip = "192.168.255.254" # Dummy IP user.bypass_restrict_ip_check_if_2fa_enabled = 0 user.save() enable_2fa(bypass_restrict_ip_check=0) with self.assertRaises(frappe.AuthenticationError): validate_ip_address(self.user) - #2 + # 2 enable_2fa(bypass_restrict_ip_check=1) self.assertIsNone(validate_ip_address(self.user)) - #3 - user = frappe.get_doc('User', self.user) + # 3 + user = frappe.get_doc("User", self.user) user.bypass_restrict_ip_check_if_2fa_enabled = 1 user.save() enable_2fa() self.assertIsNone(validate_ip_address(self.user)) def test_otp_attempt_tracker(self): - """Check that OTP login attempts are tracked. - """ + """Check that OTP login attempts are tracked.""" authenticate_for_2factor(self.user) - tmp_id = frappe.local.response['tmp_id'] - otp = 'wrongotp' + tmp_id = frappe.local.response["tmp_id"] + otp = "wrongotp" with self.assertRaises(frappe.AuthenticationError): - confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) + confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id) with self.assertRaises(frappe.AuthenticationError): - confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) + confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id) # REMOVE ME: current logic allows allow_consecutive_login_attempts+1 attempts # before raising security exception, remove below line when that is fixed. with self.assertRaises(frappe.AuthenticationError): - confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) + confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id) with self.assertRaises(frappe.SecurityException): - confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) + confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id) # Remove tracking cache so that user can try loging in again tracker = get_login_attempt_tracker(self.user, raise_locked_exception=False) tracker.add_success_attempt() otp = get_otp(self.user) - self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)) + self.assertTrue(confirm_otp_token(self.login_manager, otp=otp, tmp_id=tmp_id)) + def create_http_request(): - '''Get http request object.''' - set_request(method='POST', path='login') + """Get http request object.""" + set_request(method="POST", path="login") enable_2fa() - frappe.form_dict['usr'] = 'test@example.com' - frappe.form_dict['pwd'] = 'Eastern_43A1W' - frappe.local.form_dict['cmd'] = 'login' + frappe.form_dict["usr"] = "test@example.com" + frappe.form_dict["pwd"] = "Eastern_43A1W" + frappe.local.form_dict["cmd"] = "login" http_requests = HTTPRequest() return http_requests + def enable_2fa(bypass_two_factor_auth=0, bypass_restrict_ip_check=0): - '''Enable Two factor in system settings.''' - system_settings = frappe.get_doc('System Settings') + """Enable Two factor in system settings.""" + system_settings = frappe.get_doc("System Settings") system_settings.enable_two_factor_auth = 1 system_settings.bypass_2fa_for_retricted_ip_users = cint(bypass_two_factor_auth) system_settings.bypass_restrict_ip_check_if_2fa_enabled = cint(bypass_restrict_ip_check) - system_settings.two_factor_method = 'OTP App' + system_settings.two_factor_method = "OTP App" system_settings.flags.ignore_mandatory = True system_settings.save(ignore_permissions=True) frappe.db.commit() + def disable_2fa(): - system_settings = frappe.get_doc('System Settings') + system_settings = frappe.get_doc("System Settings") system_settings.enable_two_factor_auth = 0 system_settings.flags.ignore_mandatory = True system_settings.save(ignore_permissions=True) frappe.db.commit() + def toggle_2fa_all_role(state=None): - '''Enable or disable 2fa for 'all' role on the system.''' - all_role = frappe.get_doc('Role','All') + """Enable or disable 2fa for 'all' role on the system.""" + all_role = frappe.get_doc("Role", "All") state = state if state is not None else False if type(state) != bool: return @@ -230,6 +239,7 @@ def toggle_2fa_all_role(state=None): all_role.save(ignore_permissions=True) frappe.db.commit() + def get_otp(user): otp_secret = get_otpsecret_for_(user) otp = pyotp.TOTP(otp_secret) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index fa6b5a3820..a56d404fac 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import io -import os import json +import os import unittest from datetime import date, datetime, time, timedelta from decimal import Decimal @@ -15,73 +15,127 @@ import pytz from PIL import Image import frappe -from frappe.utils import ceil, evaluate_filters, floor, format_timedelta, get_bench_path -from frappe.utils import get_url, money_in_words, parse_timedelta, scrub_urls -from frappe.utils import validate_email_address, validate_url -from frappe.utils.data import cast, get_time, get_timedelta, nowtime, now_datetime, validate_python_code +from frappe.installer import parse_app_name +from frappe.utils import ( + ceil, + evaluate_filters, + floor, + format_timedelta, + get_bench_path, + get_url, + money_in_words, + parse_timedelta, + scrub_urls, + validate_email_address, + validate_url, +) +from frappe.utils.data import ( + cast, + get_time, + get_timedelta, + now_datetime, + nowtime, + validate_python_code, +) from frappe.utils.diff import _get_value_from_version, get_version_diff, version_query from frappe.utils.image import optimize_image, strip_exif_data from frappe.utils.response import json_handler -from frappe.installer import parse_app_name class TestFilters(unittest.TestCase): def test_simple_dict(self): - self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Open'})) - self.assertFalse(evaluate_filters({'doctype': 'User', 'status': 'Open'}, {'status': 'Closed'})) + self.assertTrue(evaluate_filters({"doctype": "User", "status": "Open"}, {"status": "Open"})) + self.assertFalse(evaluate_filters({"doctype": "User", "status": "Open"}, {"status": "Closed"})) def test_multiple_dict(self): - self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open', 'name': 'Test 1'}, - {'status': 'Open', 'name':'Test 1'})) - self.assertFalse(evaluate_filters({'doctype': 'User', 'status': 'Open', 'name': 'Test 1'}, - {'status': 'Closed', 'name': 'Test 1'})) + self.assertTrue( + evaluate_filters( + {"doctype": "User", "status": "Open", "name": "Test 1"}, {"status": "Open", "name": "Test 1"} + ) + ) + self.assertFalse( + evaluate_filters( + {"doctype": "User", "status": "Open", "name": "Test 1"}, {"status": "Closed", "name": "Test 1"} + ) + ) def test_list_filters(self): - self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open', 'name': 'Test 1'}, - [{'status': 'Open'}, {'name':'Test 1'}])) - self.assertFalse(evaluate_filters({'doctype': 'User', 'status': 'Open', 'name': 'Test 1'}, - [{'status': 'Open'}, {'name':'Test 2'}])) + self.assertTrue( + evaluate_filters( + {"doctype": "User", "status": "Open", "name": "Test 1"}, + [{"status": "Open"}, {"name": "Test 1"}], + ) + ) + self.assertFalse( + evaluate_filters( + {"doctype": "User", "status": "Open", "name": "Test 1"}, + [{"status": "Open"}, {"name": "Test 2"}], + ) + ) def test_list_filters_as_list(self): - self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open', 'name': 'Test 1'}, - [['status', '=', 'Open'], ['name', '=', 'Test 1']])) - self.assertFalse(evaluate_filters({'doctype': 'User', 'status': 'Open', 'name': 'Test 1'}, - [['status', '=', 'Open'], ['name', '=', 'Test 2']])) + self.assertTrue( + evaluate_filters( + {"doctype": "User", "status": "Open", "name": "Test 1"}, + [["status", "=", "Open"], ["name", "=", "Test 1"]], + ) + ) + self.assertFalse( + evaluate_filters( + {"doctype": "User", "status": "Open", "name": "Test 1"}, + [["status", "=", "Open"], ["name", "=", "Test 2"]], + ) + ) def test_lt_gt(self): - self.assertTrue(evaluate_filters({'doctype': 'User', 'status': 'Open', 'age': 20}, - {'status': 'Open', 'age': ('>', 10)})) - self.assertFalse(evaluate_filters({'doctype': 'User', 'status': 'Open', 'age': 20}, - {'status': 'Open', 'age': ('>', 30)})) + self.assertTrue( + evaluate_filters( + {"doctype": "User", "status": "Open", "age": 20}, {"status": "Open", "age": (">", 10)} + ) + ) + self.assertFalse( + evaluate_filters( + {"doctype": "User", "status": "Open", "age": 20}, {"status": "Open", "age": (">", 30)} + ) + ) + class TestMoney(unittest.TestCase): def test_money_in_words(self): nums_bhd = [ - (5000, "BHD Five Thousand only."), (5000.0, "BHD Five Thousand only."), - (0.1, "One Hundred Fils only."), (0, "BHD Zero only."), ("Fail", "") + (5000, "BHD Five Thousand only."), + (5000.0, "BHD Five Thousand only."), + (0.1, "One Hundred Fils only."), + (0, "BHD Zero only."), + ("Fail", ""), ] nums_ngn = [ - (5000, "NGN Five Thousand only."), (5000.0, "NGN Five Thousand only."), - (0.1, "Ten Kobo only."), (0, "NGN Zero only."), ("Fail", "") + (5000, "NGN Five Thousand only."), + (5000.0, "NGN Five Thousand only."), + (0.1, "Ten Kobo only."), + (0, "NGN Zero only."), + ("Fail", ""), ] for num in nums_bhd: self.assertEqual( money_in_words(num[0], "BHD"), num[1], - "{0} is not the same as {1}".format(money_in_words(num[0], "BHD"), num[1]) + "{0} is not the same as {1}".format(money_in_words(num[0], "BHD"), num[1]), ) for num in nums_ngn: self.assertEqual( - money_in_words(num[0], "NGN"), num[1], - "{0} is not the same as {1}".format(money_in_words(num[0], "NGN"), num[1]) + money_in_words(num[0], "NGN"), + num[1], + "{0} is not the same as {1}".format(money_in_words(num[0], "NGN"), num[1]), ) + class TestDataManipulation(unittest.TestCase): def test_scrub_urls(self): - html = ''' + html = """

You have a new message from: John

Hey, wassup!

@@ -93,7 +147,7 @@ class TestDataManipulation(unittest.TestCase):
Please mail us at email
- ''' + """ html = scrub_urls(html) url = get_url() @@ -102,13 +156,23 @@ class TestDataManipulation(unittest.TestCase): self.assertTrue('Test link 2'.format(url) in html) self.assertTrue('Test link 3'.format(url) in html) self.assertTrue(''.format(url) in html) - self.assertTrue('style="background-image: url(\'{0}/assets/frappe/bg.jpg\') !important"'.format(url) in html) + self.assertTrue( + "style=\"background-image: url('{0}/assets/frappe/bg.jpg') !important\"".format(url) in html + ) self.assertTrue('email' in html) + class TestFieldCasting(unittest.TestCase): def test_str_types(self): STR_TYPES = ( - "Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link" + "Data", + "Text", + "Small Text", + "Long Text", + "Text Editor", + "Select", + "Link", + "Dynamic Link", ) for fieldtype in STR_TYPES: self.assertIsInstance(cast(fieldtype, value=None), str) @@ -144,97 +208,101 @@ class TestFieldCasting(unittest.TestCase): self.assertIsInstance(cast("Time", value=None), timedelta) self.assertIsInstance(cast("Time", value="12:03:34"), timedelta) + class TestMathUtils(unittest.TestCase): def test_floor(self): from decimal import Decimal - self.assertEqual(floor(2), 2) - self.assertEqual(floor(12.32904), 12) - self.assertEqual(floor(22.7330), 22) - self.assertEqual(floor('24.7'), 24) - self.assertEqual(floor('26.7'), 26) + + self.assertEqual(floor(2), 2) + self.assertEqual(floor(12.32904), 12) + self.assertEqual(floor(22.7330), 22) + self.assertEqual(floor("24.7"), 24) + self.assertEqual(floor("26.7"), 26) self.assertEqual(floor(Decimal(29.45)), 29) def test_ceil(self): from decimal import Decimal - self.assertEqual(ceil(2), 2) - self.assertEqual(ceil(12.32904), 13) - self.assertEqual(ceil(22.7330), 23) - self.assertEqual(ceil('24.7'), 25) - self.assertEqual(ceil('26.7'), 27) - self.assertEqual(ceil(Decimal(29.45)), 30) + + self.assertEqual(ceil(2), 2) + self.assertEqual(ceil(12.32904), 13) + self.assertEqual(ceil(22.7330), 23) + self.assertEqual(ceil("24.7"), 25) + self.assertEqual(ceil("26.7"), 27) + self.assertEqual(ceil(Decimal(29.45)), 30) + class TestHTMLUtils(unittest.TestCase): def test_clean_email_html(self): from frappe.utils.html_utils import clean_email_html - sample = '''

Hello

Para

''' + + sample = """

Hello

Para

""" clean = clean_email_html(sample) - self.assertFalse('", content) self.assertIn("background-color: var(--bg-color);", content) def test_raw_assets_are_loaded(self): - content = get_response_content('/_test/assets/js_asset.min.js') + content = get_response_content("/_test/assets/js_asset.min.js") # minified js files should not be passed through jinja renderer self.assertEqual("//{% if title %} {{title}} {% endif %}\nconsole.log('in');", content) - content = get_response_content('/_test/assets/css_asset.css') + content = get_response_content("/_test/assets/css_asset.css") self.assertEqual("""body{color:red}""", content) def test_breadcrumbs(self): - content = get_response_content('/_test/_test_folder/_test_page') + content = get_response_content("/_test/_test_folder/_test_page") self.assertIn('Test Folder', content) self.assertIn(' Test Page', content) - content = get_response_content('/_test/_test_folder/index') + content = get_response_content("/_test/_test_folder/index") self.assertIn(' Test', content) self.assertIn('Test Folder', content) def test_get_context_without_context_object(self): - content = get_response_content('/_test/_test_no_context') + content = get_response_content("/_test/_test_no_context") self.assertIn("Custom Content", content) def test_caching(self): @@ -287,21 +293,21 @@ class TestWebsite(unittest.TestCase): clear_website_cache() # first response no-cache - response = get_response('/_test/_test_folder/_test_page') - self.assertIn(('X-From-Cache', 'False'), list(response.headers)) + response = get_response("/_test/_test_folder/_test_page") + self.assertIn(("X-From-Cache", "False"), list(response.headers)) # first response returned from cache - response = get_response('/_test/_test_folder/_test_page') - self.assertIn(('X-From-Cache', 'True'), list(response.headers)) + response = get_response("/_test/_test_folder/_test_page") + self.assertIn(("X-From-Cache", "True"), list(response.headers)) frappe.flags.force_website_cache = False def test_safe_render(self): - content = get_response_content('/_test/_test_safe_render_on') + content = get_response_content("/_test/_test_safe_render_on") self.assertNotIn("Safe Render On", content) self.assertIn("frappe.exceptions.ValidationError: Illegal template", content) - content = get_response_content('/_test/_test_safe_render_off') + content = get_response_content("/_test/_test_safe_render_off") self.assertIn("Safe Render Off", content) self.assertIn("test.__test", content) self.assertNotIn("frappe.exceptions.ValidationError: Illegal template", content) @@ -309,22 +315,29 @@ class TestWebsite(unittest.TestCase): def set_home_page_hook(key, value): from frappe import hooks + # reset home_page hooks - for hook in ('get_website_user_home_page','website_user_home_page','role_home_page','home_page'): + for hook in ( + "get_website_user_home_page", + "website_user_home_page", + "role_home_page", + "home_page", + ): if hasattr(hooks, hook): delattr(hooks, hook) setattr(hooks, key, value) - frappe.cache().delete_key('app_hooks') + frappe.cache().delete_key("app_hooks") + -class CustomPageRenderer(): +class CustomPageRenderer: def __init__(self, path, status_code=None): self.path = path # custom status code self.status_code = 3984 def can_render(self): - if self.path in ('new', 'custom'): + if self.path in ("new", "custom"): return True def render(self): diff --git a/frappe/tests/tests_geo_utils.py b/frappe/tests/tests_geo_utils.py index 28b987e21e..2ac03e67e0 100644 --- a/frappe/tests/tests_geo_utils.py +++ b/frappe/tests/tests_geo_utils.py @@ -11,30 +11,46 @@ from frappe.geo.utils import get_coords class TestGeoUtils(unittest.TestCase): def setUp(self): self.todo = frappe.get_doc( - dict(doctype='ToDo', description='Test description', assigned_by='Administrator')).insert() - - self.test_location_dict = {'type': 'FeatureCollection', 'features': [ - {'type': 'Feature', 'properties': {}, "geometry": {'type': 'Point', 'coordinates': [49.20433, 55.753395]}}]} - self.test_location = frappe.get_doc({'name': 'Test Location', 'doctype': 'Location', - 'location': str(self.test_location_dict)}) - - self.test_filter_exists = [['Location', 'name', 'like', '%Test Location%']] - self.test_filter_not_exists = [['Location', 'name', 'like', '%Test Location Not exists%']] - self.test_filter_todo = [['ToDo', 'description', 'like', '%Test description%']] + dict(doctype="ToDo", description="Test description", assigned_by="Administrator") + ).insert() + + self.test_location_dict = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": {"type": "Point", "coordinates": [49.20433, 55.753395]}, + } + ], + } + self.test_location = frappe.get_doc( + {"name": "Test Location", "doctype": "Location", "location": str(self.test_location_dict)} + ) + + self.test_filter_exists = [["Location", "name", "like", "%Test Location%"]] + self.test_filter_not_exists = [["Location", "name", "like", "%Test Location Not exists%"]] + self.test_filter_todo = [["ToDo", "description", "like", "%Test description%"]] def test_get_coords_location_with_filter_exists(self): - coords = get_coords('Location', self.test_filter_exists, 'location_field') - self.assertEqual(self.test_location_dict['features'][0]['geometry'], coords['features'][0]['geometry']) + coords = get_coords("Location", self.test_filter_exists, "location_field") + self.assertEqual( + self.test_location_dict["features"][0]["geometry"], coords["features"][0]["geometry"] + ) def test_get_coords_location_with_filter_not_exists(self): - coords = get_coords('Location', self.test_filter_not_exists, 'location_field') - self.assertEqual(coords, {'type': 'FeatureCollection', 'features': []}) + coords = get_coords("Location", self.test_filter_not_exists, "location_field") + self.assertEqual(coords, {"type": "FeatureCollection", "features": []}) def test_get_coords_from_not_existable_location(self): - self.assertRaises(frappe.ValidationError, get_coords, 'ToDo', self.test_filter_todo, 'location_field') + self.assertRaises( + frappe.ValidationError, get_coords, "ToDo", self.test_filter_todo, "location_field" + ) def test_get_coords_from_not_existable_coords(self): - self.assertRaises(frappe.ValidationError, get_coords, 'ToDo', self.test_filter_todo, 'coordinates') + self.assertRaises( + frappe.ValidationError, get_coords, "ToDo", self.test_filter_todo, "coordinates" + ) def tearDown(self): self.todo.delete() diff --git a/frappe/tests/ui_test_helpers.py b/frappe/tests/ui_test_helpers.py index 063451825e..4cb3e95a6d 100644 --- a/frappe/tests/ui_test_helpers.py +++ b/frappe/tests/ui_test_helpers.py @@ -2,16 +2,17 @@ import frappe from frappe import _ from frappe.utils import add_to_date, now + @frappe.whitelist() def create_if_not_exists(doc): - '''Create records if they dont exist. + """Create records if they dont exist. Will check for uniqueness by checking if a record exists with these field value pairs :param doc: dict of field value pairs. can be a list of dict for multiple records. - ''' + """ if not frappe.local.dev_server: - frappe.throw(_('This method can only be accessed in development'), frappe.PermissionError) + frappe.throw(_("This method can only be accessed in development"), frappe.PermissionError) doc = frappe.parse_json(doc) @@ -24,7 +25,7 @@ def create_if_not_exists(doc): for doc in docs: doc = frappe._dict(doc) filters = doc.copy() - filters.pop('doctype') + filters.pop("doctype") name = frappe.db.exists(doc.doctype, filters) if not name: d = frappe.get_doc(doc) @@ -37,215 +38,225 @@ def create_if_not_exists(doc): @frappe.whitelist() def create_todo_records(): - if frappe.db.get_all('ToDo', {'description': 'this is first todo'}): + if frappe.db.get_all("ToDo", {"description": "this is first todo"}): return - frappe.get_doc({ - "doctype": "ToDo", - "date": add_to_date(now(), days=7), - "description": "this is first todo" - }).insert() - frappe.get_doc({ - "doctype": "ToDo", - "date": add_to_date(now(), days=-7), - "description": "this is second todo" - }).insert() - frappe.get_doc({ - "doctype": "ToDo", - "date": add_to_date(now(), months=2), - "description": "this is third todo" - }).insert() - frappe.get_doc({ - "doctype": "ToDo", - "date": add_to_date(now(), months=-2), - "description": "this is fourth todo" - }).insert() + frappe.get_doc( + {"doctype": "ToDo", "date": add_to_date(now(), days=7), "description": "this is first todo"} + ).insert() + frappe.get_doc( + {"doctype": "ToDo", "date": add_to_date(now(), days=-7), "description": "this is second todo"} + ).insert() + frappe.get_doc( + {"doctype": "ToDo", "date": add_to_date(now(), months=2), "description": "this is third todo"} + ).insert() + frappe.get_doc( + {"doctype": "ToDo", "date": add_to_date(now(), months=-2), "description": "this is fourth todo"} + ).insert() + @frappe.whitelist() def create_communication_record(): - doc = frappe.get_doc({ - "doctype": "Communication", - "recipients": "test@gmail.com", - "subject": "Test Form Communication 1", - "communication_date": frappe.utils.now_datetime(), - }) + doc = frappe.get_doc( + { + "doctype": "Communication", + "recipients": "test@gmail.com", + "subject": "Test Form Communication 1", + "communication_date": frappe.utils.now_datetime(), + } + ) doc.insert() return doc + @frappe.whitelist() def setup_workflow(): from frappe.workflow.doctype.workflow.test_workflow import create_todo_workflow + create_todo_workflow() create_todo_records() frappe.clear_cache() + @frappe.whitelist() def create_contact_phone_nos_records(): - if frappe.db.get_all('Contact', {'first_name': 'Test Contact'}): + if frappe.db.get_all("Contact", {"first_name": "Test Contact"}): return - doc = frappe.new_doc('Contact') - doc.first_name = 'Test Contact' + doc = frappe.new_doc("Contact") + doc.first_name = "Test Contact" for index in range(1000): - doc.append('phone_nos', {'phone': '123456{}'.format(index)}) + doc.append("phone_nos", {"phone": "123456{}".format(index)}) doc.insert() + @frappe.whitelist() def create_doctype(name, fields): fields = frappe.parse_json(fields) - if frappe.db.exists('DocType', name): + if frappe.db.exists("DocType", name): return - frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "fields": fields, - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": name - }).insert() + frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "custom": 1, + "fields": fields, + "permissions": [{"role": "System Manager", "read": 1}], + "name": name, + } + ).insert() + @frappe.whitelist() def create_child_doctype(name, fields): fields = frappe.parse_json(fields) - if frappe.db.exists('DocType', name): + if frappe.db.exists("DocType", name): return - frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "istable": 1, - "custom": 1, - "fields": fields, - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": name - }).insert() + frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "istable": 1, + "custom": 1, + "fields": fields, + "permissions": [{"role": "System Manager", "read": 1}], + "name": name, + } + ).insert() + @frappe.whitelist() def create_contact_records(): - if frappe.db.get_all('Contact', {'first_name': 'Test Form Contact 1'}): + if frappe.db.get_all("Contact", {"first_name": "Test Form Contact 1"}): return - insert_contact('Test Form Contact 1', '12345') - insert_contact('Test Form Contact 2', '54321') - insert_contact('Test Form Contact 3', '12345') + insert_contact("Test Form Contact 1", "12345") + insert_contact("Test Form Contact 2", "54321") + insert_contact("Test Form Contact 3", "12345") + @frappe.whitelist() def create_multiple_todo_records(): values = [] - if frappe.db.get_all('ToDo', {'description': 'Multiple ToDo 1'}): + if frappe.db.get_all("ToDo", {"description": "Multiple ToDo 1"}): return for index in range(1, 1002): - values.append(('100{}'.format(index), 'Multiple ToDo {}'.format(index))) + values.append(("100{}".format(index), "Multiple ToDo {}".format(index))) + + frappe.db.bulk_insert("ToDo", fields=["name", "description"], values=set(values)) - frappe.db.bulk_insert('ToDo', fields=['name', 'description'], values=set(values)) def insert_contact(first_name, phone_number): - doc = frappe.get_doc({ - 'doctype': 'Contact', - 'first_name': first_name - }) - doc.append('phone_nos', {'phone': phone_number}) + doc = frappe.get_doc({"doctype": "Contact", "first_name": first_name}) + doc.append("phone_nos", {"phone": phone_number}) doc.insert() + @frappe.whitelist() def create_form_tour(): - if frappe.db.exists('Form Tour', {'name': 'Test Form Tour'}): + if frappe.db.exists("Form Tour", {"name": "Test Form Tour"}): return - tour = frappe.get_doc({ - 'doctype': 'Form Tour', - 'title': 'Test Form Tour', - 'reference_doctype': 'Contact', - 'save_on_complete': 1, - 'steps': [{ - "title": "Test Title 1", - "description": "Test Description 1", - "has_next_condition": 1, - "next_step_condition": "eval: doc.first_name", - "fieldname": "first_name", - "fieldtype": "Data" - },{ - "title": "Test Title 2", - "description": "Test Description 2", - "has_next_condition": 1, - "next_step_condition": "eval: doc.last_name", - "fieldname": "last_name", - "fieldtype": "Data" - },{ - "title": "Test Title 3", - "description": "Test Description 3", - "fieldname": "phone_nos", - "fieldtype": "Table" - },{ - "title": "Test Title 4", - "description": "Test Description 4", - "is_table_field": 1, - "parent_fieldname": "phone_nos", - "next_step_condition": "eval: doc.phone", - "has_next_condition": 1, - "fieldname": "phone", - "fieldtype": "Data" - }] - }) + tour = frappe.get_doc( + { + "doctype": "Form Tour", + "title": "Test Form Tour", + "reference_doctype": "Contact", + "save_on_complete": 1, + "steps": [ + { + "title": "Test Title 1", + "description": "Test Description 1", + "has_next_condition": 1, + "next_step_condition": "eval: doc.first_name", + "fieldname": "first_name", + "fieldtype": "Data", + }, + { + "title": "Test Title 2", + "description": "Test Description 2", + "has_next_condition": 1, + "next_step_condition": "eval: doc.last_name", + "fieldname": "last_name", + "fieldtype": "Data", + }, + { + "title": "Test Title 3", + "description": "Test Description 3", + "fieldname": "phone_nos", + "fieldtype": "Table", + }, + { + "title": "Test Title 4", + "description": "Test Description 4", + "is_table_field": 1, + "parent_fieldname": "phone_nos", + "next_step_condition": "eval: doc.phone", + "has_next_condition": 1, + "fieldname": "phone", + "fieldtype": "Data", + }, + ], + } + ) tour.insert() + @frappe.whitelist() def create_data_for_discussions(): web_page = create_web_page("Test page for discussions", "test-page-discussions", False) create_topic_and_reply(web_page) create_web_page("Test single thread discussion", "test-single-thread", True) + def create_web_page(title, route, single_thread): web_page = frappe.db.exists("Web Page", {"route": route}) if web_page: return web_page - web_page = frappe.get_doc({ - "doctype": "Web Page", - "title": title, - "route": route, - "published": True - }) + web_page = frappe.get_doc( + {"doctype": "Web Page", "title": title, "route": route, "published": True} + ) web_page.save() - web_page.append("page_blocks", { - "web_template": "Discussions", - "web_template_values": frappe.as_json({ - "title": "Discussions", - "cta_title": "New Discussion", - "docname": web_page.name, - "single_thread": single_thread - }) - }) + web_page.append( + "page_blocks", + { + "web_template": "Discussions", + "web_template_values": frappe.as_json( + { + "title": "Discussions", + "cta_title": "New Discussion", + "docname": web_page.name, + "single_thread": single_thread, + } + ), + }, + ) web_page.save() return web_page.name + def create_topic_and_reply(web_page): - topic = frappe.db.exists("Discussion Topic",{ - "reference_doctype": "Web Page", - "reference_docname": web_page - }) + topic = frappe.db.exists( + "Discussion Topic", {"reference_doctype": "Web Page", "reference_docname": web_page} + ) if not topic: - topic = frappe.get_doc({ - "doctype": "Discussion Topic", - "reference_doctype": "Web Page", - "reference_docname": web_page, - "title": "Test Topic" - }) + topic = frappe.get_doc( + { + "doctype": "Discussion Topic", + "reference_doctype": "Web Page", + "reference_docname": web_page, + "title": "Test Topic", + } + ) topic.save() - reply = frappe.get_doc({ - "doctype": "Discussion Reply", - "topic": topic.name, - "reply": "This is a test reply" - }) + reply = frappe.get_doc( + {"doctype": "Discussion Reply", "topic": topic.name, "reply": "This is a test reply"} + ) reply.save() @@ -261,58 +272,66 @@ def update_webform_to_multistep(): _doc.is_standard = False _doc.save() + @frappe.whitelist() def update_child_table(name): - doc = frappe.get_doc('DocType', name) + doc = frappe.get_doc("DocType", name) if len(doc.fields) == 1: - doc.append('fields', { - 'fieldname': 'doctype_to_link', - 'fieldtype': 'Link', - 'in_list_view': 1, - 'label': 'Doctype to Link', - 'options': 'Doctype to Link' - }) + doc.append( + "fields", + { + "fieldname": "doctype_to_link", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Doctype to Link", + "options": "Doctype to Link", + }, + ) doc.save() + @frappe.whitelist() def insert_doctype_with_child_table_record(name): - if frappe.db.get_all(name, {'title': 'Test Grid Search'}): + if frappe.db.get_all(name, {"title": "Test Grid Search"}): return def insert_child(doc, data, barcode, check, rating, duration, date): - doc.append('child_table_1', { - 'data': data, - 'barcode': barcode, - 'check': check, - 'rating': rating, - 'duration': duration, - 'date': date, - }) + doc.append( + "child_table_1", + { + "data": data, + "barcode": barcode, + "check": check, + "rating": rating, + "duration": duration, + "date": date, + }, + ) doc = frappe.new_doc(name) - doc.title = 'Test Grid Search' - doc.append('child_table', {'title': 'Test Grid Search'}) - - insert_child(doc, 'Data', '09709KJKKH2432', 1, 0.5, 266851, "2022-02-21") - insert_child(doc, 'Test', '09209KJHKH2432', 1, 0.8, 547877, "2021-05-27") - insert_child(doc, 'New', '09709KJHYH1132', 0, 0.1, 3, "2019-03-02") - insert_child(doc, 'Old', '09701KJHKH8750', 0, 0, 127455, "2022-01-11") - insert_child(doc, 'Alpha', '09204KJHKH2432', 0, 0.6, 364, "2019-12-31") - insert_child(doc, 'Delta', '09709KSPIO2432', 1, 0.9, 1242000, "2020-04-21") - insert_child(doc, 'Update', '76989KJLVA2432', 0, 1, 183845, "2022-02-10") - insert_child(doc, 'Delete', '29189KLHVA1432', 0, 0, 365647, "2021-05-07") - insert_child(doc, 'Make', '09689KJHAA2431', 0, 0.3, 24, "2020-11-11") - insert_child(doc, 'Create', '09709KLKKH2432', 1, 0.3, 264851, "2021-02-21") - insert_child(doc, 'Group', '09209KJLKH2432', 1, 0.8, 537877, "2020-03-15") - insert_child(doc, 'Slide', '01909KJHYH1132', 0, 0.5, 9, "2018-03-02") - insert_child(doc, 'Drop', '09701KJHKH8750', 1, 0, 127255, "2018-01-01") - insert_child(doc, 'Beta', '09204QJHKN2432', 0, 0.6, 354, "2017-12-30") - insert_child(doc, 'Flag', '09709KXPIP2432', 1, 0, 1241000, "2021-04-21") - insert_child(doc, 'Upgrade', '75989ZJLVA2432', 0.8, 1, 183645, "2020-08-13") - insert_child(doc, 'Down', '28189KLHRA1432', 1, 0, 362647, "2020-06-17") - insert_child(doc, 'Note', '09689DJHAA2431', 0, 0.1, 29, "2021-09-11") - insert_child(doc, 'Click', '08189DJHAA2431', 1, 0.3, 209, "2020-07-04") - insert_child(doc, 'Drag', '08189DIHAA2981', 0, 0.7, 342628, "2022-05-04") + doc.title = "Test Grid Search" + doc.append("child_table", {"title": "Test Grid Search"}) + + insert_child(doc, "Data", "09709KJKKH2432", 1, 0.5, 266851, "2022-02-21") + insert_child(doc, "Test", "09209KJHKH2432", 1, 0.8, 547877, "2021-05-27") + insert_child(doc, "New", "09709KJHYH1132", 0, 0.1, 3, "2019-03-02") + insert_child(doc, "Old", "09701KJHKH8750", 0, 0, 127455, "2022-01-11") + insert_child(doc, "Alpha", "09204KJHKH2432", 0, 0.6, 364, "2019-12-31") + insert_child(doc, "Delta", "09709KSPIO2432", 1, 0.9, 1242000, "2020-04-21") + insert_child(doc, "Update", "76989KJLVA2432", 0, 1, 183845, "2022-02-10") + insert_child(doc, "Delete", "29189KLHVA1432", 0, 0, 365647, "2021-05-07") + insert_child(doc, "Make", "09689KJHAA2431", 0, 0.3, 24, "2020-11-11") + insert_child(doc, "Create", "09709KLKKH2432", 1, 0.3, 264851, "2021-02-21") + insert_child(doc, "Group", "09209KJLKH2432", 1, 0.8, 537877, "2020-03-15") + insert_child(doc, "Slide", "01909KJHYH1132", 0, 0.5, 9, "2018-03-02") + insert_child(doc, "Drop", "09701KJHKH8750", 1, 0, 127255, "2018-01-01") + insert_child(doc, "Beta", "09204QJHKN2432", 0, 0.6, 354, "2017-12-30") + insert_child(doc, "Flag", "09709KXPIP2432", 1, 0, 1241000, "2021-04-21") + insert_child(doc, "Upgrade", "75989ZJLVA2432", 0.8, 1, 183645, "2020-08-13") + insert_child(doc, "Down", "28189KLHRA1432", 1, 0, 362647, "2020-06-17") + insert_child(doc, "Note", "09689DJHAA2431", 0, 0.1, 29, "2021-09-11") + insert_child(doc, "Click", "08189DJHAA2431", 1, 0.3, 209, "2020-07-04") + insert_child(doc, "Drag", "08189DIHAA2981", 0, 0.7, 342628, "2022-05-04") doc.insert() diff --git a/frappe/tests/utils.py b/frappe/tests/utils.py index 8699a51bf8..bad368afd0 100644 --- a/frappe/tests/utils.py +++ b/frappe/tests/utils.py @@ -8,6 +8,7 @@ import frappe class FrappeTestCase(unittest.TestCase): """Base test class for Frappe tests.""" + @classmethod def setUpClass(cls) -> None: frappe.db.commit() @@ -21,7 +22,7 @@ class FrappeTestCase(unittest.TestCase): @contextmanager def change_settings(doctype, settings_dict): - """ A context manager to ensure that settings are changed before running + """A context manager to ensure that settings are changed before running function and restored after running it regardless of exceptions occured. This is useful in tests where you want to make changes in a function but don't retain those changes. @@ -30,7 +31,7 @@ def change_settings(doctype, settings_dict): example: @change_settings("Print Settings", {"send_print_as_pdf": 1}) def test_case(self): - ... + ... """ try: @@ -46,7 +47,7 @@ def change_settings(doctype, settings_dict): settings.save() # singles are cached by default, clear to avoid flake frappe.db.value_cache[settings] = {} - yield # yield control to calling function + yield # yield control to calling function finally: # restore settings @@ -57,9 +58,10 @@ def change_settings(doctype, settings_dict): def timeout(seconds=30, error_message="Test timed out."): - """ Timeout decorator to ensure a test doesn't run for too long. + """Timeout decorator to ensure a test doesn't run for too long. + + adapted from https://stackoverflow.com/a/2282656""" - adapted from https://stackoverflow.com/a/2282656""" def decorator(func): def _handle_timeout(signum, frame): raise Exception(error_message) @@ -72,5 +74,7 @@ def timeout(seconds=30, error_message="Test timed out."): finally: signal.alarm(0) return result + return wrapper + return decorator diff --git a/frappe/translate.py b/frappe/translate.py index 6e0fefd6fa..d95c8eb3e8 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -7,48 +7,45 @@ Translation tools for frappe """ +import functools import io import itertools import json import operator -import functools import os import re from csv import reader -from typing import List, Union, Tuple +from typing import List, Tuple, Union + +from pypika.terms import PseudoColumn import frappe from frappe.model.utils import InvalidIncludePath, render_include +from frappe.query_builder import DocType, Field from frappe.utils import get_bench_path, is_html, strip, strip_html_tags -from frappe.query_builder import Field, DocType -from pypika.terms import PseudoColumn TRANSLATE_PATTERN = re.compile( r"_\([\s\n]*" # starts with literal `_(`, ignore following whitespace/newlines - # BEGIN: message search r"([\"']{,3})" # start of message string identifier - allows: ', ", """, '''; 1st capture group - r"(?P((?!\1).)*)" # Keep matching until string closing identifier is met which is same as 1st capture group - r"\1" # match exact string closing identifier + r"(?P((?!\1).)*)" # Keep matching until string closing identifier is met which is same as 1st capture group + r"\1" # match exact string closing identifier # END: message search - # BEGIN: python context search r"([\s\n]*,[\s\n]*context\s*=\s*" # capture `context=` with ignoring whitespace - r"([\"'])" # start of context string identifier; 5th capture group - r"(?P((?!\5).)*)" # capture context string till closing id is found - r"\5" # match context string closure + r"([\"'])" # start of context string identifier; 5th capture group + r"(?P((?!\5).)*)" # capture context string till closing id is found + r"\5" # match context string closure r")?" # match 0 or 1 context strings # END: python context search - # BEGIN: JS context search - r"(\s*,\s*(.)*?\s*(,\s*" # skip message format replacements: ["format", ...] | null | [] - r"([\"'])" # start of context string; 11th capture group - r"(?P((?!\11).)*)" # capture context string till closing id is found - r"\11" # match context string closure - r")*" + r"(\s*,\s*(.)*?\s*(,\s*" # skip message format replacements: ["format", ...] | null | [] + r"([\"'])" # start of context string; 11th capture group + r"(?P((?!\11).)*)" # capture context string till closing id is found + r"\11" # match context string closure + r")*" r")*" # match one or more context string # END: JS context search - r"[\s\n]*\)" # Closing function call ignore leading whitespace/newlines ) @@ -67,9 +64,7 @@ def get_language(lang_list: List = None) -> str: # fetch language from form_dict if frappe.form_dict._lang: - language = get_lang_code( - frappe.form_dict._lang or get_parent_language(frappe.form_dict._lang) - ) + language = get_lang_code(frappe.form_dict._lang or get_parent_language(frappe.form_dict._lang)) if language: return language @@ -110,12 +105,12 @@ def get_parent_language(lang: str) -> str: """If the passed language is a variant, return its parent Eg: - 1. zh-TW -> zh - 2. sr-BA -> sr + 1. zh-TW -> zh + 2. sr-BA -> sr """ is_language_variant = "-" in lang if is_language_variant: - return lang[:lang.index("-")] + return lang[: lang.index("-")] def get_user_lang(user: str = None) -> str: @@ -136,29 +131,34 @@ def get_user_lang(user: str = None) -> str: return lang + def get_lang_code(lang: str) -> Union[str, None]: - return ( - frappe.db.get_value("Language", {"name": lang}) - or frappe.db.get_value("Language", {"language_name": lang}) + return frappe.db.get_value("Language", {"name": lang}) or frappe.db.get_value( + "Language", {"language_name": lang} ) + def set_default_language(lang): """Set Global default language""" if frappe.db.get_default("lang") != lang: frappe.db.set_default("lang", lang) frappe.local.lang = lang + def get_lang_dict(): """Returns all languages in dict format, full name is the key e.g. `{"english":"en"}`""" - result = dict(frappe.get_all("Language", fields=["language_name", "name"], order_by="modified", as_list=True)) + result = dict( + frappe.get_all("Language", fields=["language_name", "name"], order_by="modified", as_list=True) + ) return result + def get_dict(fortype, name=None): """Returns translation dict for a type of object. - :param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot` - :param name: name of the document for which assets are to be returned. - """ + :param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot` + :param name: name of the document for which assets are to be returned. + """ fortype = fortype.lower() cache = frappe.cache() asset_key = fortype + ":" + (name or "-") @@ -166,17 +166,17 @@ def get_dict(fortype, name=None): if asset_key not in translation_assets: messages = [] - if fortype=="doctype": + if fortype == "doctype": messages = get_messages_from_doctype(name) - elif fortype=="page": + elif fortype == "page": messages = get_messages_from_page(name) - elif fortype=="report": + elif fortype == "report": messages = get_messages_from_report(name) - elif fortype=="include": + elif fortype == "include": messages = get_messages_from_include_files() - elif fortype=="jsfile": + elif fortype == "jsfile": messages = get_messages_from_file(name) - elif fortype=="boot": + elif fortype == "boot": apps = frappe.get_all_apps(True) for app in apps: messages.extend(get_server_messages(app)) @@ -184,30 +184,23 @@ def get_dict(fortype, name=None): messages += get_messages_from_navbar() messages += get_messages_from_include_files() messages += ( - frappe.qb.from_("Print Format") - .select(PseudoColumn("'Print Format:'"), "name")).run() - messages += ( - frappe.qb.from_("DocType") - .select(PseudoColumn("'DocType:'"), "name")).run() - messages += ( - frappe.qb.from_("Role").select(PseudoColumn("'Role:'"), "name").run() - ) - messages += ( - frappe.qb.from_("Module Def") - .select(PseudoColumn("'Module:'"), "name")).run() + frappe.qb.from_("Print Format").select(PseudoColumn("'Print Format:'"), "name") + ).run() + messages += (frappe.qb.from_("DocType").select(PseudoColumn("'DocType:'"), "name")).run() + messages += frappe.qb.from_("Role").select(PseudoColumn("'Role:'"), "name").run() + messages += (frappe.qb.from_("Module Def").select(PseudoColumn("'Module:'"), "name")).run() messages += ( frappe.qb.from_("Workspace Shortcut") .where(Field("format").isnotnull()) - .select(PseudoColumn("''"), "format")).run() - messages += ( - frappe.qb.from_("Onboarding Step") - .select(PseudoColumn("''"), "title")).run() + .select(PseudoColumn("''"), "format") + ).run() + messages += (frappe.qb.from_("Onboarding Step").select(PseudoColumn("''"), "title")).run() messages = deduplicate_messages(messages) message_dict = make_dict_from_messages(messages, load_user_translation=False) message_dict.update(get_dict_from_hooks(fortype, name)) # remove untranslated - message_dict = {k: v for k, v in message_dict.items() if k!=v} + message_dict = {k: v for k, v in message_dict.items() if k != v} translation_assets[asset_key] = message_dict cache.hset("translation_assets", frappe.local.lang, translation_assets, shared=True) @@ -229,6 +222,7 @@ def get_dict_from_hooks(fortype, name): return translated_dict + def make_dict_from_messages(messages, full_dict=None, load_user_translation=True): """Returns translated messages as a dict in Language specified in `frappe.local.lang` @@ -246,12 +240,13 @@ def make_dict_from_messages(messages, full_dict=None, load_user_translation=True out[m[1]] = full_dict[m[1]] # check if msg with context as key exist eg. msg:context if len(m) > 2 and m[2]: - key = m[1] + ':' + m[2] + key = m[1] + ":" + m[2] if full_dict.get(key): out[key] = full_dict[key] return out + def get_lang_js(fortype, name): """Returns code snippet to be appended at the end of a JS script. @@ -260,6 +255,7 @@ def get_lang_js(fortype, name): """ return "\n\n$.extend(frappe._messages, %s)" % json.dumps(get_dict(fortype, name)) + def get_full_dict(lang): """Load and return the entire translations dictionary for a language from :meth:`frape.cache` @@ -269,7 +265,7 @@ def get_full_dict(lang): return {} # found in local, return! - if getattr(frappe.local, 'lang_full_dict', None) and frappe.local.lang_full_dict.get(lang, None): + if getattr(frappe.local, "lang_full_dict", None) and frappe.local.lang_full_dict.get(lang, None): return frappe.local.lang_full_dict frappe.local.lang_full_dict = load_lang(lang) @@ -283,23 +279,24 @@ def get_full_dict(lang): return frappe.local.lang_full_dict + def load_lang(lang, apps=None): """Combine all translations from `.csv` files in all `apps`. For derivative languages (es-GT), take translations from the base language (es) and then update translations from the child (es-GT)""" - if lang=='en': + if lang == "en": return {} out = frappe.cache().hget("lang_full_dict", lang, shared=True) if not out: out = {} - for app in (apps or frappe.get_all_apps(True)): + for app in apps or frappe.get_all_apps(True): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") out.update(get_translation_dict_from_file(path, lang, app) or {}) - if '-' in lang: - parent = lang.split('-')[0] + if "-" in lang: + parent = lang.split("-")[0] parent_out = load_lang(parent) parent_out.update(out) out = parent_out @@ -308,6 +305,7 @@ def load_lang(lang, apps=None): return out or {} + def get_translation_dict_from_file(path, lang, app): """load translation dict from given path""" translation_map = {} @@ -315,36 +313,39 @@ def get_translation_dict_from_file(path, lang, app): csv_content = read_csv_file(path) for item in csv_content: - if len(item)==3 and item[2]: - key = item[0] + ':' + item[2] + if len(item) == 3 and item[2]: + key = item[0] + ":" + item[2] translation_map[key] = strip(item[1]) elif len(item) in [2, 3]: translation_map[item[0]] = strip(item[1]) elif item: - raise Exception("Bad translation in '{app}' for language '{lang}': {values}".format( - app=app, lang=lang, values=repr(item).encode("utf-8") - )) + raise Exception( + "Bad translation in '{app}' for language '{lang}': {values}".format( + app=app, lang=lang, values=repr(item).encode("utf-8") + ) + ) return translation_map + def get_user_translations(lang): if not frappe.db: frappe.connect() - out = frappe.cache().hget('lang_user_translations', lang) + out = frappe.cache().hget("lang_user_translations", lang) if out is None: out = {} - user_translations = frappe.get_all('Translation', - fields=["source_text", "translated_text", "context"], - filters={'language': lang}) + user_translations = frappe.get_all( + "Translation", fields=["source_text", "translated_text", "context"], filters={"language": lang} + ) for translation in user_translations: key = translation.source_text value = translation.translated_text if translation.context: - key += ':' + translation.context + key += ":" + translation.context out[key] = value - frappe.cache().hset('lang_user_translations', lang, out) + frappe.cache().hset("lang_user_translations", lang, out) return out @@ -360,6 +361,7 @@ def clear_cache(): cache.delete_key("translation_assets", shared=True) cache.delete_key("lang_user_translations") + def get_messages_for_app(app, deduplicate=True): """Returns all messages (list) for a specified `app`""" messages = [] @@ -369,21 +371,20 @@ def get_messages_for_app(app, deduplicate=True): if modules: if isinstance(modules, str): modules = [modules] - filtered_doctypes = frappe.qb.from_("DocType").where( - Field("module").isin(modules) - ).select("name").run(pluck=True) + filtered_doctypes = ( + frappe.qb.from_("DocType").where(Field("module").isin(modules)).select("name").run(pluck=True) + ) for name in filtered_doctypes: messages.extend(get_messages_from_doctype(name)) # pages - filtered_pages = frappe.qb.from_("Page").where( - Field("module").isin(modules) - ).select("name", "title").run() + filtered_pages = ( + frappe.qb.from_("Page").where(Field("module").isin(modules)).select("name", "title").run() + ) for name, title in filtered_pages: messages.append((None, title or name)) messages.extend(get_messages_from_page(name)) - # reports report = DocType("Report") doctype = DocType("DocType") @@ -391,7 +392,9 @@ def get_messages_for_app(app, deduplicate=True): frappe.qb.from_(doctype) .from_(report) .where((report.ref_doctype == doctype.name) & doctype.module.isin(modules)) - .select(report.name).run(pluck=True)) + .select(report.name) + .run(pluck=True) + ) for name in names: messages.append((None, name)) messages.extend(get_messages_from_report(name)) @@ -422,8 +425,8 @@ def get_messages_for_app(app, deduplicate=True): def get_messages_from_navbar(): """Return all labels from Navbar Items, as specified in Navbar Settings.""" - labels = frappe.get_all('Navbar Item', filters={'item_label': ('is', 'set')}, pluck='item_label') - return [('Navbar:', label, 'Label of a Navbar Item') for label in labels] + labels = frappe.get_all("Navbar Item", filters={"item_label": ("is", "set")}, pluck="item_label") + return [("Navbar:", label, "Label of a Navbar Item") for label in labels] def get_messages_from_doctype(name): @@ -441,11 +444,11 @@ def get_messages_from_doctype(name): for d in meta.get("fields"): messages.extend([d.label, d.description]) - if d.fieldtype=='Select' and d.options: - options = d.options.split('\n') + if d.fieldtype == "Select" and d.options: + options = d.options.split("\n") if not "icon" in options[0]: messages.extend(options) - if d.fieldtype=='HTML' and d.options: + if d.fieldtype == "HTML" and d.options: messages.append(d.options) # translations of roles @@ -454,7 +457,7 @@ def get_messages_from_doctype(name): messages.append(d.role) messages = [message for message in messages if message] - messages = [('DocType: ' + name, message) for message in messages if is_translatable(message)] + messages = [("DocType: " + name, message) for message in messages if is_translatable(message)] # extract from js, py files if not meta.custom: @@ -469,23 +472,24 @@ def get_messages_from_doctype(name): messages.extend(get_messages_from_workflow(doctype=name)) return messages + def get_messages_from_workflow(doctype=None, app_name=None): - assert doctype or app_name, 'doctype or app_name should be provided' + assert doctype or app_name, "doctype or app_name should be provided" # translations for Workflows workflows = [] if doctype: - workflows = frappe.get_all('Workflow', filters={'document_type': doctype}) + workflows = frappe.get_all("Workflow", filters={"document_type": doctype}) else: - fixtures = frappe.get_hooks('fixtures', app_name=app_name) or [] + fixtures = frappe.get_hooks("fixtures", app_name=app_name) or [] for fixture in fixtures: - if isinstance(fixture, str) and fixture == 'Worflow': - workflows = frappe.get_all('Workflow') + if isinstance(fixture, str) and fixture == "Worflow": + workflows = frappe.get_all("Workflow") break - elif isinstance(fixture, dict) and fixture.get('dt', fixture.get('doctype')) == 'Workflow': - workflows.extend(frappe.get_all('Workflow', filters=fixture.get('filters'))) + elif isinstance(fixture, dict) and fixture.get("dt", fixture.get("doctype")) == "Workflow": + workflows.extend(frappe.get_all("Workflow", filters=fixture.get("filters"))) - messages = [] + messages = [] document_state = DocType("Workflow Document State") for w in workflows: states = frappe.db.get_values( @@ -496,18 +500,28 @@ def get_messages_from_workflow(doctype=None, app_name=None): as_dict=True, order_by=None, ) - messages.extend([('Workflow: ' + w['name'], state['state']) for state in states if is_translatable(state['state'])]) + messages.extend( + [ + ("Workflow: " + w["name"], state["state"]) + for state in states + if is_translatable(state["state"]) + ] + ) states = frappe.db.get_values( document_state, - filters=(document_state.parent == w["name"]) - & (document_state.message.isnotnull()), + filters=(document_state.parent == w["name"]) & (document_state.message.isnotnull()), fieldname="message", distinct=True, order_by=None, as_dict=True, ) - messages.extend([("Workflow: " + w['name'], state['message']) - for state in states if is_translatable(state['message'])]) + messages.extend( + [ + ("Workflow: " + w["name"], state["message"]) + for state in states + if is_translatable(state["message"]) + ] + ) actions = frappe.db.get_values( "Workflow Transition", @@ -518,67 +532,91 @@ def get_messages_from_workflow(doctype=None, app_name=None): order_by=None, ) - messages.extend([("Workflow: " + w['name'], action['action']) \ - for action in actions if is_translatable(action['action'])]) + messages.extend( + [ + ("Workflow: " + w["name"], action["action"]) + for action in actions + if is_translatable(action["action"]) + ] + ) return messages def get_messages_from_custom_fields(app_name): - fixtures = frappe.get_hooks('fixtures', app_name=app_name) or [] + fixtures = frappe.get_hooks("fixtures", app_name=app_name) or [] custom_fields = [] for fixture in fixtures: - if isinstance(fixture, str) and fixture == 'Custom Field': - custom_fields = frappe.get_all('Custom Field', fields=['name','label', 'description', 'fieldtype', 'options']) + if isinstance(fixture, str) and fixture == "Custom Field": + custom_fields = frappe.get_all( + "Custom Field", fields=["name", "label", "description", "fieldtype", "options"] + ) break - elif isinstance(fixture, dict) and fixture.get('dt', fixture.get('doctype')) == 'Custom Field': - custom_fields.extend(frappe.get_all('Custom Field', filters=fixture.get('filters'), - fields=['name','label', 'description', 'fieldtype', 'options'])) + elif isinstance(fixture, dict) and fixture.get("dt", fixture.get("doctype")) == "Custom Field": + custom_fields.extend( + frappe.get_all( + "Custom Field", + filters=fixture.get("filters"), + fields=["name", "label", "description", "fieldtype", "options"], + ) + ) messages = [] for cf in custom_fields: - for prop in ('label', 'description'): + for prop in ("label", "description"): if not cf.get(prop) or not is_translatable(cf[prop]): continue - messages.append(('Custom Field - {}: {}'.format(prop, cf['name']), cf[prop])) - if cf['fieldtype'] == 'Selection' and cf.get('options'): - for option in cf['options'].split('\n'): - if option and 'icon' not in option and is_translatable(option): - messages.append(('Custom Field - Description: ' + cf['name'], option)) + messages.append(("Custom Field - {}: {}".format(prop, cf["name"]), cf[prop])) + if cf["fieldtype"] == "Selection" and cf.get("options"): + for option in cf["options"].split("\n"): + if option and "icon" not in option and is_translatable(option): + messages.append(("Custom Field - Description: " + cf["name"], option)) return messages + def get_messages_from_page(name): """Returns all translatable strings from a :class:`frappe.core.doctype.Page`""" return _get_messages_from_page_or_report("Page", name) + def get_messages_from_report(name): """Returns all translatable strings from a :class:`frappe.core.doctype.Report`""" report = frappe.get_doc("Report", name) - messages = _get_messages_from_page_or_report("Report", name, - frappe.db.get_value("DocType", report.ref_doctype, "module")) + messages = _get_messages_from_page_or_report( + "Report", name, frappe.db.get_value("DocType", report.ref_doctype, "module") + ) if report.columns: - context = "Column of report '%s'" % report.name # context has to match context in `prepare_columns` in query_report.js + context = ( + "Column of report '%s'" % report.name + ) # context has to match context in `prepare_columns` in query_report.js messages.extend([(None, report_column.label, context) for report_column in report.columns]) if report.filters: messages.extend([(None, report_filter.label) for report_filter in report.filters]) if report.query: - messages.extend([(None, message) for message in re.findall('"([^:,^"]*):', report.query) if is_translatable(message)]) + messages.extend( + [ + (None, message) + for message in re.findall('"([^:,^"]*):', report.query) + if is_translatable(message) + ] + ) - messages.append((None,report.report_name)) + messages.append((None, report.report_name)) return messages + def _get_messages_from_page_or_report(doctype, name, module=None): if not module: module = frappe.db.get_value(doctype, name, "module") doc_path = frappe.get_module_path(module, doctype, name) - messages = get_messages_from_file(os.path.join(doc_path, frappe.scrub(name) +".py")) + messages = get_messages_from_file(os.path.join(doc_path, frappe.scrub(name) + ".py")) if os.path.exists(doc_path): for filename in os.listdir(doc_path): @@ -587,14 +625,16 @@ def _get_messages_from_page_or_report(doctype, name, module=None): return messages + def get_server_messages(app): """Extracts all translatable strings (tagged with :func:`frappe._`) from Python modules - inside an app""" + inside an app""" messages = [] - file_extensions = ('.py', '.html', '.js', '.vue') + file_extensions = (".py", ".html", ".js", ".vue") for basepath, folders, files in os.walk(frappe.get_pymodule_path(app)): for dontwalk in (".git", "public", "locale"): - if dontwalk in folders: folders.remove(dontwalk) + if dontwalk in folders: + folders.remove(dontwalk) for f in files: f = frappe.as_unicode(f) @@ -603,9 +643,11 @@ def get_server_messages(app): return messages + def get_messages_from_include_files(app_name=None): """Returns messages from js files included at time of boot like desk.min.js for desk and web""" from frappe.utils.jinja_globals import bundled_asset + messages = [] app_include_js = frappe.get_hooks("app_include_js", app_name=app_name) or [] web_include_js = frappe.get_hooks("web_include_js", app_name=app_name) or [] @@ -613,33 +655,35 @@ def get_messages_from_include_files(app_name=None): for js_path in include_js: file_path = bundled_asset(js_path) - relative_path = os.path.join(frappe.local.sites_path, file_path.lstrip('/')) + relative_path = os.path.join(frappe.local.sites_path, file_path.lstrip("/")) messages_from_file = get_messages_from_file(relative_path) messages.extend(messages_from_file) return messages + def get_all_messages_from_js_files(app_name=None): """Extracts all translatable strings from app `.js` files""" messages = [] - for app in ([app_name] if app_name else frappe.get_installed_apps()): + for app in [app_name] if app_name else frappe.get_installed_apps(): if os.path.exists(frappe.get_app_path(app, "public")): for basepath, folders, files in os.walk(frappe.get_app_path(app, "public")): if "frappe/public/js/lib" in basepath: continue for fname in files: - if fname.endswith(".js") or fname.endswith(".html") or fname.endswith('.vue'): + if fname.endswith(".js") or fname.endswith(".html") or fname.endswith(".vue"): messages.extend(get_messages_from_file(os.path.join(basepath, fname))) return messages + def get_messages_from_file(path: str) -> List[Tuple[str, str, str, str]]: """Returns a list of transatable strings from a code file :param path: path of the code file """ - frappe.flags.setdefault('scanned_files', []) + frappe.flags.setdefault("scanned_files", []) # TODO: Find better alternative # To avoid duplicate scan if path in set(frappe.flags.scanned_files): @@ -649,7 +693,7 @@ def get_messages_from_file(path: str) -> List[Tuple[str, str, str, str]]: bench_path = get_bench_path() if os.path.exists(path): - with open(path, 'r') as sourcefile: + with open(path, "r") as sourcefile: try: file_contents = sourcefile.read() except Exception: @@ -663,11 +707,12 @@ def get_messages_from_file(path: str) -> List[Tuple[str, str, str, str]]: else: return [] + def extract_messages_from_code(code): """ - Extracts translatable strings from a code file - :param code: code from which translatable files are to be extracted - :param is_py: include messages in triple quotes e.g. `_('''message''')` + Extracts translatable strings from a code file + :param code: code from which translatable files are to be extracted + :param is_py: include messages in triple quotes e.g. `_('''message''')` """ from jinja2 import TemplateError @@ -682,8 +727,8 @@ def extract_messages_from_code(code): messages = [] for m in TRANSLATE_PATTERN.finditer(code): - message = m.group('message') - context = m.group('py_context') or m.group('js_context') + message = m.group("message") + context = m.group("py_context") or m.group("js_context") pos = m.start() if is_translatable(message): @@ -691,35 +736,44 @@ def extract_messages_from_code(code): return add_line_number(messages, code) + def is_translatable(m): - if re.search("[a-zA-Z]", m) and not m.startswith("fa fa-") and not m.endswith("px") and not m.startswith("eval:"): + if ( + re.search("[a-zA-Z]", m) + and not m.startswith("fa fa-") + and not m.endswith("px") + and not m.startswith("eval:") + ): return True return False + def add_line_number(messages, code): ret = [] messages = sorted(messages, key=lambda x: x[0]) - newlines = [m.start() for m in re.compile(r'\n').finditer(code)] + newlines = [m.start() for m in re.compile(r"\n").finditer(code)] line = 1 newline_i = 0 for pos, message, context in messages: while newline_i < len(newlines) and pos > newlines[newline_i]: - line+=1 - newline_i+= 1 + line += 1 + newline_i += 1 ret.append([line, message, context]) return ret + def read_csv_file(path): """Read CSV file and return as list of list :param path: File path""" - with io.open(path, mode='r', encoding='utf-8', newline='') as msgfile: + with io.open(path, mode="r", encoding="utf-8", newline="") as msgfile: data = reader(msgfile) newdata = [[val for val in row] for row in data] return newdata + def write_csv_file(path, app_messages, lang_dict): """Write translation CSV file. @@ -727,10 +781,11 @@ def write_csv_file(path, app_messages, lang_dict): :param app_messages: Translatable strings for this app. :param lang_dict: Full translated dict. """ - app_messages.sort(key = lambda x: x[1]) + app_messages.sort(key=lambda x: x[1]) from csv import writer - with open(path, 'w', newline='') as msgfile: - w = writer(msgfile, lineterminator='\n') + + with open(path, "w", newline="") as msgfile: + w = writer(msgfile, lineterminator="\n") for app_message in app_messages: context = None @@ -743,12 +798,13 @@ def write_csv_file(path, app_messages, lang_dict): else: continue - t = lang_dict.get(message, '') + t = lang_dict.get(message, "") # strip whitespaces - translated_string = re.sub(r'{\s?([0-9]+)\s?}', r"{\g<1>}", t) + translated_string = re.sub(r"{\s?([0-9]+)\s?}", r"{\g<1>}", t) if translated_string: w.writerow([message, translated_string, context]) + def get_untranslated(lang, untranslated_file, get_all=False): """Returns all untranslated strings for a language and writes in a file @@ -766,9 +822,7 @@ def get_untranslated(lang, untranslated_file, get_all=False): messages = deduplicate_messages(messages) def escape_newlines(s): - return (s.replace("\\\n", "|||||") - .replace("\\n", "||||") - .replace("\n", "|||")) + return s.replace("\\\n", "|||||").replace("\\n", "||||").replace("\n", "|||") if get_all: print(str(len(messages)) + " messages") @@ -792,6 +846,7 @@ def get_untranslated(lang, untranslated_file, get_all=False): else: print("all translated!") + def update_translations(lang, untranslated_file, translated_file): """Update translations from a source and target file for a given language. @@ -802,16 +857,20 @@ def update_translations(lang, untranslated_file, translated_file): full_dict = get_full_dict(lang) def restore_newlines(s): - return (s.replace("|||||", "\\\n") - .replace("| | | | |", "\\\n") - .replace("||||", "\\n") - .replace("| | | |", "\\n") - .replace("|||", "\n") - .replace("| | |", "\n")) + return ( + s.replace("|||||", "\\\n") + .replace("| | | | |", "\\\n") + .replace("||||", "\\n") + .replace("| | | |", "\\n") + .replace("|||", "\n") + .replace("| | |", "\n") + ) translation_dict = {} - for key, value in zip(frappe.get_file_items(untranslated_file, ignore_empty_lines=False), - frappe.get_file_items(translated_file, ignore_empty_lines=False)): + for key, value in zip( + frappe.get_file_items(untranslated_file, ignore_empty_lines=False), + frappe.get_file_items(translated_file, ignore_empty_lines=False), + ): # undo hack in get_untranslated translation_dict[restore_newlines(key)] = restore_newlines(value) @@ -821,11 +880,12 @@ def update_translations(lang, untranslated_file, translated_file): for app in frappe.get_all_apps(True): write_translations_file(app, lang, full_dict) + def import_translations(lang, path): """Import translations from file in standard format""" clear_cache() full_dict = get_full_dict(lang) - full_dict.update(get_translation_dict_from_file(path, lang, 'import')) + full_dict.update(get_translation_dict_from_file(path, lang, "import")) for app in frappe.get_all_apps(True): write_translations_file(app, lang, full_dict) @@ -837,6 +897,7 @@ def rebuild_all_translation_files(): for app in frappe.get_all_apps(): write_translations_file(app, lang) + def write_translations_file(app, lang, full_dict=None, app_messages=None): """Write a translation file for a given language. @@ -853,8 +914,8 @@ def write_translations_file(app, lang, full_dict=None, app_messages=None): tpath = frappe.get_pymodule_path(app, "translations") frappe.create_folder(tpath) - write_csv_file(os.path.join(tpath, lang + ".csv"), - app_messages, full_dict or get_full_dict(lang)) + write_csv_file(os.path.join(tpath, lang + ".csv"), app_messages, full_dict or get_full_dict(lang)) + def send_translations(translation_dict): """Append translated dict in `frappe.local.response`""" @@ -863,6 +924,7 @@ def send_translations(translation_dict): frappe.local.response["__messages"].update(translation_dict) + def deduplicate_messages(messages): ret = [] op = operator.itemgetter(1) @@ -871,16 +933,20 @@ def deduplicate_messages(messages): ret.append(next(g)) return ret + def rename_language(old_name, new_name): - if not frappe.db.exists('Language', new_name): + if not frappe.db.exists("Language", new_name): return language_in_system_settings = frappe.db.get_single_value("System Settings", "language") if language_in_system_settings == old_name: frappe.db.set_value("System Settings", "System Settings", "language", new_name) - frappe.db.sql("""update `tabUser` set language=%(new_name)s where language=%(old_name)s""", - { "old_name": old_name, "new_name": new_name }) + frappe.db.sql( + """update `tabUser` set language=%(new_name)s where language=%(old_name)s""", + {"old_name": old_name, "new_name": new_name}, + ) + @frappe.whitelist() def update_translations_for_source(source=None, translation_dict=None): @@ -893,22 +959,22 @@ def update_translations_for_source(source=None, translation_dict=None): source = strip_html_tags(source) # for existing records - translation_records = frappe.db.get_values('Translation', { - 'source_text': source - }, ['name', 'language'], as_dict=1) + translation_records = frappe.db.get_values( + "Translation", {"source_text": source}, ["name", "language"], as_dict=1 + ) for d in translation_records: if translation_dict.get(d.language, None): - doc = frappe.get_doc('Translation', d.name) + doc = frappe.get_doc("Translation", d.name) doc.translated_text = translation_dict.get(d.language) doc.save() # done with this lang value translation_dict.pop(d.language) else: - frappe.delete_doc('Translation', d.name) + frappe.delete_doc("Translation", d.name) # remaining values are to be inserted for lang, translated_text in translation_dict.items(): - doc = frappe.new_doc('Translation') + doc = frappe.new_doc("Translation") doc.language = lang doc.source_text = source doc.translated_text = translated_text @@ -916,72 +982,89 @@ def update_translations_for_source(source=None, translation_dict=None): return translation_records + @frappe.whitelist() def get_translations(source_text): if is_html(source_text): source_text = strip_html_tags(source_text) - return frappe.db.get_list('Translation', - fields = ['name', 'language', 'translated_text as translation'], - filters = { - 'source_text': source_text - } + return frappe.db.get_list( + "Translation", + fields=["name", "language", "translated_text as translation"], + filters={"source_text": source_text}, ) + @frappe.whitelist() -def get_messages(language, start=0, page_length=100, search_text=''): +def get_messages(language, start=0, page_length=100, search_text=""): from frappe.frappeclient import FrappeClient + translator = FrappeClient(get_translator_url()) - translated_dict = translator.post_api('translator.api.get_strings_for_translation', params=locals()) + translated_dict = translator.post_api( + "translator.api.get_strings_for_translation", params=locals() + ) return translated_dict @frappe.whitelist() -def get_source_additional_info(source, language=''): +def get_source_additional_info(source, language=""): from frappe.frappeclient import FrappeClient + translator = FrappeClient(get_translator_url()) - return translator.post_api('translator.api.get_source_additional_info', params=locals()) + return translator.post_api("translator.api.get_source_additional_info", params=locals()) + @frappe.whitelist() def get_contributions(language): - return frappe.get_all('Translation', fields=['*'], filters={ - 'contributed': 1, - }) + return frappe.get_all( + "Translation", + fields=["*"], + filters={ + "contributed": 1, + }, + ) + @frappe.whitelist() def get_contribution_status(message_id): from frappe.frappeclient import FrappeClient - doc = frappe.get_doc('Translation', message_id) + + doc = frappe.get_doc("Translation", message_id) translator = FrappeClient(get_translator_url()) - contributed_translation = translator.get_api('translator.api.get_contribution_status', params={ - 'translation_id': doc.contribution_docname - }) + contributed_translation = translator.get_api( + "translator.api.get_contribution_status", params={"translation_id": doc.contribution_docname} + ) return contributed_translation + def get_translator_url(): - return frappe.get_hooks()['translator_url'][0] + return frappe.get_hooks()["translator_url"][0] + @frappe.whitelist(allow_guest=True) def get_all_languages(with_language_name=False): """Returns all language codes ar, ch etc""" + def get_language_codes(): return frappe.get_all("Language", pluck="name") def get_all_language_with_name(): - return frappe.db.get_all('Language', ['language_code', 'language_name']) + return frappe.db.get_all("Language", ["language_code", "language_name"]) if not frappe.db: frappe.connect() if with_language_name: - return frappe.cache().get_value('languages_with_name', get_all_language_with_name) + return frappe.cache().get_value("languages_with_name", get_all_language_with_name) else: - return frappe.cache().get_value('languages', get_language_codes) + return frappe.cache().get_value("languages", get_language_codes) + @frappe.whitelist(allow_guest=True) def set_preferred_language_cookie(preferred_language): frappe.local.cookie_manager.set_cookie("preferred_language", preferred_language) + def get_preferred_language_cookie(): return frappe.request.cookies.get("preferred_language") diff --git a/frappe/twofactor.py b/frappe/twofactor.py index bb063faf5a..5a2799bc54 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -1,31 +1,42 @@ # Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import os +from base64 import b32encode, b64encode +from io import BytesIO + +import pyotp +from pyqrcode import create as qrcreate + import frappe from frappe import _ -import pyotp, os +from frappe.utils import cint, get_datetime, get_url, time_diff_in_seconds from frappe.utils.background_jobs import enqueue -from pyqrcode import create as qrcreate -from io import BytesIO -from base64 import b64encode, b32encode -from frappe.utils import get_url, get_datetime, time_diff_in_seconds, cint -class ExpiredLoginException(Exception): pass + +class ExpiredLoginException(Exception): + pass + def toggle_two_factor_auth(state, roles=None): - '''Enable or disable 2FA in site_config and roles''' + """Enable or disable 2FA in site_config and roles""" for role in roles or []: - role = frappe.get_doc('Role', {'role_name': role}) + role = frappe.get_doc("Role", {"role_name": role}) role.two_factor_auth = cint(state) role.save(ignore_permissions=True) + def two_factor_is_enabled(user=None): - '''Returns True if 2FA is enabled.''' - enabled = int(frappe.db.get_value('System Settings', None, 'enable_two_factor_auth') or 0) + """Returns True if 2FA is enabled.""" + enabled = int(frappe.db.get_value("System Settings", None, "enable_two_factor_auth") or 0) if enabled: - bypass_two_factor_auth = int(frappe.db.get_value('System Settings', None, 'bypass_2fa_for_retricted_ip_users') or 0) + bypass_two_factor_auth = int( + frappe.db.get_value("System Settings", None, "bypass_2fa_for_retricted_ip_users") or 0 + ) if bypass_two_factor_auth and user: user_doc = frappe.get_doc("User", user) - restrict_ip_list = user_doc.get_restricted_ip_list() #can be None or one or more than one ip address + restrict_ip_list = ( + user_doc.get_restricted_ip_list() + ) # can be None or one or more than one ip address if restrict_ip_list and frappe.local.request_ip: for ip in restrict_ip_list: if frappe.local.request_ip.startswith(ip): @@ -36,22 +47,25 @@ def two_factor_is_enabled(user=None): return enabled return two_factor_is_enabled_for_(user) + def should_run_2fa(user): - '''Check if 2fa should run.''' + """Check if 2fa should run.""" return two_factor_is_enabled(user=user) + def get_cached_user_pass(): - '''Get user and password if set.''' + """Get user and password if set.""" user = pwd = None - tmp_id = frappe.form_dict.get('tmp_id') + tmp_id = frappe.form_dict.get("tmp_id") if tmp_id: - user = frappe.safe_decode(frappe.cache().get(tmp_id+'_usr')) - pwd = frappe.safe_decode(frappe.cache().get(tmp_id+'_pwd')) + user = frappe.safe_decode(frappe.cache().get(tmp_id + "_usr")) + pwd = frappe.safe_decode(frappe.cache().get(tmp_id + "_pwd")) return (user, pwd) + def authenticate_for_2factor(user): - '''Authenticate two factor for enabled user before login.''' - if frappe.form_dict.get('otp'): + """Authenticate two factor for enabled user before login.""" + if frappe.form_dict.get("otp"): return otp_secret = get_otpsecret_for_(user) token = int(pyotp.TOTP(otp_secret).now()) @@ -59,27 +73,29 @@ def authenticate_for_2factor(user): cache_2fa_data(user, token, otp_secret, tmp_id) verification_obj = get_verification_obj(user, token, otp_secret) # Save data in local - frappe.local.response['verification'] = verification_obj - frappe.local.response['tmp_id'] = tmp_id + frappe.local.response["verification"] = verification_obj + frappe.local.response["tmp_id"] = tmp_id + def cache_2fa_data(user, token, otp_secret, tmp_id): - '''Cache and set expiry for data.''' - pwd = frappe.form_dict.get('pwd') + """Cache and set expiry for data.""" + pwd = frappe.form_dict.get("pwd") verification_method = get_verification_method() # set increased expiry time for SMS and Email - if verification_method in ['SMS', 'Email']: + if verification_method in ["SMS", "Email"]: expiry_time = frappe.flags.token_expiry or 300 - frappe.cache().set(tmp_id + '_token', token) - frappe.cache().expire(tmp_id + '_token', expiry_time) + frappe.cache().set(tmp_id + "_token", token) + frappe.cache().expire(tmp_id + "_token", expiry_time) else: expiry_time = frappe.flags.otp_expiry or 180 - for k, v in {'_usr': user, '_pwd': pwd, '_otp_secret': otp_secret}.items(): + for k, v in {"_usr": user, "_pwd": pwd, "_otp_secret": otp_secret}.items(): frappe.cache().set("{0}{1}".format(tmp_id, k), v) frappe.cache().expire("{0}{1}".format(tmp_id, k), expiry_time) + def two_factor_is_enabled_for_(user): - '''Check if 2factor is enabled for user.''' + """Check if 2factor is enabled for user.""" if user == "Administrator": return False @@ -89,8 +105,9 @@ def two_factor_is_enabled_for_(user): roles = [d.role for d in user.roles or []] + ["All"] role_doctype = frappe.qb.DocType("Role") - no_of_users = frappe.db.count(role_doctype, filters= - ((role_doctype.two_factor_auth == 1) & (role_doctype.name.isin(roles))), + no_of_users = frappe.db.count( + role_doctype, + filters=((role_doctype.two_factor_auth == 1) & (role_doctype.name.isin(roles))), ) if int(no_of_users) > 0: @@ -100,134 +117,146 @@ def two_factor_is_enabled_for_(user): def get_otpsecret_for_(user): - '''Set OTP Secret for user even if not set.''' - otp_secret = frappe.db.get_default(user + '_otpsecret') + """Set OTP Secret for user even if not set.""" + otp_secret = frappe.db.get_default(user + "_otpsecret") if not otp_secret: - otp_secret = b32encode(os.urandom(10)).decode('utf-8') - frappe.db.set_default(user + '_otpsecret', otp_secret) + otp_secret = b32encode(os.urandom(10)).decode("utf-8") + frappe.db.set_default(user + "_otpsecret", otp_secret) frappe.db.commit() return otp_secret + def get_verification_method(): - return frappe.db.get_value('System Settings', None, 'two_factor_method') + return frappe.db.get_value("System Settings", None, "two_factor_method") + def confirm_otp_token(login_manager, otp=None, tmp_id=None): - '''Confirm otp matches.''' + """Confirm otp matches.""" from frappe.auth import get_login_attempt_tracker + if not otp: - otp = frappe.form_dict.get('otp') + otp = frappe.form_dict.get("otp") if not otp: if two_factor_is_enabled_for_(login_manager.user): return False return True if not tmp_id: - tmp_id = frappe.form_dict.get('tmp_id') - hotp_token = frappe.cache().get(tmp_id + '_token') - otp_secret = frappe.cache().get(tmp_id + '_otp_secret') + tmp_id = frappe.form_dict.get("tmp_id") + hotp_token = frappe.cache().get(tmp_id + "_token") + otp_secret = frappe.cache().get(tmp_id + "_otp_secret") if not otp_secret: - raise ExpiredLoginException(_('Login session expired, refresh page to retry')) + raise ExpiredLoginException(_("Login session expired, refresh page to retry")) tracker = get_login_attempt_tracker(login_manager.user) hotp = pyotp.HOTP(otp_secret) if hotp_token: if hotp.verify(otp, int(hotp_token)): - frappe.cache().delete(tmp_id + '_token') + frappe.cache().delete(tmp_id + "_token") tracker.add_success_attempt() return True else: tracker.add_failure_attempt() - login_manager.fail(_('Incorrect Verification code'), login_manager.user) + login_manager.fail(_("Incorrect Verification code"), login_manager.user) totp = pyotp.TOTP(otp_secret) if totp.verify(otp): # show qr code only once - if not frappe.db.get_default(login_manager.user + '_otplogin'): - frappe.db.set_default(login_manager.user + '_otplogin', 1) + if not frappe.db.get_default(login_manager.user + "_otplogin"): + frappe.db.set_default(login_manager.user + "_otplogin", 1) delete_qrimage(login_manager.user) tracker.add_success_attempt() return True else: tracker.add_failure_attempt() - login_manager.fail(_('Incorrect Verification code'), login_manager.user) + login_manager.fail(_("Incorrect Verification code"), login_manager.user) def get_verification_obj(user, token, otp_secret): - otp_issuer = frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name') + otp_issuer = frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") verification_method = get_verification_method() verification_obj = None - if verification_method == 'SMS': + if verification_method == "SMS": verification_obj = process_2fa_for_sms(user, token, otp_secret) - elif verification_method == 'OTP App': - #check if this if the first time that the user is trying to login. If so, send an email - if not frappe.db.get_default(user + '_otplogin'): - verification_obj = process_2fa_for_email(user, token, otp_secret, otp_issuer, method='OTP App') + elif verification_method == "OTP App": + # check if this if the first time that the user is trying to login. If so, send an email + if not frappe.db.get_default(user + "_otplogin"): + verification_obj = process_2fa_for_email(user, token, otp_secret, otp_issuer, method="OTP App") else: verification_obj = process_2fa_for_otp_app(user, otp_secret, otp_issuer) - elif verification_method == 'Email': + elif verification_method == "Email": verification_obj = process_2fa_for_email(user, token, otp_secret, otp_issuer) return verification_obj + def process_2fa_for_sms(user, token, otp_secret): - '''Process sms method for 2fa.''' - phone = frappe.db.get_value('User', user, ['phone', 'mobile_no'], as_dict=1) + """Process sms method for 2fa.""" + phone = frappe.db.get_value("User", user, ["phone", "mobile_no"], as_dict=1) phone = phone.mobile_no or phone.phone status = send_token_via_sms(otp_secret, token=token, phone_no=phone) verification_obj = { - 'token_delivery': status, - 'prompt': status and 'Enter verification code sent to {}'.format(phone[:4] + '******' + phone[-3:]), - 'method': 'SMS', - 'setup': status + "token_delivery": status, + "prompt": status + and "Enter verification code sent to {}".format(phone[:4] + "******" + phone[-3:]), + "method": "SMS", + "setup": status, } return verification_obj + def process_2fa_for_otp_app(user, otp_secret, otp_issuer): - '''Process OTP App method for 2fa.''' + """Process OTP App method for 2fa.""" totp_uri = pyotp.TOTP(otp_secret).provisioning_uri(user, issuer_name=otp_issuer) - if frappe.db.get_default(user + '_otplogin'): + if frappe.db.get_default(user + "_otplogin"): otp_setup_completed = True else: otp_setup_completed = False - verification_obj = { - 'method': 'OTP App', - 'setup': otp_setup_completed - } + verification_obj = {"method": "OTP App", "setup": otp_setup_completed} return verification_obj -def process_2fa_for_email(user, token, otp_secret, otp_issuer, method='Email'): - '''Process Email method for 2fa.''' + +def process_2fa_for_email(user, token, otp_secret, otp_issuer, method="Email"): + """Process Email method for 2fa.""" subject = None message = None status = True - prompt = '' - if method == 'OTP App' and not frappe.db.get_default(user + '_otplogin'): - '''Sending one-time email for OTP App''' + prompt = "" + if method == "OTP App" and not frappe.db.get_default(user + "_otplogin"): + """Sending one-time email for OTP App""" totp_uri = pyotp.TOTP(otp_secret).provisioning_uri(user, issuer_name=otp_issuer) qrcode_link = get_link_for_qrcode(user, totp_uri) - message = get_email_body_for_qr_code({'qrcode_link': qrcode_link}) - subject = get_email_subject_for_qr_code({'qrcode_link': qrcode_link}) - prompt = _('Please check your registered email address for instructions on how to proceed. Do not close this window as you will have to return to it.') + message = get_email_body_for_qr_code({"qrcode_link": qrcode_link}) + subject = get_email_subject_for_qr_code({"qrcode_link": qrcode_link}) + prompt = _( + "Please check your registered email address for instructions on how to proceed. Do not close this window as you will have to return to it." + ) else: - '''Sending email verification''' - prompt = _('Verification code has been sent to your registered email address.') - status = send_token_via_email(user, token, otp_secret, otp_issuer, subject=subject, message=message) + """Sending email verification""" + prompt = _("Verification code has been sent to your registered email address.") + status = send_token_via_email( + user, token, otp_secret, otp_issuer, subject=subject, message=message + ) verification_obj = { - 'token_delivery': status, - 'prompt': status and prompt, - 'method': 'Email', - 'setup': status + "token_delivery": status, + "prompt": status and prompt, + "method": "Email", + "setup": status, } return verification_obj + def get_email_subject_for_2fa(kwargs_dict): - '''Get email subject for 2fa.''' - subject_template = _('Login Verification Code from {}').format(frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')) + """Get email subject for 2fa.""" + subject_template = _("Login Verification Code from {}").format( + frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + ) subject = frappe.render_template(subject_template, kwargs_dict) return subject + def get_email_body_for_2fa(kwargs_dict): - '''Get email body for 2fa.''' + """Get email body for 2fa.""" body_template = """ Enter this code to complete your login:

@@ -236,30 +265,38 @@ def get_email_body_for_2fa(kwargs_dict): body = frappe.render_template(body_template, kwargs_dict) return body + def get_email_subject_for_qr_code(kwargs_dict): - '''Get QRCode email subject.''' - subject_template = _('One Time Password (OTP) Registration Code from {}').format(frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name')) + """Get QRCode email subject.""" + subject_template = _("One Time Password (OTP) Registration Code from {}").format( + frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + ) subject = frappe.render_template(subject_template, kwargs_dict) return subject + def get_email_body_for_qr_code(kwargs_dict): - '''Get QRCode email body.''' - body_template = 'Please click on the following link and follow the instructions on the page.

{{qrcode_link}}' + """Get QRCode email body.""" + body_template = "Please click on the following link and follow the instructions on the page.

{{qrcode_link}}" body = frappe.render_template(body_template, kwargs_dict) return body + def get_link_for_qrcode(user, totp_uri): - '''Get link to temporary page showing QRCode.''' + """Get link to temporary page showing QRCode.""" key = frappe.generate_hash(length=20) key_user = "{}_user".format(key) key_uri = "{}_uri".format(key) - lifespan = int(frappe.db.get_value('System Settings', 'System Settings', 'lifespan_qrcode_image')) or 240 + lifespan = ( + int(frappe.db.get_value("System Settings", "System Settings", "lifespan_qrcode_image")) or 240 + ) frappe.cache().set_value(key_uri, totp_uri, expires_in_sec=lifespan) frappe.cache().set_value(key_user, user, expires_in_sec=lifespan) - return get_url('/qrcode?k={}'.format(key)) + return get_url("/qrcode?k={}".format(key)) + def send_token_via_sms(otpsecret, token=None, phone_no=None): - '''Send token as sms to user.''' + """Send token as sms to user.""" try: from frappe.core.doctype.sms_settings.sms_settings import send_request except: @@ -268,152 +305,187 @@ def send_token_via_sms(otpsecret, token=None, phone_no=None): if not phone_no: return False - ss = frappe.get_doc('SMS Settings', 'SMS Settings') + ss = frappe.get_doc("SMS Settings", "SMS Settings") if not ss.sms_gateway_url: return False hotp = pyotp.HOTP(otpsecret) - args = { - ss.message_parameter: 'Your verification code is {}'.format(hotp.at(int(token))) - } + args = {ss.message_parameter: "Your verification code is {}".format(hotp.at(int(token)))} for d in ss.get("parameters"): args[d.parameter] = d.value args[ss.receiver_parameter] = phone_no - sms_args = { - 'params': args, - 'gateway_url': ss.sms_gateway_url, - 'use_post': ss.use_post - } - enqueue(method=send_request, queue='short', timeout=300, event=None, - is_async=True, job_name=None, now=False, **sms_args) + sms_args = {"params": args, "gateway_url": ss.sms_gateway_url, "use_post": ss.use_post} + enqueue( + method=send_request, + queue="short", + timeout=300, + event=None, + is_async=True, + job_name=None, + now=False, + **sms_args + ) return True + def send_token_via_email(user, token, otp_secret, otp_issuer, subject=None, message=None): - '''Send token to user as email.''' - user_email = frappe.db.get_value('User', user, 'email') + """Send token to user as email.""" + user_email = frappe.db.get_value("User", user, "email") if not user_email: return False hotp = pyotp.HOTP(otp_secret) otp = hotp.at(int(token)) - template_args = {'otp': otp, 'otp_issuer': otp_issuer} + template_args = {"otp": otp, "otp_issuer": otp_issuer} if not subject: subject = get_email_subject_for_2fa(template_args) if not message: message = get_email_body_for_2fa(template_args) email_args = { - 'recipients': user_email, - 'sender': None, - 'subject': subject, - 'message': message, - 'header': [_('Verfication Code'), 'blue'], - 'delayed': False, - 'retry':3 + "recipients": user_email, + "sender": None, + "subject": subject, + "message": message, + "header": [_("Verfication Code"), "blue"], + "delayed": False, + "retry": 3, } - enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, - is_async=True, job_name=None, now=False, **email_args) + enqueue( + method=frappe.sendmail, + queue="short", + timeout=300, + event=None, + is_async=True, + job_name=None, + now=False, + **email_args + ) return True + def get_qr_svg_code(totp_uri): - '''Get SVG code to display Qrcode for OTP.''' + """Get SVG code to display Qrcode for OTP.""" url = qrcreate(totp_uri) - svg = '' + svg = "" stream = BytesIO() try: url.svg(stream, scale=4, background="#eee", module_color="#222") - svg = stream.getvalue().decode().replace('\n', '') + svg = stream.getvalue().decode().replace("\n", "") svg = b64encode(svg.encode()) finally: stream.close() return svg + def qrcode_as_png(user, totp_uri): - '''Save temporary Qrcode to server.''' + """Save temporary Qrcode to server.""" folder = create_barcode_folder() - png_file_name = '{}.png'.format(frappe.generate_hash(length=20)) - _file = frappe.get_doc({ - "doctype": "File", - "file_name": png_file_name, - "attached_to_doctype": 'User', - "attached_to_name": user, - "folder": folder, - "content": png_file_name}) + png_file_name = "{}.png".format(frappe.generate_hash(length=20)) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": png_file_name, + "attached_to_doctype": "User", + "attached_to_name": user, + "folder": folder, + "content": png_file_name, + } + ) _file.save() frappe.db.commit() file_url = get_url(_file.file_url) - file_path = os.path.join(frappe.get_site_path('public', 'files'), _file.file_name) + file_path = os.path.join(frappe.get_site_path("public", "files"), _file.file_name) url = qrcreate(totp_uri) - with open(file_path, 'w') as png_file: - url.png(png_file, scale=8, module_color=[0, 0, 0, 180], background=[0xff, 0xff, 0xcc]) + with open(file_path, "w") as png_file: + url.png(png_file, scale=8, module_color=[0, 0, 0, 180], background=[0xFF, 0xFF, 0xCC]) return file_url + def create_barcode_folder(): - '''Get Barcodes folder.''' - folder_name = 'Barcodes' - folder = frappe.db.exists('File', {'file_name': folder_name}) + """Get Barcodes folder.""" + folder_name = "Barcodes" + folder = frappe.db.exists("File", {"file_name": folder_name}) if folder: return folder - folder = frappe.get_doc({ - 'doctype': 'File', - 'file_name': folder_name, - 'is_folder':1, - 'folder': 'Home' - }) + folder = frappe.get_doc( + {"doctype": "File", "file_name": folder_name, "is_folder": 1, "folder": "Home"} + ) folder.insert(ignore_permissions=True) return folder.name + def delete_qrimage(user, check_expiry=False): - '''Delete Qrimage when user logs in.''' - user_barcodes = frappe.get_all('File', {'attached_to_doctype': 'User', - 'attached_to_name': user, 'folder': 'Home/Barcodes'}) + """Delete Qrimage when user logs in.""" + user_barcodes = frappe.get_all( + "File", {"attached_to_doctype": "User", "attached_to_name": user, "folder": "Home/Barcodes"} + ) for barcode in user_barcodes: if check_expiry and not should_remove_barcode_image(barcode): continue - barcode = frappe.get_doc('File', barcode.name) - frappe.delete_doc('File', barcode.name, ignore_permissions=True) + barcode = frappe.get_doc("File", barcode.name) + frappe.delete_doc("File", barcode.name, ignore_permissions=True) + def delete_all_barcodes_for_users(): - '''Task to delete all barcodes for user.''' + """Task to delete all barcodes for user.""" - users = frappe.get_all('User', {'enabled':1}) + users = frappe.get_all("User", {"enabled": 1}) for user in users: if not two_factor_is_enabled(user=user.name): continue delete_qrimage(user.name, check_expiry=True) + def should_remove_barcode_image(barcode): - '''Check if it's time to delete barcode image from server. ''' + """Check if it's time to delete barcode image from server.""" if isinstance(barcode, str): - barcode = frappe.get_doc('File', barcode) - lifespan = frappe.db.get_value('System Settings', 'System Settings', 'lifespan_qrcode_image') or 240 + barcode = frappe.get_doc("File", barcode) + lifespan = ( + frappe.db.get_value("System Settings", "System Settings", "lifespan_qrcode_image") or 240 + ) if time_diff_in_seconds(get_datetime(), barcode.creation) > int(lifespan): return True return False + def disable(): - frappe.db.set_value('System Settings', None, 'enable_two_factor_auth', 0) + frappe.db.set_value("System Settings", None, "enable_two_factor_auth", 0) + @frappe.whitelist() def reset_otp_secret(user): - otp_issuer = frappe.db.get_value('System Settings', 'System Settings', 'otp_issuer_name') - user_email = frappe.db.get_value('User', user, 'email') - if frappe.session.user in ["Administrator", user] : - frappe.defaults.clear_default(user + '_otplogin') - frappe.defaults.clear_default(user + '_otpsecret') + otp_issuer = frappe.db.get_value("System Settings", "System Settings", "otp_issuer_name") + user_email = frappe.db.get_value("User", user, "email") + if frappe.session.user in ["Administrator", user]: + frappe.defaults.clear_default(user + "_otplogin") + frappe.defaults.clear_default(user + "_otpsecret") email_args = { - 'recipients': user_email, - 'sender': None, - 'subject': _('OTP Secret Reset - {0}').format(otp_issuer or "Frappe Framework"), - 'message': _('

Your OTP secret on {0} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.

').format(otp_issuer or "Frappe Framework"), - 'delayed':False, - 'retry':3 + "recipients": user_email, + "sender": None, + "subject": _("OTP Secret Reset - {0}").format(otp_issuer or "Frappe Framework"), + "message": _( + "

Your OTP secret on {0} has been reset. If you did not perform this reset and did not request it, please contact your System Administrator immediately.

" + ).format(otp_issuer or "Frappe Framework"), + "delayed": False, + "retry": 3, } - enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, is_async=True, job_name=None, now=False, **email_args) - return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login.")) + enqueue( + method=frappe.sendmail, + queue="short", + timeout=300, + event=None, + is_async=True, + job_name=None, + now=False, + **email_args + ) + return frappe.msgprint( + _("OTP Secret has been reset. Re-registration will be required on next login.") + ) else: - return frappe.throw(_("OTP secret can only be reset by the Administrator.")) \ No newline at end of file + return frappe.throw(_("OTP secret can only be reset by the Administrator.")) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 62b8df40b6..3e62589664 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -10,16 +10,18 @@ import re import sys import traceback import typing +from collections.abc import MutableMapping, MutableSequence, Sequence from email.header import decode_header, make_header from email.utils import formataddr, parseaddr from gzip import GzipFile from typing import Generator, Iterable from urllib.parse import quote, urlparse -from werkzeug.test import Client + from redis.exceptions import ConnectionError -from collections.abc import MutableMapping, MutableSequence, Sequence +from werkzeug.test import Client import frappe + # utility functions like cint, int, flt, etc. from frappe.utils.data import * from frappe.utils.html_utils import sanitize_html @@ -36,13 +38,15 @@ def get_fullname(user=None): if not frappe.local.fullnames.get(user): p = frappe.db.get_value("User", user, ["first_name", "last_name"], as_dict=True) if p: - frappe.local.fullnames[user] = " ".join(filter(None, - [p.get('first_name'), p.get('last_name')])) or user + frappe.local.fullnames[user] = ( + " ".join(filter(None, [p.get("first_name"), p.get("last_name")])) or user + ) else: frappe.local.fullnames[user] = user return frappe.local.fullnames.get(user) + def get_email_address(user=None): """get the email address of the user from User""" if not user: @@ -50,10 +54,11 @@ def get_email_address(user=None): return frappe.db.get_value("User", user, "email") + def get_formatted_email(user, mail=None): """get Email Address of user formatted as: `John Doe `""" fullname = get_fullname(user) - method = get_hook_method('get_sender_details') + method = get_hook_method("get_sender_details") if method: sender_name, mail = method() @@ -64,10 +69,11 @@ def get_formatted_email(user, mail=None): mail = get_email_address(user) or validate_email_address(user) if not mail: - return '' + return "" else: return cstr(make_header(decode_header(formataddr((fullname, mail))))) + def extract_email_id(email): """fetch only the email part of the Email Address""" email_id = parse_addr(email)[1] @@ -75,6 +81,7 @@ def extract_email_id(email): email_id = email_id.decode("utf-8", "ignore") return email_id + def validate_phone_number(phone_number, throw=False): """Returns True if valid phone number""" if not phone_number: @@ -84,10 +91,13 @@ def validate_phone_number(phone_number, throw=False): match = re.match(r"([0-9\ \+\_\-\,\.\*\#\(\)]){1,20}$", phone_number) if not match and throw: - frappe.throw(frappe._("{0} is not a valid Phone Number").format(phone_number), frappe.InvalidPhoneNumberError) + frappe.throw( + frappe._("{0} is not a valid Phone Number").format(phone_number), frappe.InvalidPhoneNumberError + ) return bool(match) + def validate_name(name, throw=False): """Returns True if the name is valid valid names may have unicode and ascii characters, dash, quotes, numbers @@ -104,6 +114,7 @@ def validate_name(name, throw=False): return bool(match) + def validate_email_address(email_str, throw=False): """Validates the email string""" email = email_str = (email_str or "").strip() @@ -113,7 +124,7 @@ def validate_email_address(email_str, throw=False): if not e: _valid = False - if 'undisclosed-recipient' in e: + if "undisclosed-recipient" in e: return False elif " " in e and "<" not in e: @@ -122,40 +133,47 @@ def validate_email_address(email_str, throw=False): else: email_id = extract_email_id(e) - match = re.match( - r"[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", - email_id.lower() - ) if email_id else None + match = ( + re.match( + r"[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", + email_id.lower(), + ) + if email_id + else None + ) if not match: _valid = False else: matched = match.group(0) if match: - match = matched==email_id.lower() + match = matched == email_id.lower() if not _valid: if throw: invalid_email = frappe.utils.escape_html(e) - frappe.throw(frappe._("{0} is not a valid Email Address").format(invalid_email), - frappe.InvalidEmailAddressError) + frappe.throw( + frappe._("{0} is not a valid Email Address").format(invalid_email), + frappe.InvalidEmailAddressError, + ) return None else: return matched out = [] - for e in email_str.split(','): + for e in email_str.split(","): email = _check(e.strip()) if email: out.append(email) - return ', '.join(out) + return ", ".join(out) + def split_emails(txt): email_list = [] # emails can be separated by comma or newline - s = re.sub(r'[\t\n\r]', ' ', cstr(txt)) + s = re.sub(r"[\t\n\r]", " ", cstr(txt)) for email in re.split(r'[,\n](?=(?:[^"]|"[^"]*")*$)', s): email = strip(cstr(email)) if email: @@ -163,16 +181,17 @@ def split_emails(txt): return email_list + def validate_url(txt, throw=False, valid_schemes=None): """ - Checks whether `txt` has a valid URL string + Checks whether `txt` has a valid URL string - Parameters: - throw (`bool`): throws a validationError if URL is not valid - valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this + Parameters: + throw (`bool`): throws a validationError if URL is not valid + valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this - Returns: - bool: if `txt` represents a valid URL + Returns: + bool: if `txt` represents a valid URL """ url = urlparse(txt) is_valid = bool(url.netloc) @@ -184,44 +203,46 @@ def validate_url(txt, throw=False, valid_schemes=None): is_valid = is_valid and (url.scheme in valid_schemes) if not is_valid and throw: - frappe.throw( - frappe._("'{0}' is not a valid URL").format(frappe.bold(txt)) - ) + frappe.throw(frappe._("'{0}' is not a valid URL").format(frappe.bold(txt))) return is_valid + def random_string(length): """generate a random string""" import string from random import choice - return ''.join(choice(string.ascii_letters + string.digits) for i in range(length)) + + return "".join(choice(string.ascii_letters + string.digits) for i in range(length)) def has_gravatar(email): - '''Returns gravatar url if user has set an avatar at gravatar.com''' + """Returns gravatar url if user has set an avatar at gravatar.com""" import requests - if (frappe.flags.in_import - or frappe.flags.in_install - or frappe.flags.in_test): + if frappe.flags.in_import or frappe.flags.in_install or frappe.flags.in_test: # no gravatar if via upload # since querying gravatar for every item will be slow - return '' + return "" - hexdigest = hashlib.md5(frappe.as_unicode(email).encode('utf-8')).hexdigest() + hexdigest = hashlib.md5(frappe.as_unicode(email).encode("utf-8")).hexdigest() gravatar_url = "https://secure.gravatar.com/avatar/{hash}?d=404&s=200".format(hash=hexdigest) try: res = requests.get(gravatar_url) - if res.status_code==200: + if res.status_code == 200: return gravatar_url else: - return '' + return "" except requests.exceptions.ConnectionError: - return '' + return "" + def get_gravatar_url(email): - return "https://secure.gravatar.com/avatar/{hash}?d=mm&s=200".format(hash=hashlib.md5(email.encode('utf-8')).hexdigest()) + return "https://secure.gravatar.com/avatar/{hash}?d=mm&s=200".format( + hash=hashlib.md5(email.encode("utf-8")).hexdigest() + ) + def get_gravatar(email): from frappe.utils.identicon import Identicon @@ -233,9 +254,10 @@ def get_gravatar(email): return gravatar_url + def get_traceback() -> str: """ - Returns the traceback of the Exception + Returns the traceback of the Exception """ exc_type, exc_value, exc_tb = sys.exc_info() @@ -247,43 +269,50 @@ def get_traceback() -> str: return "".join(cstr(t) for t in trace_list).replace(bench_path, "") + def log(event, details): frappe.logger().info(details) -def dict_to_str(args, sep = '&'): + +def dict_to_str(args, sep="&"): """ Converts a dictionary to URL """ t = [] for k in list(args): - t.append(str(k)+'='+quote(str(args[k] or ''))) + t.append(str(k) + "=" + quote(str(args[k] or ""))) return sep.join(t) -def list_to_str(seq, sep = ', '): + +def list_to_str(seq, sep=", "): """Convert a sequence into a string using seperator. Same as str.join, but does type conversion and strip extra spaces. """ return sep.join(map(str.strip, map(str, seq))) + # Get Defaults # ============================================================================== + def get_defaults(key=None): """ Get dictionary of default values from the defaults, or a value if key is passed """ return frappe.db.get_defaults(key) + def set_default(key, val): """ Set / add a default value to defaults` """ return frappe.db.set_default(key, val) + def remove_blanks(d): """ - Returns d with empty ('' or None) values stripped + Returns d with empty ('' or None) values stripped """ empty_keys = [] for key in d: @@ -295,52 +324,58 @@ def remove_blanks(d): return d + def strip_html_tags(text): """Remove html tags from text""" return re.sub(r"\<[^>]*\>", "", text) + def get_file_timestamp(fn): """ - Returns timestamp of the given file + Returns timestamp of the given file """ from frappe.utils import cint try: return str(cint(os.stat(fn).st_mtime)) except OSError as e: - if e.args[0]!=2: + if e.args[0] != 2: raise else: return None + # to be deprecated def make_esc(esc_chars): """ - Function generator for Escaping special characters + Function generator for Escaping special characters """ - return lambda s: ''.join('\\' + c if c in esc_chars else c for c in s) + return lambda s: "".join("\\" + c if c in esc_chars else c for c in s) + # esc / unescape characters -- used for command line def esc(s, esc_chars): """ - Escape special characters + Escape special characters """ if not s: return "" for c in esc_chars: - esc_str = '\\' + c + esc_str = "\\" + c s = s.replace(c, esc_str) return s + def unesc(s, esc_chars): """ - UnEscape special characters + UnEscape special characters """ for c in esc_chars: - esc_str = '\\' + c + esc_str = "\\" + c s = s.replace(esc_str, c) return s + def execute_in_shell(cmd, verbose=0, low_priority=False): # using Popen instead of os.system - as recommended by python docs import tempfile @@ -348,11 +383,7 @@ def execute_in_shell(cmd, verbose=0, low_priority=False): with tempfile.TemporaryFile() as stdout: with tempfile.TemporaryFile() as stderr: - kwargs = { - "shell": True, - "stdout": stdout, - "stderr": stderr - } + kwargs = {"shell": True, "stdout": stdout, "stderr": stderr} if low_priority: kwargs["preexec_fn"] = lambda: os.nice(10) @@ -367,46 +398,56 @@ def execute_in_shell(cmd, verbose=0, low_priority=False): err = stderr.read() if verbose: - if err: print(err) - if out: print(out) + if err: + print(err) + if out: + print(out) return err, out + def get_path(*path, **kwargs): - base = kwargs.get('base') + base = kwargs.get("base") if not base: base = frappe.local.site_path return os.path.join(base, *path) + def get_site_base_path(): return frappe.local.site_path + def get_site_path(*path): return get_path(base=get_site_base_path(), *path) + def get_files_path(*path, **kwargs): return get_site_path("private" if kwargs.get("is_private") else "public", "files", *path) + def get_bench_path(): - return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) + return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), "..", "..", "..")) + def get_bench_id(): - return frappe.get_conf().get('bench_id', get_bench_path().strip('/').replace('/', '-')) + return frappe.get_conf().get("bench_id", get_bench_path().strip("/").replace("/", "-")) + def get_site_id(site=None): return f"{site or frappe.local.site}@{get_bench_id()}" + def get_backups_path(): return get_site_path("private", "backups") + def get_request_site_address(full_address=False): return get_url(full_address=full_address) + def get_site_url(site): - return 'http://{site}:{port}'.format( - site=site, - port=frappe.get_conf(site).webserver_port - ) + return "http://{site}:{port}".format(site=site, port=frappe.get_conf(site).webserver_port) + def encode_dict(d, encoding="utf-8"): for key in d: @@ -415,15 +456,18 @@ def encode_dict(d, encoding="utf-8"): return d + def decode_dict(d, encoding="utf-8"): for key in d: if isinstance(d[key], str) and not isinstance(d[key], str): d[key] = d[key].decode(encoding, "ignore") return d + @functools.lru_cache() def get_site_name(hostname): - return hostname.split(':')[0] + return hostname.split(":")[0] + def get_disk_usage(): """get disk usage of files folder""" @@ -433,16 +477,20 @@ def get_disk_usage(): err, out = execute_in_shell("du -hsm {files_path}".format(files_path=files_path)) return cint(out.split("\n")[-2].split("\t")[0]) + def touch_file(path): - with open(path, 'a'): + with open(path, "a"): os.utime(path, None) return path + def get_test_client() -> Client: """Returns an test instance of the Frappe WSGI""" from frappe.app import application + return Client(application) + def get_hook_method(hook_name, fallback=None): method = frappe.get_hooks().get(hook_name) if method: @@ -451,6 +499,7 @@ def get_hook_method(hook_name, fallback=None): if fallback: return fallback + def call_hook_method(hook, *args, **kwargs): out = None for method_name in frappe.get_hooks(hook): @@ -458,9 +507,9 @@ def call_hook_method(hook, *args, **kwargs): return out + def is_cli() -> bool: - """Returns True if current instance is being run via a terminal - """ + """Returns True if current instance is being run via a terminal""" invoked_from_terminal = False try: invoked_from_terminal = bool(os.get_terminal_size()) @@ -468,6 +517,7 @@ def is_cli() -> bool: invoked_from_terminal = sys.stdin.isatty() return invoked_from_terminal + def update_progress_bar(txt, i, l): if os.environ.get("CI"): if i == 0: @@ -477,7 +527,7 @@ def update_progress_bar(txt, i, l): sys.stdout.flush() return - if not getattr(frappe.local, 'request', None) or is_cli(): + if not getattr(frappe.local, "request", None) or is_cli(): lt = len(txt) try: col = 40 if os.get_terminal_size().columns > 80 else 20 @@ -486,14 +536,15 @@ def update_progress_bar(txt, i, l): col = 40 if lt < 36: - txt = txt + " "*(36-lt) + txt = txt + " " * (36 - lt) - complete = int(float(i+1) / l * col) - completion_bar = ("=" * complete).ljust(col, ' ') - percent_complete = str(int(float(i+1) / l * 100)) + complete = int(float(i + 1) / l * col) + completion_bar = ("=" * complete).ljust(col, " ") + percent_complete = str(int(float(i + 1) / l * 100)) sys.stdout.write("\r{0}: [{1}] {2}%".format(txt, completion_bar, percent_complete)) sys.stdout.flush() + def get_html_format(print_path): html_format = None if os.path.exists(print_path): @@ -510,6 +561,7 @@ def get_html_format(print_path): return html_format + def is_markdown(text): if "" in text: return True @@ -518,34 +570,41 @@ def is_markdown(text): else: return not re.search(r"|", text) + def get_sites(sites_path=None): if not sites_path: - sites_path = getattr(frappe.local, 'sites_path', None) or '.' + sites_path = getattr(frappe.local, "sites_path", None) or "." sites = [] for site in os.listdir(sites_path): path = os.path.join(sites_path, site) - if (os.path.isdir(path) + if ( + os.path.isdir(path) and not os.path.islink(path) - and os.path.exists(os.path.join(path, 'site_config.json'))): + and os.path.exists(os.path.join(path, "site_config.json")) + ): # is a dir and has site_config.json sites.append(site) return sorted(sites) + def get_request_session(max_retries=5): import requests from urllib3.util import Retry session = requests.Session() - http_adapter = requests.adapters.HTTPAdapter(max_retries=Retry(total=max_retries, status_forcelist=[500])) + http_adapter = requests.adapters.HTTPAdapter( + max_retries=Retry(total=max_retries, status_forcelist=[500]) + ) session.mount("http://", http_adapter) session.mount("https://", http_adapter) return session + def markdown(text, sanitize=True, linkify=True): html = text if is_html(text) else frappe.utils.md_to_html(text) @@ -555,6 +614,7 @@ def markdown(text, sanitize=True, linkify=True): return html + def sanitize_email(emails): sanitized = [] for e in split_emails(emails): @@ -566,6 +626,7 @@ def sanitize_email(emails): return ", ".join(sanitized) + def parse_addr(email_string): """ Return email_id and user_name based on email string @@ -579,12 +640,13 @@ def parse_addr(email_string): email_regex = re.compile(r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") email_list = re.findall(email_regex, email_string) if len(email_list) > 0 and check_format(email_list[0]): - #take only first email address + # take only first email address email = email_list[0] name = get_name_from_email_string(email_string, email, name) return (name, email) return (None, email) + def check_format(email_id): """ Check if email_id is valid. valid email:text@example.com @@ -596,75 +658,87 @@ def check_format(email_id): pos = email_id.rindex("@") is_valid = pos > 0 and (email_id.rindex(".") > pos) and (len(email_id) - pos > 4) except Exception: - #print(e) + # print(e) pass return is_valid + def get_name_from_email_string(email_string, email_id, name): - name = email_string.replace(email_id, '') - name = re.sub(r'[^A-Za-z0-9\u00C0-\u024F\/\_\' ]+', '', name).strip() + name = email_string.replace(email_id, "") + name = re.sub(r"[^A-Za-z0-9\u00C0-\u024F\/\_\' ]+", "", name).strip() if not name: name = email_id return name + def get_installed_apps_info(): out = [] from frappe.utils.change_log import get_versions for app, version_details in get_versions().items(): - out.append({ - 'app_name': app, - 'version': version_details.get('branch_version') or version_details.get('version'), - 'branch': version_details.get('branch') - }) + out.append( + { + "app_name": app, + "version": version_details.get("branch_version") or version_details.get("version"), + "branch": version_details.get("branch"), + } + ) return out + def get_site_info(): from frappe.email.queue import get_emails_sent_this_month from frappe.utils.user import get_system_managers # only get system users - users = frappe.get_all('User', filters={'user_type': 'System User', 'name': ('not in', frappe.STANDARD_USERS)}, - fields=['name', 'enabled', 'last_login', 'last_active', 'language', 'time_zone']) + users = frappe.get_all( + "User", + filters={"user_type": "System User", "name": ("not in", frappe.STANDARD_USERS)}, + fields=["name", "enabled", "last_login", "last_active", "language", "time_zone"], + ) system_managers = get_system_managers(only_name=True) for u in users: # tag system managers u.is_system_manager = 1 if u.name in system_managers else 0 u.full_name = get_fullname(u.name) u.email = u.name - del u['name'] + del u["name"] - system_settings = frappe.db.get_singles_dict('System Settings') - space_usage = frappe._dict((frappe.local.conf.limits or {}).get('space_usage', {})) + system_settings = frappe.db.get_singles_dict("System Settings") + space_usage = frappe._dict((frappe.local.conf.limits or {}).get("space_usage", {})) - kwargs = {"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"} + kwargs = { + "fields": ["user", "creation", "full_name"], + "filters": {"Operation": "Login", "Status": "Success"}, + "limit": "10", + } site_info = { - 'installed_apps': get_installed_apps_info(), - 'users': users, - 'country': system_settings.country, - 'language': system_settings.language or 'english', - 'time_zone': system_settings.time_zone, - 'setup_complete': cint(system_settings.setup_complete), - 'scheduler_enabled': system_settings.enable_scheduler, - + "installed_apps": get_installed_apps_info(), + "users": users, + "country": system_settings.country, + "language": system_settings.language or "english", + "time_zone": system_settings.time_zone, + "setup_complete": cint(system_settings.setup_complete), + "scheduler_enabled": system_settings.enable_scheduler, # usage - 'emails_sent': get_emails_sent_this_month(), - 'space_used': flt((space_usage.total or 0) / 1024.0, 2), - 'database_size': space_usage.database_size, - 'backup_size': space_usage.backup_size, - 'files_size': space_usage.files_size, - 'last_logins': frappe.get_all("Activity Log", **kwargs) + "emails_sent": get_emails_sent_this_month(), + "space_used": flt((space_usage.total or 0) / 1024.0, 2), + "database_size": space_usage.database_size, + "backup_size": space_usage.backup_size, + "files_size": space_usage.files_size, + "last_logins": frappe.get_all("Activity Log", **kwargs), } # from other apps - for method_name in frappe.get_hooks('get_site_info'): + for method_name in frappe.get_hooks("get_site_info"): site_info.update(frappe.get_attr(method_name)(site_info) or {}) # dumps -> loads to prevent datatype conflicts return json.loads(frappe.as_json(site_info)) + def parse_json(val): """ Parses json if string else return @@ -675,18 +749,19 @@ def parse_json(val): val = frappe._dict(val) return val + def get_db_count(*args): """ Pass a doctype or a series of doctypes to get the count of docs in them Parameters: - *args: Variable length argument list of doctype names whose doc count you need + *args: Variable length argument list of doctype names whose doc count you need Returns: - dict: A dict with the count values. + dict: A dict with the count values. Example: - via terminal: - bench --site erpnext.local execute frappe.utils.get_db_count --args "['DocType', 'Communication']" + via terminal: + bench --site erpnext.local execute frappe.utils.get_db_count --args "['DocType', 'Communication']" """ db_count = {} for doctype in args: @@ -694,21 +769,23 @@ def get_db_count(*args): return json.loads(frappe.as_json(db_count)) + def call(fn, *args, **kwargs): """ Pass a doctype or a series of doctypes to get the count of docs in them Parameters: - fn: frappe function to be called + fn: frappe function to be called Returns: - based on the function you call: output of the function you call + based on the function you call: output of the function you call Example: - via terminal: - bench --site erpnext.local execute frappe.utils.call --args '''["frappe.get_all", "Activity Log"]''' --kwargs '''{"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"}''' + via terminal: + bench --site erpnext.local execute frappe.utils.call --args '''["frappe.get_all", "Activity Log"]''' --kwargs '''{"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"}''' """ return json.loads(frappe.as_json(frappe.call(fn, *args, **kwargs))) + # Following methods are aken as-is from Python 3 codebase # since gzip.compress and gzip.decompress are not available in Python 2.7 def gzip_compress(data, compresslevel=9): @@ -716,10 +793,11 @@ def gzip_compress(data, compresslevel=9): Optional argument is the compression level, in range of 0-9. """ buf = io.BytesIO() - with GzipFile(fileobj=buf, mode='wb', compresslevel=compresslevel) as f: + with GzipFile(fileobj=buf, mode="wb", compresslevel=compresslevel) as f: f.write(data) return buf.getvalue() + def gzip_decompress(data): """Decompress a gzip compressed string in one shot. Return the decompressed string. @@ -727,6 +805,7 @@ def gzip_decompress(data): with GzipFile(fileobj=io.BytesIO(data)) as f: return f.read() + def get_safe_filters(filters): try: filters = json.loads(filters) @@ -740,56 +819,64 @@ def get_safe_filters(filters): return filters + def create_batch(iterable: Iterable, size: int) -> Generator[Iterable, None, None]: """Convert an iterable to multiple batches of constant size of batch_size Args: - iterable (Iterable): Iterable object which is subscriptable - size (int): Maximum size of batches to be generated + iterable (Iterable): Iterable object which is subscriptable + size (int): Maximum size of batches to be generated Yields: - Generator[List]: Batched iterable of maximum length `size` + Generator[List]: Batched iterable of maximum length `size` """ total_count = len(iterable) for i in range(0, total_count, size): yield iterable[i : min(i + size, total_count)] + def set_request(**kwargs): from werkzeug.test import EnvironBuilder from werkzeug.wrappers import Request + builder = EnvironBuilder(**kwargs) frappe.local.request = Request(builder.get_environ()) + def get_html_for_route(route): from frappe.website.serve import get_response - set_request(method='GET', path=route) + + set_request(method="GET", path=route) response = get_response() html = frappe.safe_decode(response.get_data()) return html + def get_file_size(path, format=False): num = os.path.getsize(path) if not format: return num - suffix = 'B' + suffix = "B" - for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: if abs(num) < 1024: return "{0:3.1f}{1}{2}".format(num, unit, suffix) num /= 1024 - return "{0:.1f}{1}{2}".format(num, 'Yi', suffix) + return "{0:.1f}{1}{2}".format(num, "Yi", suffix) + def get_build_version(): try: - return str(os.path.getmtime(os.path.join(frappe.local.sites_path, '.build'))) + return str(os.path.getmtime(os.path.join(frappe.local.sites_path, ".build"))) except OSError: # .build can sometimes not exist # this is not a major problem so send fallback return frappe.utils.random_string(8) + def get_assets_json(): def _get_assets(): # get merged assets.json and assets-rtl.json @@ -818,44 +905,44 @@ def get_bench_relative_path(file_path): """Fixes paths relative to the bench root directory if exists and returns the absolute path Args: - file_path (str, Path): Path of a file that exists on the file system + file_path (str, Path): Path of a file that exists on the file system Returns: - str: Absolute path of the file_path + str: Absolute path of the file_path """ if not os.path.exists(file_path): - base_path = '..' + base_path = ".." elif file_path.startswith(os.sep): base_path = os.sep else: - base_path = '.' + base_path = "." file_path = os.path.join(base_path, file_path) if not os.path.exists(file_path): - print('Invalid path {0}'.format(file_path[3:])) + print("Invalid path {0}".format(file_path[3:])) sys.exit(1) return os.path.abspath(file_path) def groupby_metric(iterable: typing.Dict[str, list], key: str): - """ Group records by a metric. + """Group records by a metric. Usecase: Lets assume we got country wise players list with the ranking given for each player(multiple players in a country can have same ranking aswell). We can group the players by ranking(can be any other metric) using this function. >>> d = { - 'india': [{'id':1, 'name': 'iplayer-1', 'ranking': 1}, {'id': 2, 'ranking': 1, 'name': 'iplayer-2'}, {'id': 2, 'ranking': 2, 'name': 'iplayer-3'}], - 'Aus': [{'id':1, 'name': 'aplayer-1', 'ranking': 1}, {'id': 2, 'ranking': 1, 'name': 'aplayer-2'}, {'id': 2, 'ranking': 2, 'name': 'aplayer-3'}] + 'india': [{'id':1, 'name': 'iplayer-1', 'ranking': 1}, {'id': 2, 'ranking': 1, 'name': 'iplayer-2'}, {'id': 2, 'ranking': 2, 'name': 'iplayer-3'}], + 'Aus': [{'id':1, 'name': 'aplayer-1', 'ranking': 1}, {'id': 2, 'ranking': 1, 'name': 'aplayer-2'}, {'id': 2, 'ranking': 2, 'name': 'aplayer-3'}] } >>> groupby(d, key='ranking') {1: {'Aus': [{'id': 1, 'name': 'aplayer-1', 'ranking': 1}, - {'id': 2, 'name': 'aplayer-2', 'ranking': 1}], - 'india': [{'id': 1, 'name': 'iplayer-1', 'ranking': 1}, - {'id': 2, 'name': 'iplayer-2', 'ranking': 1}]}, + {'id': 2, 'name': 'aplayer-2', 'ranking': 1}], + 'india': [{'id': 1, 'name': 'iplayer-1', 'ranking': 1}, + {'id': 2, 'name': 'iplayer-2', 'ranking': 1}]}, 2: {'Aus': [{'id': 2, 'name': 'aplayer-3', 'ranking': 2}], - 'india': [{'id': 2, 'name': 'iplayer-3', 'ranking': 2}]}} + 'india': [{'id': 2, 'name': 'iplayer-3', 'ranking': 2}]}} """ records = {} for category, items in iterable.items(): @@ -863,15 +950,18 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str): records.setdefault(item[key], {}).setdefault(category, []).append(item) return records + def get_table_name(table_name: str) -> str: return f"tab{table_name}" if not table_name.startswith("__") else table_name + def squashify(what): if isinstance(what, Sequence) and len(what) == 1: return what[0] return what + def safe_json_loads(*args): results = [] @@ -885,6 +975,7 @@ def safe_json_loads(*args): return squashify(results) + def dictify(arg): if isinstance(arg, MutableSequence): for i, a in enumerate(arg): @@ -894,18 +985,24 @@ def dictify(arg): return arg + def add_user_info(user, user_info): if user not in user_info: - info = frappe.db.get_value("User", - user, ["full_name", "user_image", "name", 'email', 'time_zone'], as_dict=True) or frappe._dict() + info = ( + frappe.db.get_value( + "User", user, ["full_name", "user_image", "name", "email", "time_zone"], as_dict=True + ) + or frappe._dict() + ) user_info[user] = frappe._dict( - fullname = info.full_name or user, - image = info.user_image, - name = user, - email = info.email, - time_zone = info.time_zone + fullname=info.full_name or user, + image=info.user_image, + name=user, + email=info.email, + time_zone=info.time_zone, ) + def is_git_url(url: str) -> bool: # modified to allow without the tailing .git from https://github.com/jonschlinkert/is-git-url.git pattern = r"(?:git|ssh|https?|\w*@[-\w.]+):(\/\/)?(.*?)(\.git)?(\/?|\#[-\d\w._]+?)$" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 600a1fd4a9..b2795e16c3 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -1,11 +1,10 @@ import os import socket import time -from functools import lru_cache -from uuid import uuid4 from collections import defaultdict +from functools import lru_cache from typing import List - +from uuid import uuid4 import redis from redis.exceptions import BusyLoadingError, ConnectionError @@ -14,12 +13,11 @@ from rq.logutils import setup_loghandlers from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed import frappe -from frappe import _ import frappe.monitor +from frappe import _ from frappe.utils import cstr, get_bench_id -from frappe.utils.redis_queue import RedisQueue from frappe.utils.commands import log - +from frappe.utils.redis_queue import RedisQueue @lru_cache() @@ -35,14 +33,16 @@ def get_queues_timeout(): **{ worker: config.get("timeout", default_timeout) for worker, config in custom_workers_config.items() - } + }, } + redis_connection = None + def enqueue( method, - queue='default', + queue="default", timeout=None, event=None, is_async=True, @@ -51,25 +51,29 @@ def enqueue( enqueue_after_commit=False, *, at_front=False, - **kwargs + **kwargs, ): - ''' - Enqueue method to be executed using a background worker - - :param method: method string or method object - :param queue: should be either long, default or short - :param timeout: should be set according to the functions - :param event: this is passed to enable clearing of jobs from queues - :param is_async: if is_async=False, the method is executed immediately, else via a worker - :param job_name: can be used to name an enqueue call, which can be used to prevent duplicate calls - :param now: if now=True, the method is executed via frappe.call - :param kwargs: keyword arguments to be passed to the method - ''' + """ + Enqueue method to be executed using a background worker + + :param method: method string or method object + :param queue: should be either long, default or short + :param timeout: should be set according to the functions + :param event: this is passed to enable clearing of jobs from queues + :param is_async: if is_async=False, the method is executed immediately, else via a worker + :param job_name: can be used to name an enqueue call, which can be used to prevent duplicate calls + :param now: if now=True, the method is executed via frappe.call + :param kwargs: keyword arguments to be passed to the method + """ # To handle older implementations - is_async = kwargs.pop('async', is_async) + is_async = kwargs.pop("async", is_async) if not is_async and not frappe.flags.in_test: - print(_("Using enqueue with is_async=False outside of tests is not recommended, use now=True instead.")) + print( + _( + "Using enqueue with is_async=False outside of tests is not recommended, use now=True instead." + ) + ) call_directly = now or frappe.flags.in_migrate or (not is_async and not frappe.flags.in_test) if call_directly: @@ -85,36 +89,45 @@ def enqueue( "event": event, "job_name": job_name or cstr(method), "is_async": is_async, - "kwargs": kwargs + "kwargs": kwargs, } if enqueue_after_commit: if not frappe.flags.enqueue_after_commit: frappe.flags.enqueue_after_commit = [] - frappe.flags.enqueue_after_commit.append({ - "queue": queue, - "is_async": is_async, - "timeout": timeout, - "queue_args":queue_args - }) + frappe.flags.enqueue_after_commit.append( + {"queue": queue, "is_async": is_async, "timeout": timeout, "queue_args": queue_args} + ) return frappe.flags.enqueue_after_commit return q.enqueue_call(execute_job, timeout=timeout, kwargs=queue_args, at_front=at_front) -def enqueue_doc(doctype, name=None, method=None, queue='default', timeout=300, - now=False, **kwargs): - '''Enqueue a method to be run on a document''' - return enqueue('frappe.utils.background_jobs.run_doc_method', doctype=doctype, name=name, - doc_method=method, queue=queue, timeout=timeout, now=now, **kwargs) + +def enqueue_doc( + doctype, name=None, method=None, queue="default", timeout=300, now=False, **kwargs +): + """Enqueue a method to be run on a document""" + return enqueue( + "frappe.utils.background_jobs.run_doc_method", + doctype=doctype, + name=name, + doc_method=method, + queue=queue, + timeout=timeout, + now=now, + **kwargs, + ) + def run_doc_method(doctype, name, doc_method, **kwargs): getattr(frappe.get_doc(doctype, name), doc_method)(**kwargs) + def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, retry=0): - '''Executes job in a worker, performs commit/rollback and logs if there is any error''' + """Executes job in a worker, performs commit/rollback and logs if there is any error""" if is_async: frappe.connect(site) - if os.environ.get('CI'): + if os.environ.get("CI"): frappe.flags.in_test = True if user: @@ -133,18 +146,18 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, except (frappe.db.InternalError, frappe.RetryBackgroundJobError) as e: frappe.db.rollback() - if (retry < 5 and - (isinstance(e, frappe.RetryBackgroundJobError) or - (frappe.db.is_deadlocked(e) or frappe.db.is_timedout(e)))): + if retry < 5 and ( + isinstance(e, frappe.RetryBackgroundJobError) + or (frappe.db.is_deadlocked(e) or frappe.db.is_timedout(e)) + ): # retry the job if # 1213 = deadlock # 1205 = lock wait timeout # or RetryBackgroundJobError is explicitly raised frappe.destroy() - time.sleep(retry+1) + time.sleep(retry + 1) - return execute_job(site, method, event, job_name, kwargs, - is_async=is_async, retry=retry+1) + return execute_job(site, method, event, job_name, kwargs, is_async=is_async, retry=retry + 1) else: frappe.log_error(title=method_name) @@ -165,64 +178,66 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, if is_async: frappe.destroy() -def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): - '''Wrapper to start rq worker. Connects to redis and monitors these queues.''' + +def start_worker(queue=None, quiet=False, rq_username=None, rq_password=None): + """Wrapper to start rq worker. Connects to redis and monitors these queues.""" with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json redis_connection = get_redis_conn(username=rq_username, password=rq_password) queues = get_queue_list(queue, build_queue_name=True) queue_name = queue and generate_qname(queue) - if os.environ.get('CI'): - setup_loghandlers('ERROR') + if os.environ.get("CI"): + setup_loghandlers("ERROR") with Connection(redis_connection): logging_level = "INFO" if quiet: logging_level = "WARNING" - Worker(queues, name=get_worker_name(queue_name)).work(logging_level = logging_level) + Worker(queues, name=get_worker_name(queue_name)).work(logging_level=logging_level) + def get_worker_name(queue): - '''When limiting worker to a specific queue, also append queue name to default worker name''' + """When limiting worker to a specific queue, also append queue name to default worker name""" name = None if queue: # hostname.pid is the default worker name - name = '{uuid}.{hostname}.{pid}.{queue}'.format( - uuid=uuid4().hex, - hostname=socket.gethostname(), - pid=os.getpid(), - queue=queue) + name = "{uuid}.{hostname}.{pid}.{queue}".format( + uuid=uuid4().hex, hostname=socket.gethostname(), pid=os.getpid(), queue=queue + ) return name -def get_jobs(site=None, queue=None, key='method'): - '''Gets jobs per queue or per site or both''' + +def get_jobs(site=None, queue=None, key="method"): + """Gets jobs per queue or per site or both""" jobs_per_site = defaultdict(list) def add_to_dict(job): if key in job.kwargs: - jobs_per_site[job.kwargs['site']].append(job.kwargs[key]) + jobs_per_site[job.kwargs["site"]].append(job.kwargs[key]) - elif key in job.kwargs.get('kwargs', {}): + elif key in job.kwargs.get("kwargs", {}): # optional keyword arguments are stored in 'kwargs' of 'kwargs' - jobs_per_site[job.kwargs['site']].append(job.kwargs['kwargs'][key]) + jobs_per_site[job.kwargs["site"]].append(job.kwargs["kwargs"][key]) for queue in get_queue_list(queue): q = get_queue(queue) jobs = q.jobs + get_running_jobs_in_queue(q) for job in jobs: - if job.kwargs.get('site'): + if job.kwargs.get("site"): # if job belongs to current site, or if all jobs are requested - if (job.kwargs['site'] == site) or site is None: + if (job.kwargs["site"] == site) or site is None: add_to_dict(job) else: - print('No site found in job', job.__dict__) + print("No site found in job", job.__dict__) return jobs_per_site + def get_queue_list(queue_list=None, build_queue_name=False): - '''Defines possible queues. Also wraps a given queue in a list after validating.''' + """Defines possible queues. Also wraps a given queue in a list after validating.""" default_queue_list = list(get_queues_timeout()) if queue_list: if isinstance(queue_list, str): @@ -234,15 +249,17 @@ def get_queue_list(queue_list=None, build_queue_name=False): queue_list = default_queue_list return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list + def get_workers(queue=None): - '''Returns a list of Worker objects tied to a queue object if queue is passed, else returns a list of all workers''' + """Returns a list of Worker objects tied to a queue object if queue is passed, else returns a list of all workers""" if queue: return Worker.all(queue=queue) else: return Worker.all(get_redis_conn()) + def get_running_jobs_in_queue(queue): - '''Returns a list of Jobs objects that are tied to a queue object and are currently running''' + """Returns a list of Jobs objects that are tied to a queue object and are currently running""" jobs = [] workers = get_workers(queue) for worker in workers: @@ -251,63 +268,69 @@ def get_running_jobs_in_queue(queue): jobs.append(current_job) return jobs + def get_queue(qtype, is_async=True): - '''Returns a Queue object tied to a redis connection''' + """Returns a Queue object tied to a redis connection""" validate_queue(qtype) return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async) + def validate_queue(queue, default_queue_list=None): if not default_queue_list: default_queue_list = list(get_queues_timeout()) if queue not in default_queue_list: - frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) + frappe.throw(_("Queue should be one of {0}").format(", ".join(default_queue_list))) + @retry( retry=retry_if_exception_type(BusyLoadingError) | retry_if_exception_type(ConnectionError), stop=stop_after_attempt(10), - wait=wait_fixed(1) + wait=wait_fixed(1), ) def get_redis_conn(username=None, password=None): - if not hasattr(frappe.local, 'conf'): - raise Exception('You need to call frappe.init') + if not hasattr(frappe.local, "conf"): + raise Exception("You need to call frappe.init") elif not frappe.local.conf.redis_queue: - raise Exception('redis_queue missing in common_site_config.json') + raise Exception("redis_queue missing in common_site_config.json") global redis_connection cred = frappe._dict() - if frappe.conf.get('use_rq_auth'): + if frappe.conf.get("use_rq_auth"): if username: - cred['username'] = username - cred['password'] = password + cred["username"] = username + cred["password"] = password else: - cred['username'] = frappe.get_site_config().rq_username or get_bench_id() - cred['password'] = frappe.get_site_config().rq_password + cred["username"] = frappe.get_site_config().rq_username or get_bench_id() + cred["password"] = frappe.get_site_config().rq_password - elif os.environ.get('RQ_ADMIN_PASWORD'): - cred['username'] = 'default' - cred['password'] = os.environ.get('RQ_ADMIN_PASWORD') + elif os.environ.get("RQ_ADMIN_PASWORD"): + cred["username"] = "default" + cred["password"] = os.environ.get("RQ_ADMIN_PASWORD") try: redis_connection = RedisQueue.get_connection(**cred) except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): - log(f'Wrong credentials used for {cred.username or "default user"}. ' - 'You can reset credentials using `bench create-rq-users` CLI and restart the server', - colour='red') + log( + f'Wrong credentials used for {cred.username or "default user"}. ' + "You can reset credentials using `bench create-rq-users` CLI and restart the server", + colour="red", + ) raise except Exception: - log(f'Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}', colour='red') + log(f"Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}", colour="red") raise return redis_connection + def get_queues() -> List[Queue]: - """Get all the queues linked to the current bench. - """ + """Get all the queues linked to the current bench.""" queues = Queue.all(connection=get_redis_conn()) return [q for q in queues if is_queue_accessible(q)] + def generate_qname(qtype: str) -> str: """Generate qname by combining bench ID and queue type. @@ -315,16 +338,19 @@ def generate_qname(qtype: str) -> str: """ return f"{get_bench_id()}:{qtype}" + def is_queue_accessible(qobj: Queue) -> bool: - """Checks whether queue is relate to current bench or not. - """ + """Checks whether queue is relate to current bench or not.""" accessible_queues = [generate_qname(q) for q in list(get_queues_timeout())] return qobj.name in accessible_queues + def enqueue_test_job(): - enqueue('frappe.utils.background_jobs.test_job', s=100) + enqueue("frappe.utils.background_jobs.test_job", s=100) + def test_job(s): import time - print('sleeping...') + + print("sleeping...") time.sleep(s) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 5197b20bd3..e2579444bd 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -9,14 +9,13 @@ from datetime import datetime from glob import glob from shutil import which - # imports - third party imports import click # imports - module imports import frappe from frappe import conf -from frappe.utils import get_file_size, get_url, now, now_datetime, cint +from frappe.utils import cint, get_file_size, get_url, now, now_datetime from frappe.utils.password import get_encryption_key # backup variable for backwards compatibility @@ -234,9 +233,7 @@ class BackupGenerator: paths = (self.backup_path_db, self.backup_path_files, self.backup_path_private_files) for path in paths: if os.path.exists(path): - cmd_string = ( - "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -c {filelocation}" - ) + cmd_string = "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -c {filelocation}" try: command = cmd_string.format( passphrase=get_encryption_key(), @@ -248,21 +245,23 @@ class BackupGenerator: except Exception as err: print(err) - click.secho("Error occurred during encryption. Files are stored without encryption.", fg="red") + click.secho( + "Error occurred during encryption. Files are stored without encryption.", fg="red" + ) def get_recent_backup(self, older_than, partial=False): backup_path = get_backup_path() if not frappe.get_system_settings("encrypt_backup"): file_type_slugs = { - "database": "*-{{}}-{}database.sql.gz".format('*' if partial else ''), + "database": "*-{{}}-{}database.sql.gz".format("*" if partial else ""), "public": "*-{}-files.tar", "private": "*-{}-private-files.tar", "config": "*-{}-site_config_backup.json", } else: file_type_slugs = { - "database": "*-{{}}-{}database.enc.sql.gz".format('*' if partial else ''), + "database": "*-{{}}-{}database.enc.sql.gz".format("*" if partial else ""), "public": "*-{}-files.enc.tar", "private": "*-{}-private-files.enc.tar", "config": "*-{}-site_config_backup.json", @@ -303,8 +302,7 @@ class BackupGenerator: def zip_files(self): # For backwards compatibility - pre v13 click.secho( - "BackupGenerator.zip_files has been deprecated in favour of" - " BackupGenerator.backup_files", + "BackupGenerator.zip_files has been deprecated in favour of" " BackupGenerator.backup_files", fg="yellow", ) return self.backup_files() @@ -321,9 +319,7 @@ class BackupGenerator: }, } - if os.path.exists(self.backup_path_files) and os.path.exists( - self.backup_path_private_files - ): + if os.path.exists(self.backup_path_files) and os.path.exists(self.backup_path_private_files): summary.update( { "public": { @@ -353,9 +349,7 @@ class BackupGenerator: def backup_files(self): for folder in ("public", "private"): files_path = frappe.get_site_path(folder, "files") - backup_path = ( - self.backup_path_files if folder == "public" else self.backup_path_private_files - ) + backup_path = self.backup_path_files if folder == "public" else self.backup_path_private_files if self.compress_files: cmd_string = "tar cf - {1} | gzip > {0}" @@ -363,9 +357,7 @@ class BackupGenerator: cmd_string = "tar -cf {0} {1}" frappe.utils.execute_in_shell( - cmd_string.format(backup_path, files_path), - verbose=self.verbose, - low_priority=True + cmd_string.format(backup_path, files_path), verbose=self.verbose, low_priority=True ) def copy_site_config(self): @@ -388,8 +380,7 @@ class BackupGenerator: if not (gzip_exc and db_exc[1]): _exc = "gzip" if not gzip_exc else db_exc[0] frappe.throw( - f"{_exc} not found in PATH! This is required to take a backup.", - exc=frappe.ExecutableNotFound + f"{_exc} not found in PATH! This is required to take a backup.", exc=frappe.ExecutableNotFound ) db_exc = db_exc[0] @@ -400,8 +391,7 @@ class BackupGenerator: # escape reserved characters args = frappe._dict( - [item[0], frappe.utils.esc(str(item[1]), "$ ")] - for item in self.__dict__.copy().items() + [item[0], frappe.utils.esc(str(item[1]), "$ ")] for item in self.__dict__.copy().items() ) if self.backup_includes: @@ -411,12 +401,14 @@ class BackupGenerator: if self.partial: if self.verbose: - print(''.join(backup_info), "\n") - database_header_content.extend([ - f"Partial Backup of Frappe Site {frappe.local.site}", - ("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1], - "", - ]) + print("".join(backup_info), "\n") + database_header_content.extend( + [ + f"Partial Backup of Frappe Site {frappe.local.site}", + ("Backup contains: " if self.backup_includes else "Backup excludes: ") + backup_info[1], + "", + ] + ) generated_header = "\n".join(f"-- {x}" for x in database_header_content) + "\n" @@ -480,12 +472,8 @@ class BackupGenerator: from frappe.email import get_system_managers recipient_list = get_system_managers() - db_backup_url = get_url( - os.path.join("backups", os.path.basename(self.backup_path_db)) - ) - files_backup_url = get_url( - os.path.join("backups", os.path.basename(self.backup_path_files)) - ) + db_backup_url = get_url(os.path.join("backups", os.path.basename(self.backup_path_db))) + files_backup_url = get_url(os.path.join("backups", os.path.basename(self.backup_path_files))) msg = """Hello, @@ -501,9 +489,7 @@ download only after 24 hours.""" % { } datetime_str = datetime.fromtimestamp(os.stat(self.backup_path_db).st_ctime) - subject = ( - datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded""" - ) + subject = datetime_str.strftime("%d/%m/%Y %H:%M:%S") + """ - Backup ready to be downloaded""" frappe.sendmail(recipients=recipient_list, message=msg, subject=subject) return recipient_list @@ -515,7 +501,7 @@ def fetch_latest_backups(partial=False): Only for: System Managers Returns: - dict: relative Backup Paths + dict: relative Backup Paths """ frappe.only_for("System Manager") odb = BackupGenerator( @@ -649,16 +635,18 @@ def get_backup_path(): backup_path = frappe.utils.get_site_path(conf.get("backup_path", "private/backups")) return backup_path + @frappe.whitelist() def get_backup_encryption_key(): frappe.only_for("System Manager") return frappe.conf.encryption_key + class Backup: def __init__(self, file_path): self.file_path = file_path - def backup_decryption(self,passphrase): + def backup_decryption(self, passphrase): """ Decrypts backup at the given path using the passphrase. """ @@ -669,9 +657,7 @@ class Backup: os.rename(self.file_path, self.file_path + ".gpg") file_path = self.file_path + ".gpg" - cmd_string = ( - "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -o {decrypted_file} -d {file_location}" - ) + cmd_string = "gpg --yes --passphrase {passphrase} --pinentry-mode loopback -o {decrypted_file} -d {file_location}" command = cmd_string.format( passphrase=passphrase, file_location=file_path, @@ -679,14 +665,13 @@ class Backup: ) frappe.utils.execute_in_shell(command) - def decryption_rollback(self): """ Checks if the decrypted file exists at the given path. if exists - Renames the orginal encrypted file. + Renames the orginal encrypted file. else - Removes the decrypted file and rename the original file. + Removes the decrypted file and rename the original file. """ if os.path.exists(self.file_path + ".gpg"): if os.path.exists(self.file_path): diff --git a/frappe/utils/bench_helper.py b/frappe/utils/bench_helper.py index b406c7e427..a0b011acc1 100644 --- a/frappe/utils/bench_helper.py +++ b/frappe/utils/bench_helper.py @@ -1,73 +1,74 @@ -import click -import frappe -import os -import json import importlib -import frappe.utils +import json +import os import traceback import warnings +import click + +import frappe +import frappe.utils + click.disable_unicode_literals_warning = True + def main(): commands = get_app_groups() - commands.update({ - 'get-frappe-commands': get_frappe_commands, - 'get-frappe-help': get_frappe_help - }) - click.Group(commands=commands)(prog_name='bench') + commands.update({"get-frappe-commands": get_frappe_commands, "get-frappe-help": get_frappe_help}) + click.Group(commands=commands)(prog_name="bench") + def get_app_groups(): - '''Get all app groups, put them in main group "frappe" since bench is - designed to only handle that''' + """Get all app groups, put them in main group "frappe" since bench is + designed to only handle that""" commands = dict() for app in get_apps(): app_commands = get_app_commands(app) if app_commands: commands.update(app_commands) - ret = dict(frappe=click.group(name='frappe', commands=commands)(app_group)) + ret = dict(frappe=click.group(name="frappe", commands=commands)(app_group)) return ret + def get_app_group(app): app_commands = get_app_commands(app) if app_commands: return click.group(name=app, commands=app_commands)(app_group) -@click.option('--site') -@click.option('--profile', is_flag=True, default=False, help='Profile') -@click.option('--verbose', is_flag=True, default=False, help='Verbose') -@click.option('--force', is_flag=True, default=False, help='Force') + +@click.option("--site") +@click.option("--profile", is_flag=True, default=False, help="Profile") +@click.option("--verbose", is_flag=True, default=False, help="Verbose") +@click.option("--force", is_flag=True, default=False, help="Force") @click.pass_context def app_group(ctx, site=False, force=False, verbose=False, profile=False): - ctx.obj = { - 'sites': get_sites(site), - 'force': force, - 'verbose': verbose, - 'profile': profile - } - if ctx.info_name == 'frappe': - ctx.info_name = '' + ctx.obj = {"sites": get_sites(site), "force": force, "verbose": verbose, "profile": profile} + if ctx.info_name == "frappe": + ctx.info_name = "" + def get_sites(site_arg): - if site_arg == 'all': + if site_arg == "all": return frappe.utils.get_sites() elif site_arg: return [site_arg] - elif os.environ.get('FRAPPE_SITE'): - return [os.environ.get('FRAPPE_SITE')] - elif os.path.exists('currentsite.txt'): - with open('currentsite.txt') as f: + elif os.environ.get("FRAPPE_SITE"): + return [os.environ.get("FRAPPE_SITE")] + elif os.path.exists("currentsite.txt"): + with open("currentsite.txt") as f: site = f.read().strip() if site: return [site] return [] + def get_app_commands(app): - if os.path.exists(os.path.join('..', 'apps', app, app, 'commands.py'))\ - or os.path.exists(os.path.join('..', 'apps', app, app, 'commands', '__init__.py')): + if os.path.exists(os.path.join("..", "apps", app, app, "commands.py")) or os.path.exists( + os.path.join("..", "apps", app, app, "commands", "__init__.py") + ): try: - app_command_module = importlib.import_module(app + '.commands') + app_command_module = importlib.import_module(app + ".commands") except Exception: traceback.print_exc() return [] @@ -75,13 +76,14 @@ def get_app_commands(app): return [] ret = {} - for command in getattr(app_command_module, 'commands', []): + for command in getattr(app_command_module, "commands", []): ret[command.name] = command return ret -@click.command('get-frappe-commands') + +@click.command("get-frappe-commands") def get_frappe_commands(): - commands = list(get_app_commands('frappe')) + commands = list(get_app_commands("frappe")) for app in get_apps(): app_commands = get_app_commands(app) @@ -90,15 +92,18 @@ def get_frappe_commands(): print(json.dumps(commands)) -@click.command('get-frappe-help') + +@click.command("get-frappe-help") def get_frappe_help(): - print(click.Context(get_app_groups()['frappe']).get_help()) + print(click.Context(get_app_groups()["frappe"]).get_help()) + def get_apps(): - return frappe.get_all_apps(with_internal_apps=False, sites_path='.') + return frappe.get_all_apps(with_internal_apps=False, sites_path=".") + if __name__ == "__main__": if not frappe._dev_server: - warnings.simplefilter('ignore') + warnings.simplefilter("ignore") main() diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index cb9ce27a09..f5243ecc95 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -1,12 +1,14 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import git import os import re +import git + import frappe -from frappe.utils import touch_file, cstr +from frappe.utils import cstr, touch_file + def make_boilerplate(dest, app_name, no_git=False): if not os.path.exists(dest): @@ -19,11 +21,15 @@ def make_boilerplate(dest, app_name, no_git=False): hooks = frappe._dict() hooks.app_name = app_name app_title = hooks.app_name.replace("_", " ").title() - for key in ("App Title (default: {0})".format(app_title), - "App Description", "App Publisher", "App Email", + for key in ( + "App Title (default: {0})".format(app_title), + "App Description", + "App Publisher", + "App Email", "App Icon (default 'octicon octicon-file-directory')", "App Color (default 'grey')", - "App License (default 'MIT')"): + "App License (default 'MIT')", + ): hook_key = key.split(" (")[0].lower().replace(" ", "_") hook_val = None while not hook_val: @@ -34,38 +40,43 @@ def make_boilerplate(dest, app_name, no_git=False): "app_title": app_title, "app_icon": "octicon octicon-file-directory", "app_color": "grey", - "app_license": "MIT" + "app_license": "MIT", } if hook_key in defaults: hook_val = defaults[hook_key] - if hook_key=="app_name" and hook_val.lower().replace(" ", "_") != hook_val: + if hook_key == "app_name" and hook_val.lower().replace(" ", "_") != hook_val: print("App Name must be all lowercase and without spaces") hook_val = "" - elif hook_key=="app_title" and not re.match(r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE): - print("App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores") + elif hook_key == "app_title" and not re.match( + r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE + ): + print( + "App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores" + ) hook_val = "" hooks[hook_key] = hook_val - frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, frappe.scrub(hooks.app_title)), - with_init=True) - frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "templates"), with_init=True) + frappe.create_folder( + os.path.join(dest, hooks.app_name, hooks.app_name, frappe.scrub(hooks.app_title)), with_init=True + ) + frappe.create_folder( + os.path.join(dest, hooks.app_name, hooks.app_name, "templates"), with_init=True + ) frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "www")) - frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "templates", - "pages"), with_init=True) - frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "templates", - "includes")) + frappe.create_folder( + os.path.join(dest, hooks.app_name, hooks.app_name, "templates", "pages"), with_init=True + ) + frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "templates", "includes")) frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "config"), with_init=True) - frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "public", - "css")) - frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "public", - "js")) + frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "public", "css")) + frappe.create_folder(os.path.join(dest, hooks.app_name, hooks.app_name, "public", "js")) # add .gitkeep file so that public folder is committed to git # this is needed because if public doesn't exist, bench build doesn't symlink the apps assets with open(os.path.join(dest, hooks.app_name, hooks.app_name, "public", ".gitkeep"), "w") as f: - f.write('') + f.write("") with open(os.path.join(dest, hooks.app_name, hooks.app_name, "__init__.py"), "w") as f: f.write(frappe.as_unicode(init_template)) @@ -77,8 +88,13 @@ def make_boilerplate(dest, app_name, no_git=False): f.write("# frappe -- https://github.com/frappe/frappe is installed via 'bench init'") with open(os.path.join(dest, hooks.app_name, "README.md"), "w") as f: - f.write(frappe.as_unicode("## {0}\n\n{1}\n\n#### License\n\n{2}".format(hooks.app_title, - hooks.app_description, hooks.app_license))) + f.write( + frappe.as_unicode( + "## {0}\n\n{1}\n\n#### License\n\n{2}".format( + hooks.app_title, hooks.app_description, hooks.app_license + ) + ) + ) with open(os.path.join(dest, hooks.app_name, "license.txt"), "w") as f: f.write(frappe.as_unicode("License: " + hooks.app_license)) @@ -89,7 +105,7 @@ def make_boilerplate(dest, app_name, no_git=False): # These values could contain quotes and can break string declarations # So escaping them before setting variables in setup.py and hooks.py for key in ("app_publisher", "app_description", "app_license"): - hooks[key] = hooks[key].replace("\\", "\\\\").replace("'", "\\'").replace("\"", "\\\"") + hooks[key] = hooks[key].replace("\\", "\\\\").replace("'", "\\'").replace('"', '\\"') with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f: f.write(frappe.as_unicode(setup_template.format(**hooks))) @@ -109,7 +125,7 @@ def make_boilerplate(dest, app_name, no_git=False): if not no_git: with open(os.path.join(dest, hooks.app_name, ".gitignore"), "w") as f: - f.write(frappe.as_unicode(gitignore_template.format(app_name = hooks.app_name))) + f.write(frappe.as_unicode(gitignore_template.format(app_name=hooks.app_name))) # initialize git repository app_repo = git.Repo.init(app_directory) diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index 5888166d5d..95a17f9dc1 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -14,10 +14,12 @@ from frappe.utils import cstr def get_change_log(user=None): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - last_known_versions = frappe._dict(json.loads(frappe.db.get_value("User", - user, "last_known_versions") or "{}")) + last_known_versions = frappe._dict( + json.loads(frappe.db.get_value("User", user, "last_known_versions") or "{}") + ) current_versions = get_versions() if not last_known_versions: @@ -25,6 +27,7 @@ def get_change_log(user=None): return [] change_log = [] + def set_in_change_log(app, opts, change_log): from_version = last_known_versions.get(app, {}).get("version") or "0.0.1" to_version = opts["version"] @@ -33,12 +36,14 @@ def get_change_log(user=None): app_change_log = get_change_log_for_app(app, from_version=from_version, to_version=to_version) if app_change_log: - change_log.append({ - "title": opts["title"], - "description": opts["description"], - "version": to_version, - "change_log": app_change_log - }) + change_log.append( + { + "title": opts["title"], + "description": opts["description"], + "version": to_version, + "change_log": app_change_log, + } + ) for app, opts in current_versions.items(): if app != "frappe": @@ -49,6 +54,7 @@ def get_change_log(user=None): return change_log + def get_change_log_for_app(app, from_version, to_version): change_log_folder = os.path.join(frappe.get_app_path(app), "change_log") if not os.path.exists(change_log_folder): @@ -59,7 +65,9 @@ def get_change_log_for_app(app, from_version, to_version): # remove pre-release part to_version.prerelease = None - major_version_folders = ["v{0}".format(i) for i in range(from_version.major, to_version.major + 1)] + major_version_folders = [ + "v{0}".format(i) for i in range(from_version.major, to_version.major + 1) + ] app_change_log = [] for folder in os.listdir(change_log_folder): @@ -77,10 +85,17 @@ def get_change_log_for_app(app, from_version, to_version): # convert version to string and send return [[cstr(d[0]), d[1]] for d in app_change_log] + @frappe.whitelist() def update_last_known_versions(): - frappe.db.set_value("User", frappe.session.user, "last_known_versions", - json.dumps(get_versions()), update_modified=False) + frappe.db.set_value( + "User", + frappe.session.user, + "last_known_versions", + json.dumps(get_versions()), + update_modified=False, + ) + @frappe.whitelist() def get_versions(): @@ -88,55 +103,68 @@ def get_versions(): Example: - { - "frappe": { - "title": "Frappe Framework", - "version": "5.0.0" - } - }""" + { + "frappe": { + "title": "Frappe Framework", + "version": "5.0.0" + } + }""" versions = {} for app in frappe.get_installed_apps(sort=True): app_hooks = frappe.get_hooks(app_name=app) versions[app] = { "title": app_hooks.get("app_title")[0], "description": app_hooks.get("app_description")[0], - "branch": get_app_branch(app) + "branch": get_app_branch(app), } - if versions[app]['branch'] != 'master': - branch_version = app_hooks.get('{0}_version'.format(versions[app]['branch'])) + if versions[app]["branch"] != "master": + branch_version = app_hooks.get("{0}_version".format(versions[app]["branch"])) if branch_version: - versions[app]['branch_version'] = branch_version[0] + ' ({0})'.format(get_app_last_commit_ref(app)) + versions[app]["branch_version"] = branch_version[0] + " ({0})".format( + get_app_last_commit_ref(app) + ) try: versions[app]["version"] = frappe.get_attr(app + ".__version__") except AttributeError: - versions[app]["version"] = '0.0.1' + versions[app]["version"] = "0.0.1" return versions + def get_app_branch(app): - '''Returns branch of an app''' + """Returns branch of an app""" try: - with open(os.devnull, 'wb') as null_stream: - result = subprocess.check_output(f'cd ../apps/{app} && git rev-parse --abbrev-ref HEAD', - shell=True, stdin=null_stream, stderr=null_stream) + with open(os.devnull, "wb") as null_stream: + result = subprocess.check_output( + f"cd ../apps/{app} && git rev-parse --abbrev-ref HEAD", + shell=True, + stdin=null_stream, + stderr=null_stream, + ) result = safe_decode(result) result = result.strip() return result except Exception: - return '' + return "" + def get_app_last_commit_ref(app): try: - with open(os.devnull, 'wb') as null_stream: - result = subprocess.check_output(f'cd ../apps/{app} && git rev-parse HEAD --short 7', - shell=True, stdin=null_stream, stderr=null_stream) + with open(os.devnull, "wb") as null_stream: + result = subprocess.check_output( + f"cd ../apps/{app} && git rev-parse HEAD --short 7", + shell=True, + stdin=null_stream, + stderr=null_stream, + ) result = safe_decode(result) result = result.strip() return result except Exception: - return '' + return "" + def check_for_update(): updates = frappe._dict(major=[], minor=[], patch=[]) @@ -144,25 +172,31 @@ def check_for_update(): for app in apps: app_details = check_release_on_github(app) - if not app_details: continue + if not app_details: + continue github_version, org_name = app_details # Get local instance's current version or the app - branch_version = apps[app]['branch_version'].split(' ')[0] if apps[app].get('branch_version', '') else '' - instance_version = Version(branch_version or apps[app].get('version')) + branch_version = ( + apps[app]["branch_version"].split(" ")[0] if apps[app].get("branch_version", "") else "" + ) + instance_version = Version(branch_version or apps[app].get("version")) # Compare and popup update message for update_type in updates: if github_version.__dict__[update_type] > instance_version.__dict__[update_type]: - updates[update_type].append(frappe._dict( - current_version=str(instance_version), - available_version=str(github_version), - org_name=org_name, - app_name=app, - title=apps[app]['title'], - )) + updates[update_type].append( + frappe._dict( + current_version=str(instance_version), + available_version=str(github_version), + org_name=org_name, + app_name=app, + title=apps[app]["title"], + ) + ) + break + if github_version.__dict__[update_type] < instance_version.__dict__[update_type]: break - if github_version.__dict__[update_type] < instance_version.__dict__[update_type]: break add_message_to_redis(updates) @@ -177,7 +211,9 @@ def parse_latest_non_beta_release(response): Returns json : json object pertaining to the latest non-beta release """ - version_list = [release.get('tag_name').strip('v') for release in response if not release.get('prerelease')] + version_list = [ + release.get("tag_name").strip("v") for release in response if not release.get("prerelease") + ] if version_list: return sorted(version_list, key=Version, reverse=True)[0] @@ -190,11 +226,11 @@ def check_release_on_github(app: str): Check the latest release for a given Frappe application hosted on Github. Args: - app (str): The name of the Frappe application. + app (str): The name of the Frappe application. Returns: - tuple(Version, str): The semantic version object of the latest release and the - organization name, if the application exists, otherwise None. + tuple(Version, str): The semantic version object of the latest release and the + organization name, if the application exists, otherwise None. """ from giturlparse import parse @@ -202,7 +238,9 @@ def check_release_on_github(app: str): try: # Check if repo remote is on github - remote_url = subprocess.check_output("cd ../apps/{} && git ls-remote --get-url".format(app), shell=True) + remote_url = subprocess.check_output( + "cd ../apps/{} && git ls-remote --get-url".format(app), shell=True + ) except subprocess.CalledProcessError: # Passing this since some apps may not have git initialized in them return @@ -236,9 +274,10 @@ def add_message_to_redis(update_json): cache = frappe.cache() cache.set_value("update-info", json.dumps(update_json)) user_list = [x.name for x in frappe.get_all("User", filters={"enabled": True})] - system_managers = [user for user in user_list if 'System Manager' in frappe.get_roles(user)] + system_managers = [user for user in user_list if "System Manager" in frappe.get_roles(user)] cache.sadd("update-user-set", *system_managers) + @frappe.whitelist() def show_update_popup(): cache = frappe.cache() @@ -261,12 +300,16 @@ def show_update_popup(): available_version=app.available_version, org_name=app.org_name, app_name=app.app_name, - title=app.title + title=app.title, ) if release_links: message = _("New {} releases for the following apps are available").format(_(update_type)) - update_message += "
{0}
{1}
".format(message, release_links) + update_message += ( + "
{0}
{1}
".format( + message, release_links + ) + ) if update_message: - frappe.msgprint(update_message, title=_("New updates are available"), indicator='green') + frappe.msgprint(update_message, title=_("New updates are available"), indicator="green") cache.srem("update-user-set", user) diff --git a/frappe/utils/commands.py b/frappe/utils/commands.py index 3cf0d553b6..a610872f03 100644 --- a/frappe/utils/commands.py +++ b/frappe/utils/commands.py @@ -1,5 +1,6 @@ import functools + @functools.lru_cache(maxsize=1024) def get_first_party_apps(): """Get list of all apps under orgs: frappe. erpnext from GitHub""" @@ -7,7 +8,9 @@ def get_first_party_apps(): apps = [] for org in ["frappe", "erpnext"]: - req = requests.get(f"https://api.github.com/users/{org}/repos", {"type": "sources", "per_page": 200}) + req = requests.get( + f"https://api.github.com/users/{org}/repos", {"type": "sources", "per_page": 200} + ) if req.ok: apps.extend([x["name"] for x in req.json()]) return apps @@ -21,34 +24,38 @@ def render_table(data): def add_line_after(function): """Adds an extra line to STDOUT after the execution of a function this decorates""" + def empty_line(*args, **kwargs): result = function(*args, **kwargs) print() return result + return empty_line def add_line_before(function): """Adds an extra line to STDOUT before the execution of a function this decorates""" + def empty_line(*args, **kwargs): print() result = function(*args, **kwargs) return result + return empty_line -def log(message, colour=''): +def log(message, colour=""): """Coloured log outputs to STDOUT""" colours = { - "nc": '\033[0m', - "blue": '\033[94m', - "green": '\033[92m', - "yellow": '\033[93m', - "red": '\033[91m', - "silver": '\033[90m' + "nc": "\033[0m", + "blue": "\033[94m", + "green": "\033[92m", + "yellow": "\033[93m", + "red": "\033[91m", + "silver": "\033[90m", } colour = colours.get(colour, "") - end_line = '\033[0m' + end_line = "\033[0m" print(colour + message + end_line) diff --git a/frappe/utils/connections.py b/frappe/utils/connections.py index 5640da666c..bf0a338681 100644 --- a/frappe/utils/connections.py +++ b/frappe/utils/connections.py @@ -1,9 +1,9 @@ import socket - from urllib.parse import urlparse + from frappe import get_conf -REDIS_KEYS = ('redis_cache', 'redis_queue', 'redis_socketio') +REDIS_KEYS = ("redis_cache", "redis_queue", "redis_socketio") def is_open(ip, port, timeout=10): diff --git a/frappe/utils/csvutils.py b/frappe/utils/csvutils.py index 2fb212ea3c..547372778b 100644 --- a/frappe/utils/csvutils.py +++ b/frappe/utils/csvutils.py @@ -1,18 +1,26 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -from frappe import msgprint, _ -import json import csv -import requests +import json from io import StringIO -from frappe.utils import cstr, cint, flt, comma_or + +import requests + +import frappe +from frappe import _, msgprint +from frappe.utils import cint, comma_or, cstr, flt + def read_csv_content_from_attached_file(doc): - fileid = frappe.get_all("File", fields = ["name"], filters = {"attached_to_doctype": doc.doctype, - "attached_to_name":doc.name}, order_by="creation desc") + fileid = frappe.get_all( + "File", + fields=["name"], + filters={"attached_to_doctype": doc.doctype, "attached_to_name": doc.name}, + order_by="creation desc", + ) - if fileid : fileid = fileid[0].name + if fileid: + fileid = fileid[0].name if not fileid: msgprint(_("File not attached")) @@ -21,9 +29,12 @@ def read_csv_content_from_attached_file(doc): try: _file = frappe.get_doc("File", fileid) fcontent = _file.get_content() - return read_csv_content(fcontent, frappe.form_dict.get('ignore_encoding_errors')) + return read_csv_content(fcontent, frappe.form_dict.get("ignore_encoding_errors")) except Exception: - frappe.throw(_("Unable to open attached file. Did you export it as CSV?"), title=_('Invalid CSV Format')) + frappe.throw( + _("Unable to open attached file. Did you export it as CSV?"), title=_("Invalid CSV Format") + ) + def read_csv_content(fcontent, ignore_encoding=False): rows = [] @@ -39,10 +50,12 @@ def read_csv_content(fcontent, ignore_encoding=False): continue if not decoded: - frappe.msgprint(_("Unknown file encoding. Tried utf-8, windows-1250, windows-1252."), raise_exception=True) + frappe.msgprint( + _("Unknown file encoding. Tried utf-8, windows-1250, windows-1252."), raise_exception=True + ) fcontent = fcontent.encode("utf-8") - content = [ ] + content = [] for line in fcontent.splitlines(True): content.append(frappe.safe_decode(line)) @@ -54,7 +67,7 @@ def read_csv_content(fcontent, ignore_encoding=False): # decode everything val = val.strip() - if val=="": + if val == "": # reason: in maraidb strict config, one cannot have blank strings for non string datatypes r.append(None) else: @@ -68,6 +81,7 @@ def read_csv_content(fcontent, ignore_encoding=False): frappe.msgprint(_("Not a valid Comma Separated Value (CSV File)")) raise + @frappe.whitelist() def send_csv_to_client(args): if isinstance(args, str): @@ -79,6 +93,7 @@ def send_csv_to_client(args): frappe.response["doctype"] = args.filename frappe.response["type"] = "csv" + def to_csv(data): writer = UnicodeWriter() for row in data: @@ -86,11 +101,13 @@ def to_csv(data): return writer.getvalue() + def build_csv_response(data, filename): frappe.response["result"] = cstr(to_csv(data)) frappe.response["doctype"] = filename frappe.response["type"] = "csv" + class UnicodeWriter: def __init__(self, encoding="utf-8", quoting=csv.QUOTE_NONNUMERIC): self.encoding = encoding @@ -103,34 +120,39 @@ class UnicodeWriter: def getvalue(self): return self.queue.getvalue() + def check_record(d): """check for mandatory, select options, dates. these should ideally be in doclist""" from frappe.utils.dateutils import parse_date + doc = frappe.get_doc(d) for key in d: docfield = doc.meta.get_field(key) val = d[key] if docfield: - if docfield.reqd and (val=='' or val is None): + if docfield.reqd and (val == "" or val is None): frappe.msgprint(_("{0} is required").format(docfield.label), raise_exception=1) - if docfield.fieldtype=='Select' and val and docfield.options: - if val not in docfield.options.split('\n'): - frappe.throw(_("{0} must be one of {1}").format(_(docfield.label), comma_or(docfield.options.split("\n")))) + if docfield.fieldtype == "Select" and val and docfield.options: + if val not in docfield.options.split("\n"): + frappe.throw( + _("{0} must be one of {1}").format(_(docfield.label), comma_or(docfield.options.split("\n"))) + ) - if val and docfield.fieldtype=='Date': + if val and docfield.fieldtype == "Date": d[key] = parse_date(val) elif val and docfield.fieldtype in ["Int", "Check"]: d[key] = cint(val) elif val and docfield.fieldtype in ["Currency", "Float", "Percent"]: d[key] = flt(val) + def import_doc(d, doctype, overwrite, row_idx, submit=False, ignore_links=False): """import main (non child) document""" - if d.get("name") and frappe.db.exists(doctype, d['name']): + if d.get("name") and frappe.db.exists(doctype, d["name"]): if overwrite: - doc = frappe.get_doc(doctype, d['name']) + doc = frappe.get_doc(doctype, d["name"]) doc.flags.ignore_links = ignore_links doc.update(d) if d.get("docstatus") == 1: @@ -139,10 +161,9 @@ def import_doc(d, doctype, overwrite, row_idx, submit=False, ignore_links=False) doc.submit() else: doc.save() - return 'Updated row (#%d) %s' % (row_idx + 1, getlink(doctype, d['name'])) + return "Updated row (#%d) %s" % (row_idx + 1, getlink(doctype, d["name"])) else: - return 'Ignored row (#%d) %s (exists)' % (row_idx + 1, - getlink(doctype, d['name'])) + return "Ignored row (#%d) %s (exists)" % (row_idx + 1, getlink(doctype, d["name"])) else: doc = frappe.get_doc(d) doc.flags.ignore_links = ignore_links @@ -151,45 +172,48 @@ def import_doc(d, doctype, overwrite, row_idx, submit=False, ignore_links=False) if submit: doc.submit() - return 'Inserted row (#%d) %s' % (row_idx + 1, getlink(doctype, - doc.get('name'))) + return "Inserted row (#%d) %s" % (row_idx + 1, getlink(doctype, doc.get("name"))) + def getlink(doctype, name): return '%(name)s' % locals() + def get_csv_content_from_google_sheets(url): # https://docs.google.com/spreadsheets/d/{sheetid}}/edit#gid={gid} validate_google_sheets_url(url) # get gid, defaults to first sheet if "gid=" in url: - gid = url.rsplit('gid=', 1)[1] + gid = url.rsplit("gid=", 1)[1] else: gid = 0 # remove /edit path - url = url.rsplit('/edit', 1)[0] + url = url.rsplit("/edit", 1)[0] # add /export path, - url = url + '/export?format=csv&gid={0}'.format(gid) + url = url + "/export?format=csv&gid={0}".format(gid) - headers = { - 'Accept': 'text/csv' - } + headers = {"Accept": "text/csv"} response = requests.get(url, headers=headers) if response.ok: # if it returns html, it couldn't find the CSV content # because of invalid url or no access - if response.text.strip().endswith(''): + if response.text.strip().endswith(""): frappe.throw( - _('Google Sheets URL is invalid or not publicly accessible.'), - title=_("Invalid URL") + _("Google Sheets URL is invalid or not publicly accessible."), title=_("Invalid URL") ) return response.content elif response.status_code == 400: - frappe.throw(_('Google Sheets URL must end with "gid={number}". Copy and paste the URL from the browser address bar and try again.'), - title=_("Incorrect URL")) + frappe.throw( + _( + 'Google Sheets URL must end with "gid={number}". Copy and paste the URL from the browser address bar and try again.' + ), + title=_("Incorrect URL"), + ) else: response.raise_for_status() + def validate_google_sheets_url(url): if "docs.google.com/spreadsheets" not in url: frappe.throw( diff --git a/frappe/utils/dashboard.py b/frappe/utils/dashboard.py index 3b5378ef90..9a53b93de2 100644 --- a/frappe/utils/dashboard.py +++ b/frappe/utils/dashboard.py @@ -1,24 +1,25 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import os +from functools import wraps +from os.path import join + import frappe from frappe import _ -from functools import wraps -from frappe.utils import add_to_date, cint, get_link_to_form from frappe.modules.import_file import import_file_by_path -import os -from os.path import join +from frappe.utils import add_to_date, cint, get_link_to_form def cache_source(function): @wraps(function) def wrapper(*args, **kwargs): if kwargs.get("chart_name"): - chart = frappe.get_doc('Dashboard Chart', kwargs.get("chart_name")) + chart = frappe.get_doc("Dashboard Chart", kwargs.get("chart_name")) else: chart = kwargs.get("chart") no_cache = kwargs.get("no_cache") if no_cache: - return function(chart = chart, no_cache = no_cache) + return function(chart=chart, no_cache=no_cache) chart_name = frappe.parse_json(chart).name cache_key = "chart-data:{}".format(chart_name) if int(kwargs.get("refresh") or 0): @@ -30,19 +31,21 @@ def cache_source(function): else: results = generate_and_cache_results(kwargs, function, cache_key, chart) return results + return wrapper + def generate_and_cache_results(args, function, cache_key, chart): try: args = frappe._dict(args) results = function( - chart_name = args.chart_name, - filters = args.filters or None, - from_date = args.from_date or None, - to_date = args.to_date or None, - time_interval = args.time_interval or None, - timespan = args.timespan or None, - heatmap_year = args.heatmap_year or None + chart_name=args.chart_name, + filters=args.filters or None, + from_date=args.from_date or None, + to_date=args.to_date or None, + time_interval=args.time_interval or None, + timespan=args.timespan or None, + heatmap_year=args.heatmap_year or None, ) except TypeError as e: if str(e) == "'NoneType' object is not iterable": @@ -51,38 +54,38 @@ def generate_and_cache_results(args, function, cache_key, chart): # Note: Do not try to find the right way of doing this because # it results in an inelegant & inefficient solution # ref: https://github.com/frappe/frappe/pull/9403 - frappe.throw(_('Please check the filter values set for Dashboard Chart: {}').format( - get_link_to_form(chart.doctype, chart.name)), title=_('Invalid Filter Value')) + frappe.throw( + _("Please check the filter values set for Dashboard Chart: {}").format( + get_link_to_form(chart.doctype, chart.name) + ), + title=_("Invalid Filter Value"), + ) return else: raise - frappe.db.set_value("Dashboard Chart", args.chart_name, "last_synced_on", frappe.utils.now(), update_modified = False) + frappe.db.set_value( + "Dashboard Chart", args.chart_name, "last_synced_on", frappe.utils.now(), update_modified=False + ) return results + def get_dashboards_with_link(docname, doctype): dashboards = [] links = [] - if doctype == 'Dashboard Chart': - links = frappe.get_all('Dashboard Chart Link', - fields = ['parent'], - filters = { - 'chart': docname - }) - elif doctype == 'Number Card': - links = frappe.get_all('Number Card Link', - fields = ['parent'], - filters = { - 'card': docname - }) + if doctype == "Dashboard Chart": + links = frappe.get_all("Dashboard Chart Link", fields=["parent"], filters={"chart": docname}) + elif doctype == "Number Card": + links = frappe.get_all("Number Card Link", fields=["parent"], filters={"card": docname}) dashboards = [link.parent for link in links] return dashboards + def sync_dashboards(app=None): """Import, overwrite fixtures from `[app]/fixtures`""" - if not cint(frappe.db.get_single_value('System Settings', 'setup_complete')): + if not cint(frappe.db.get_single_value("System Settings", "setup_complete")): return if app: apps = [app] @@ -96,6 +99,7 @@ def sync_dashboards(app=None): make_records_in_module(app_name, module_name) frappe.flags.in_import = False + def make_records_in_module(app, module): dashboards_path = frappe.get_module_path(module, "{module}_dashboard".format(module=module)) charts_path = frappe.get_module_path(module, "dashboard chart") @@ -105,10 +109,11 @@ def make_records_in_module(app, module): for path in paths: make_records(path) + def make_records(path, filters=None): if os.path.isdir(path): for fname in os.listdir(path): if os.path.isdir(join(path, fname)): - if fname == '__pycache__': + if fname == "__pycache__": continue import_file_by_path("{path}/{fname}/{fname}.json".format(path=path, fname=fname)) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index e9f029d293..05ae7034eb 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -22,6 +22,7 @@ DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S.%f" DATETIME_FORMAT = DATE_FORMAT + " " + TIME_FORMAT + class Weekday(Enum): Sunday = 0 Monday = 1 @@ -31,15 +32,21 @@ class Weekday(Enum): Friday = 5 Saturday = 6 + def get_first_day_of_the_week(): - return frappe.get_system_settings('first_day_of_the_week') or "Sunday" + return frappe.get_system_settings("first_day_of_the_week") or "Sunday" + def get_start_of_week_index(): return Weekday[get_first_day_of_the_week()].value + def is_invalid_date_string(date_string): # dateutil parser does not agree with dates like "0001-01-01" or "0000-00-00" - return not isinstance(date_string, str) or ((not date_string) or (date_string or "").startswith(("0001-01-01", "0000-00-00"))) + return not isinstance(date_string, str) or ( + (not date_string) or (date_string or "").startswith(("0001-01-01", "0000-00-00")) + ) + # datetime functions def getdate(string_date: Optional[str] = None): @@ -63,9 +70,11 @@ def getdate(string_date: Optional[str] = None): try: return parser.parse(string_date).date() except ParserError: - frappe.throw(frappe._('{} is not a valid date string.').format( - frappe.bold(string_date) - ), title=frappe._('Invalid Date')) + frappe.throw( + frappe._("{} is not a valid date string.").format(frappe.bold(string_date)), + title=frappe._("Invalid Date"), + ) + def get_datetime(datetime_str=None): from dateutil import parser @@ -90,18 +99,19 @@ def get_datetime(datetime_str=None): except ValueError: return parser.parse(datetime_str) + def get_timedelta(time: Optional[str] = None) -> Optional[datetime.timedelta]: """Return `datetime.timedelta` object from string value of a valid time format. Returns None if `time` is not a valid format Args: - time (str): A valid time representation. This string is parsed - using `dateutil.parser.parse`. Examples of valid inputs are: - '0:0:0', '17:21:00', '2012-01-19 17:21:00'. Checkout - https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse + time (str): A valid time representation. This string is parsed + using `dateutil.parser.parse`. Examples of valid inputs are: + '0:0:0', '17:21:00', '2012-01-19 17:21:00'. Checkout + https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.parse Returns: - datetime.timedelta: Timedelta object equivalent of the passed `time` string + datetime.timedelta: Timedelta object equivalent of the passed `time` string """ from dateutil import parser from dateutil.parser import ParserError @@ -121,6 +131,7 @@ def get_timedelta(time: Optional[str] = None) -> Optional[datetime.timedelta]: except Exception: return None + def to_timedelta(time_str): from dateutil import parser @@ -129,12 +140,26 @@ def to_timedelta(time_str): if isinstance(time_str, str): t = parser.parse(time_str) - return datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond) + return datetime.timedelta( + hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond + ) else: return time_str -def add_to_date(date, years=0, months=0, weeks=0, days=0, hours=0, minutes=0, seconds=0, as_string=False, as_datetime=False): + +def add_to_date( + date, + years=0, + months=0, + weeks=0, + days=0, + hours=0, + minutes=0, + seconds=0, + as_string=False, + as_datetime=False, +): """Adds `days` to the given date""" from dateutil import parser from dateutil.parser._parser import ParserError @@ -155,7 +180,9 @@ def add_to_date(date, years=0, months=0, weeks=0, days=0, hours=0, minutes=0, se except ParserError: frappe.throw(frappe._("Please select a valid date filter"), title=frappe._("Invalid Date")) - date = date + relativedelta(years=years, months=months, weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds) + date = date + relativedelta( + years=years, months=months, weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds + ) if as_string: if as_datetime: @@ -165,45 +192,58 @@ def add_to_date(date, years=0, months=0, weeks=0, days=0, hours=0, minutes=0, se else: return date + def add_days(date, days): return add_to_date(date, days=days) + def add_months(date, months): return add_to_date(date, months=months) + def add_years(date, years): return add_to_date(date, years=years) + def date_diff(string_ed_date, string_st_date): return (getdate(string_ed_date) - getdate(string_st_date)).days + def month_diff(string_ed_date, string_st_date): ed_date = getdate(string_ed_date) st_date = getdate(string_st_date) return (ed_date.year - st_date.year) * 12 + ed_date.month - st_date.month + 1 + def time_diff(string_ed_date, string_st_date): return get_datetime(string_ed_date) - get_datetime(string_st_date) + def time_diff_in_seconds(string_ed_date, string_st_date): return time_diff(string_ed_date, string_st_date).total_seconds() + def time_diff_in_hours(string_ed_date, string_st_date): return round(float(time_diff(string_ed_date, string_st_date).total_seconds()) / 3600, 6) + def now_datetime(): dt = convert_utc_to_user_timezone(datetime.datetime.utcnow()) return dt.replace(tzinfo=None) + def get_timestamp(date): return time.mktime(getdate(date).timetuple()) + def get_eta(from_time, percent_complete): diff = time_diff(now_datetime(), from_time).total_seconds() return str(datetime.timedelta(seconds=(100 - percent_complete) / percent_complete * diff)) + def _get_time_zone(): - return frappe.db.get_system_setting('time_zone') or 'Asia/Kolkata' # Default to India ?! + return frappe.db.get_system_setting("time_zone") or "Asia/Kolkata" # Default to India ?! + def get_time_zone(): if frappe.local.flags.in_test: @@ -211,53 +251,66 @@ def get_time_zone(): return frappe.cache().get_value("time_zone", _get_time_zone) + def convert_utc_to_timezone(utc_timestamp, time_zone): from pytz import UnknownTimeZoneError, timezone - utcnow = timezone('UTC').localize(utc_timestamp) + + utcnow = timezone("UTC").localize(utc_timestamp) try: return utcnow.astimezone(timezone(time_zone)) except UnknownTimeZoneError: return utcnow + def get_datetime_in_timezone(time_zone): utc_timestamp = datetime.datetime.utcnow() return convert_utc_to_timezone(utc_timestamp, time_zone) + def convert_utc_to_user_timezone(utc_timestamp): time_zone = get_time_zone() return convert_utc_to_timezone(utc_timestamp, time_zone) + def now(): """return current datetime as yyyy-mm-dd hh:mm:ss""" if frappe.flags.current_date: - return getdate(frappe.flags.current_date).strftime(DATE_FORMAT) + " " + \ - now_datetime().strftime(TIME_FORMAT) + return ( + getdate(frappe.flags.current_date).strftime(DATE_FORMAT) + + " " + + now_datetime().strftime(TIME_FORMAT) + ) else: return now_datetime().strftime(DATETIME_FORMAT) + def nowdate(): """return current date as yyyy-mm-dd""" return now_datetime().strftime(DATE_FORMAT) + def today(): return nowdate() + def get_abbr(string, max_len=2): - abbr='' - for part in string.split(' '): + abbr = "" + for part in string.split(" "): if len(abbr) < max_len and part: abbr += part[0] - return abbr or '?' + return abbr or "?" + def nowtime(): """return current time in hh:mm""" return now_datetime().strftime(TIME_FORMAT) + def get_first_day(dt, d_years=0, d_months=0, as_str=False): """ - Returns the first day of the month for the date specified by date object - Also adds `d_years` and `d_months` if specified + Returns the first day of the month for the date specified by date object + Also adds `d_years` and `d_months` if specified """ dt = getdate(dt) @@ -265,7 +318,12 @@ def get_first_day(dt, d_years=0, d_months=0, as_str=False): overflow_years, month = divmod(dt.month + d_months - 1, 12) year = dt.year + d_years + overflow_years - return datetime.date(year, month + 1, 1).strftime(DATE_FORMAT) if as_str else datetime.date(year, month + 1, 1) + return ( + datetime.date(year, month + 1, 1).strftime(DATE_FORMAT) + if as_str + else datetime.date(year, month + 1, 1) + ) + def get_quarter_start(dt, as_str=False): date = getdate(dt) @@ -273,11 +331,13 @@ def get_quarter_start(dt, as_str=False): first_date_of_quarter = datetime.date(date.year, ((quarter - 1) * 3) + 1, 1) return first_date_of_quarter.strftime(DATE_FORMAT) if as_str else first_date_of_quarter + def get_first_day_of_week(dt, as_str=False): dt = getdate(dt) date = dt - datetime.timedelta(days=get_week_start_offset_days(dt)) return date.strftime(DATE_FORMAT) if as_str else date + def get_week_start_offset_days(dt): current_day_index = get_normalized_weekday_index(dt) start_of_week_index = get_start_of_week_index() @@ -287,33 +347,38 @@ def get_week_start_offset_days(dt): else: return 7 - (start_of_week_index - current_day_index) + def get_normalized_weekday_index(dt): # starts Sunday with 0 return (dt.weekday() + 1) % 7 + def get_year_start(dt, as_str=False): dt = getdate(dt) date = datetime.date(dt.year, 1, 1) return date.strftime(DATE_FORMAT) if as_str else date + def get_last_day_of_week(dt): dt = get_first_day_of_week(dt) return dt + datetime.timedelta(days=6) + def get_last_day(dt): """ - Returns last day of the month using: - `get_first_day(dt, 0, 1) + datetime.timedelta(-1)` + Returns last day of the month using: + `get_first_day(dt, 0, 1) + datetime.timedelta(-1)` """ return get_first_day(dt, 0, 1) + datetime.timedelta(-1) + def get_quarter_ending(date): date = getdate(date) # find the earliest quarter ending date that is after # the given date for month in (3, 6, 9, 12): - quarter_end_month = getdate('{}-{}-01'.format(date.year, month)) + quarter_end_month = getdate("{}-{}-01".format(date.year, month)) quarter_end_date = getdate(get_last_day(quarter_end_month)) if date <= quarter_end_date: date = quarter_end_date @@ -321,14 +386,16 @@ def get_quarter_ending(date): return date + def get_year_ending(date): - ''' returns year ending of the given date ''' + """returns year ending of the given date""" date = getdate(date) # first day of next year (note year starts from 1) - date = add_to_date('{}-01-01'.format(date.year), months = 12) + date = add_to_date("{}-01-01".format(date.year), months=12) # last day of this month return add_to_date(date, days=-1) + def get_time(time_str: str) -> datetime.time: from dateutil import parser from dateutil.parser import ParserError @@ -343,21 +410,22 @@ def get_time(time_str: str) -> datetime.time: return parser.parse(time_str).time() except ParserError as e: if "day" in e.args[1] or "hour must be in" in e.args[0]: - return ( - datetime.datetime.min + parse_timedelta(time_str) - ).time() + return (datetime.datetime.min + parse_timedelta(time_str)).time() raise e + def get_datetime_str(datetime_obj): if isinstance(datetime_obj, str): datetime_obj = get_datetime(datetime_obj) return datetime_obj.strftime(DATETIME_FORMAT) + def get_date_str(date_obj): if isinstance(date_obj, str): date_obj = get_datetime(date_obj) return date_obj.strftime(DATE_FORMAT) + def get_time_str(timedelta_obj): if isinstance(timedelta_obj, str): timedelta_obj = to_timedelta(timedelta_obj) @@ -366,6 +434,7 @@ def get_time_str(timedelta_obj): minutes, seconds = divmod(remainder, 60) return "{0}:{1}:{2}".format(hours, minutes, seconds) + def get_user_date_format(): """Get the current user date format. The result will be cached.""" if getattr(frappe.local, "user_date_format", None) is None: @@ -373,8 +442,10 @@ def get_user_date_format(): return frappe.local.user_date_format or "yyyy-mm-dd" + get_user_format = get_user_date_format # for backwards compatibility + def get_user_time_format(): """Get the current user time format. The result will be cached.""" if getattr(frappe.local, "user_time_format", None) is None: @@ -382,6 +453,7 @@ def get_user_time_format(): return frappe.local.user_time_format or "HH:mm:ss" + def format_date(string_date=None, format_string=None): """Converts the given string date to :data:`user_date_format` User format specified in defaults @@ -396,7 +468,7 @@ def format_date(string_date=None, format_string=None): from babel.core import UnknownLocaleError if not string_date: - return '' + return "" date = getdate(string_date) if not format_string: @@ -404,15 +476,17 @@ def format_date(string_date=None, format_string=None): format_string = format_string.replace("mm", "MM").replace("Y", "y") try: formatted_date = babel.dates.format_date( - date, format_string, - locale=(frappe.local.lang or "").replace("-", "_")) + date, format_string, locale=(frappe.local.lang or "").replace("-", "_") + ) except UnknownLocaleError: format_string = format_string.replace("MM", "%m").replace("dd", "%d").replace("yyyy", "%Y") formatted_date = date.strftime(format_string) return formatted_date + formatdate = format_date # For backwards compatibility + def format_time(time_string=None, format_string=None): """Converts the given string time to :data:`user_time_format` User format specified in defaults @@ -426,19 +500,20 @@ def format_time(time_string=None, format_string=None): from babel.core import UnknownLocaleError if not time_string: - return '' + return "" time_ = get_time(time_string) if not format_string: format_string = get_user_time_format() try: formatted_time = babel.dates.format_time( - time_, format_string, - locale=(frappe.local.lang or "").replace("-", "_")) + time_, format_string, locale=(frappe.local.lang or "").replace("-", "_") + ) except UnknownLocaleError: formatted_time = time_.strftime("%H:%M:%S") return formatted_time + def format_datetime(datetime_string, format_string=None): """Converts the given string time to :data:`user_datetime_format` User format specified in defaults @@ -456,16 +531,17 @@ def format_datetime(datetime_string, format_string=None): datetime = get_datetime(datetime_string) if not format_string: - format_string = ( - get_user_date_format().replace("mm", "MM") - + ' ' + get_user_time_format()) + format_string = get_user_date_format().replace("mm", "MM") + " " + get_user_time_format() try: - formatted_datetime = babel.dates.format_datetime(datetime, format_string, locale=(frappe.local.lang or "").replace("-", "_")) + formatted_datetime = babel.dates.format_datetime( + datetime, format_string, locale=(frappe.local.lang or "").replace("-", "_") + ) except UnknownLocaleError: - formatted_datetime = datetime.strftime('%Y-%m-%d %H:%M:%S') + formatted_datetime = datetime.strftime("%Y-%m-%d %H:%M:%S") return formatted_datetime + def format_duration(seconds, hide_days=False): """Converts the given duration value in float(seconds) to duration format @@ -475,32 +551,33 @@ def format_duration(seconds, hide_days=False): seconds = cint(seconds) total_duration = { - 'days': math.floor(seconds / (3600 * 24)), - 'hours': math.floor(seconds % (3600 * 24) / 3600), - 'minutes': math.floor(seconds % 3600 / 60), - 'seconds': math.floor(seconds % 60) + "days": math.floor(seconds / (3600 * 24)), + "hours": math.floor(seconds % (3600 * 24) / 3600), + "minutes": math.floor(seconds % 3600 / 60), + "seconds": math.floor(seconds % 60), } if hide_days: - total_duration['hours'] = math.floor(seconds / 3600) - total_duration['days'] = 0 + total_duration["hours"] = math.floor(seconds / 3600) + total_duration["days"] = 0 - duration = '' + duration = "" if total_duration: - if total_duration.get('days'): - duration += str(total_duration.get('days')) + 'd' - if total_duration.get('hours'): - duration += ' ' if len(duration) else '' - duration += str(total_duration.get('hours')) + 'h' - if total_duration.get('minutes'): - duration += ' ' if len(duration) else '' - duration += str(total_duration.get('minutes')) + 'm' - if total_duration.get('seconds'): - duration += ' ' if len(duration) else '' - duration += str(total_duration.get('seconds')) + 's' + if total_duration.get("days"): + duration += str(total_duration.get("days")) + "d" + if total_duration.get("hours"): + duration += " " if len(duration) else "" + duration += str(total_duration.get("hours")) + "h" + if total_duration.get("minutes"): + duration += " " if len(duration) else "" + duration += str(total_duration.get("minutes")) + "m" + if total_duration.get("seconds"): + duration += " " if len(duration) else "" + duration += str(total_duration.get("seconds")) + "s" return duration + def duration_to_seconds(duration): """Converts the given duration formatted value to duration value in seconds @@ -508,36 +585,44 @@ def duration_to_seconds(duration): """ validate_duration_format(duration) value = 0 - if 'd' in duration: - val = duration.split('d') + if "d" in duration: + val = duration.split("d") days = val[0] value += cint(days) * 24 * 60 * 60 duration = val[1] - if 'h' in duration: - val = duration.split('h') + if "h" in duration: + val = duration.split("h") hours = val[0] value += cint(hours) * 60 * 60 duration = val[1] - if 'm' in duration: - val = duration.split('m') + if "m" in duration: + val = duration.split("m") mins = val[0] value += cint(mins) * 60 duration = val[1] - if 's' in duration: - val = duration.split('s') + if "s" in duration: + val = duration.split("s") secs = val[0] value += cint(secs) return value + def validate_duration_format(duration): import re + is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", duration) if not is_valid_duration: - frappe.throw(frappe._("Value {0} must be in the valid duration format: d h m s").format(frappe.bold(duration))) + frappe.throw( + frappe._("Value {0} must be in the valid duration format: d h m s").format( + frappe.bold(duration) + ) + ) + def get_weekdays(): - return ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] + return ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"] + def get_weekday(datetime=None): if not datetime: @@ -545,14 +630,30 @@ def get_weekday(datetime=None): weekdays = get_weekdays() return weekdays[datetime.weekday()] + def get_timespan_date_range(timespan): today = nowdate() date_range_map = { - "last week": lambda: (get_first_day_of_week(add_to_date(today, days=-7)), get_last_day_of_week(add_to_date(today, days=-7))), - "last month": lambda: (get_first_day(add_to_date(today, months=-1)), get_last_day(add_to_date(today, months=-1))), - "last quarter": lambda: (get_quarter_start(add_to_date(today, months=-3)), get_quarter_ending(add_to_date(today, months=-3))), - "last 6 months": lambda: (get_quarter_start(add_to_date(today, months=-6)), get_quarter_ending(add_to_date(today, months=-3))), - "last year": lambda: (get_year_start(add_to_date(today, years=-1)), get_year_ending(add_to_date(today, years=-1))), + "last week": lambda: ( + get_first_day_of_week(add_to_date(today, days=-7)), + get_last_day_of_week(add_to_date(today, days=-7)), + ), + "last month": lambda: ( + get_first_day(add_to_date(today, months=-1)), + get_last_day(add_to_date(today, months=-1)), + ), + "last quarter": lambda: ( + get_quarter_start(add_to_date(today, months=-3)), + get_quarter_ending(add_to_date(today, months=-3)), + ), + "last 6 months": lambda: ( + get_quarter_start(add_to_date(today, months=-6)), + get_quarter_ending(add_to_date(today, months=-3)), + ), + "last year": lambda: ( + get_year_start(add_to_date(today, years=-1)), + get_year_ending(add_to_date(today, years=-1)), + ), "yesterday": lambda: (add_to_date(today, days=-1),) * 2, "today": lambda: (today, today), "tomorrow": lambda: (add_to_date(today, days=1),) * 2, @@ -560,28 +661,48 @@ def get_timespan_date_range(timespan): "this month": lambda: (get_first_day(today), get_last_day(today)), "this quarter": lambda: (get_quarter_start(today), get_quarter_ending(today)), "this year": lambda: (get_year_start(today), get_year_ending(today)), - "next week": lambda: (get_first_day_of_week(add_to_date(today, days=7)), get_last_day_of_week(add_to_date(today, days=7))), - "next month": lambda: (get_first_day(add_to_date(today, months=1)), get_last_day(add_to_date(today, months=1))), - "next quarter": lambda: (get_quarter_start(add_to_date(today, months=3)), get_quarter_ending(add_to_date(today, months=3))), - "next 6 months": lambda: (get_quarter_start(add_to_date(today, months=3)), get_quarter_ending(add_to_date(today, months=6))), - "next year": lambda: (get_year_start(add_to_date(today, years=1)), get_year_ending(add_to_date(today, years=1))), + "next week": lambda: ( + get_first_day_of_week(add_to_date(today, days=7)), + get_last_day_of_week(add_to_date(today, days=7)), + ), + "next month": lambda: ( + get_first_day(add_to_date(today, months=1)), + get_last_day(add_to_date(today, months=1)), + ), + "next quarter": lambda: ( + get_quarter_start(add_to_date(today, months=3)), + get_quarter_ending(add_to_date(today, months=3)), + ), + "next 6 months": lambda: ( + get_quarter_start(add_to_date(today, months=3)), + get_quarter_ending(add_to_date(today, months=6)), + ), + "next year": lambda: ( + get_year_start(add_to_date(today, years=1)), + get_year_ending(add_to_date(today, years=1)), + ), } if timespan in date_range_map: return date_range_map[timespan]() + def global_date_format(date, format="long"): """returns localized date in the form of January 1, 2012""" import babel.dates date = getdate(date) - formatted_date = babel.dates.format_date(date, locale=(frappe.local.lang or "en").replace("-", "_"), format=format) + formatted_date = babel.dates.format_date( + date, locale=(frappe.local.lang or "en").replace("-", "_"), format=format + ) return formatted_date + def has_common(l1, l2): """Returns truthy value if there are common elements in lists l1 and l2""" return set(l1) & set(l2) + def cast_fieldtype(fieldtype, value, show_warning=True): if show_warning: message = ( @@ -596,8 +717,16 @@ def cast_fieldtype(fieldtype, value, show_warning=True): elif fieldtype in ("Int", "Check"): value = cint(value) - elif fieldtype in ("Data", "Text", "Small Text", "Long Text", - "Text Editor", "Select", "Link", "Dynamic Link"): + elif fieldtype in ( + "Data", + "Text", + "Small Text", + "Long Text", + "Text Editor", + "Select", + "Link", + "Dynamic Link", + ): value = cstr(value) elif fieldtype == "Date": @@ -611,18 +740,19 @@ def cast_fieldtype(fieldtype, value, show_warning=True): return value + def cast(fieldtype, value=None): """Cast the value to the Python native object of the Frappe fieldtype provided. If value is None, the first/lowest value of the `fieldtype` will be returned. If value can't be cast as fieldtype due to an invalid input, None will be returned. Mapping of Python types => Frappe types: - * str => ("Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link") - * float => ("Currency", "Float", "Percent") - * int => ("Int", "Check") - * datetime.datetime => ("Datetime",) - * datetime.date => ("Date",) - * datetime.time => ("Time",) + * str => ("Data", "Text", "Small Text", "Long Text", "Text Editor", "Select", "Link", "Dynamic Link") + * float => ("Currency", "Float", "Percent") + * int => ("Int", "Check") + * datetime.datetime => ("Datetime",) + * datetime.date => ("Date",) + * datetime.time => ("Time",) """ if fieldtype in ("Currency", "Float", "Percent"): value = flt(value) @@ -630,8 +760,16 @@ def cast(fieldtype, value=None): elif fieldtype in ("Int", "Check"): value = cint(sbool(value)) - elif fieldtype in ("Data", "Text", "Small Text", "Long Text", - "Text Editor", "Select", "Link", "Dynamic Link"): + elif fieldtype in ( + "Data", + "Text", + "Small Text", + "Long Text", + "Text Editor", + "Select", + "Link", + "Dynamic Link", + ): value = cstr(value) elif fieldtype == "Date": @@ -651,28 +789,29 @@ def cast(fieldtype, value=None): return value + def flt(s, precision=None): """Convert to float (ignoring commas in string) - :param s: Number in string or other numeric format. - :param precision: optional argument to specify precision for rounding. - :returns: Converted number in python float type. + :param s: Number in string or other numeric format. + :param precision: optional argument to specify precision for rounding. + :returns: Converted number in python float type. - Returns 0 if input can not be converted to float. + Returns 0 if input can not be converted to float. - Examples: + Examples: - >>> flt("43.5", precision=0) - 44 - >>> flt("42.5", precision=0) - 42 - >>> flt("10,500.5666", precision=2) - 10500.57 - >>> flt("a") - 0.0 + >>> flt("43.5", precision=0) + 44 + >>> flt("42.5", precision=0) + 42 + >>> flt("10,500.5666", precision=2) + 10500.57 + >>> flt("a") + 0.0 """ if isinstance(s, str): - s = s.replace(',','') + s = s.replace(",", "") try: num = float(s) @@ -683,19 +822,20 @@ def flt(s, precision=None): return num + def cint(s, default=0): """Convert to integer - :param s: Number in string or other numeric format. - :returns: Converted number in python integer type. + :param s: Number in string or other numeric format. + :returns: Converted number in python integer type. - Returns default if input can not be converted to integer. + Returns default if input can not be converted to integer. - Examples: - >>> cint("100") - 100 - >>> cint("a") - 0 + Examples: + >>> cint("100") + 100 + >>> cint("a") + 0 """ try: @@ -703,6 +843,7 @@ def cint(s, default=0): except Exception: return default + def floor(s): """ A number representing the largest integer less than or equal to the specified number @@ -710,18 +851,21 @@ def floor(s): Parameters ---------- s : int or str or Decimal object - The mathematical value to be floored + The mathematical value to be floored Returns ------- int - number representing the largest integer less than or equal to the specified number + number representing the largest integer less than or equal to the specified number """ - try: num = cint(math.floor(flt(s))) - except: num = 0 + try: + num = cint(math.floor(flt(s))) + except: + num = 0 return num + def ceil(s): """ The smallest integer greater than or equal to the given number @@ -729,48 +873,53 @@ def ceil(s): Parameters ---------- s : int or str or Decimal object - The mathematical value to be ceiled + The mathematical value to be ceiled Returns ------- int - smallest integer greater than or equal to the given number + smallest integer greater than or equal to the given number """ - try: num = cint(math.ceil(flt(s))) - except: num = 0 + try: + num = cint(math.ceil(flt(s))) + except: + num = 0 return num -def cstr(s, encoding='utf-8'): + +def cstr(s, encoding="utf-8"): return frappe.as_unicode(s, encoding) + def sbool(x: str) -> Union[bool, Any]: """Converts str object to Boolean if possible. Example: - "true" becomes True - "1" becomes True - "{}" remains "{}" + "true" becomes True + "1" becomes True + "{}" remains "{}" Args: - x (str): String to be converted to Bool + x (str): String to be converted to Bool Returns: - object: Returns Boolean or x + object: Returns Boolean or x """ try: val = x.lower() - if val in ('true', '1'): + if val in ("true", "1"): return True - elif val in ('false', '0'): + elif val in ("false", "0"): return False return x except Exception: return x + def rounded(num, precision=0): """round method for round halfs to nearest even algorithm aka banker's rounding - compatible with python3""" precision = cint(precision) - multiplier = 10 ** precision + multiplier = 10**precision # avoid rounding errors num = round(num * multiplier if precision else num, 8) @@ -788,9 +937,10 @@ def rounded(num, precision=0): return (num / multiplier) if precision else num + def remainder(numerator, denominator, precision=2): precision = cint(precision) - multiplier = 10 ** precision + multiplier = 10**precision if precision: _remainder = ((numerator * multiplier) % (denominator * multiplier)) / multiplier @@ -799,6 +949,7 @@ def remainder(numerator, denominator, precision=2): return flt(_remainder, precision) + def safe_div(numerator, denominator, precision=2): """ SafeMath division that returns zero when divided by zero. @@ -812,9 +963,11 @@ def safe_div(numerator, denominator, precision=2): return flt(_res, precision) + def round_based_on_smallest_currency_fraction(value, currency, precision=2): - smallest_currency_fraction_value = flt(frappe.db.get_value("Currency", - currency, "smallest_currency_fraction_value", cache=True)) + smallest_currency_fraction_value = flt( + frappe.db.get_value("Currency", currency, "smallest_currency_fraction_value", cache=True) + ) if smallest_currency_fraction_value: remainder_val = remainder(value, smallest_currency_fraction_value, precision) @@ -827,6 +980,7 @@ def round_based_on_smallest_currency_fraction(value, currency, precision=2): return flt(value, precision) + def encode(obj, encoding="utf-8"): if isinstance(obj, list): out = [] @@ -841,6 +995,7 @@ def encode(obj, encoding="utf-8"): else: return obj + def parse_val(v): """Converts to simple datatypes from SQL query results""" if isinstance(v, (datetime.date, datetime.datetime)): @@ -851,13 +1006,14 @@ def parse_val(v): v = int(v) return v + def fmt_money(amount, precision=None, currency=None, format=None): """ Convert to string with commas for thousands, millions etc """ number_format = format or frappe.db.get_default("number_format") or "#,###.##" if precision is None: - precision = cint(frappe.db.get_default('currency_precision')) or None + precision = cint(frappe.db.get_default("currency_precision")) or None decimal_str, comma_str, number_format_precision = get_number_format_info(number_format) @@ -873,38 +1029,38 @@ def fmt_money(amount, precision=None, currency=None, format=None): if decimal_str: decimals_after = str(round(amount % 1, precision)) - parts = decimals_after.split('.') + parts = decimals_after.split(".") parts = parts[1] if len(parts) > 1 else parts[0] decimals = parts if precision > 2: if len(decimals) < 3: if currency: - fraction = frappe.db.get_value("Currency", currency, "fraction_units", cache=True) or 100 + fraction = frappe.db.get_value("Currency", currency, "fraction_units", cache=True) or 100 precision = len(cstr(fraction)) - 1 else: precision = number_format_precision elif len(decimals) < precision: precision = len(decimals) - amount = '%.*f' % (precision, round(flt(amount), precision)) + amount = "%.*f" % (precision, round(flt(amount), precision)) - if amount.find('.') == -1: - decimals = '' + if amount.find(".") == -1: + decimals = "" else: - decimals = amount.split('.')[1] + decimals = amount.split(".")[1] parts = [] - minus = '' + minus = "" if flt(amount) < 0: - minus = '-' + minus = "-" - amount = cstr(abs(flt(amount))).split('.')[0] + amount = cstr(abs(flt(amount))).split(".")[0] if len(amount) > 3: parts.append(amount[-3:]) amount = amount[:-3] - val = number_format=="#,##,###.##" and 2 or 3 + val = number_format == "#,##,###.##" and 2 or 3 while len(amount) > val: parts.append(amount[-val:]) @@ -915,7 +1071,7 @@ def fmt_money(amount, precision=None, currency=None, format=None): parts.reverse() amount = comma_str.join(parts) + ((precision and decimal_str) and (decimal_str + decimals) or "") - if amount != '0': + if amount != "0": amount = minus + amount if currency and frappe.defaults.get_global_default("hide_currency_symbol") != "Yes": @@ -924,6 +1080,7 @@ def fmt_money(amount, precision=None, currency=None, format=None): return amount + number_format_info = { "#,###.##": (".", ",", 2), "#.###,##": (",", ".", 2), @@ -935,20 +1092,25 @@ number_format_info = { "#,###.###": (".", ",", 3), "#.###": ("", ".", 0), "#,###": ("", ",", 0), - "#.########": (".", "", 8) + "#.########": (".", "", 8), } + def get_number_format_info(format: str) -> Tuple[str, str, int]: return number_format_info.get(format) or (".", ",", 2) + # # convert currency to words # -def money_in_words(number: str, main_currency: Optional[str] = None, fraction_currency: Optional[str] = None): +def money_in_words( + number: str, main_currency: Optional[str] = None, fraction_currency: Optional[str] = None +): """ Returns string in words with currency and fraction currency. """ from frappe.utils import get_defaults + _ = frappe._ try: @@ -963,39 +1125,54 @@ def money_in_words(number: str, main_currency: Optional[str] = None, fraction_cu d = get_defaults() if not main_currency: - main_currency = d.get('currency', 'INR') + main_currency = d.get("currency", "INR") if not fraction_currency: - fraction_currency = frappe.db.get_value("Currency", main_currency, "fraction", cache=True) or _("Cent") + fraction_currency = frappe.db.get_value("Currency", main_currency, "fraction", cache=True) or _( + "Cent" + ) - number_format = frappe.db.get_value("Currency", main_currency, "number_format", cache=True) or \ - frappe.db.get_default("number_format") or "#,###.##" + number_format = ( + frappe.db.get_value("Currency", main_currency, "number_format", cache=True) + or frappe.db.get_default("number_format") + or "#,###.##" + ) fraction_length = get_number_format_info(number_format)[2] n = "%.{0}f".format(fraction_length) % number - numbers = n.split('.') - main, fraction = numbers if len(numbers) > 1 else [n, '00'] + numbers = n.split(".") + main, fraction = numbers if len(numbers) > 1 else [n, "00"] if len(fraction) < fraction_length: - zeros = '0' * (fraction_length - len(fraction)) + zeros = "0" * (fraction_length - len(fraction)) fraction += zeros in_million = True - if number_format == "#,##,###.##": in_million = False + if number_format == "#,##,###.##": + in_million = False # 0.00 - if main == '0' and fraction in ['00', '000']: - out = "{0} {1}".format(main_currency, _('Zero')) + if main == "0" and fraction in ["00", "000"]: + out = "{0} {1}".format(main_currency, _("Zero")) # 0.XX - elif main == '0': - out = _(in_words(fraction, in_million).title()) + ' ' + fraction_currency + elif main == "0": + out = _(in_words(fraction, in_million).title()) + " " + fraction_currency else: - out = main_currency + ' ' + _(in_words(main, in_million).title()) + out = main_currency + " " + _(in_words(main, in_million).title()) if cint(fraction): - out = out + ' ' + _('and') + ' ' + _(in_words(fraction, in_million).title()) + ' ' + fraction_currency + out = ( + out + + " " + + _("and") + + " " + + _(in_words(fraction, in_million).title()) + + " " + + fraction_currency + ) + + return out + " " + _("only.") - return out + ' ' + _('only.') # # convert number to words @@ -1006,28 +1183,31 @@ def in_words(integer, in_million=True): """ from num2words import num2words - locale = 'en_IN' if not in_million else frappe.local.lang + locale = "en_IN" if not in_million else frappe.local.lang integer = int(integer) try: ret = num2words(integer, lang=locale) except NotImplementedError: - ret = num2words(integer, lang='en') + ret = num2words(integer, lang="en") except OverflowError: - ret = num2words(integer, lang='en') - return ret.replace('-', ' ') + ret = num2words(integer, lang="en") + return ret.replace("-", " ") + def is_html(text): if not isinstance(text, str): return False - return re.search('<[^>]+>', text) + return re.search("<[^>]+>", text) + def is_image(filepath): from mimetypes import guess_type # filepath can be https://example.com/bed.jpg?v=129 - filepath = (filepath or "").split('?')[0] + filepath = (filepath or "").split("?")[0] return (guess_type(filepath)[0] or "").startswith("image/") + def get_thumbnail_base64_for_image(src): from os.path import exists as file_exists @@ -1037,12 +1217,12 @@ def get_thumbnail_base64_for_image(src): from frappe.core.doctype.file.file import get_local_image if not src: - frappe.throw('Invalid source for image: {0}'.format(src)) + frappe.throw("Invalid source for image: {0}".format(src)) - if not src.startswith('/files') or '..' in src: + if not src.startswith("/files") or ".." in src: return - if src.endswith('.svg'): + if src.endswith(".svg"): return def _get_base64(): @@ -1061,44 +1241,50 @@ def get_thumbnail_base64_for_image(src): base64_string = image_to_base64(image, extn) return { - 'base64': 'data:image/{0};base64,{1}'.format(extn, safe_decode(base64_string)), - 'width': original_size[0], - 'height': original_size[1] + "base64": "data:image/{0};base64,{1}".format(extn, safe_decode(base64_string)), + "width": original_size[0], + "height": original_size[1], } - return cache().hget('thumbnail_base64', src, generator=_get_base64) + return cache().hget("thumbnail_base64", src, generator=_get_base64) + def image_to_base64(image, extn): from io import BytesIO buffered = BytesIO() - if extn.lower() in ('jpg', 'jpe'): - extn = 'JPEG' + if extn.lower() in ("jpg", "jpe"): + extn = "JPEG" image.save(buffered, extn) img_str = base64.b64encode(buffered.getvalue()) return img_str + def pdf_to_base64(filename): from frappe.utils.file_manager import get_file_path - if '../' in filename or filename.rsplit('.')[-1] not in ['pdf', 'PDF']: + if "../" in filename or filename.rsplit(".")[-1] not in ["pdf", "PDF"]: return file_path = get_file_path(filename) if not file_path: return - with open(file_path, 'rb') as pdf_file: + with open(file_path, "rb") as pdf_file: base64_string = base64.b64encode(pdf_file.read()) return base64_string + # from Jinja2 code -_striptags_re = re.compile(r'(|<[^>]*>)') +_striptags_re = re.compile(r"(|<[^>]*>)") + + def strip_html(text): """removes anything enclosed in and including <>""" return _striptags_re.sub("", text) + def escape_html(text): if not isinstance(text, str): return text @@ -1111,16 +1297,19 @@ def escape_html(text): "<": "<", } - return "".join(html_escape_table.get(c,c) for c in text) + return "".join(html_escape_table.get(c, c) for c in text) + def pretty_date(iso_datetime): """ - Takes an ISO time and returns a string representing how - long ago the date represents. - Ported from PrettyDate by John Resig + Takes an ISO time and returns a string representing how + long ago the date represents. + Ported from PrettyDate by John Resig """ from frappe import _ - if not iso_datetime: return '' + + if not iso_datetime: + return "" import math if isinstance(iso_datetime, str): @@ -1137,38 +1326,41 @@ def pretty_date(iso_datetime): # differnt cases if dt_diff_seconds < 60.0: - return _('just now') + return _("just now") elif dt_diff_seconds < 120.0: - return _('1 minute ago') + return _("1 minute ago") elif dt_diff_seconds < 3600.0: - return _('{0} minutes ago').format(cint(math.floor(dt_diff_seconds / 60.0))) + return _("{0} minutes ago").format(cint(math.floor(dt_diff_seconds / 60.0))) elif dt_diff_seconds < 7200.0: - return _('1 hour ago') + return _("1 hour ago") elif dt_diff_seconds < 86400.0: - return _('{0} hours ago').format(cint(math.floor(dt_diff_seconds / 3600.0))) + return _("{0} hours ago").format(cint(math.floor(dt_diff_seconds / 3600.0))) elif dt_diff_days == 1.0: - return _('Yesterday') + return _("Yesterday") elif dt_diff_days < 7.0: - return _('{0} days ago').format(cint(dt_diff_days)) + return _("{0} days ago").format(cint(dt_diff_days)) elif dt_diff_days < 12: - return _('1 week ago') + return _("1 week ago") elif dt_diff_days < 31.0: - return _('{0} weeks ago').format(cint(math.ceil(dt_diff_days / 7.0))) + return _("{0} weeks ago").format(cint(math.ceil(dt_diff_days / 7.0))) elif dt_diff_days < 46: - return _('1 month ago') + return _("1 month ago") elif dt_diff_days < 365.0: - return _('{0} months ago').format(cint(math.ceil(dt_diff_days / 30.0))) + return _("{0} months ago").format(cint(math.ceil(dt_diff_days / 30.0))) elif dt_diff_days < 550.0: - return _('1 year ago') + return _("1 year ago") else: - return '{0} years ago'.format(cint(math.floor(dt_diff_days / 365.0))) + return "{0} years ago".format(cint(math.floor(dt_diff_days / 365.0))) + def comma_or(some_list, add_quotes=True): return comma_sep(some_list, frappe._("{0} or {1}"), add_quotes) -def comma_and(some_list ,add_quotes=True): + +def comma_and(some_list, add_quotes=True): return comma_sep(some_list, frappe._("{0} and {1}"), add_quotes) + def comma_sep(some_list, pattern, add_quotes=True): if isinstance(some_list, (list, tuple)): # list(some_list) is done to preserve the existing list @@ -1183,6 +1375,7 @@ def comma_sep(some_list, pattern, add_quotes=True): else: return some_list + def new_line_sep(some_list): if isinstance(some_list, (list, tuple)): # list(some_list) is done to preserve the existing list @@ -1202,6 +1395,7 @@ def filter_strip_join(some_list, sep): """given a list, filter None values, strip spaces and join""" return (cstr(sep)).join((cstr(a).strip() for a in filter(None, some_list))) + def get_url(uri=None, full_address=False): """get app url from request""" host_name = frappe.local.conf.host_name or frappe.local.conf.hostname @@ -1216,21 +1410,24 @@ def get_url(uri=None, full_address=False): host_name = request_host_name elif frappe.local.site: - protocol = 'http://' + protocol = "http://" if frappe.local.conf.ssl_certificate: - protocol = 'https://' + protocol = "https://" elif frappe.local.conf.wildcard: - domain = frappe.local.conf.wildcard.get('domain') - if domain and frappe.local.site.endswith(domain) and frappe.local.conf.wildcard.get('ssl_certificate'): - protocol = 'https://' + domain = frappe.local.conf.wildcard.get("domain") + if ( + domain + and frappe.local.site.endswith(domain) + and frappe.local.conf.wildcard.get("ssl_certificate") + ): + protocol = "https://" host_name = protocol + frappe.local.site else: - host_name = frappe.db.get_value("Website Settings", "Website Settings", - "subdomain") + host_name = frappe.db.get_value("Website Settings", "Website Settings", "subdomain") if not host_name: host_name = "http://localhost" @@ -1243,77 +1440,99 @@ def get_url(uri=None, full_address=False): port = frappe.conf.http_port or frappe.conf.webserver_port - if not (frappe.conf.restart_supervisor_on_update or frappe.conf.restart_systemd_on_update) and host_name and not url_contains_port(host_name) and port: - host_name = host_name + ':' + str(port) + if ( + not (frappe.conf.restart_supervisor_on_update or frappe.conf.restart_systemd_on_update) + and host_name + and not url_contains_port(host_name) + and port + ): + host_name = host_name + ":" + str(port) url = urljoin(host_name, uri) if uri else host_name return url + def get_host_name_from_request(): if hasattr(frappe.local, "request") and frappe.local.request and frappe.local.request.host: - protocol = 'https://' if 'https' == frappe.get_request_header('X-Forwarded-Proto', "") else 'http://' + protocol = ( + "https://" if "https" == frappe.get_request_header("X-Forwarded-Proto", "") else "http://" + ) return protocol + frappe.local.request.host + def url_contains_port(url): - parts = url.split(':') + parts = url.split(":") return len(parts) > 2 + def get_host_name(): return get_url().rsplit("//", 1)[-1] + def get_link_to_form(doctype, name, label=None): - if not label: label = name + if not label: + label = name return """{1}""".format(get_url_to_form(doctype, name), label) + def get_link_to_report(name, label=None, report_type=None, doctype=None, filters=None): - if not label: label = name + if not label: + label = name if filters: conditions = [] - for k,v in filters.items(): + for k, v in filters.items(): if isinstance(v, list): for value in v: - conditions.append(str(k)+'='+'["'+str(value[0]+'"'+','+'"'+str(value[1])+'"]')) + conditions.append( + str(k) + "=" + '["' + str(value[0] + '"' + "," + '"' + str(value[1]) + '"]') + ) else: - conditions.append(str(k)+"="+str(v)) + conditions.append(str(k) + "=" + str(v)) filters = "&".join(conditions) - return """{1}""".format(get_url_to_report_with_filters(name, filters, report_type, doctype), label) + return """{1}""".format( + get_url_to_report_with_filters(name, filters, report_type, doctype), label + ) else: return """{1}""".format(get_url_to_report(name, report_type, doctype), label) + def get_absolute_url(doctype, name): return "/app/{0}/{1}".format(quoted(slug(doctype)), quoted(name)) + def get_url_to_form(doctype, name): - return get_url(uri = "/app/{0}/{1}".format(quoted(slug(doctype)), quoted(name))) + return get_url(uri="/app/{0}/{1}".format(quoted(slug(doctype)), quoted(name))) + def get_url_to_list(doctype): - return get_url(uri = "/app/{0}".format(quoted(slug(doctype)))) + return get_url(uri="/app/{0}".format(quoted(slug(doctype)))) + -def get_url_to_report(name, report_type = None, doctype = None): +def get_url_to_report(name, report_type=None, doctype=None): if report_type == "Report Builder": - return get_url(uri = "/app/{0}/view/report/{1}".format(quoted(slug(doctype)), quoted(name))) + return get_url(uri="/app/{0}/view/report/{1}".format(quoted(slug(doctype)), quoted(name))) else: - return get_url(uri = "/app/query-report/{0}".format(quoted(name))) + return get_url(uri="/app/query-report/{0}".format(quoted(name))) + -def get_url_to_report_with_filters(name, filters, report_type = None, doctype = None): +def get_url_to_report_with_filters(name, filters, report_type=None, doctype=None): if report_type == "Report Builder": - return get_url(uri = "/app/{0}/view/report?{1}".format(quoted(doctype), filters)) + return get_url(uri="/app/{0}/view/report?{1}".format(quoted(doctype), filters)) else: - return get_url(uri = "/app/query-report/{0}?{1}".format(quoted(name), filters)) + return get_url(uri="/app/query-report/{0}?{1}".format(quoted(name), filters)) + operator_map = { # startswith "^": lambda a, b: (a or "").startswith(b), - # in or not in a list "in": lambda a, b: operator.contains(b, a), "not in": lambda a, b: not operator.contains(b, a), - # comparison operators "=": lambda a, b: operator.eq(a, b), "!=": lambda a, b: operator.ne(a, b), @@ -1322,14 +1541,15 @@ operator_map = { ">=": lambda a, b: operator.ge(a, b), "<=": lambda a, b: operator.le(a, b), "not None": lambda a, b: a and True or False, - "None": lambda a, b: (not a) and True or False + "None": lambda a, b: (not a) and True or False, } + def evaluate_filters(doc, filters: Union[Dict, List, Tuple]): - '''Returns true if doc matches filters''' + """Returns true if doc matches filters""" if isinstance(filters, dict): for key, value in filters.items(): - f = get_filter(None, {key:value}) + f = get_filter(None, {key: value}) if not compare(doc.get(f.fieldname), f.operator, f.value, f.fieldtype): return False @@ -1351,18 +1571,19 @@ def compare(val1: Any, condition: str, val2: Any, fieldtype: Optional[str] = Non return ret + def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -> "frappe._dict": """Returns a _dict like - { - "doctype": - "fieldname": - "operator": - "value": - "fieldtype": - } + { + "doctype": + "fieldname": + "operator": + "value": + "fieldtype": + } """ - from frappe.model import default_fields, optional_fields, child_table_fields + from frappe.model import child_table_fields, default_fields, optional_fields if isinstance(f, dict): key, value = next(iter(f.items())) @@ -1376,7 +1597,9 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) - elif len(f) > 4: f = f[0:4] elif len(f) != 4: - frappe.throw(frappe._("Filter must have 4 values (doctype, fieldname, operator, value): {0}").format(str(f))) + frappe.throw( + frappe._("Filter must have 4 values (doctype, fieldname, operator, value): {0}").format(str(f)) + ) f = frappe._dict(doctype=f[0], fieldname=f[1], operator=f[2], value=f[3]) @@ -1386,9 +1609,27 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) - # if operator is missing f.operator = "=" - valid_operators = ("=", "!=", ">", "<", ">=", "<=", "like", "not like", "in", "not in", "is", - "between", "descendants of", "ancestors of", "not descendants of", "not ancestors of", - "timespan", "previous", "next") + valid_operators = ( + "=", + "!=", + ">", + "<", + ">=", + "<=", + "like", + "not like", + "in", + "not in", + "is", + "between", + "descendants of", + "ancestors of", + "not descendants of", + "not ancestors of", + "timespan", + "previous", + "next", + ) if filters_config: additional_operators = [] @@ -1399,7 +1640,6 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) - if f.operator.lower() not in valid_operators: frappe.throw(frappe._("Operator must be one of {0}").format(", ".join(valid_operators))) - if f.doctype and (f.fieldname not in default_fields + optional_fields + child_table_fields): # verify fieldname belongs to the doctype meta = frappe.get_meta(f.doctype) @@ -1420,23 +1660,26 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) - return f + def make_filter_tuple(doctype, key, value): - '''return a filter tuple like [doctype, key, operator, value]''' + """return a filter tuple like [doctype, key, operator, value]""" if isinstance(value, (list, tuple)): return [doctype, key, value[0], value[1]] else: return [doctype, key, "=", value] + def make_filter_dict(filters): - '''convert this [[doctype, key, operator, value], ..] + """convert this [[doctype, key, operator, value], ..] to this { key: (operator, value), .. } - ''' + """ _filter = frappe._dict() for f in filters: _filter[f[1]] = (f[2], f[3]) return _filter + def sanitize_column(column_name): import sqlparse @@ -1444,15 +1687,25 @@ def sanitize_column(column_name): regex = re.compile("^.*[,'();].*") column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower") - blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or'] + blacklisted_keywords = [ + "select", + "create", + "insert", + "delete", + "drop", + "update", + "case", + "and", + "or", + ] def _raise_exception(): frappe.throw(_("Invalid field name {0}").format(column_name), frappe.DataError) - if 'ifnull' in column_name: + if "ifnull" in column_name: if regex.match(column_name): # to avoid and, or - if any(' {0} '.format(keyword) in column_name.split() for keyword in blacklisted_keywords): + if any(" {0} ".format(keyword) in column_name.split() for keyword in blacklisted_keywords): _raise_exception() # to avoid select, delete, drop, update and case @@ -1462,47 +1715,55 @@ def sanitize_column(column_name): elif regex.match(column_name): _raise_exception() + def scrub_urls(html): html = expand_relative_urls(html) # encoding should be responsibility of the composer # html = quote_urls(html) return html + def expand_relative_urls(html): # expand relative urls url = get_url() - if url.endswith("/"): url = url[:-1] + if url.endswith("/"): + url = url[:-1] def _expand_relative_urls(match): to_expand = list(match.groups()) - if not to_expand[2].startswith('mailto') and not to_expand[2].startswith('data:'): + if not to_expand[2].startswith("mailto") and not to_expand[2].startswith("data:"): if not to_expand[2].startswith("/"): to_expand[2] = "/" + to_expand[2] to_expand.insert(2, url) - if 'url' in to_expand[0] and to_expand[1].startswith('(') and to_expand[-1].endswith(')'): + if "url" in to_expand[0] and to_expand[1].startswith("(") and to_expand[-1].endswith(")"): # background-image: url('/assets/...') - workaround for wkhtmltopdf print-media-type - to_expand.append(' !important') + to_expand.append(" !important") return "".join(to_expand) - html = re.sub(r'(href|src){1}([\s]*=[\s]*[\'"]?)((?!http)[^\'" >]+)([\'"]?)', _expand_relative_urls, html) + html = re.sub( + r'(href|src){1}([\s]*=[\s]*[\'"]?)((?!http)[^\'" >]+)([\'"]?)', _expand_relative_urls, html + ) # background-image: url('/assets/...') html = re.sub(r'(:[\s]?url)(\([\'"]?)((?!http)[^\'" >]+)([\'"]?\))', _expand_relative_urls, html) return html + def quoted(url): return cstr(quote(encode(cstr(url)), safe=b"~@#$&()*!+=:;,.?/'")) + def quote_urls(html): def _quote_url(match): groups = list(match.groups()) groups[2] = quoted(groups[2]) return "".join(groups) - return re.sub(r'(href|src){1}([\s]*=[\s]*[\'"]?)((?:http)[^\'">]+)([\'"]?)', - _quote_url, html) + + return re.sub(r'(href|src){1}([\s]*=[\s]*[\'"]?)((?:http)[^\'">]+)([\'"]?)', _quote_url, html) + def unique(seq): """use this instead of list(set()) to preserve order of the original list. @@ -1510,12 +1771,14 @@ def unique(seq): seen = set() seen_add = seen.add - return [ x for x in seq if not (x in seen or seen_add(x)) ] + return [x for x in seq if not (x in seen or seen_add(x))] + def strip(val, chars=None): # \ufeff is no-width-break, \u200b is no-width-space return (val or "").replace("\ufeff", "").replace("\u200b", "").strip(chars) + def to_markdown(html): from html.parser import HTMLParser @@ -1523,46 +1786,48 @@ def to_markdown(html): text = None try: - text = html2text(html or '') + text = html2text(html or "") except HTMLParser.HTMLParseError: pass return text + def md_to_html(markdown_text): from markdown2 import MarkdownError from markdown2 import markdown as _markdown extras = { - 'fenced-code-blocks': None, - 'tables': None, - 'header-ids': None, - 'toc': None, - 'highlightjs-lang': None, - 'html-classes': { - 'table': 'table table-bordered', - 'img': 'screenshot' - } + "fenced-code-blocks": None, + "tables": None, + "header-ids": None, + "toc": None, + "highlightjs-lang": None, + "html-classes": {"table": "table table-bordered", "img": "screenshot"}, } html = None try: - html = UnicodeWithAttrs(_markdown(markdown_text or '', extras=extras)) + html = UnicodeWithAttrs(_markdown(markdown_text or "", extras=extras)) except MarkdownError: pass return html + def markdown(markdown_text): return md_to_html(markdown_text) + def is_subset(list_a: List, list_b: List) -> bool: - '''Returns whether list_a is a subset of list_b''' + """Returns whether list_a is a subset of list_b""" return len(list(set(list_a) & set(list_b))) == len(list_a) + def generate_hash(*args, **kwargs) -> str: return frappe.generate_hash(*args, **kwargs) + def guess_date_format(date_string: str) -> str: DATE_FORMATS = [ r"%d/%b/%y", @@ -1632,11 +1897,12 @@ def guess_date_format(date_string: str) -> str: if " " in date_string: date_str, time_str = date_string.split(" ", 1) - date_format = _get_date_format(date_str) or '' - time_format = _get_time_format(time_str) or '' + date_format = _get_date_format(date_str) or "" + time_format = _get_time_format(time_str) or "" if date_format and time_format: - return (date_format + ' ' + time_format).strip() + return (date_format + " " + time_format).strip() + def validate_json_string(string: str) -> None: try: @@ -1644,30 +1910,23 @@ def validate_json_string(string: str) -> None: except (TypeError, ValueError): raise frappe.ValidationError + def get_user_info_for_avatar(user_id: str) -> Dict: try: user = frappe.get_cached_doc("User", user_id) - return { - "email": user.email, - "image": user.user_image, - "name": user.full_name - } + return {"email": user.email, "image": user.user_image, "name": user.full_name} except frappe.DoesNotExistError: frappe.clear_last_message() - return { - "email": user_id, - "image": "", - "name": user_id - } + return {"email": user_id, "image": "", "name": user_id} def validate_python_code(string: str, fieldname=None, is_expression: bool = True) -> None: - """ Validate python code fields by using compile_command to ensure that expression is valid python. + """Validate python code fields by using compile_command to ensure that expression is valid python. args: - fieldname: name of field being validated. - is_expression: true for validating simple single line python expression, else validated as script. + fieldname: name of field being validated. + is_expression: true for validating simple single line python expression, else validated as script. """ if not string: @@ -1679,15 +1938,18 @@ def validate_python_code(string: str, fieldname=None, is_expression: bool = True line_no = se.lineno - 1 or 0 offset = se.offset - 1 or 0 error_line = string if is_expression else string.split("\n")[line_no] - msg = (frappe._("{} Invalid python code on line {}") - .format(fieldname + ":" if fieldname else "", line_no+1)) + msg = frappe._("{} Invalid python code on line {}").format( + fieldname + ":" if fieldname else "", line_no + 1 + ) msg += f"
{error_line}
" msg += f"
{' ' * offset}^
" frappe.throw(msg, title=frappe._("Syntax Error")) except Exception as e: - frappe.msgprint(frappe._("{} Possibly invalid python code.
{}") - .format(fieldname + ": " or "", str(e)), indicator="orange") + frappe.msgprint( + frappe._("{} Possibly invalid python code.
{}").format(fieldname + ": " or "", str(e)), + indicator="orange", + ) class UnicodeWithAttrs(str): @@ -1715,8 +1977,10 @@ def format_timedelta(o: datetime.timedelta) -> str: def parse_timedelta(s: str) -> datetime.timedelta: # ref: https://stackoverflow.com/a/21074460/10309266 - if 'day' in s: - m = re.match(r"(?P[-\d]+) day[s]*, (?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) + if "day" in s: + m = re.match( + r"(?P[-\d]+) day[s]*, (?P\d+):(?P\d+):(?P\d[\.\d+]*)", s + ) else: m = re.match(r"(?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) diff --git a/frappe/utils/dateutils.py b/frappe/utils/dateutils.py index 83ef046f2d..b8f22e7ed7 100644 --- a/frappe/utils/dateutils.py +++ b/frappe/utils/dateutils.py @@ -1,39 +1,50 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import datetime + import frappe import frappe.defaults -import datetime -from frappe.utils import get_datetime, add_to_date, getdate -from frappe.utils.data import get_first_day, get_first_day_of_week, get_quarter_start, get_year_start,\ - get_last_day, get_last_day_of_week, get_quarter_ending, get_year_ending +from frappe.utils import add_to_date, get_datetime, getdate +from frappe.utils.data import ( + get_first_day, + get_first_day_of_week, + get_last_day, + get_last_day_of_week, + get_quarter_ending, + get_quarter_start, + get_year_ending, + get_year_start, +) # global values -- used for caching dateformats = { - 'yyyy-mm-dd': '%Y-%m-%d', - 'mm/dd/yyyy': '%m/%d/%Y', - 'mm-dd-yyyy': '%m-%d-%Y', + "yyyy-mm-dd": "%Y-%m-%d", + "mm/dd/yyyy": "%m/%d/%Y", + "mm-dd-yyyy": "%m-%d-%Y", "mm/dd/yy": "%m/%d/%y", - 'dd-mmm-yyyy': '%d-%b-%Y', # numbers app format - 'dd/mm/yyyy': '%d/%m/%Y', - 'dd.mm.yyyy': '%d.%m.%Y', - 'dd.mm.yy': '%d.%m.%y', - 'dd-mm-yyyy': '%d-%m-%Y', + "dd-mmm-yyyy": "%d-%b-%Y", # numbers app format + "dd/mm/yyyy": "%d/%m/%Y", + "dd.mm.yyyy": "%d.%m.%Y", + "dd.mm.yy": "%d.%m.%y", + "dd-mm-yyyy": "%d-%m-%Y", "dd/mm/yy": "%d/%m/%y", } + def user_to_str(date, date_format=None): - if not date: return date + if not date: + return date if not date_format: date_format = get_user_date_format() try: - return datetime.datetime.strptime(date, - dateformats[date_format]).strftime('%Y-%m-%d') + return datetime.datetime.strptime(date, dateformats[date_format]).strftime("%Y-%m-%d") except ValueError: raise ValueError("Date %s must be in format %s" % (date, date_format)) + def parse_date(date): """tries to parse given date to system's format i.e. yyyy-mm-dd. returns a string""" parsed_date = None @@ -43,8 +54,9 @@ def parse_date(date): date = date.split(" ")[0] # why the sorting? checking should be done in a predictable order - check_formats = [None] + sorted(list(dateformats), - reverse=not get_user_date_format().startswith("dd")) + check_formats = [None] + sorted( + list(dateformats), reverse=not get_user_date_format().startswith("dd") + ) for f in check_formats: try: @@ -55,26 +67,32 @@ def parse_date(date): pass if not parsed_date: - raise Exception("""Cannot understand date - '%s'. - Try formatting it like your default format - '%s'""" % (date, get_user_date_format()) + raise Exception( + """Cannot understand date - '%s'. + Try formatting it like your default format - '%s'""" + % (date, get_user_date_format()) ) return parsed_date + def get_user_date_format(): if getattr(frappe.local, "user_date_format", None) is None: frappe.local.user_date_format = frappe.defaults.get_global_default("date_format") or "yyyy-mm-dd" return frappe.local.user_date_format + def datetime_in_user_format(date_time): if not date_time: return "" if isinstance(date_time, str): date_time = get_datetime(date_time) from frappe.utils import formatdate + return formatdate(date_time.date()) + " " + date_time.strftime("%H:%M") + def get_dates_from_timegrain(from_date, to_date, timegrain="Daily"): from_date = getdate(from_date) to_date = getdate(to_date) @@ -98,10 +116,13 @@ def get_dates_from_timegrain(from_date, to_date, timegrain="Daily"): if "Weekly" == timegrain: date = get_last_day_of_week(add_to_date(dates[-1], years=years, months=months, days=days)) else: - date = get_period_ending(add_to_date(dates[-1], years=years, months=months, days=days), timegrain) + date = get_period_ending( + add_to_date(dates[-1], years=years, months=months, days=days), timegrain + ) dates.append(date) return dates + def get_from_date_from_timespan(to_date, timespan): days = months = years = 0 if timespan == "Last Week": @@ -114,38 +135,44 @@ def get_from_date_from_timespan(to_date, timespan): years = -1 elif timespan == "All Time": years = -50 - return add_to_date(to_date, years=years, months=months, days=days, - as_datetime=True) + return add_to_date(to_date, years=years, months=months, days=days, as_datetime=True) + -def get_period(date, interval='Monthly'): +def get_period(date, interval="Monthly"): date = getdate(date) months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] return { - 'Daily': date.strftime('%d-%m-%y'), - 'Weekly': date.strftime('%d-%m-%y'), - 'Monthly': str(months[date.month - 1]) + ' ' + str(date.year), - 'Quarterly': 'Quarter ' + str(((date.month-1)//3)+1) + ' ' + str(date.year), - 'Yearly': str(date.year) + "Daily": date.strftime("%d-%m-%y"), + "Weekly": date.strftime("%d-%m-%y"), + "Monthly": str(months[date.month - 1]) + " " + str(date.year), + "Quarterly": "Quarter " + str(((date.month - 1) // 3) + 1) + " " + str(date.year), + "Yearly": str(date.year), }[interval] + def get_period_beginning(date, timegrain, as_str=True): - return getdate({ - 'Daily': date, - 'Weekly': get_first_day_of_week(date), - 'Monthly': get_first_day(date), - 'Quarterly': get_quarter_start(date), - 'Yearly': get_year_start(date) - }[timegrain]) + return getdate( + { + "Daily": date, + "Weekly": get_first_day_of_week(date), + "Monthly": get_first_day(date), + "Quarterly": get_quarter_start(date), + "Yearly": get_year_start(date), + }[timegrain] + ) + def get_period_ending(date, timegrain): date = getdate(date) - if timegrain == 'Daily': + if timegrain == "Daily": return date else: - return getdate({ - 'Daily': date, - 'Weekly': get_last_day_of_week(date), - 'Monthly': get_last_day(date), - 'Quarterly': get_quarter_ending(date), - 'Yearly': get_year_ending(date) - }[timegrain]) + return getdate( + { + "Daily": date, + "Weekly": get_last_day_of_week(date), + "Monthly": get_last_day(date), + "Quarterly": get_quarter_ending(date), + "Yearly": get_year_ending(date), + }[timegrain] + ) diff --git a/frappe/utils/diff.py b/frappe/utils/diff.py index 2574f47fbd..02fa1ef4c7 100644 --- a/frappe/utils/diff.py +++ b/frappe/utils/diff.py @@ -33,9 +33,7 @@ def get_version_diff( def _get_value_from_version(version_name: Union[int, str], fieldname: str): - version = frappe.get_list( - "Version", fields=["data", "modified"], filters={"name": version_name} - ) + version = frappe.get_list("Version", fields=["data", "modified"], filters={"name": version_name}) if version: data = json.loads(version[0].data) changed_fields = data.get("changed", []) diff --git a/frappe/utils/doctor.py b/frappe/utils/doctor.py index 9dafc4dd21..5ad16edee0 100644 --- a/frappe/utils/doctor.py +++ b/frappe/utils/doctor.py @@ -1,7 +1,9 @@ -import frappe.utils from collections import defaultdict -from rq import Worker, Connection -from frappe.utils.background_jobs import get_redis_conn, get_queue, get_queue_list + +from rq import Connection, Worker + +import frappe.utils +from frappe.utils.background_jobs import get_queue, get_queue_list, get_redis_conn from frappe.utils.scheduler import is_scheduler_disabled, is_scheduler_inactive @@ -21,25 +23,25 @@ def purge_pending_jobs(event=None, site=None, queue=None): for queue in get_queue_list(queue): q = get_queue(queue) for job in q.jobs: - if (site and event): - if job.kwargs['site'] == site and job.kwargs['event'] == event: + if site and event: + if job.kwargs["site"] == site and job.kwargs["event"] == event: job.delete() - purged_task_count+=1 + purged_task_count += 1 elif site: - if job.kwargs['site'] == site: + if job.kwargs["site"] == site: job.delete() - purged_task_count+=1 + purged_task_count += 1 elif event: - if job.kwargs['event'] == event: + if job.kwargs["event"] == event: job.delete() - purged_task_count+=1 + purged_task_count += 1 else: purged_task_count += q.count q.empty() - return purged_task_count + def get_jobs_by_queue(site=None): jobs_per_queue = defaultdict(list) job_count = consolidated_methods = {} @@ -47,9 +49,9 @@ def get_jobs_by_queue(site=None): q = get_queue(queue) for job in q.jobs: if not site: - jobs_per_queue[queue].append(job.kwargs.get('method') or job.description) - elif job.kwargs['site'] == site: - jobs_per_queue[queue].append(job.kwargs.get('method') or job.description) + jobs_per_queue[queue].append(job.kwargs.get("method") or job.description) + elif job.kwargs["site"] == site: + jobs_per_queue[queue].append(job.kwargs.get("method") or job.description) consolidated_methods = {} @@ -62,7 +64,6 @@ def get_jobs_by_queue(site=None): job_count[queue] = len(jobs_per_queue[queue]) jobs_per_queue[queue] = consolidated_methods - return jobs_per_queue, job_count @@ -71,18 +72,17 @@ def get_pending_jobs(site=None): for queue in get_queue_list(): q = get_queue(queue) for job in q.jobs: - method_kwargs = job.kwargs['kwargs'] if job.kwargs['kwargs'] else "" - if job.kwargs['site'] == site: - jobs_per_queue[queue].append("{0} {1}". - format(job.kwargs['method'], method_kwargs)) + method_kwargs = job.kwargs["kwargs"] if job.kwargs["kwargs"] else "" + if job.kwargs["site"] == site: + jobs_per_queue[queue].append("{0} {1}".format(job.kwargs["method"], method_kwargs)) return jobs_per_queue - def check_number_of_workers(): return len(get_workers()) + def get_running_tasks(): for worker in get_workers(): return worker.get_current_job() @@ -134,11 +134,11 @@ def doctor(site=None): return True + def pending_jobs(site=None): print("-----Pending Jobs-----") pending_jobs = get_pending_jobs(site) for queue in get_queue_list(): - if(pending_jobs[queue]): + if pending_jobs[queue]: print("-----Queue :{0}-----".format(queue)) print("\n".join(pending_jobs[queue])) - diff --git a/frappe/utils/error.py b/frappe/utils/error.py index ba0fbf1605..26a05e9443 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -2,19 +2,19 @@ # Copyright (c) 2015, Maxwell Morais and contributors # License: MIT. See LICENSE +import cgitb +import datetime +import functools +import inspect +import json +import linecache import os +import pydoc import sys import traceback -import functools import frappe from frappe.utils import cstr, encode -import inspect -import linecache -import pydoc -import cgitb -import datetime -import json def make_error_snapshot(exception): @@ -24,10 +24,10 @@ def make_error_snapshot(exception): logger = frappe.logger(with_more_info=True) try: - error_id = '{timestamp:s}-{ip:s}-{hash:s}'.format( + error_id = "{timestamp:s}-{ip:s}-{hash:s}".format( timestamp=cstr(datetime.datetime.now()), - ip=frappe.local.request_ip or '127.0.0.1', - hash=frappe.generate_hash(length=3) + ip=frappe.local.request_ip or "127.0.0.1", + hash=frappe.generate_hash(length=3), ) snapshot_folder = get_error_snapshot_path() frappe.create_folder(snapshot_folder) @@ -35,13 +35,14 @@ def make_error_snapshot(exception): snapshot_file_path = os.path.join(snapshot_folder, "{0}.json".format(error_id)) snapshot = get_snapshot(exception) - with open(encode(snapshot_file_path), 'wb') as error_file: + with open(encode(snapshot_file_path), "wb") as error_file: error_file.write(encode(frappe.as_json(snapshot))) - logger.error('New Exception collected with id: {}'.format(error_id)) + logger.error("New Exception collected with id: {}".format(error_id)) except Exception as e: - logger.error('Could not take error snapshot: {0}'.format(e), exc_info=True) + logger.error("Could not take error snapshot: {0}".format(e), exc_info=True) + def get_snapshot(exception, context=10): """ @@ -55,33 +56,33 @@ def get_snapshot(exception, context=10): # creates a snapshot dict with some basic information s = { - 'pyver': 'Python {version:s}: {executable:s} (prefix: {prefix:s})'.format( - version = sys.version.split()[0], - executable = sys.executable, - prefix = sys.prefix + "pyver": "Python {version:s}: {executable:s} (prefix: {prefix:s})".format( + version=sys.version.split()[0], executable=sys.executable, prefix=sys.prefix ), - 'timestamp': cstr(datetime.datetime.now()), - 'traceback': traceback.format_exc(), - 'frames': [], - 'etype': cstr(etype), - 'evalue': cstr(repr(evalue)), - 'exception': {}, - 'locals': {} + "timestamp": cstr(datetime.datetime.now()), + "traceback": traceback.format_exc(), + "frames": [], + "etype": cstr(etype), + "evalue": cstr(repr(evalue)), + "exception": {}, + "locals": {}, } # start to process frames records = inspect.getinnerframes(etb, 5) for frame, file, lnum, func, lines, index in records: - file = file and os.path.abspath(file) or '?' + file = file and os.path.abspath(file) or "?" args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' + call = "" - if func != '?': - call = inspect.formatargvalues(args, varargs, varkw, locals, formatvalue=lambda value: '={}'.format(pydoc.text.repr(value))) + if func != "?": + call = inspect.formatargvalues( + args, varargs, varkw, locals, formatvalue=lambda value: "={}".format(pydoc.text.repr(value)) + ) # basic frame information - f = {'file': file, 'func': func, 'call': call, 'lines': {}, 'lnum': lnum} + f = {"file": file, "func": func, "call": call, "lines": {}, "lnum": lnum} def reader(lnum=[lnum]): try: @@ -101,38 +102,39 @@ def get_snapshot(exception, context=10): if index is not None: i = lnum - index for line in lines: - f['lines'][i] = line.rstrip() + f["lines"][i] = line.rstrip() i += 1 # dump local variable (referenced in current line only) - f['dump'] = {} + f["dump"] = {} for name, where, value in vars: - if name in f['dump']: + if name in f["dump"]: continue if value is not cgitb.__UNDEF__: - if where == 'global': - name = 'global {name:s}'.format(name=name) - elif where != 'local': - name = where + ' ' + name.split('.')[-1] - f['dump'][name] = pydoc.text.repr(value) + if where == "global": + name = "global {name:s}".format(name=name) + elif where != "local": + name = where + " " + name.split(".")[-1] + f["dump"][name] = pydoc.text.repr(value) else: - f['dump'][name] = 'undefined' + f["dump"][name] = "undefined" - s['frames'].append(f) + s["frames"].append(f) # add exception type, value and attributes if isinstance(evalue, BaseException): for name in dir(evalue): - if name != 'messages' and not name.startswith('__'): + if name != "messages" and not name.startswith("__"): value = pydoc.text.repr(getattr(evalue, name)) - s['exception'][name] = encode(value) + s["exception"][name] = encode(value) # add all local values (of last frame) to the snapshot for name, value in locals.items(): - s['locals'][name] = value if isinstance(value, str) else pydoc.text.repr(value) + s["locals"][name] = value if isinstance(value, str) else pydoc.text.repr(value) return s + def collect_error_snapshots(): """Scheduled task to collect error snapshots from files and push into Error Snapshot table""" if frappe.conf.disable_error_snapshot: @@ -147,7 +149,7 @@ def collect_error_snapshots(): fullpath = os.path.join(path, fname) try: - with open(fullpath, 'r') as filedata: + with open(fullpath, "r") as filedata: data = json.load(filedata) except ValueError: @@ -155,10 +157,10 @@ def collect_error_snapshots(): os.remove(fullpath) continue - for field in ['locals', 'exception', 'frames']: + for field in ["locals", "exception", "frames"]: data[field] = frappe.as_json(data[field]) - doc = frappe.new_doc('Error Snapshot') + doc = frappe.new_doc("Error Snapshot") doc.update(data) doc.save() @@ -174,15 +176,14 @@ def collect_error_snapshots(): # prevent creation of unlimited error snapshots raise + def clear_old_snapshots(): """Clear snapshots that are older than a month""" from frappe.query_builder import DocType, Interval from frappe.query_builder.functions import Now ErrorSnapshot = DocType("Error Snapshot") - frappe.db.delete(ErrorSnapshot, filters=( - ErrorSnapshot.creation < (Now() - Interval(months=1)) - )) + frappe.db.delete(ErrorSnapshot, filters=(ErrorSnapshot.creation < (Now() - Interval(months=1)))) path = get_error_snapshot_path() today = datetime.datetime.now() @@ -193,15 +194,18 @@ def clear_old_snapshots(): if (today - ctime).days > 31: os.remove(os.path.join(path, p)) + def get_error_snapshot_path(): - return frappe.get_site_path('error-snapshots') + return frappe.get_site_path("error-snapshots") + def get_default_args(func): - """Get default arguments of a function from its signature. - """ + """Get default arguments of a function from its signature.""" signature = inspect.signature(func) - return {k: v.default - for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty} + return { + k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty + } + def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None): """Decorate any function to throw error incase of missing output. @@ -221,6 +225,7 @@ def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None): >>> get_ingradients() `Exception Name`: Ingradients missing """ + def decorator_raise_error_on_no_output(func): @functools.wraps(func) def wrapper_raise_error_on_no_output(*args, **kwargs): @@ -229,11 +234,13 @@ def raise_error_on_no_output(error_message, error_type=None, keep_quiet=None): return response default_kwargs = get_default_args(func) - default_raise_error = default_kwargs.get('_raise_error') - raise_error = kwargs.get('_raise_error') if '_raise_error' in kwargs else default_raise_error + default_raise_error = default_kwargs.get("_raise_error") + raise_error = kwargs.get("_raise_error") if "_raise_error" in kwargs else default_raise_error if (not response) and raise_error: frappe.throw(error_message, error_type or Exception) return response + return wrapper_raise_error_on_no_output + return decorator_raise_error_on_no_output diff --git a/frappe/utils/file_lock.py b/frappe/utils/file_lock.py index 9ffe5150fc..fc399f7c96 100644 --- a/frappe/utils/file_lock.py +++ b/frappe/utils/file_lock.py @@ -1,29 +1,34 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -''' +""" File based locking utility -''' +""" import os from time import time + from frappe.utils import get_site_path, touch_file + class LockTimeoutError(Exception): - pass + pass + def create_lock(name): - '''Creates a file in the /locks folder by the given name''' + """Creates a file in the /locks folder by the given name""" lock_path = get_lock_path(name) if not check_lock(lock_path): return touch_file(lock_path) else: return False + def lock_exists(name): - '''Returns True if lock of the given name exists''' + """Returns True if lock of the given name exists""" return os.path.exists(get_lock_path(name)) + def check_lock(path, timeout=600): if not os.path.exists(path): return False @@ -31,6 +36,7 @@ def check_lock(path, timeout=600): raise LockTimeoutError(path) return True + def delete_lock(name): lock_path = get_lock_path(name) try: @@ -39,8 +45,9 @@ def delete_lock(name): pass return True + def get_lock_path(name): name = name.lower() - locks_dir = 'locks' - lock_path = get_site_path(locks_dir, name + '.lock') + locks_dir = "locks" + lock_path = get_site_path(locks_dir, name + ".lock") return lock_path diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index ce5985f619..67c51864d3 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -1,19 +1,31 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import os, base64, re, json +import base64 import hashlib -import mimetypes import io -from frappe.query_builder.utils import DocType -from frappe.utils import get_hook_method, get_files_path, random_string, encode, cstr, call_hook_method, cint -from frappe import _ -from frappe import conf +import json +import mimetypes +import os +import re from copy import copy from urllib.parse import unquote + +import frappe +from frappe import _, conf +from frappe.query_builder.utils import DocType +from frappe.utils import ( + call_hook_method, + cint, + cstr, + encode, + get_files_path, + get_hook_method, + random_string, +) from frappe.utils.image import optimize_image + class MaxFileSizeReachedError(frappe.ValidationError): pass @@ -26,8 +38,8 @@ def safe_b64decode(binary: bytes) -> bytes: be an indication of corrupted data. Refs: - * https://en.wikipedia.org/wiki/Base64 - * https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding + * https://en.wikipedia.org/wiki/Base64 + * https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding """ return base64.b64decode(binary + b"===") @@ -46,39 +58,50 @@ def upload(): frappe.form_dict.is_private = cint(frappe.form_dict.is_private) if not filename and not file_url: - frappe.msgprint(_("Please select a file or url"), - raise_exception=True) + frappe.msgprint(_("Please select a file or url"), raise_exception=True) file_doc = get_file_doc() comment = {} if dt and dn: - comment = frappe.get_doc(dt, dn).add_comment("Attachment", - _("added {0}").format("{file_name}{icon}".format(**{ - "icon": ' ' \ - if file_doc.is_private else "", - "file_url": file_doc.file_url.replace("#", "%23") \ - if file_doc.file_name else file_doc.file_url, - "file_name": file_doc.file_name or file_doc.file_url - }))) + comment = frappe.get_doc(dt, dn).add_comment( + "Attachment", + _("added {0}").format( + "{file_name}{icon}".format( + **{ + "icon": ' ' if file_doc.is_private else "", + "file_url": file_doc.file_url.replace("#", "%23") + if file_doc.file_name + else file_doc.file_url, + "file_name": file_doc.file_name or file_doc.file_url, + } + ) + ), + ) return { "name": file_doc.name, "file_name": file_doc.file_name, "file_url": file_doc.file_url, "is_private": file_doc.is_private, - "comment": comment.as_dict() if comment else {} + "comment": comment.as_dict() if comment else {}, } + def get_file_doc(dt=None, dn=None, folder=None, is_private=None, df=None): - '''returns File object (Document) from given parameters or form_dict''' + """returns File object (Document) from given parameters or form_dict""" r = frappe.form_dict - if dt is None: dt = r.doctype - if dn is None: dn = r.docname - if df is None: df = r.docfield - if folder is None: folder = r.folder - if is_private is None: is_private = r.is_private + if dt is None: + dt = r.doctype + if dn is None: + dn = r.docname + if df is None: + df = r.docfield + if folder is None: + folder = r.folder + if is_private is None: + is_private = r.is_private if r.filedata: file_doc = save_uploaded(dt, dn, folder, is_private, df) @@ -88,6 +111,7 @@ def get_file_doc(dt=None, dn=None, folder=None, is_private=None, df=None): return file_doc + def save_uploaded(dt, dn, folder, is_private, df=None): fname, content = get_uploaded_content() if content: @@ -95,6 +119,7 @@ def save_uploaded(dt, dn, folder, is_private, df=None): else: raise Exception + def save_url(file_url, filename, dt, dn, folder, is_private, df=None): # if not (file_url.startswith("http://") or file_url.startswith("https://")): # frappe.msgprint("URL must start with 'http://' or 'https://'") @@ -103,17 +128,19 @@ def save_url(file_url, filename, dt, dn, folder, is_private, df=None): file_url = unquote(file_url) file_size = frappe.form_dict.file_size - f = frappe.get_doc({ - "doctype": "File", - "file_url": file_url, - "file_name": filename, - "attached_to_doctype": dt, - "attached_to_name": dn, - "attached_to_field": df, - "folder": folder, - "file_size": file_size, - "is_private": is_private - }) + f = frappe.get_doc( + { + "doctype": "File", + "file_url": file_url, + "file_name": filename, + "attached_to_doctype": dt, + "attached_to_name": dn, + "attached_to_field": df, + "folder": folder, + "file_size": file_size, + "is_private": is_private, + } + ) f.flags.ignore_permissions = True try: f.insert() @@ -124,16 +151,17 @@ def save_url(file_url, filename, dt, dn, folder, is_private, df=None): def get_uploaded_content(): # should not be unicode when reading a file, hence using frappe.form - if 'filedata' in frappe.form_dict: + if "filedata" in frappe.form_dict: if "," in frappe.form_dict.filedata: frappe.form_dict.filedata = frappe.form_dict.filedata.rsplit(",", 1)[1] frappe.uploaded_content = safe_b64decode(frappe.form_dict.filedata) frappe.uploaded_filename = frappe.form_dict.filename return frappe.uploaded_filename, frappe.uploaded_content else: - frappe.msgprint(_('No file attached')) + frappe.msgprint(_("No file attached")) return None, None + def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, df=None): if decode: if isinstance(content, str): @@ -151,20 +179,22 @@ def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, d if not file_data: call_hook_method("before_write_file", file_size=file_size) - write_file_method = get_hook_method('write_file', fallback=save_file_on_filesystem) + write_file_method = get_hook_method("write_file", fallback=save_file_on_filesystem) file_data = write_file_method(fname, content, content_type=content_type, is_private=is_private) file_data = copy(file_data) - file_data.update({ - "doctype": "File", - "attached_to_doctype": dt, - "attached_to_name": dn, - "attached_to_field": df, - "folder": folder, - "file_size": file_size, - "content_hash": content_hash, - "is_private": is_private - }) + file_data.update( + { + "doctype": "File", + "attached_to_doctype": dt, + "attached_to_name": dn, + "attached_to_field": df, + "folder": folder, + "file_size": file_size, + "content_hash": content_hash, + "is_private": is_private, + } + ) f = frappe.get_doc(file_data) f.flags.ignore_permissions = True @@ -177,9 +207,11 @@ def save_file(fname, content, dt, dn, folder=None, decode=False, is_private=0, d def get_file_data_from_hash(content_hash, is_private=0): - for name in frappe.get_all("File", {"content_hash": content_hash, "is_private": is_private}, pluck="name"): - b = frappe.get_doc('File', name) - return {k: b.get(k) for k in frappe.get_hooks()['write_file_keys']} + for name in frappe.get_all( + "File", {"content_hash": content_hash, "is_private": is_private}, pluck="name" + ): + b = frappe.get_doc("File", name) + return {k: b.get(k) for k in frappe.get_hooks()["write_file_keys"]} return False @@ -191,14 +223,11 @@ def save_file_on_filesystem(fname, content, content_type=None, is_private=0): else: file_url = "/files/{0}".format(fname) - return { - 'file_name': os.path.basename(fpath), - 'file_url': file_url - } + return {"file_name": os.path.basename(fpath), "file_url": file_url} def get_max_file_size(): - return conf.get('max_file_size') or 10485760 + return conf.get("max_file_size") or 10485760 def check_max_file_size(content): @@ -206,9 +235,10 @@ def check_max_file_size(content): file_size = len(content) if file_size > max_file_size: - frappe.msgprint(_("File size exceeded the maximum allowed size of {0} MB").format( - max_file_size / 1048576), - raise_exception=MaxFileSizeReachedError) + frappe.msgprint( + _("File size exceeded the maximum allowed size of {0} MB").format(max_file_size / 1048576), + raise_exception=MaxFileSizeReachedError, + ) return file_size @@ -222,7 +252,7 @@ def write_file(content, fname, is_private=0): # write the file if isinstance(content, str): content = content.encode() - with open(os.path.join(file_path.encode('utf-8'), fname.encode('utf-8')), 'wb+') as f: + with open(os.path.join(file_path.encode("utf-8"), fname.encode("utf-8")), "wb+") as f: f.write(content) return get_files_path(fname, is_private=is_private) @@ -231,23 +261,39 @@ def write_file(content, fname, is_private=0): def remove_all(dt, dn, from_delete=False, delete_permanently=False): """remove all files in a transaction""" try: - for fid in frappe.get_all("File", {"attached_to_doctype": dt, "attached_to_name": dn}, pluck="name"): + for fid in frappe.get_all( + "File", {"attached_to_doctype": dt, "attached_to_name": dn}, pluck="name" + ): if from_delete: # If deleting a doc, directly delete files frappe.delete_doc("File", fid, ignore_permissions=True, delete_permanently=delete_permanently) else: # Removes file and adds a comment in the document it is attached to - remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn, - from_delete=from_delete, delete_permanently=delete_permanently) + remove_file( + fid=fid, + attached_to_doctype=dt, + attached_to_name=dn, + from_delete=from_delete, + delete_permanently=delete_permanently, + ) except Exception as e: - if e.args[0]!=1054: raise # (temp till for patched) + if e.args[0] != 1054: + raise # (temp till for patched) -def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_delete=False, delete_permanently=False): + +def remove_file( + fid=None, + attached_to_doctype=None, + attached_to_name=None, + from_delete=False, + delete_permanently=False, +): """Remove file and File entry""" file_name = None if not (attached_to_doctype and attached_to_name): - attached = frappe.db.get_value("File", fid, - ["attached_to_doctype", "attached_to_name", "file_name"]) + attached = frappe.db.get_value( + "File", fid, ["attached_to_doctype", "attached_to_name", "file_name"] + ) if attached: attached_to_doctype, attached_to_name, file_name = attached @@ -260,13 +306,15 @@ def remove_file(fid=None, attached_to_doctype=None, attached_to_name=None, from_ if not file_name: file_name = frappe.db.get_value("File", fid, "file_name") comment = doc.add_comment("Attachment Removed", _("Removed {0}").format(file_name)) - frappe.delete_doc("File", fid, ignore_permissions=ignore_permissions, delete_permanently=delete_permanently) + frappe.delete_doc( + "File", fid, ignore_permissions=ignore_permissions, delete_permanently=delete_permanently + ) return comment def delete_file_data_content(doc, only_thumbnail=False): - method = get_hook_method('delete_file_data_content', fallback=delete_file_from_filesystem) + method = get_hook_method("delete_file_data_content", fallback=delete_file_from_filesystem) method(doc, only_thumbnail=only_thumbnail) @@ -283,10 +331,12 @@ def delete_file(path): """Delete file from `public folder`""" if path: if ".." in path.split("/"): - frappe.msgprint(_("It is risky to delete this file: {0}. Please contact your System Manager.").format(path)) + frappe.msgprint( + _("It is risky to delete this file: {0}. Please contact your System Manager.").format(path) + ) parts = os.path.split(path.strip("/")) - if parts[0]=="files": + if parts[0] == "files": path = frappe.utils.get_site_path("public", "files", parts[-1]) else: @@ -302,7 +352,7 @@ def get_file(fname): file_path = get_file_path(fname) # read the file - with io.open(encode(file_path), mode='rb') as f: + with io.open(encode(file_path), mode="rb") as f: content = f.read() try: # for plain text files @@ -316,12 +366,17 @@ def get_file(fname): def get_file_path(file_name): """Returns file path from given file name""" - if '../' in file_name: + if "../" in file_name: return File = DocType("File") - f = frappe.qb.from_(File).where((File.name == file_name) | (File.file_name == file_name)).select(File.file_url).run() + f = ( + frappe.qb.from_(File) + .where((File.name == file_name) | (File.file_name == file_name)) + .select(File.file_url) + .run() + ) if f: file_name = f[0][0] @@ -355,12 +410,12 @@ def get_file_name(fname, optional_suffix): n_records = frappe.get_all("File", {"file_name": fname}, pluck="name") if len(n_records) > 0 or os.path.exists(encode(get_files_path(fname))): - f = fname.rsplit('.', 1) + f = fname.rsplit(".", 1) if len(f) == 1: partial, extn = f[0], "" else: partial, extn = f[0], "." + f[1] - return '{partial}{suffix}{extn}'.format(partial=partial, extn=extn, suffix=optional_suffix) + return "{partial}{suffix}{extn}".format(partial=partial, extn=extn, suffix=optional_suffix) return fname @@ -374,7 +429,7 @@ def download_file(file_url): Endpoint : frappe.utils.file_manager.download_file URL Params : file_name = /path/to/file relative to site path """ - file_doc = frappe.get_doc("File", {"file_url":file_url}) + file_doc = frappe.get_doc("File", {"file_url": file_url}) file_doc.check_permission("read") path = os.path.join(get_files_path(), os.path.basename(file_url)) @@ -384,18 +439,23 @@ def download_file(file_url): frappe.local.response.filecontent = filedata frappe.local.response.type = "download" + @frappe.whitelist() def add_attachments(doctype, name, attachments): - '''Add attachments to the given DocType''' + """Add attachments to the given DocType""" if isinstance(attachments, str): attachments = json.loads(attachments) # loop through attachments - files =[] + files = [] for a in attachments: if isinstance(a, str): - attach = frappe.db.get_value("File", {"name":a}, ["file_name", "file_url", "is_private"], as_dict=1) + attach = frappe.db.get_value( + "File", {"name": a}, ["file_name", "file_url", "is_private"], as_dict=1 + ) # save attachments to new doc - f = save_url(attach.file_url, attach.file_name, doctype, name, "Home/Attachments", attach.is_private) + f = save_url( + attach.file_url, attach.file_name, doctype, name, "Home/Attachments", attach.is_private + ) files.append(f) return files diff --git a/frappe/utils/fixtures.py b/frappe/utils/fixtures.py index fe05a4dc30..f00d310c9d 100644 --- a/frappe/utils/fixtures.py +++ b/frappe/utils/fixtures.py @@ -35,8 +35,7 @@ def import_custom_scripts(app): if os.path.exists(frappe.get_app_path(app, "fixtures", "custom_scripts")): for fname in os.listdir(frappe.get_app_path(app, "fixtures", "custom_scripts")): if fname.endswith(".js"): - with open(frappe.get_app_path(app, "fixtures", - "custom_scripts") + os.path.sep + fname) as f: + with open(frappe.get_app_path(app, "fixtures", "custom_scripts") + os.path.sep + fname) as f: doctype = fname.rsplit(".", 1)[0] script = f.read() if frappe.db.exists("Client Script", {"dt": doctype}): @@ -44,11 +43,7 @@ def import_custom_scripts(app): custom_script.script = script custom_script.save() else: - frappe.get_doc({ - "doctype": "Client Script", - "dt": doctype, - "script": script - }).insert() + frappe.get_doc({"doctype": "Client Script", "dt": doctype, "script": script}).insert() def export_fixtures(app=None): @@ -65,9 +60,16 @@ def export_fixtures(app=None): filters = fixture.get("filters") or_filters = fixture.get("or_filters") fixture = fixture.get("doctype") or fixture.get("dt") - print("Exporting {0} app {1} filters {2}".format(fixture, app, (filters if filters else or_filters))) + print( + "Exporting {0} app {1} filters {2}".format(fixture, app, (filters if filters else or_filters)) + ) if not os.path.exists(frappe.get_app_path(app, "fixtures")): os.mkdir(frappe.get_app_path(app, "fixtures")) - export_json(fixture, frappe.get_app_path(app, "fixtures", frappe.scrub(fixture) + ".json"), - filters=filters, or_filters=or_filters, order_by="idx asc, creation asc") + export_json( + fixture, + frappe.get_app_path(app, "fixtures", frappe.scrub(fixture) + ".json"), + filters=filters, + or_filters=or_filters, + order_by="idx asc, creation asc", + ) diff --git a/frappe/utils/formatters.py b/frappe/utils/formatters.py index ae925a0ab2..adf551580c 100644 --- a/frappe/utils/formatters.py +++ b/frappe/utils/formatters.py @@ -1,36 +1,48 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import datetime -from frappe.utils import formatdate, fmt_money, flt, cstr, cint, format_datetime, format_time, format_duration, format_timedelta -from frappe.model.meta import get_field_currency, get_field_precision import re + from dateutil.parser import ParserError +import frappe +from frappe.model.meta import get_field_currency, get_field_precision +from frappe.utils import ( + cint, + cstr, + flt, + fmt_money, + format_datetime, + format_duration, + format_time, + format_timedelta, + formatdate, +) + def format_value(value, df=None, doc=None, currency=None, translated=False, format=None): - '''Format value based on given fieldtype, document reference, currency reference. - If docfield info (df) is not given, it will try and guess based on the datatype of the value''' + """Format value based on given fieldtype, document reference, currency reference. + If docfield info (df) is not given, it will try and guess based on the datatype of the value""" if isinstance(df, str): df = frappe._dict(fieldtype=df) if not df: df = frappe._dict() if isinstance(value, datetime.datetime): - df.fieldtype = 'Datetime' + df.fieldtype = "Datetime" elif isinstance(value, datetime.date): - df.fieldtype = 'Date' + df.fieldtype = "Date" elif isinstance(value, datetime.timedelta): - df.fieldtype = 'Time' + df.fieldtype = "Time" elif isinstance(value, int): - df.fieldtype = 'Int' + df.fieldtype = "Int" elif isinstance(value, float): - df.fieldtype = 'Float' + df.fieldtype = "Float" else: - df.fieldtype = 'Data' + df.fieldtype = "Data" - elif (isinstance(df, dict)): + elif isinstance(df, dict): # Convert dict to object if necessary df = frappe._dict(df) @@ -42,19 +54,23 @@ def format_value(value, df=None, doc=None, currency=None, translated=False, form if not df: return value - elif df.get("fieldtype")=="Date": + elif df.get("fieldtype") == "Date": return formatdate(value) - elif df.get("fieldtype")=="Datetime": + elif df.get("fieldtype") == "Datetime": return format_datetime(value) - elif df.get("fieldtype")=="Time": + elif df.get("fieldtype") == "Time": try: return format_time(value) except ParserError: return format_timedelta(value) - elif value==0 and df.get("fieldtype") in ("Int", "Float", "Currency", "Percent") and df.get("print_hide_if_no_value"): + elif ( + value == 0 + and df.get("fieldtype") in ("Int", "Float", "Currency", "Percent") + and df.get("print_hide_if_no_value") + ): # this is required to show 0 as blank in table columns return "" @@ -72,7 +88,7 @@ def format_value(value, df=None, doc=None, currency=None, translated=False, form # options should not specified if not df.options and value is not None: temp = cstr(value).split(".") - if len(temp)==1 or cint(temp[1])==0: + if len(temp) == 1 or cint(temp[1]) == 0: precision = 0 return fmt_money(value, precision=precision, currency=currency) @@ -90,13 +106,13 @@ def format_value(value, df=None, doc=None, currency=None, translated=False, form elif df.get("fieldtype") == "Table MultiSelect": values = [] meta = frappe.get_meta(df.options) - link_field = [df for df in meta.fields if df.fieldtype == 'Link'][0] + link_field = [df for df in meta.fields if df.fieldtype == "Link"][0] for v in value: - v.update({'__link_titles': doc.get('__link_titles')}) - formatted_value = frappe.format_value(v.get(link_field.fieldname, ''), link_field, v) + v.update({"__link_titles": doc.get("__link_titles")}) + formatted_value = frappe.format_value(v.get(link_field.fieldname, ""), link_field, v) values.append(formatted_value) - return ', '.join(values) + return ", ".join(values) elif df.get("fieldtype") == "Duration": hide_days = df.hide_days diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index f2bc9946a1..b121a9b46f 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -1,15 +1,17 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import re -import redis import json import os -from frappe.utils import cint, strip_html_tags -from frappe.utils.html_utils import unescape_html +import re + +import redis + +import frappe from frappe.model.base_document import get_controller +from frappe.utils import cint, strip_html_tags from frappe.utils.data import cstr +from frappe.utils.html_utils import unescape_html def setup_global_search_table(): @@ -27,18 +29,20 @@ def reset(): """ frappe.db.delete("__global_search") + def get_doctypes_with_global_search(with_child_tables=True): """ Return doctypes with global search fields :param with_child_tables: :return: """ + def _get(): global_search_doctypes = [] filters = {} if not with_child_tables: filters = {"istable": ["!=", 1], "issingle": ["!=", 1]} - for d in frappe.get_all('DocType', fields=['name', 'module'], filters=filters): + for d in frappe.get_all("DocType", fields=["name", "module"], filters=filters): meta = frappe.get_meta(d.name) if len(meta.get_global_search_fields()) > 0: global_search_doctypes.append(d) @@ -47,14 +51,15 @@ def get_doctypes_with_global_search(with_child_tables=True): module_app = frappe.local.module_app doctypes = [ - d.name for d in global_search_doctypes + d.name + for d in global_search_doctypes if module_app.get(frappe.scrub(d.module)) and module_app[frappe.scrub(d.module)] in installed_apps ] return doctypes - return frappe.cache().get_value('doctypes_with_global_search', _get) + return frappe.cache().get_value("doctypes_with_global_search", _get) def rebuild_for_doctype(doctype): @@ -63,14 +68,14 @@ def rebuild_for_doctype(doctype): searchable fields :param doctype: Doctype """ - if frappe.local.conf.get('disable_global_search'): + if frappe.local.conf.get("disable_global_search"): return - if frappe.local.conf.get('disable_global_search'): + if frappe.local.conf.get("disable_global_search"): return def _get_filters(): - filters = frappe._dict({ "docstatus": ["!=", 2] }) + filters = frappe._dict({"docstatus": ["!=", 2]}) if meta.has_field("enabled"): filters.enabled = 1 if meta.has_field("disabled"): @@ -84,10 +89,11 @@ def rebuild_for_doctype(doctype): return if cint(meta.istable) == 1: - parent_doctypes = frappe.get_all("DocField", fields="parent", filters={ - "fieldtype": ["in", frappe.model.table_fields], - "options": doctype - }) + parent_doctypes = frappe.get_all( + "DocField", + fields="parent", + filters={"fieldtype": ["in", frappe.model.table_fields], "options": doctype}, + ) for p in parent_doctypes: rebuild_for_doctype(p.parent) @@ -134,26 +140,27 @@ def rebuild_for_doctype(doctype): # some doctypes has been deleted via future patch, hence controller does not exists pass - all_contents.append({ - "doctype": frappe.db.escape(doctype), - "name": frappe.db.escape(doc.name), - "content": frappe.db.escape(' ||| '.join(content or '')), - "published": published, - "title": frappe.db.escape((title or '')[:int(frappe.db.VARCHAR_LEN)]), - "route": frappe.db.escape((route or '')[:int(frappe.db.VARCHAR_LEN)]) - }) + all_contents.append( + { + "doctype": frappe.db.escape(doctype), + "name": frappe.db.escape(doc.name), + "content": frappe.db.escape(" ||| ".join(content or "")), + "published": published, + "title": frappe.db.escape((title or "")[: int(frappe.db.VARCHAR_LEN)]), + "route": frappe.db.escape((route or "")[: int(frappe.db.VARCHAR_LEN)]), + } + ) if all_contents: insert_values_for_multiple_docs(all_contents) def delete_global_search_records_for_doctype(doctype): - frappe.db.delete("__global_search", { - "doctype": doctype - }) + frappe.db.delete("__global_search", {"doctype": doctype}) + def get_selected_fields(meta, global_search_fields): fieldnames = [df.fieldname for df in global_search_fields] - if meta.istable==1: + if meta.istable == 1: fieldnames.append("parent") elif "name" not in fieldnames: fieldnames.append("name") @@ -166,18 +173,18 @@ def get_selected_fields(meta, global_search_fields): def get_children_data(doctype, meta): """ - Get all records from all the child tables of a doctype - - all_children = { - "parent1": { - "child_doctype1": [ - { - "field1": val1, - "field2": val2 - } - ] - } - } + Get all records from all the child tables of a doctype + + all_children = { + "parent1": { + "child_doctype1": [ + { + "field1": val1, + "field2": val2 + } + ] + } + } """ all_children = frappe._dict() @@ -189,14 +196,14 @@ def get_children_data(doctype, meta): if search_fields: child_search_fields.setdefault(child.options, search_fields) child_fieldnames = get_selected_fields(child_meta, search_fields) - child_records = frappe.get_all(child.options, fields=child_fieldnames, filters={ - "docstatus": ["!=", 1], - "parenttype": doctype - }) + child_records = frappe.get_all( + child.options, fields=child_fieldnames, filters={"docstatus": ["!=", 1], "parenttype": doctype} + ) for record in child_records: - all_children.setdefault(record.parent, frappe._dict())\ - .setdefault(child.options, []).append(record) + all_children.setdefault(record.parent, frappe._dict()).setdefault(child.options, []).append( + record + ) return all_children, child_search_fields @@ -204,22 +211,27 @@ def get_children_data(doctype, meta): def insert_values_for_multiple_docs(all_contents): values = [] for content in all_contents: - values.append("({doctype}, {name}, {content}, {published}, {title}, {route})" - .format(**content)) + values.append("({doctype}, {name}, {content}, {published}, {title}, {route})".format(**content)) batch_size = 50000 for i in range(0, len(values), batch_size): - batch_values = values[i:i + batch_size] + batch_values = values[i : i + batch_size] # ignoring duplicate keys for doctype_name - frappe.db.multisql({ - 'mariadb': '''INSERT IGNORE INTO `__global_search` + frappe.db.multisql( + { + "mariadb": """INSERT IGNORE INTO `__global_search` (doctype, name, content, published, title, route) - VALUES {0} '''.format(", ".join(batch_values)), - 'postgres': '''INSERT INTO `__global_search` + VALUES {0} """.format( + ", ".join(batch_values) + ), + "postgres": """INSERT INTO `__global_search` (doctype, name, content, published, title, route) VALUES {0} - ON CONFLICT("name", "doctype") DO NOTHING'''.format(", ".join(batch_values)) - }) + ON CONFLICT("name", "doctype") DO NOTHING""".format( + ", ".join(batch_values) + ), + } + ) def update_global_search(doc): @@ -228,12 +240,15 @@ def update_global_search(doc): `global_search_queue` from given doc :param doc: Document to be added to global search """ - if frappe.local.conf.get('disable_global_search'): + if frappe.local.conf.get("disable_global_search"): return - if doc.docstatus > 1 or (doc.meta.has_field("enabled") and not doc.get("enabled")) \ - or doc.get("disabled"): - return + if ( + doc.docstatus > 1 + or (doc.meta.has_field("enabled") and not doc.get("enabled")) + or doc.get("disabled") + ): + return content = [] for field in doc.meta.get_global_search_fields(): @@ -250,28 +265,29 @@ def update_global_search(doc): if content: published = 0 - if hasattr(doc, 'is_website_published') and doc.meta.allow_guest_to_view: + if hasattr(doc, "is_website_published") and doc.meta.allow_guest_to_view: published = 1 if doc.is_website_published() else 0 - title = (cstr(doc.get_title()) or '')[:int(frappe.db.VARCHAR_LEN)] - route = doc.get('route') if doc else '' + title = (cstr(doc.get_title()) or "")[: int(frappe.db.VARCHAR_LEN)] + route = doc.get("route") if doc else "" value = dict( doctype=doc.doctype, name=doc.name, - content=' ||| '.join(content or ''), + content=" ||| ".join(content or ""), published=published, title=title, - route=route + route=route, ) sync_value_in_queue(value) + def update_global_search_for_all_web_pages(): - if frappe.conf.get('disable_global_search'): + if frappe.conf.get("disable_global_search"): return - print('Update global search for all web pages...') + print("Update global search for all web pages...") routes_to_index = get_routes_to_index() for route in routes_to_index: add_route_to_global_search(route) @@ -283,19 +299,19 @@ def get_routes_to_index(): routes_to_index = [] for app in apps: - base = frappe.get_app_path(app, 'www') - path_to_index = frappe.get_app_path(app, 'www') + base = frappe.get_app_path(app, "www") + path_to_index = frappe.get_app_path(app, "www") for dirpath, _, filenames in os.walk(path_to_index, topdown=True): for f in filenames: - if f.endswith(('.md', '.html')): + if f.endswith((".md", ".html")): filepath = os.path.join(dirpath, f) route = os.path.relpath(filepath, base) - route = route.split('.')[0] + route = route.split(".")[0] - if route.endswith('index'): - route = route.rsplit('index', 1)[0] + if route.endswith("index"): + route = route.rsplit("index", 1)[0] routes_to_index.append(route) @@ -304,32 +320,34 @@ def get_routes_to_index(): def add_route_to_global_search(route): from bs4 import BeautifulSoup - from frappe.website.serve import get_response_content + from frappe.utils import set_request - frappe.set_user('Guest') + from frappe.website.serve import get_response_content + + frappe.set_user("Guest") frappe.local.no_cache = True try: - set_request(method='GET', path=route) + set_request(method="GET", path=route) content = get_response_content(route) - soup = BeautifulSoup(content, 'html.parser') - page_content = soup.find(class_='page_content') - text_content = page_content.text if page_content else '' + soup = BeautifulSoup(content, "html.parser") + page_content = soup.find(class_="page_content") + text_content = page_content.text if page_content else "" title = soup.title.text.strip() if soup.title else route value = dict( - doctype='Static Web Page', + doctype="Static Web Page", name=route, content=text_content, published=1, title=title, - route=route + route=route, ) sync_value_in_queue(value) except Exception: pass - frappe.set_user('Administrator') + frappe.set_user("Administrator") def get_formatted_value(value, field): @@ -340,10 +358,10 @@ def get_formatted_value(value, field): :return: """ - if getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]: + if getattr(field, "fieldtype", None) in ["Text", "Text Editor"]: value = unescape_html(frappe.safe_decode(value)) - value = (re.subn(r'(?s)<[\s]*(script|style).*?', '', str(value))[0]) - value = ' '.join(value.split()) + value = re.subn(r"(?s)<[\s]*(script|style).*?", "", str(value))[0] + value = " ".join(value.split()) return field.label + " : " + strip_html_tags(str(value)) @@ -354,28 +372,31 @@ def sync_global_search(): :param flags: :return: """ - while frappe.cache().llen('global_search_queue') > 0: + while frappe.cache().llen("global_search_queue") > 0: # rpop to follow FIFO # Last one should override all previous contents of same document - value = json.loads(frappe.cache().rpop('global_search_queue').decode('utf-8')) + value = json.loads(frappe.cache().rpop("global_search_queue").decode("utf-8")) sync_value(value) + def sync_value_in_queue(value): try: # append to search queue if connected - frappe.cache().lpush('global_search_queue', json.dumps(value)) + frappe.cache().lpush("global_search_queue", json.dumps(value)) except redis.exceptions.ConnectionError: # not connected, sync directly sync_value(value) + def sync_value(value): - ''' + """ Sync a given document to global search :param value: dict of { doctype, name, content, published, title, route } - ''' + """ - frappe.db.multisql({ - 'mariadb': '''INSERT INTO `__global_search` + frappe.db.multisql( + { + "mariadb": """INSERT INTO `__global_search` (`doctype`, `name`, `content`, `published`, `title`, `route`) VALUES (%(doctype)s, %(name)s, %(content)s, %(published)s, %(title)s, %(route)s) ON DUPLICATE key UPDATE @@ -383,8 +404,8 @@ def sync_value(value): `published`=%(published)s, `title`=%(title)s, `route`=%(route)s - ''', - 'postgres': '''INSERT INTO `__global_search` + """, + "postgres": """INSERT INTO `__global_search` (`doctype`, `name`, `content`, `published`, `title`, `route`) VALUES (%(doctype)s, %(name)s, %(content)s, %(published)s, %(title)s, %(route)s) ON CONFLICT("doctype", "name") DO UPDATE SET @@ -392,8 +413,11 @@ def sync_value(value): `published`=%(published)s, `title`=%(title)s, `route`=%(route)s - ''' - }, value) + """, + }, + value, + ) + def delete_for_document(doc): """ @@ -401,10 +425,8 @@ def delete_for_document(doc): been deleted :param doc: Deleted document """ - frappe.db.delete("__global_search", { - "doctype": doc.doctype, - "name": doc.name - }) + frappe.db.delete("__global_search", {"doctype": doc.doctype, "name": doc.name}) + @frappe.whitelist() def search(text, start=0, limit=20, doctype=""): @@ -434,9 +456,7 @@ def search(text, start=0, limit=20, doctype=""): rank = Match(global_search.content).Against(text).as_("rank") query = ( frappe.qb.from_(global_search) - .select( - global_search.doctype, global_search.name, global_search.content, rank - ) + .select(global_search.doctype, global_search.name, global_search.content, rank) .orderby("rank", order=frappe.qb.desc) .limit(limit) ) @@ -460,9 +480,7 @@ def search(text, start=0, limit=20, doctype=""): try: meta = frappe.get_meta(r.doctype) if meta.image_field: - r.image = frappe.db.get_value( - r.doctype, r.name, meta.image_field - ) + r.image = frappe.db.get_value(r.doctype, r.name, meta.image_field) except Exception: frappe.clear_messages() @@ -470,6 +488,7 @@ def search(text, start=0, limit=20, doctype=""): return sorted_results + @frappe.whitelist(allow_guest=True) def web_search(text, scope=None, start=0, limit=20): """ @@ -482,31 +501,35 @@ def web_search(text, scope=None, start=0, limit=20): """ results = [] - texts = text.split('&') + texts = text.split("&") for text in texts: - common_query = ''' SELECT `doctype`, `name`, `content`, `title`, `route` + common_query = """ SELECT `doctype`, `name`, `content`, `title`, `route` FROM `__global_search` WHERE {conditions} - LIMIT %(limit)s OFFSET %(start)s''' + LIMIT %(limit)s OFFSET %(start)s""" - scope_condition = '`route` like %(scope)s AND ' if scope else '' - published_condition = '`published` = 1 AND ' - mariadb_conditions = postgres_conditions = ' '.join([published_condition, scope_condition]) + scope_condition = "`route` like %(scope)s AND " if scope else "" + published_condition = "`published` = 1 AND " + mariadb_conditions = postgres_conditions = " ".join([published_condition, scope_condition]) # https://mariadb.com/kb/en/library/full-text-index-overview/#in-boolean-mode - mariadb_conditions += 'MATCH(`content`) AGAINST ({} IN BOOLEAN MODE)'.format(frappe.db.escape('+' + text + '*')) - postgres_conditions += 'TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({})'.format(frappe.db.escape(text)) - - values = { - "scope": "".join([scope, "%"]) if scope else '', - "limit": limit, - "start": start - } - - result = frappe.db.multisql({ - 'mariadb': common_query.format(conditions=mariadb_conditions), - 'postgres': common_query.format(conditions=postgres_conditions) - }, values=values, as_dict=True) + mariadb_conditions += "MATCH(`content`) AGAINST ({} IN BOOLEAN MODE)".format( + frappe.db.escape("+" + text + "*") + ) + postgres_conditions += 'TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({})'.format( + frappe.db.escape(text) + ) + + values = {"scope": "".join([scope, "%"]) if scope else "", "limit": limit, "start": start} + + result = frappe.db.multisql( + { + "mariadb": common_query.format(conditions=mariadb_conditions), + "postgres": common_query.format(conditions=postgres_conditions), + }, + values=values, + as_dict=True, + ) tmp_result = [] for i in result: if i in results or not results: @@ -524,7 +547,8 @@ def web_search(text, scope=None, start=0, limit=20): results = sorted(results, key=lambda x: x.relevance, reverse=True) return results + def get_distinct_words(text): - text = text.replace('"', '') - text = text.replace("'", '') - return [w.strip().lower() for w in text.split(' ')] + text = text.replace('"', "") + text = text.replace("'", "") + return [w.strip().lower() for w in text.split(" ")] diff --git a/frappe/utils/goal.py b/frappe/utils/goal.py index 0a9116f0e5..fb348496da 100644 --- a/frappe/utils/goal.py +++ b/frappe/utils/goal.py @@ -1,6 +1,7 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from contextlib import suppress from typing import Dict, Optional import frappe @@ -9,7 +10,7 @@ from frappe.query_builder.functions import DateFormat, Function from frappe.query_builder.utils import DocType from frappe.utils.data import add_to_date, cstr, flt, now_datetime from frappe.utils.formatters import format_value -from contextlib import suppress + def get_monthly_results( goal_doctype: str, @@ -69,7 +70,9 @@ def get_monthly_goal_graph_data( :return: dict of graph data """ if isinstance(filter_str, str): - frappe.throw("String filters have been deprecated. Pass Dict filters instead.", exc=DeprecationWarning) # nosemgrep + frappe.throw( + "String filters have been deprecated. Pass Dict filters instead.", exc=DeprecationWarning + ) # nosemgrep doc = frappe.get_doc(doctype, docname) doc.check_permission() @@ -80,9 +83,7 @@ def get_monthly_goal_graph_data( current_month_value = doc.get(goal_total_field) current_month_year = today_date.strftime("%m-%Y") # eg: "02-2022" - formatted_value = format_value( - current_month_value, meta.get_field(goal_total_field), doc - ) + formatted_value = format_value(current_month_value, meta.get_field(goal_total_field), doc) history = doc.get(goal_history_field) month_to_value_dict = None @@ -90,7 +91,7 @@ def get_monthly_goal_graph_data( with suppress(ValueError): month_to_value_dict = frappe.parse_json(history) - if month_to_value_dict is None: # nosemgrep + if month_to_value_dict is None: # nosemgrep doc_filter = {} with suppress(ValueError): doc_filter = frappe.parse_json(filters or "{}") @@ -122,9 +123,7 @@ def get_monthly_goal_graph_data( "value": f"{int(round(flt(current_month_value) / flt(goal) * 100))}%", }, ] - y_markers = { - "yMarkers": [{"label": _("Goal"), "lineType": "dashed", "value": flt(goal)}] - } + y_markers = {"yMarkers": [{"label": _("Goal"), "lineType": "dashed", "value": flt(goal)}]} for i in range(12): date_value = add_to_date(today_date, months=-i, as_datetime=True) @@ -134,9 +133,7 @@ def get_monthly_goal_graph_data( month_value = date_value.strftime("%m-%Y") # eg: "02-2022" val = month_to_value_dict.get(month_value, 0) dataset_values.insert(0, val) - values_formatted.insert( - 0, format_value(val, meta.get_field(goal_total_field), doc) - ) + values_formatted.insert(0, format_value(val, meta.get_field(goal_total_field), doc)) return { "title": title, diff --git a/frappe/utils/html_utils.py b/frappe/utils/html_utils.py index ccb374fbcc..8eac761220 100644 --- a/frappe/utils/html_utils.py +++ b/frappe/utils/html_utils.py @@ -12,12 +12,32 @@ def clean_html(html): if not isinstance(html, str): return html - return bleach.clean(clean_script_and_style(html), - tags=['div', 'p', 'br', 'ul', 'ol', 'li', 'strong', 'b', 'em', 'i', 'u', - 'table', 'thead', 'tbody', 'td', 'tr'], + return bleach.clean( + clean_script_and_style(html), + tags=[ + "div", + "p", + "br", + "ul", + "ol", + "li", + "strong", + "b", + "em", + "i", + "u", + "table", + "thead", + "tbody", + "td", + "tr", + ], attributes=[], - styles=['color', 'border', 'border-color'], - strip=True, strip_comments=True) + styles=["color", "border", "border-color"], + strip=True, + strip_comments=True, + ) + def clean_email_html(html): import bleach @@ -25,32 +45,88 @@ def clean_email_html(html): if not isinstance(html, str): return html - return bleach.clean(clean_script_and_style(html), - tags=['div', 'p', 'br', 'ul', 'ol', 'li', 'strong', 'b', 'em', 'i', 'u', 'a', - 'table', 'thead', 'tbody', 'td', 'tr', 'th', 'pre', 'code', - 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'button', 'img'], - attributes=['border', 'colspan', 'rowspan', - 'src', 'href', 'style', 'id'], - styles=['color', 'border-color', 'width', 'height', 'max-width', - 'background-color', 'border-collapse', 'border-radius', - 'border', 'border-top', 'border-bottom', 'border-left', 'border-right', - 'margin', 'margin-top', 'margin-bottom', 'margin-left', 'margin-right', - 'padding', 'padding-top', 'padding-bottom', 'padding-left', 'padding-right', - 'font-size', 'font-weight', 'font-family', 'text-decoration', - 'line-height', 'text-align', 'vertical-align', 'display' + return bleach.clean( + clean_script_and_style(html), + tags=[ + "div", + "p", + "br", + "ul", + "ol", + "li", + "strong", + "b", + "em", + "i", + "u", + "a", + "table", + "thead", + "tbody", + "td", + "tr", + "th", + "pre", + "code", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "button", + "img", + ], + attributes=["border", "colspan", "rowspan", "src", "href", "style", "id"], + styles=[ + "color", + "border-color", + "width", + "height", + "max-width", + "background-color", + "border-collapse", + "border-radius", + "border", + "border-top", + "border-bottom", + "border-left", + "border-right", + "margin", + "margin-top", + "margin-bottom", + "margin-left", + "margin-right", + "padding", + "padding-top", + "padding-bottom", + "padding-left", + "padding-right", + "font-size", + "font-weight", + "font-family", + "text-decoration", + "line-height", + "text-align", + "vertical-align", + "display", ], - protocols=['cid', 'http', 'https', 'mailto', 'data'], - strip=True, strip_comments=True) + protocols=["cid", "http", "https", "mailto", "data"], + strip=True, + strip_comments=True, + ) + def clean_script_and_style(html): # remove script and style from bs4 import BeautifulSoup - soup = BeautifulSoup(html, 'html5lib') - for s in soup(['script', 'style']): + soup = BeautifulSoup(html, "html5lib") + for s in soup(["script", "style"]): s.decompose() return frappe.as_unicode(soup) + def sanitize_html(html, linkify=False): """ Sanitize HTML tags, attributes and style to prevent XSS attacks @@ -67,21 +143,32 @@ def sanitize_html(html, linkify=False): elif is_json(html): return html - if not bool(BeautifulSoup(html, 'html.parser').find()): + if not bool(BeautifulSoup(html, "html.parser").find()): return html - tags = (acceptable_elements + svg_elements + mathml_elements - + ["html", "head", "meta", "link", "body", "style", "o:p"]) - attributes = {"*": acceptable_attributes, 'svg': svg_attributes} + tags = ( + acceptable_elements + + svg_elements + + mathml_elements + + ["html", "head", "meta", "link", "body", "style", "o:p"] + ) + attributes = {"*": acceptable_attributes, "svg": svg_attributes} styles = bleach_allowlist.all_styles strip_comments = False # returns html with escaped tags, escaped orphan >, <, etc. - escaped_html = bleach.clean(html, tags=tags, attributes=attributes, styles=styles, - strip_comments=strip_comments, protocols=['cid', 'http', 'https', 'mailto']) + escaped_html = bleach.clean( + html, + tags=tags, + attributes=attributes, + styles=styles, + strip_comments=strip_comments, + protocols=["cid", "http", "https", "mailto"], + ) return escaped_html + def is_json(text): try: json.loads(text) @@ -90,136 +177,564 @@ def is_json(text): else: return True + def get_icon_html(icon, small=False): from frappe.utils import is_image emoji_pattern = re.compile( - u"(\ud83d[\ude00-\ude4f])|" - u"(\ud83c[\udf00-\uffff])|" - u"(\ud83d[\u0000-\uddff])|" - u"(\ud83d[\ude80-\udeff])|" - u"(\ud83c[\udde0-\uddff])" - "+", flags=re.UNICODE) + "(\ud83d[\ude00-\ude4f])|" + "(\ud83c[\udf00-\uffff])|" + "(\ud83d[\u0000-\uddff])|" + "(\ud83d[\ude80-\udeff])|" + "(\ud83c[\udde0-\uddff])" + "+", + flags=re.UNICODE, + ) icon = icon or "" if icon and emoji_pattern.match(icon): - return '' + icon + '' + return '' + icon + "" if is_image(icon): - return \ - ''.format(icon=icon) \ - if small else \ - ''.format(icon=icon) + return ( + ''.format(icon=icon) + if small + else ''.format(icon=icon) + ) else: return "".format(icon=icon) + def unescape_html(value): from html import unescape + return unescape(value) + # adapted from https://raw.githubusercontent.com/html5lib/html5lib-python/4aa79f113e7486c7ec5d15a6e1777bfe546d3259/html5lib/sanitizer.py acceptable_elements = [ - 'a', 'abbr', 'acronym', 'address', 'area', - 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', - 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', - 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', - 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', - 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', - 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', - 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', - 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', - 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', - 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', - 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video' + "a", + "abbr", + "acronym", + "address", + "area", + "article", + "aside", + "audio", + "b", + "big", + "blockquote", + "br", + "button", + "canvas", + "caption", + "center", + "cite", + "code", + "col", + "colgroup", + "command", + "datagrid", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "dir", + "div", + "dl", + "dt", + "em", + "event-source", + "fieldset", + "figcaption", + "figure", + "footer", + "font", + "form", + "header", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "i", + "img", + "input", + "ins", + "keygen", + "kbd", + "label", + "legend", + "li", + "m", + "map", + "menu", + "meter", + "multicol", + "nav", + "nextid", + "ol", + "output", + "optgroup", + "option", + "p", + "pre", + "progress", + "q", + "s", + "samp", + "section", + "select", + "small", + "sound", + "source", + "spacer", + "span", + "strike", + "strong", + "sub", + "sup", + "table", + "tbody", + "td", + "textarea", + "time", + "tfoot", + "th", + "thead", + "tr", + "tt", + "u", + "ul", + "var", + "video", ] mathml_elements = [ - 'maction', 'math', 'merror', 'mfrac', 'mi', - 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', - 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', - 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', - 'munderover', 'none' + "maction", + "math", + "merror", + "mfrac", + "mi", + "mmultiscripts", + "mn", + "mo", + "mover", + "mpadded", + "mphantom", + "mprescripts", + "mroot", + "mrow", + "mspace", + "msqrt", + "mstyle", + "msub", + "msubsup", + "msup", + "mtable", + "mtd", + "mtext", + "mtr", + "munder", + "munderover", + "none", ] svg_elements = [ - 'a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', - 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', - 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use' + "a", + "animate", + "animateColor", + "animateMotion", + "animateTransform", + "clipPath", + "circle", + "defs", + "desc", + "ellipse", + "font-face", + "font-face-name", + "font-face-src", + "g", + "glyph", + "hkern", + "linearGradient", + "line", + "marker", + "metadata", + "missing-glyph", + "mpath", + "path", + "polygon", + "polyline", + "radialGradient", + "rect", + "set", + "stop", + "svg", + "switch", + "text", + "title", + "tspan", + "use", ] acceptable_attributes = [ - 'abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', - 'background', 'balance', 'bgcolor', 'bgproperties', 'border', - 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', - 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', - 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', - 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', - 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', - 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', - 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', - 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', - 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', - 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', - 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', - 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', - 'optimum', 'pattern', 'ping', 'point-size', 'poster', 'pqg', 'preload', - 'prompt', 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', - 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', - 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', - 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', - 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', - 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', - 'width', 'wrap', 'xml:lang', 'data-row', 'data-list', 'data-language', - 'data-value', 'role', 'frameborder', 'allowfullscreen', 'spellcheck', - 'data-mode', 'data-gramm', 'data-placeholder', 'data-comment', - 'data-id', 'data-denotation-char', 'itemprop', 'itemscope', - 'itemtype', 'itemid', 'itemref', 'datetime', 'data-is-group' + "abbr", + "accept", + "accept-charset", + "accesskey", + "action", + "align", + "alt", + "autocomplete", + "autofocus", + "axis", + "background", + "balance", + "bgcolor", + "bgproperties", + "border", + "bordercolor", + "bordercolordark", + "bordercolorlight", + "bottompadding", + "cellpadding", + "cellspacing", + "ch", + "challenge", + "char", + "charoff", + "choff", + "charset", + "checked", + "cite", + "class", + "clear", + "color", + "cols", + "colspan", + "compact", + "contenteditable", + "controls", + "coords", + "data", + "datafld", + "datapagesize", + "datasrc", + "datetime", + "default", + "delay", + "dir", + "disabled", + "draggable", + "dynsrc", + "enctype", + "end", + "face", + "for", + "form", + "frame", + "galleryimg", + "gutter", + "headers", + "height", + "hidefocus", + "hidden", + "high", + "href", + "hreflang", + "hspace", + "icon", + "id", + "inputmode", + "ismap", + "keytype", + "label", + "leftspacing", + "lang", + "list", + "longdesc", + "loop", + "loopcount", + "loopend", + "loopstart", + "low", + "lowsrc", + "max", + "maxlength", + "media", + "method", + "min", + "multiple", + "name", + "nohref", + "noshade", + "nowrap", + "open", + "optimum", + "pattern", + "ping", + "point-size", + "poster", + "pqg", + "preload", + "prompt", + "radiogroup", + "readonly", + "rel", + "repeat-max", + "repeat-min", + "replace", + "required", + "rev", + "rightspacing", + "rows", + "rowspan", + "rules", + "scope", + "selected", + "shape", + "size", + "span", + "src", + "start", + "step", + "style", + "summary", + "suppress", + "tabindex", + "target", + "template", + "title", + "toppadding", + "type", + "unselectable", + "usemap", + "urn", + "valign", + "value", + "variable", + "volume", + "vspace", + "vrml", + "width", + "wrap", + "xml:lang", + "data-row", + "data-list", + "data-language", + "data-value", + "role", + "frameborder", + "allowfullscreen", + "spellcheck", + "data-mode", + "data-gramm", + "data-placeholder", + "data-comment", + "data-id", + "data-denotation-char", + "itemprop", + "itemscope", + "itemtype", + "itemid", + "itemref", + "datetime", + "data-is-group", ] mathml_attributes = [ - 'actiontype', 'align', 'columnalign', 'columnalign', - 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', - 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', - 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', - 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', - 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', - 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', - 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', - 'xlink:type', 'xmlns', 'xmlns:xlink' + "actiontype", + "align", + "columnalign", + "columnalign", + "columnalign", + "columnlines", + "columnspacing", + "columnspan", + "depth", + "display", + "displaystyle", + "equalcolumns", + "equalrows", + "fence", + "fontstyle", + "fontweight", + "frame", + "height", + "linethickness", + "lspace", + "mathbackground", + "mathcolor", + "mathvariant", + "mathvariant", + "maxsize", + "minsize", + "other", + "rowalign", + "rowalign", + "rowalign", + "rowlines", + "rowspacing", + "rowspan", + "rspace", + "scriptlevel", + "selection", + "separator", + "stretchy", + "width", + "width", + "xlink:href", + "xlink:show", + "xlink:type", + "xmlns", + "xmlns:xlink", ] svg_attributes = [ - 'accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', - 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', - 'fill-opacity', 'fill-rule', 'font-family', 'font-size', - 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', - 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', - 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', - 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', - 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', - 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', - 'opacity', 'orient', 'origin', 'overline-position', - 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', - 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', - 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', - 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', - 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', - 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', - 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', - 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', - 'transform', 'type', 'u1', 'u2', 'underline-position', - 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', - 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', - 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', - 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', - 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', - 'y1', 'y2', 'zoomAndPan' + "accent-height", + "accumulate", + "additive", + "alphabetic", + "arabic-form", + "ascent", + "attributeName", + "attributeType", + "baseProfile", + "bbox", + "begin", + "by", + "calcMode", + "cap-height", + "class", + "clip-path", + "color", + "color-rendering", + "content", + "cx", + "cy", + "d", + "dx", + "dy", + "descent", + "display", + "dur", + "end", + "fill", + "fill-opacity", + "fill-rule", + "font-family", + "font-size", + "font-stretch", + "font-style", + "font-variant", + "font-weight", + "from", + "fx", + "fy", + "g1", + "g2", + "glyph-name", + "gradientUnits", + "hanging", + "height", + "horiz-adv-x", + "horiz-origin-x", + "id", + "ideographic", + "k", + "keyPoints", + "keySplines", + "keyTimes", + "lang", + "marker-end", + "marker-mid", + "marker-start", + "markerHeight", + "markerUnits", + "markerWidth", + "mathematical", + "max", + "min", + "name", + "offset", + "opacity", + "orient", + "origin", + "overline-position", + "overline-thickness", + "panose-1", + "path", + "pathLength", + "points", + "preserveAspectRatio", + "r", + "refX", + "refY", + "repeatCount", + "repeatDur", + "requiredExtensions", + "requiredFeatures", + "restart", + "rotate", + "rx", + "ry", + "slope", + "stemh", + "stemv", + "stop-color", + "stop-opacity", + "strikethrough-position", + "strikethrough-thickness", + "stroke", + "stroke-dasharray", + "stroke-dashoffset", + "stroke-linecap", + "stroke-linejoin", + "stroke-miterlimit", + "stroke-opacity", + "stroke-width", + "systemLanguage", + "target", + "text-anchor", + "to", + "transform", + "type", + "u1", + "u2", + "underline-position", + "underline-thickness", + "unicode", + "unicode-range", + "units-per-em", + "values", + "version", + "viewBox", + "visibility", + "width", + "widths", + "x", + "x-height", + "x1", + "x2", + "xlink:actuate", + "xlink:arcrole", + "xlink:href", + "xlink:role", + "xlink:show", + "xlink:title", + "xlink:type", + "xml:base", + "xml:lang", + "xml:space", + "xmlns", + "xmlns:xlink", + "y", + "y1", + "y2", + "zoomAndPan", ] diff --git a/frappe/utils/identicon.py b/frappe/utils/identicon.py index e570875b4a..28df486d03 100644 --- a/frappe/utils/identicon.py +++ b/frappe/utils/identicon.py @@ -1,23 +1,23 @@ - -from PIL import Image, ImageDraw -from hashlib import md5 import base64 import random +from hashlib import md5 from io import StringIO +from PIL import Image, ImageDraw + GRID_SIZE = 5 BORDER_SIZE = 20 SQUARE_SIZE = 40 class Identicon(object): - def __init__(self, str_, background='#fafbfc'): + def __init__(self, str_, background="#fafbfc"): """ `str_` is the string used to generate the identicon. `background` is the background of the identicon. """ w = h = BORDER_SIZE * 2 + SQUARE_SIZE * GRID_SIZE - self.image = Image.new('RGB', (w, h), background) + self.image = Image.new("RGB", (w, h), background) self.draw = ImageDraw.Draw(self.image) self.hash = self.digest(str_) @@ -25,7 +25,7 @@ class Identicon(object): """ Returns a md5 numeric hash """ - return int(md5(str_.encode('utf-8')).hexdigest(), 16) + return int(md5(str_.encode("utf-8")).hexdigest(), 16) def calculate(self): """ @@ -34,26 +34,28 @@ class Identicon(object): remaining bytes are used to create the drawing """ # color = (self.hash & 0xff, self.hash >> 8 & 0xff, self.hash >> 16 & 0xff) - color = random.choice(( - (254, 196, 197), - (253, 138, 139), - (254, 231, 206), - (254, 208, 159), - (210, 211, 253), - (163, 165, 252), - (247, 213, 247), - (242, 172, 238), - (235, 247, 206), - (217, 241, 157), - (211, 248, 237), - (167, 242, 221), - (255, 249, 207), - (254, 245, 161), - (211, 241, 254), - (168, 228, 254), - (207, 245, 210), - (159, 235, 164), - )) + color = random.choice( + ( + (254, 196, 197), + (253, 138, 139), + (254, 231, 206), + (254, 208, 159), + (210, 211, 253), + (163, 165, 252), + (247, 213, 247), + (242, 172, 238), + (235, 247, 206), + (217, 241, 157), + (211, 248, 237), + (167, 242, 221), + (255, 249, 207), + (254, 245, 161), + (211, 241, 254), + (168, 228, 254), + (207, 245, 210), + (159, 235, 164), + ) + ) # print color # color = (254, 232, 206) @@ -63,18 +65,10 @@ class Identicon(object): if self.hash & 1: x = BORDER_SIZE + square_x * SQUARE_SIZE y = BORDER_SIZE + square_y * SQUARE_SIZE - self.draw.rectangle( - (x, y, x + SQUARE_SIZE, y + SQUARE_SIZE), - fill=color, - outline=color - ) + self.draw.rectangle((x, y, x + SQUARE_SIZE, y + SQUARE_SIZE), fill=color, outline=color) # following is just for mirroring x = BORDER_SIZE + (GRID_SIZE - 1 - square_x) * SQUARE_SIZE - self.draw.rectangle( - (x, y, x + SQUARE_SIZE, y + SQUARE_SIZE), - fill=color, - outline=color - ) + self.draw.rectangle((x, y, x + SQUARE_SIZE, y + SQUARE_SIZE), fill=color, outline=color) self.hash >>= 1 # shift to right square_y += 1 if square_y == GRID_SIZE: # done with first column @@ -86,18 +80,18 @@ class Identicon(object): Save and show calculated identicon """ self.calculate() - with open('identicon.png', 'wb') as out: - self.image.save(out, 'PNG') + with open("identicon.png", "wb") as out: + self.image.save(out, "PNG") self.image.show() - def base64(self, format='PNG'): - ''' + def base64(self, format="PNG"): + """ usage: i = Identicon('xx') - print(i.base64()) + print(i.base64()) return: this image's base64 code created by: liuzheng712 bug report: https://github.com/liuzheng712/identicons/issues - ''' + """ self.calculate() fp = StringIO() self.image.encoderinfo = {} @@ -107,7 +101,7 @@ class Identicon(object): Image.init() save_handler = Image.SAVE[format.upper()] try: - save_handler(self.image, fp, '') + save_handler(self.image, fp, "") finally: fp.seek(0) return "data:image/png;base64,{0}".format(base64.b64encode(fp.read())) diff --git a/frappe/utils/image.py b/frappe/utils/image.py index 3b0daf565c..0cbc02fb31 100644 --- a/frappe/utils/image.py +++ b/frappe/utils/image.py @@ -1,11 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import io import os + from PIL import Image -import io + def resize_images(path, maxdim=700): from PIL import Image + size = (maxdim, maxdim) for basepath, folders, files in os.walk(path): for fname in files: @@ -18,14 +21,15 @@ def resize_images(path, maxdim=700): print("resized {0}".format(os.path.join(basepath, fname))) + def strip_exif_data(content, content_type): - """ Strips EXIF from image files which support it. + """Strips EXIF from image files which support it. Works by creating a new Image object which ignores exif by default and then extracts the binary data back into content. Returns: - Bytes: Stripped image content + Bytes: Stripped image content """ original_image = Image.open(io.BytesIO(content)) @@ -36,23 +40,32 @@ def strip_exif_data(content, content_type): new_image = Image.new(original_image.mode, original_image.size) new_image.putdata(list(original_image.getdata())) - new_image.save(output, format=content_type.split('/')[1]) + new_image.save(output, format=content_type.split("/")[1]) content = output.getvalue() return content -def optimize_image(content, content_type, max_width=1920, max_height=1080, optimize=True, quality=85): - if content_type == 'image/svg+xml': + +def optimize_image( + content, content_type, max_width=1920, max_height=1080, optimize=True, quality=85 +): + if content_type == "image/svg+xml": return content image = Image.open(io.BytesIO(content)) - image_format = content_type.split('/')[1] + image_format = content_type.split("/")[1] size = max_width, max_height image.thumbnail(size, Image.LANCZOS) output = io.BytesIO() - image.save(output, format=image_format, optimize=optimize, quality=quality, save_all=True if image_format=='gif' else None) + image.save( + output, + format=image_format, + optimize=optimize, + quality=quality, + save_all=True if image_format == "gif" else None, + ) optimized_content = output.getvalue() - return optimized_content if len(optimized_content) < len(content) else content \ No newline at end of file + return optimized_content if len(optimized_content) < len(content) else content diff --git a/frappe/utils/install.py b/frappe/utils/install.py index d197304c98..08a399e5fe 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -1,9 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import getpass + +import frappe from frappe.utils.password import update_password + def before_install(): frappe.reload_doc("core", "doctype", "doctype_state") frappe.reload_doc("core", "doctype", "docfield") @@ -14,6 +16,7 @@ def before_install(): frappe.reload_doc("desk", "doctype", "form_tour") frappe.reload_doc("core", "doctype", "doctype") + def after_install(): # reset installed apps for re-install frappe.db.set_global("installed_apps", '["frappe"]') @@ -22,11 +25,13 @@ def after_install(): install_basic_docs() from frappe.core.doctype.file.file import make_home_folder + make_home_folder() import_country_and_currency() from frappe.core.doctype.language.language import sync_languages + sync_languages() # save default print setting @@ -40,52 +45,94 @@ def after_install(): update_password("Administrator", get_admin_password()) if not frappe.conf.skip_setup_wizard: - frappe.db.set_default('desktop:home_page', 'setup-wizard') + frappe.db.set_default("desktop:home_page", "setup-wizard") # clear test log - with open(frappe.get_site_path('.test_log'), 'w') as f: - f.write('') + with open(frappe.get_site_path(".test_log"), "w") as f: + f.write("") add_standard_navbar_items() frappe.db.commit() + def create_user_type(): - for user_type in ['System User', 'Website User']: - if not frappe.db.exists('User Type', user_type): - frappe.get_doc({ - 'doctype': 'User Type', - 'name': user_type, - 'is_standard': 1 - }).insert(ignore_permissions=True) + for user_type in ["System User", "Website User"]: + if not frappe.db.exists("User Type", user_type): + frappe.get_doc({"doctype": "User Type", "name": user_type, "is_standard": 1}).insert( + ignore_permissions=True + ) + def install_basic_docs(): # core users / roles install_docs = [ - {'doctype':'User', 'name':'Administrator', 'first_name':'Administrator', - 'email':'admin@example.com', 'enabled':1, "is_admin": 1, - 'roles': [{'role': 'Administrator'}], - 'thread_notify': 0, 'send_me_a_copy': 0 + { + "doctype": "User", + "name": "Administrator", + "first_name": "Administrator", + "email": "admin@example.com", + "enabled": 1, + "is_admin": 1, + "roles": [{"role": "Administrator"}], + "thread_notify": 0, + "send_me_a_copy": 0, }, - {'doctype':'User', 'name':'Guest', 'first_name':'Guest', - 'email':'guest@example.com', 'enabled':1, "is_guest": 1, - 'roles': [{'role': 'Guest'}], - 'thread_notify': 0, 'send_me_a_copy': 0 + { + "doctype": "User", + "name": "Guest", + "first_name": "Guest", + "email": "guest@example.com", + "enabled": 1, + "is_guest": 1, + "roles": [{"role": "Guest"}], + "thread_notify": 0, + "send_me_a_copy": 0, + }, + {"doctype": "Role", "role_name": "Report Manager"}, + {"doctype": "Role", "role_name": "Translator"}, + { + "doctype": "Workflow State", + "workflow_state_name": "Pending", + "icon": "question-sign", + "style": "", + }, + { + "doctype": "Workflow State", + "workflow_state_name": "Approved", + "icon": "ok-sign", + "style": "Success", + }, + { + "doctype": "Workflow State", + "workflow_state_name": "Rejected", + "icon": "remove", + "style": "Danger", + }, + {"doctype": "Workflow Action Master", "workflow_action_name": "Approve"}, + {"doctype": "Workflow Action Master", "workflow_action_name": "Reject"}, + {"doctype": "Workflow Action Master", "workflow_action_name": "Review"}, + { + "doctype": "Email Domain", + "domain_name": "example.com", + "email_id": "account@example.com", + "password": "pass", + "email_server": "imap.example.com", + "use_imap": 1, + "smtp_server": "smtp.example.com", + }, + { + "doctype": "Email Account", + "domain": "example.com", + "email_id": "notifications@example.com", + "default_outgoing": 1, + }, + { + "doctype": "Email Account", + "domain": "example.com", + "email_id": "replies@example.com", + "default_incoming": 1, }, - {'doctype': "Role", "role_name": "Report Manager"}, - {'doctype': "Role", "role_name": "Translator"}, - {'doctype': "Workflow State", "workflow_state_name": "Pending", - "icon": "question-sign", "style": ""}, - {'doctype': "Workflow State", "workflow_state_name": "Approved", - "icon": "ok-sign", "style": "Success"}, - {'doctype': "Workflow State", "workflow_state_name": "Rejected", - "icon": "remove", "style": "Danger"}, - {'doctype': "Workflow Action Master", "workflow_action_name": "Approve"}, - {'doctype': "Workflow Action Master", "workflow_action_name": "Reject"}, - {'doctype': "Workflow Action Master", "workflow_action_name": "Review"}, - {'doctype': "Email Domain", "domain_name":"example.com", "email_id": "account@example.com", "password": "pass", "email_server": "imap.example.com","use_imap": 1, "smtp_server": "smtp.example.com"}, - {'doctype': "Email Account", "domain":"example.com", "email_id": "notifications@example.com", "default_outgoing": 1}, - {'doctype': "Email Account", "domain":"example.com", "email_id": "replies@example.com", "default_incoming": 1} ] for d in install_docs: @@ -94,6 +141,7 @@ def install_basic_docs(): except frappe.NameError: pass + def get_admin_password(): def ask_admin_password(): admin_password = getpass.getpass("Set Administrator password: ") @@ -120,23 +168,28 @@ def before_tests(): frappe.clear_cache() # complete setup if missing - if not int(frappe.db.get_single_value('System Settings', 'setup_complete') or 0): + if not int(frappe.db.get_single_value("System Settings", "setup_complete") or 0): complete_setup_wizard() frappe.db.commit() frappe.clear_cache() + def complete_setup_wizard(): from frappe.desk.page.setup_wizard.setup_wizard import setup_complete - setup_complete({ - "language" :"English", - "email" :"test@erpnext.com", - "full_name" :"Test User", - "password" :"test", - "country" :"United States", - "timezone" :"America/New_York", - "currency" :"USD" - }) + + setup_complete( + { + "language": "English", + "email": "test@erpnext.com", + "full_name": "Test User", + "password": "test", + "country": "United States", + "timezone": "America/New_York", + "currency": "USD", + } + ) + def import_country_and_currency(): from frappe.geo.country_info import get_all @@ -155,122 +208,125 @@ def import_country_and_currency(): for currency in ("INR", "USD", "GBP", "EUR", "AED", "AUD", "JPY", "CNY", "CHF"): frappe.db.set_value("Currency", currency, "enabled", 1) + def add_country_and_currency(name, country): if not frappe.db.exists("Country", name): - frappe.get_doc({ - "doctype": "Country", - "country_name": name, - "code": country.code, - "date_format": country.date_format or "dd-mm-yyyy", - "time_format": country.time_format or "HH:mm:ss", - "time_zones": "\n".join(country.timezones or []), - "docstatus": 0 - }).db_insert() + frappe.get_doc( + { + "doctype": "Country", + "country_name": name, + "code": country.code, + "date_format": country.date_format or "dd-mm-yyyy", + "time_format": country.time_format or "HH:mm:ss", + "time_zones": "\n".join(country.timezones or []), + "docstatus": 0, + } + ).db_insert() if country.currency and not frappe.db.exists("Currency", country.currency): - frappe.get_doc({ - "doctype": "Currency", - "currency_name": country.currency, - "fraction": country.currency_fraction, - "symbol": country.currency_symbol, - "fraction_units": country.currency_fraction_units, - "smallest_currency_fraction_value": country.smallest_currency_fraction_value, - "number_format": country.number_format, - "docstatus": 0 - }).db_insert() + frappe.get_doc( + { + "doctype": "Currency", + "currency_name": country.currency, + "fraction": country.currency_fraction, + "symbol": country.currency_symbol, + "fraction_units": country.currency_fraction_units, + "smallest_currency_fraction_value": country.smallest_currency_fraction_value, + "number_format": country.number_format, + "docstatus": 0, + } + ).db_insert() + def add_standard_navbar_items(): navbar_settings = frappe.get_single("Navbar Settings") standard_navbar_items = [ { - 'item_label': 'My Profile', - 'item_type': 'Route', - 'route': '/app/user-profile', - 'is_standard': 1 + "item_label": "My Profile", + "item_type": "Route", + "route": "/app/user-profile", + "is_standard": 1, }, { - 'item_label': 'My Settings', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.route_to_user()', - 'is_standard': 1 + "item_label": "My Settings", + "item_type": "Action", + "action": "frappe.ui.toolbar.route_to_user()", + "is_standard": 1, }, { - 'item_label': 'Session Defaults', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.setup_session_defaults()', - 'is_standard': 1 + "item_label": "Session Defaults", + "item_type": "Action", + "action": "frappe.ui.toolbar.setup_session_defaults()", + "is_standard": 1, }, { - 'item_label': 'Reload', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.clear_cache()', - 'is_standard': 1 + "item_label": "Reload", + "item_type": "Action", + "action": "frappe.ui.toolbar.clear_cache()", + "is_standard": 1, }, { - 'item_label': 'View Website', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.view_website()', - 'is_standard': 1 + "item_label": "View Website", + "item_type": "Action", + "action": "frappe.ui.toolbar.view_website()", + "is_standard": 1, }, { - 'item_label': 'Toggle Full Width', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.toggle_full_width()', - 'is_standard': 1 + "item_label": "Toggle Full Width", + "item_type": "Action", + "action": "frappe.ui.toolbar.toggle_full_width()", + "is_standard": 1, }, { - 'item_label': 'Toggle Theme', - 'item_type': 'Action', - 'action': 'new frappe.ui.ThemeSwitcher().show()', - 'is_standard': 1 + "item_label": "Toggle Theme", + "item_type": "Action", + "action": "new frappe.ui.ThemeSwitcher().show()", + "is_standard": 1, }, { - 'item_label': 'Background Jobs', - 'item_type': 'Route', - 'route': '/app/background_jobs', - 'is_standard': 1 + "item_label": "Background Jobs", + "item_type": "Route", + "route": "/app/background_jobs", + "is_standard": 1, }, + {"item_type": "Separator", "is_standard": 1}, { - 'item_type': 'Separator', - 'is_standard': 1 + "item_label": "Log out", + "item_type": "Action", + "action": "frappe.app.logout()", + "is_standard": 1, }, - { - 'item_label': 'Log out', - 'item_type': 'Action', - 'action': 'frappe.app.logout()', - 'is_standard': 1 - } ] standard_help_items = [ { - 'item_label': 'About', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.show_about()', - 'is_standard': 1 + "item_label": "About", + "item_type": "Action", + "action": "frappe.ui.toolbar.show_about()", + "is_standard": 1, }, { - 'item_label': 'Keyboard Shortcuts', - 'item_type': 'Action', - 'action': 'frappe.ui.toolbar.show_shortcuts(event)', - 'is_standard': 1 + "item_label": "Keyboard Shortcuts", + "item_type": "Action", + "action": "frappe.ui.toolbar.show_shortcuts(event)", + "is_standard": 1, }, { - 'item_label': 'Frappe Support', - 'item_type': 'Route', - 'route': 'https://frappe.io/support', - 'is_standard': 1 - } + "item_label": "Frappe Support", + "item_type": "Route", + "route": "https://frappe.io/support", + "is_standard": 1, + }, ] navbar_settings.settings_dropdown = [] navbar_settings.help_dropdown = [] for item in standard_navbar_items: - navbar_settings.append('settings_dropdown', item) + navbar_settings.append("settings_dropdown", item) for item in standard_help_items: - navbar_settings.append('help_dropdown', item) + navbar_settings.append("help_dropdown", item) navbar_settings.save() diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 4dd297b5b3..c7902e7096 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -4,15 +4,12 @@ def get_jenv(): import frappe from frappe.utils.safe_exec import get_safe_globals - if not getattr(frappe.local, 'jenv', None): + if not getattr(frappe.local, "jenv", None): from jinja2 import DebugUndefined from jinja2.sandbox import SandboxedEnvironment # frappe will be loaded last, so app templates will get precedence - jenv = SandboxedEnvironment( - loader=get_jloader(), - undefined=DebugUndefined - ) + jenv = SandboxedEnvironment(loader=get_jloader(), undefined=DebugUndefined) set_filters(jenv) jenv.globals.update(get_safe_globals()) @@ -25,54 +22,61 @@ def get_jenv(): return frappe.local.jenv + def get_template(path): return get_jenv().get_template(path) + def get_email_from_template(name, args): from jinja2 import TemplateNotFound args = args or {} try: - message = get_template('templates/emails/' + name + '.html').render(args) + message = get_template("templates/emails/" + name + ".html").render(args) except TemplateNotFound as e: raise e try: - text_content = get_template('templates/emails/' + name + '.txt').render(args) + text_content = get_template("templates/emails/" + name + ".txt").render(args) except TemplateNotFound: text_content = None return (message, text_content) + def validate_template(html): """Throws exception if there is a syntax error in the Jinja Template""" - import frappe from jinja2 import TemplateSyntaxError + + import frappe + if not html: return jenv = get_jenv() try: jenv.from_string(html) except TemplateSyntaxError as e: - frappe.msgprint('Line {}: {}'.format(e.lineno, e.message)) + frappe.msgprint("Line {}: {}".format(e.lineno, e.message)) frappe.throw(frappe._("Syntax error in template")) + def render_template(template, context, is_path=None, safe_render=True): - '''Render a template using Jinja + """Render a template using Jinja :param template: path or HTML containing the jinja template :param context: dict of properties to pass to the template :param is_path: (optional) assert that the `template` parameter is a path :param safe_render: (optional) prevent server side scripting via jinja templating - ''' + """ - from frappe import _, get_traceback, throw from jinja2 import TemplateError + from frappe import _, get_traceback, throw + if not template: return "" - if (is_path or guess_is_path(template)): + if is_path or guess_is_path(template): return get_jenv().get_template(template).render(context) else: if safe_render and ".__" in template: @@ -80,14 +84,18 @@ def render_template(template, context, is_path=None, safe_render=True): try: return get_jenv().from_string(template).render(context) except TemplateError: - throw(title="Jinja Template Error", msg="
{template}
{tb}
".format(template=template, tb=get_traceback())) + throw( + title="Jinja Template Error", + msg="
{template}
{tb}
".format(template=template, tb=get_traceback()), + ) + def guess_is_path(template): # template can be passed as a path or content # if its single line and ends with a html, then its probably a path - if '\n' not in template and '.' in template: - extn = template.rsplit('.')[-1] - if extn in ('html', 'css', 'scss', 'py', 'md', 'json', 'js', 'xml'): + if "\n" not in template and "." in template: + extn = template.rsplit(".")[-1] + if extn in ("html", "css", "scss", "py", "md", "json", "js", "xml"): return True return False @@ -95,40 +103,42 @@ def guess_is_path(template): def get_jloader(): import frappe - if not getattr(frappe.local, 'jloader', None): + + if not getattr(frappe.local, "jloader", None): from jinja2 import ChoiceLoader, PackageLoader, PrefixLoader - apps = frappe.get_hooks('template_apps') + apps = frappe.get_hooks("template_apps") if not apps: apps = frappe.local.flags.web_pages_apps or frappe.get_installed_apps(sort=True) apps.reverse() if "frappe" not in apps: - apps.append('frappe') + apps.append("frappe") frappe.local.jloader = ChoiceLoader( # search for something like app/templates/... - [PrefixLoader(dict( - (app, PackageLoader(app, ".")) for app in apps - ))] - + [PrefixLoader(dict((app, PackageLoader(app, ".")) for app in apps))] # search for something like templates/... + [PackageLoader(app, ".") for app in apps] ) return frappe.local.jloader + def set_filters(jenv): import frappe from frappe.utils import cint, cstr, flt - jenv.filters.update({ - "json": frappe.as_json, - "len": len, - "int": cint, - "str": cstr, - "flt": flt, - }) + jenv.filters.update( + { + "json": frappe.as_json, + "len": len, + "int": cint, + "str": cstr, + "flt": flt, + } + ) + def get_jinja_hooks(): """Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair.""" @@ -137,8 +147,8 @@ def get_jinja_hooks(): if not getattr(frappe.local, "site", None): return (None, None) - from types import FunctionType, ModuleType from inspect import getmembers, isfunction + from types import FunctionType, ModuleType def get_obj_dict_from_paths(object_paths): out = {} diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index a4f522558b..ae0523b82c 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -36,7 +36,7 @@ def web_block(template, values=None, **kwargs): def web_blocks(blocks): - from frappe import throw, _dict, _ + from frappe import _, _dict, throw from frappe.website.doctype.web_page.web_page import get_web_blocks_html web_blocks = [] @@ -70,9 +70,10 @@ def web_blocks(blocks): def get_dom_id(seed=None): from frappe import generate_hash + if not seed: - seed = 'DOM' - return 'id-' + generate_hash(seed, 12) + seed = "DOM" + return "id-" + generate_hash(seed, 12) def include_script(path): @@ -91,14 +92,16 @@ def bundled_asset(path, rtl=None): if ".bundle." in path and not path.startswith("/assets"): bundled_assets = get_assets_json() - if path.endswith('.css') and is_rtl(rtl): + if path.endswith(".css") and is_rtl(rtl): path = f"rtl_{path}" path = bundled_assets.get(path) or path return abs_url(path) + def is_rtl(rtl=None): from frappe import local + if rtl is None: return local.lang in ["ar", "he", "fa", "ps"] return rtl diff --git a/frappe/utils/lazy_loader.py b/frappe/utils/lazy_loader.py index ae4cba66a1..c22d679edd 100644 --- a/frappe/utils/lazy_loader.py +++ b/frappe/utils/lazy_loader.py @@ -1,6 +1,7 @@ import importlib.util import sys + def lazy_import(name, package=None): """Import a module lazily. @@ -24,7 +25,7 @@ def lazy_import(name, package=None): # Find the spec if not loaded spec = importlib.util.find_spec(name, package) if not spec: - raise ImportError(f'Module {name} Not found.') + raise ImportError(f"Module {name} Not found.") loader = importlib.util.LazyLoader(spec.loader) spec.loader = loader diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 4699eb1949..bdae92da51 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -3,30 +3,38 @@ import logging import os from logging.handlers import RotatingFileHandler -# imports - third party imports - # imports - module imports import frappe from frappe.utils import get_sites +# imports - third party imports + default_log_level = logging.DEBUG -def get_logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20, stream_only=False): +def get_logger( + module=None, + with_more_info=False, + allow_site=True, + filter=None, + max_size=100_000, + file_count=20, + stream_only=False, +): """Application Logger for your given module Args: - module (str, optional): Name of your logger and consequently your log file. Defaults to None. - with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. - allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. - filter (function, optional): Add a filter function for your logger. Defaults to None. - max_size (int, optional): Max file size of each log file in bytes. Defaults to 100_000. - file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. - stream_only (bool, optional): Whether to stream logs only to stderr (True) or use log files (False). Defaults to False. + module (str, optional): Name of your logger and consequently your log file. Defaults to None. + with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. + allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. + filter (function, optional): Add a filter function for your logger. Defaults to None. + max_size (int, optional): Max file size of each log file in bytes. Defaults to 100_000. + file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. + stream_only (bool, optional): Whether to stream logs only to stderr (True) or use log files (False). Defaults to False. Returns: - : Returns a Python logger object with Site and Bench level logging capabilities. + : Returns a Python logger object with Site and Bench level logging capabilities. """ if allow_site is True: diff --git a/frappe/utils/make_random.py b/frappe/utils/make_random.py index 0db26dfcd2..41d6f89d72 100644 --- a/frappe/utils/make_random.py +++ b/frappe/utils/make_random.py @@ -1,12 +1,14 @@ +import random -import frappe, random +import frappe settings = frappe._dict( - prob = { - "default": { "make": 0.6, "qty": (1,5) }, + prob={ + "default": {"make": 0.6, "qty": (1, 5)}, } ) + def add_random_children(doc, fieldname, rows, randomize, unique=None): nrows = rows if rows > 1: @@ -21,27 +23,32 @@ def add_random_children(doc, fieldname, rows, randomize, unique=None): d[key] = random.randrange(*val) if unique: - if not doc.get(fieldname, {unique:d[unique]}): + if not doc.get(fieldname, {unique: d[unique]}): doc.append(fieldname, d) else: doc.append(fieldname, d) + def get_random(doctype, filters=None, doc=False): condition = [] if filters: for key, val in filters.items(): - condition.append("%s='%s'" % (key, str(val).replace("'", "\'"))) + condition.append("%s='%s'" % (key, str(val).replace("'", "'"))) if condition: condition = " where " + " and ".join(condition) else: condition = "" - out = frappe.db.multisql({ - 'mariadb': """select name from `tab%s` %s - order by RAND() limit 1 offset 0""" % (doctype, condition), - 'postgres': """select name from `tab%s` %s - order by RANDOM() limit 1 offset 0""" % (doctype, condition) - }) + out = frappe.db.multisql( + { + "mariadb": """select name from `tab%s` %s + order by RAND() limit 1 offset 0""" + % (doctype, condition), + "postgres": """select name from `tab%s` %s + order by RANDOM() limit 1 offset 0""" + % (doctype, condition), + } + ) out = out and out[0][0] or None @@ -50,8 +57,10 @@ def get_random(doctype, filters=None, doc=False): else: return out + def can_make(doctype): return random.random() < settings.prob.get(doctype, settings.prob["default"])["make"] + def how_many(doctype): return random.randrange(*settings.prob.get(doctype, settings.prob["default"])["qty"]) diff --git a/frappe/utils/momentjs.py b/frappe/utils/momentjs.py index 18df9903a7..96272445fd 100644 --- a/frappe/utils/momentjs.py +++ b/frappe/utils/momentjs.py @@ -12,23 +12,19 @@ def update(tz, out): if parts[1] in data["rules"]: out["rules"][parts[1]] = data["rules"][parts[1]] + def get_all_timezones(): return sorted(list(data["zones"])) + data = { "zones": { - "Africa/Abidjan": [ - "-0:16:8 - LMT 1912 -0:16:8", - "0 - GMT" - ], - "Africa/Accra": [ - "-0:0:52 - LMT 1918 -0:0:52", - "0 Ghana %s" - ], + "Africa/Abidjan": ["-0:16:8 - LMT 1912 -0:16:8", "0 - GMT"], + "Africa/Accra": ["-0:0:52 - LMT 1918 -0:0:52", "0 Ghana %s"], "Africa/Addis_Ababa": [ "2:34:48 - LMT 1870 2:34:48", "2:35:20 - ADMT 1936_4_5 2:35:20", - "3 - EAT" + "3 - EAT", ], "Africa/Algiers": [ "0:12:12 - LMT 1891_2_15_0_1 0:12:12", @@ -40,56 +36,37 @@ data = { "0 Algeria WE%sT 1977_9_21 1", "1 Algeria CE%sT 1979_9_26 1", "0 Algeria WE%sT 1981_4", - "1 - CET" + "1 - CET", ], "Africa/Asmara": [ "2:35:32 - LMT 1870 2:35:32", "2:35:32 - AMT 1890 2:35:32", "2:35:20 - ADMT 1936_4_5 2:35:20", - "3 - EAT" + "3 - EAT", ], "Africa/Bamako": [ "-0:32 - LMT 1912 -0:32", "0 - GMT 1934_1_26", "-1 - WAT 1960_5_20 -1", - "0 - GMT" - ], - "Africa/Bangui": [ - "1:14:20 - LMT 1912 1:14:20", - "1 - WAT" + "0 - GMT", ], + "Africa/Bangui": ["1:14:20 - LMT 1912 1:14:20", "1 - WAT"], "Africa/Banjul": [ "-1:6:36 - LMT 1912 -1:6:36", "-1:6:36 - BMT 1935 -1:6:36", "-1 - WAT 1964 -1", - "0 - GMT" - ], - "Africa/Bissau": [ - "-1:2:20 - LMT 1911_4_26 -1:2:20", - "-1 - WAT 1975 -1", - "0 - GMT" - ], - "Africa/Blantyre": [ - "2:20 - LMT 1903_2 2:20", - "2 - CAT" - ], - "Africa/Brazzaville": [ - "1:1:8 - LMT 1912 1:1:8", - "1 - WAT" - ], - "Africa/Bujumbura": [ - "1:57:28 - LMT 1890 1:57:28", - "2 - CAT" - ], - "Africa/Cairo": [ - "2:5:9 - LMT 1900_9 2:5:9", - "2 Egypt EE%sT" + "0 - GMT", ], + "Africa/Bissau": ["-1:2:20 - LMT 1911_4_26 -1:2:20", "-1 - WAT 1975 -1", "0 - GMT"], + "Africa/Blantyre": ["2:20 - LMT 1903_2 2:20", "2 - CAT"], + "Africa/Brazzaville": ["1:1:8 - LMT 1912 1:1:8", "1 - WAT"], + "Africa/Bujumbura": ["1:57:28 - LMT 1890 1:57:28", "2 - CAT"], + "Africa/Cairo": ["2:5:9 - LMT 1900_9 2:5:9", "2 Egypt EE%sT"], "Africa/Casablanca": [ "-0:30:20 - LMT 1913_9_26 -0:30:20", "0 Morocco WE%sT 1984_2_16", "1 - CET 1986 1", - "0 Morocco WE%sT" + "0 Morocco WE%sT", ], "Africa/Ceuta": [ "-0:21:16 - LMT 1901 -0:21:16", @@ -99,180 +76,105 @@ data = { "0 Spain WE%sT 1929", "0 SpainAfrica WE%sT 1984_2_16", "1 - CET 1986 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Africa/Conakry": [ "-0:54:52 - LMT 1912 -0:54:52", "0 - GMT 1934_1_26", "-1 - WAT 1960 -1", - "0 - GMT" - ], - "Africa/Dakar": [ - "-1:9:44 - LMT 1912 -1:9:44", - "-1 - WAT 1941_5 -1", - "0 - GMT" + "0 - GMT", ], + "Africa/Dakar": ["-1:9:44 - LMT 1912 -1:9:44", "-1 - WAT 1941_5 -1", "0 - GMT"], "Africa/Dar_es_Salaam": [ "2:37:8 - LMT 1931 2:37:8", "3 - EAT 1948 3", "2:45 - BEAUT 1961 2:45", - "3 - EAT" - ], - "Africa/Djibouti": [ - "2:52:36 - LMT 1911_6 2:52:36", - "3 - EAT" - ], - "Africa/Douala": [ - "0:38:48 - LMT 1912 0:38:48", - "1 - WAT" - ], - "Africa/El_Aaiun": [ - "-0:52:48 - LMT 1934_0 -0:52:48", - "-1 - WAT 1976_3_14 -1", - "0 - WET" + "3 - EAT", ], + "Africa/Djibouti": ["2:52:36 - LMT 1911_6 2:52:36", "3 - EAT"], + "Africa/Douala": ["0:38:48 - LMT 1912 0:38:48", "1 - WAT"], + "Africa/El_Aaiun": ["-0:52:48 - LMT 1934_0 -0:52:48", "-1 - WAT 1976_3_14 -1", "0 - WET"], "Africa/Freetown": [ "-0:53 - LMT 1882 -0:53", "-0:53 - FMT 1913_5 -0:53", "-1 SL %s 1957 -1", - "0 SL %s" + "0 SL %s", ], "Africa/Gaborone": [ "1:43:40 - LMT 1885 1:43:40", "1:30 - SAST 1903_2 1:30", "2 - CAT 1943_8_19_2 2", "3 - CAST 1944_2_19_2 3", - "2 - CAT" - ], - "Africa/Harare": [ - "2:4:12 - LMT 1903_2 2:4:12", - "2 - CAT" - ], - "Africa/Johannesburg": [ - "1:52 - LMT 1892_1_8 1:52", - "1:30 - SAST 1903_2 1:30", - "2 SA SAST" - ], - "Africa/Juba": [ - "2:6:24 - LMT 1931 2:6:24", - "2 Sudan CA%sT 2000_0_15_12 2", - "3 - EAT" + "2 - CAT", ], + "Africa/Harare": ["2:4:12 - LMT 1903_2 2:4:12", "2 - CAT"], + "Africa/Johannesburg": ["1:52 - LMT 1892_1_8 1:52", "1:30 - SAST 1903_2 1:30", "2 SA SAST"], + "Africa/Juba": ["2:6:24 - LMT 1931 2:6:24", "2 Sudan CA%sT 2000_0_15_12 2", "3 - EAT"], "Africa/Kampala": [ "2:9:40 - LMT 1928_6 2:9:40", "3 - EAT 1930 3", "2:30 - BEAT 1948 2:30", "2:45 - BEAUT 1957 2:45", - "3 - EAT" - ], - "Africa/Khartoum": [ - "2:10:8 - LMT 1931 2:10:8", - "2 Sudan CA%sT 2000_0_15_12 2", - "3 - EAT" - ], - "Africa/Kigali": [ - "2:0:16 - LMT 1935_5 2:0:16", - "2 - CAT" - ], - "Africa/Kinshasa": [ - "1:1:12 - LMT 1897_10_9 1:1:12", - "1 - WAT" - ], - "Africa/Lagos": [ - "0:13:36 - LMT 1919_8 0:13:36", - "1 - WAT" - ], - "Africa/Libreville": [ - "0:37:48 - LMT 1912 0:37:48", - "1 - WAT" - ], - "Africa/Lome": [ - "0:4:52 - LMT 1893 0:4:52", - "0 - GMT" - ], - "Africa/Luanda": [ - "0:52:56 - LMT 1892 0:52:56", - "0:52:4 - AOT 1911_4_26 0:52:4", - "1 - WAT" - ], - "Africa/Lubumbashi": [ - "1:49:52 - LMT 1897_10_9 1:49:52", - "2 - CAT" - ], - "Africa/Lusaka": [ - "1:53:8 - LMT 1903_2 1:53:8", - "2 - CAT" - ], - "Africa/Malabo": [ - "0:35:8 - LMT 1912 0:35:8", - "0 - GMT 1963_11_15", - "1 - WAT" - ], - "Africa/Maputo": [ - "2:10:20 - LMT 1903_2 2:10:20", - "2 - CAT" - ], + "3 - EAT", + ], + "Africa/Khartoum": ["2:10:8 - LMT 1931 2:10:8", "2 Sudan CA%sT 2000_0_15_12 2", "3 - EAT"], + "Africa/Kigali": ["2:0:16 - LMT 1935_5 2:0:16", "2 - CAT"], + "Africa/Kinshasa": ["1:1:12 - LMT 1897_10_9 1:1:12", "1 - WAT"], + "Africa/Lagos": ["0:13:36 - LMT 1919_8 0:13:36", "1 - WAT"], + "Africa/Libreville": ["0:37:48 - LMT 1912 0:37:48", "1 - WAT"], + "Africa/Lome": ["0:4:52 - LMT 1893 0:4:52", "0 - GMT"], + "Africa/Luanda": ["0:52:56 - LMT 1892 0:52:56", "0:52:4 - AOT 1911_4_26 0:52:4", "1 - WAT"], + "Africa/Lubumbashi": ["1:49:52 - LMT 1897_10_9 1:49:52", "2 - CAT"], + "Africa/Lusaka": ["1:53:8 - LMT 1903_2 1:53:8", "2 - CAT"], + "Africa/Malabo": ["0:35:8 - LMT 1912 0:35:8", "0 - GMT 1963_11_15", "1 - WAT"], + "Africa/Maputo": ["2:10:20 - LMT 1903_2 2:10:20", "2 - CAT"], "Africa/Maseru": [ "1:50 - LMT 1903_2 1:50", "2 - SAST 1943_8_19_2 2", "3 - SAST 1944_2_19_2 3", - "2 - SAST" - ], - "Africa/Mbabane": [ - "2:4:24 - LMT 1903_2 2:4:24", - "2 - SAST" + "2 - SAST", ], + "Africa/Mbabane": ["2:4:24 - LMT 1903_2 2:4:24", "2 - SAST"], "Africa/Mogadishu": [ "3:1:28 - LMT 1893_10 3:1:28", "3 - EAT 1931 3", "2:30 - BEAT 1957 2:30", - "3 - EAT" + "3 - EAT", ], "Africa/Monrovia": [ "-0:43:8 - LMT 1882 -0:43:8", "-0:43:8 - MMT 1919_2 -0:43:8", "-0:44:30 - LRT 1972_4 -0:44:30", - "0 - GMT" + "0 - GMT", ], "Africa/Nairobi": [ "2:27:16 - LMT 1928_6 2:27:16", "3 - EAT 1930 3", "2:30 - BEAT 1940 2:30", "2:45 - BEAUT 1960 2:45", - "3 - EAT" + "3 - EAT", ], "Africa/Ndjamena": [ "1:0:12 - LMT 1912 1:0:12", "1 - WAT 1979_9_14 1", "2 - WAST 1980_2_8 2", - "1 - WAT" + "1 - WAT", ], "Africa/Niamey": [ "0:8:28 - LMT 1912 0:8:28", "-1 - WAT 1934_1_26 -1", "0 - GMT 1960", - "1 - WAT" + "1 - WAT", ], "Africa/Nouakchott": [ "-1:3:48 - LMT 1912 -1:3:48", "0 - GMT 1934_1_26", "-1 - WAT 1960_10_28 -1", - "0 - GMT" - ], - "Africa/Ouagadougou": [ - "-0:6:4 - LMT 1912 -0:6:4", - "0 - GMT" - ], - "Africa/Porto-Novo": [ - "0:10:28 - LMT 1912 0:10:28", - "0 - GMT 1934_1_26", - "1 - WAT" - ], - "Africa/Sao_Tome": [ - "0:26:56 - LMT 1884 0:26:56", - "-0:36:32 - LMT 1912 -0:36:32", - "0 - GMT" + "0 - GMT", ], + "Africa/Ouagadougou": ["-0:6:4 - LMT 1912 -0:6:4", "0 - GMT"], + "Africa/Porto-Novo": ["0:10:28 - LMT 1912 0:10:28", "0 - GMT 1934_1_26", "1 - WAT"], + "Africa/Sao_Tome": ["0:26:56 - LMT 1884 0:26:56", "-0:36:32 - LMT 1912 -0:36:32", "0 - GMT"], "Africa/Tripoli": [ "0:52:44 - LMT 1920 0:52:44", "1 Libya CE%sT 1959 1", @@ -281,12 +183,12 @@ data = { "2 - EET 1996_8_30 2", "1 Libya CE%sT 1997_9_4 2", "2 - EET 2012_10_10_2 2", - "1 Libya CE%sT" + "1 Libya CE%sT", ], "Africa/Tunis": [ "0:40:44 - LMT 1881_4_12 0:40:44", "0:9:21 - PMT 1911_2_11 0:9:21", - "1 Tunisia CE%sT" + "1 Tunisia CE%sT", ], "Africa/Windhoek": [ "1:8:24 - LMT 1892_1_8 1:8:24", @@ -295,7 +197,7 @@ data = { "3 - SAST 1943_2_21_2 3", "2 - SAST 1990_2_21 2", "2 - CAT 1994_3_3 2", - "1 Namibia WA%sT" + "1 Namibia WA%sT", ], "America/Adak": [ "12:13:21 - LMT 1867_9_18 12:13:21", @@ -306,7 +208,7 @@ data = { "-11 - BST 1969 -11", "-11 US B%sT 1983_9_30_2 -10", "-10 US AH%sT 1983_10_30 -10", - "-10 US HA%sT" + "-10 US HA%sT", ], "America/Anchorage": [ "14:0:24 - LMT 1867_9_18 14:0:24", @@ -318,24 +220,17 @@ data = { "-10 - AHST 1969 -10", "-10 US AH%sT 1983_9_30_2 -9", "-9 US Y%sT 1983_10_30 -9", - "-9 US AK%sT" - ], - "America/Anguilla": [ - "-4:12:16 - LMT 1912_2_2 -4:12:16", - "-4 - AST" - ], - "America/Antigua": [ - "-4:7:12 - LMT 1912_2_2 -4:7:12", - "-5 - EST 1951 -5", - "-4 - AST" + "-9 US AK%sT", ], + "America/Anguilla": ["-4:12:16 - LMT 1912_2_2 -4:12:16", "-4 - AST"], + "America/Antigua": ["-4:7:12 - LMT 1912_2_2 -4:7:12", "-5 - EST 1951 -5", "-4 - AST"], "America/Araguaina": [ "-3:12:48 - LMT 1914 -3:12:48", "-3 Brazil BR%sT 1990_8_17 -3", "-3 - BRT 1995_8_14 -3", "-3 Brazil BR%sT 2003_8_24 -3", "-3 - BRT 2012_9_21 -3", - "-3 Brazil BR%sT" + "-3 Brazil BR%sT", ], "America/Argentina/Buenos_Aires": [ "-3:53:48 - LMT 1894_9_31 -3:53:48", @@ -344,7 +239,7 @@ data = { "-4 Arg AR%sT 1969_9_5 -4", "-3 Arg AR%sT 1999_9_3 -3", "-4 Arg AR%sT 2000_2_3 -3", - "-3 Arg AR%sT" + "-3 Arg AR%sT", ], "America/Argentina/Catamarca": [ "-4:23:8 - LMT 1894_9_31 -4:23:8", @@ -358,7 +253,7 @@ data = { "-3 - ART 2004_5_1 -3", "-4 - WART 2004_5_20 -4", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/Cordoba": [ "-4:16:48 - LMT 1894_9_31 -4:16:48", @@ -369,7 +264,7 @@ data = { "-4 - WART 1991_9_20 -4", "-3 Arg AR%sT 1999_9_3 -3", "-4 Arg AR%sT 2000_2_3 -3", - "-3 Arg AR%sT" + "-3 Arg AR%sT", ], "America/Argentina/Jujuy": [ "-4:21:12 - LMT 1894_9_31 -4:21:12", @@ -384,7 +279,7 @@ data = { "-3 Arg AR%sT 1999_9_3 -3", "-4 Arg AR%sT 2000_2_3 -3", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/La_Rioja": [ "-4:27:24 - LMT 1894_9_31 -4:27:24", @@ -398,7 +293,7 @@ data = { "-3 - ART 2004_5_1 -3", "-4 - WART 2004_5_20 -4", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/Mendoza": [ "-4:35:16 - LMT 1894_9_31 -4:35:16", @@ -416,7 +311,7 @@ data = { "-3 - ART 2004_4_23 -3", "-4 - WART 2004_8_26 -4", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/Rio_Gallegos": [ "-4:36:52 - LMT 1894_9_31 -4:36:52", @@ -428,7 +323,7 @@ data = { "-3 - ART 2004_5_1 -3", "-4 - WART 2004_5_20 -4", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/Salta": [ "-4:21:40 - LMT 1894_9_31 -4:21:40", @@ -440,7 +335,7 @@ data = { "-3 Arg AR%sT 1999_9_3 -3", "-4 Arg AR%sT 2000_2_3 -3", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/San_Juan": [ "-4:34:4 - LMT 1894_9_31 -4:34:4", @@ -454,7 +349,7 @@ data = { "-3 - ART 2004_4_31 -3", "-4 - WART 2004_6_25 -4", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" + "-3 - ART", ], "America/Argentina/San_Luis": [ "-4:25:24 - LMT 1894_9_31 -4:25:24", @@ -471,7 +366,7 @@ data = { "-3 - ART 2004_4_31 -3", "-4 - WART 2004_6_25 -4", "-3 Arg AR%sT 2008_0_21 -2", - "-4 SanLuis WAR%sT" + "-4 SanLuis WAR%sT", ], "America/Argentina/Tucuman": [ "-4:20:52 - LMT 1894_9_31 -4:20:52", @@ -484,7 +379,7 @@ data = { "-4 Arg AR%sT 2000_2_3 -3", "-3 - ART 2004_5_1 -3", "-4 - WART 2004_5_13 -4", - "-3 Arg AR%sT" + "-3 Arg AR%sT", ], "America/Argentina/Ushuaia": [ "-4:33:12 - LMT 1894_9_31 -4:33:12", @@ -496,33 +391,29 @@ data = { "-3 - ART 2004_4_30 -3", "-4 - WART 2004_5_20 -4", "-3 Arg AR%sT 2008_9_18 -3", - "-3 - ART" - ], - "America/Aruba": [ - "-4:40:24 - LMT 1912_1_12 -4:40:24", - "-4:30 - ANT 1965 -4:30", - "-4 - AST" + "-3 - ART", ], + "America/Aruba": ["-4:40:24 - LMT 1912_1_12 -4:40:24", "-4:30 - ANT 1965 -4:30", "-4 - AST"], "America/Asuncion": [ "-3:50:40 - LMT 1890 -3:50:40", "-3:50:40 - AMT 1931_9_10 -3:50:40", "-4 - PYT 1972_9 -4", "-3 - PYT 1974_3 -3", - "-4 Para PY%sT" + "-4 Para PY%sT", ], "America/Atikokan": [ "-6:6:28 - LMT 1895 -6:6:28", "-6 Canada C%sT 1940_8_29 -6", "-5 - CDT 1942_1_9_2 -6", "-6 Canada C%sT 1945_8_30_2 -5", - "-5 - EST" + "-5 - EST", ], "America/Bahia": [ "-2:34:4 - LMT 1914 -2:34:4", "-3 Brazil BR%sT 2003_8_24 -3", "-3 - BRT 2011_9_16 -3", "-3 Brazil BR%sT 2012_9_21 -3", - "-3 - BRT" + "-3 - BRT", ], "America/Bahia_Banderas": [ "-7:1 - LMT 1921_11_31_23_59 -7:1", @@ -535,45 +426,34 @@ data = { "-7 - MST 1949_0_14 -7", "-8 - PST 1970 -8", "-7 Mexico M%sT 2010_3_4_2 -7", - "-6 Mexico C%sT" + "-6 Mexico C%sT", ], "America/Barbados": [ "-3:58:29 - LMT 1924 -3:58:29", "-3:58:29 - BMT 1932 -3:58:29", - "-4 Barb A%sT" - ], - "America/Belem": [ - "-3:13:56 - LMT 1914 -3:13:56", - "-3 Brazil BR%sT 1988_8_12 -3", - "-3 - BRT" - ], - "America/Belize": [ - "-5:52:48 - LMT 1912_3 -5:52:48", - "-6 Belize C%sT" - ], - "America/Blanc-Sablon": [ - "-3:48:28 - LMT 1884 -3:48:28", - "-4 Canada A%sT 1970 -4", - "-4 - AST" + "-4 Barb A%sT", ], + "America/Belem": ["-3:13:56 - LMT 1914 -3:13:56", "-3 Brazil BR%sT 1988_8_12 -3", "-3 - BRT"], + "America/Belize": ["-5:52:48 - LMT 1912_3 -5:52:48", "-6 Belize C%sT"], + "America/Blanc-Sablon": ["-3:48:28 - LMT 1884 -3:48:28", "-4 Canada A%sT 1970 -4", "-4 - AST"], "America/Boa_Vista": [ "-4:2:40 - LMT 1914 -4:2:40", "-4 Brazil AM%sT 1988_8_12 -4", "-4 - AMT 1999_8_30 -4", "-4 Brazil AM%sT 2000_9_15 -3", - "-4 - AMT" + "-4 - AMT", ], "America/Bogota": [ "-4:56:16 - LMT 1884_2_13 -4:56:16", "-4:56:16 - BMT 1914_10_23 -4:56:16", - "-5 CO CO%sT" + "-5 CO CO%sT", ], "America/Boise": [ "-7:44:49 - LMT 1883_10_18_12_15_11 -7:44:49", "-8 US P%sT 1923_4_13_2 -8", "-7 US M%sT 1974 -7", "-7 - MST 1974_1_3_2 -7", - "-7 US M%sT" + "-7 US M%sT", ], "America/Cambridge_Bay": [ "0 - zzz 1920", @@ -581,35 +461,24 @@ data = { "-6 Canada C%sT 2000_9_29_2 -5", "-5 - EST 2000_10_5_0 -5", "-6 - CST 2001_3_1_3 -6", - "-7 Canada M%sT" - ], - "America/Campo_Grande": [ - "-3:38:28 - LMT 1914 -3:38:28", - "-4 Brazil AM%sT" + "-7 Canada M%sT", ], + "America/Campo_Grande": ["-3:38:28 - LMT 1914 -3:38:28", "-4 Brazil AM%sT"], "America/Cancun": [ "-5:47:4 - LMT 1922_0_1_0_12_56 -5:47:4", "-6 - CST 1981_11_23 -6", "-5 Mexico E%sT 1998_7_2_2 -4", - "-6 Mexico C%sT" + "-6 Mexico C%sT", ], "America/Caracas": [ "-4:27:44 - LMT 1890 -4:27:44", "-4:27:40 - CMT 1912_1_12 -4:27:40", "-4:30 - VET 1965 -4:30", "-4 - VET 2007_11_9_03 -4", - "-4:30 - VET" - ], - "America/Cayenne": [ - "-3:29:20 - LMT 1911_6 -3:29:20", - "-4 - GFT 1967_9 -4", - "-3 - GFT" - ], - "America/Cayman": [ - "-5:25:32 - LMT 1890 -5:25:32", - "-5:7:12 - KMT 1912_1 -5:7:12", - "-5 - EST" + "-4:30 - VET", ], + "America/Cayenne": ["-3:29:20 - LMT 1911_6 -3:29:20", "-4 - GFT 1967_9 -4", "-3 - GFT"], + "America/Cayman": ["-5:25:32 - LMT 1890 -5:25:32", "-5:7:12 - KMT 1912_1 -5:7:12", "-5 - EST"], "America/Chicago": [ "-5:50:36 - LMT 1883_10_18_12_9_24 -5:50:36", "-6 US C%sT 1920 -6", @@ -618,7 +487,7 @@ data = { "-6 Chicago C%sT 1942 -6", "-6 US C%sT 1946 -6", "-6 Chicago C%sT 1967 -6", - "-6 US C%sT" + "-6 US C%sT", ], "America/Chihuahua": [ "-7:4:20 - LMT 1921_11_31_23_55_40 -7:4:20", @@ -630,47 +499,43 @@ data = { "-6 - CST 1996 -6", "-6 Mexico C%sT 1998 -6", "-6 - CST 1998_3_5_3 -6", - "-7 Mexico M%sT" + "-7 Mexico M%sT", ], "America/Costa_Rica": [ "-5:36:13 - LMT 1890 -5:36:13", "-5:36:13 - SJMT 1921_0_15 -5:36:13", - "-6 CR C%sT" + "-6 CR C%sT", ], "America/Creston": [ "-7:46:4 - LMT 1884 -7:46:4", "-7 - MST 1916_9_1 -7", "-8 - PST 1918_5_2 -8", - "-7 - MST" + "-7 - MST", ], "America/Cuiaba": [ "-3:44:20 - LMT 1914 -3:44:20", "-4 Brazil AM%sT 2003_8_24 -4", "-4 - AMT 2004_9_1 -4", - "-4 Brazil AM%sT" - ], - "America/Curacao": [ - "-4:35:47 - LMT 1912_1_12 -4:35:47", - "-4:30 - ANT 1965 -4:30", - "-4 - AST" + "-4 Brazil AM%sT", ], + "America/Curacao": ["-4:35:47 - LMT 1912_1_12 -4:35:47", "-4:30 - ANT 1965 -4:30", "-4 - AST"], "America/Danmarkshavn": [ "-1:14:40 - LMT 1916_6_28 -1:14:40", "-3 - WGT 1980_3_6_2 -3", "-3 EU WG%sT 1996 -3", - "0 - GMT" + "0 - GMT", ], "America/Dawson": [ "-9:17:40 - LMT 1900_7_20 -9:17:40", "-9 NT_YK Y%sT 1973_9_28_0 -9", "-8 NT_YK P%sT 1980 -8", - "-8 Canada P%sT" + "-8 Canada P%sT", ], "America/Dawson_Creek": [ "-8:0:56 - LMT 1884 -8:0:56", "-8 Canada P%sT 1947 -8", "-8 Vanc P%sT 1972_7_30_2 -7", - "-7 - MST" + "-7 - MST", ], "America/Denver": [ "-6:59:56 - LMT 1883_10_18_12_0_4 -6:59:56", @@ -678,7 +543,7 @@ data = { "-7 Denver M%sT 1942 -7", "-7 US M%sT 1946 -7", "-7 Denver M%sT 1967 -7", - "-7 US M%sT" + "-7 US M%sT", ], "America/Detroit": [ "-5:32:11 - LMT 1905 -5:32:11", @@ -688,29 +553,19 @@ data = { "-5 Detroit E%sT 1973 -5", "-5 US E%sT 1975 -5", "-5 - EST 1975_3_27_2 -5", - "-5 US E%sT" - ], - "America/Dominica": [ - "-4:5:36 - LMT 1911_6_1_0_1 -4:5:36", - "-4 - AST" - ], - "America/Edmonton": [ - "-7:33:52 - LMT 1906_8 -7:33:52", - "-7 Edm M%sT 1987 -7", - "-7 Canada M%sT" + "-5 US E%sT", ], + "America/Dominica": ["-4:5:36 - LMT 1911_6_1_0_1 -4:5:36", "-4 - AST"], + "America/Edmonton": ["-7:33:52 - LMT 1906_8 -7:33:52", "-7 Edm M%sT 1987 -7", "-7 Canada M%sT"], "America/Eirunepe": [ "-4:39:28 - LMT 1914 -4:39:28", "-5 Brazil AC%sT 1988_8_12 -5", "-5 - ACT 1993_8_28 -5", "-5 Brazil AC%sT 1994_8_22 -5", "-5 - ACT 2008_5_24_00 -5", - "-4 - AMT" - ], - "America/El_Salvador": [ - "-5:56:48 - LMT 1921 -5:56:48", - "-6 Salv C%sT" + "-4 - AMT", ], + "America/El_Salvador": ["-5:56:48 - LMT 1921 -5:56:48", "-6 Salv C%sT"], "America/Fortaleza": [ "-2:34 - LMT 1914 -2:34", "-3 Brazil BR%sT 1990_8_17 -3", @@ -718,7 +573,7 @@ data = { "-3 Brazil BR%sT 2000_9_22 -2", "-3 - BRT 2001_8_13 -3", "-3 Brazil BR%sT 2002_9_1 -3", - "-3 - BRT" + "-3 - BRT", ], "America/Glace_Bay": [ "-3:59:48 - LMT 1902_5_15 -3:59:48", @@ -726,12 +581,12 @@ data = { "-4 Halifax A%sT 1954 -4", "-4 - AST 1972 -4", "-4 Halifax A%sT 1974 -4", - "-4 Canada A%sT" + "-4 Canada A%sT", ], "America/Godthab": [ "-3:26:56 - LMT 1916_6_28 -3:26:56", "-3 - WGT 1980_3_6_2 -3", - "-3 EU WG%sT" + "-3 EU WG%sT", ], "America/Goose_Bay": [ "-4:1:40 - LMT 1884 -4:1:40", @@ -743,36 +598,23 @@ data = { "-3:30 Canada N%sT 1946 -3:30", "-3:30 StJohns N%sT 1966_2_15_2 -3:30", "-4 StJohns A%sT 2011_10 -3", - "-4 Canada A%sT" + "-4 Canada A%sT", ], "America/Grand_Turk": [ "-4:44:32 - LMT 1890 -4:44:32", "-5:7:12 - KMT 1912_1 -5:7:12", - "-5 TC E%sT" - ], - "America/Grenada": [ - "-4:7 - LMT 1911_6 -4:7", - "-4 - AST" - ], - "America/Guadeloupe": [ - "-4:6:8 - LMT 1911_5_8 -4:6:8", - "-4 - AST" - ], - "America/Guatemala": [ - "-6:2:4 - LMT 1918_9_5 -6:2:4", - "-6 Guat C%sT" - ], - "America/Guayaquil": [ - "-5:19:20 - LMT 1890 -5:19:20", - "-5:14 - QMT 1931 -5:14", - "-5 - ECT" + "-5 TC E%sT", ], + "America/Grenada": ["-4:7 - LMT 1911_6 -4:7", "-4 - AST"], + "America/Guadeloupe": ["-4:6:8 - LMT 1911_5_8 -4:6:8", "-4 - AST"], + "America/Guatemala": ["-6:2:4 - LMT 1918_9_5 -6:2:4", "-6 Guat C%sT"], + "America/Guayaquil": ["-5:19:20 - LMT 1890 -5:19:20", "-5:14 - QMT 1931 -5:14", "-5 - ECT"], "America/Guyana": [ "-3:52:40 - LMT 1915_2 -3:52:40", "-3:45 - GBGT 1966_4_26 -3:45", "-3:45 - GYT 1975_6_31 -3:45", "-3 - GYT 1991 -3", - "-4 - GYT" + "-4 - GYT", ], "America/Halifax": [ "-4:14:24 - LMT 1902_5_15 -4:14:24", @@ -781,12 +623,12 @@ data = { "-4 Halifax A%sT 1942_1_9_2 -4", "-4 Canada A%sT 1946 -4", "-4 Halifax A%sT 1974 -4", - "-4 Canada A%sT" + "-4 Canada A%sT", ], "America/Havana": [ "-5:29:28 - LMT 1890 -5:29:28", "-5:29:36 - HMT 1925_6_19_12 -5:29:36", - "-5 Cuba C%sT" + "-5 Cuba C%sT", ], "America/Hermosillo": [ "-7:23:52 - LMT 1921_11_31_23_36_8 -7:23:52", @@ -799,7 +641,7 @@ data = { "-7 - MST 1949_0_14 -7", "-8 - PST 1970 -8", "-7 Mexico M%sT 1999 -7", - "-7 - MST" + "-7 - MST", ], "America/Indiana/Indianapolis": [ "-5:44:38 - LMT 1883_10_18_12_15_22 -5:44:38", @@ -812,7 +654,7 @@ data = { "-5 - EST 1969 -5", "-5 US E%sT 1971 -5", "-5 - EST 2006 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Indiana/Knox": [ "-5:46:30 - LMT 1883_10_18_12_13_30 -5:46:30", @@ -821,7 +663,7 @@ data = { "-5 - EST 1963_9_27_2 -5", "-6 US C%sT 1991_9_27_2 -5", "-5 - EST 2006_3_2_2 -5", - "-6 US C%sT" + "-6 US C%sT", ], "America/Indiana/Marengo": [ "-5:45:23 - LMT 1883_10_18_12_14_37 -5:45:23", @@ -832,7 +674,7 @@ data = { "-5 - CDT 1974_9_27_2 -5", "-5 US E%sT 1976 -5", "-5 - EST 2006 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Indiana/Petersburg": [ "-5:49:7 - LMT 1883_10_18_12_10_53 -5:49:7", @@ -842,7 +684,7 @@ data = { "-6 US C%sT 1977_9_30_2 -5", "-5 - EST 2006_3_2_2 -5", "-6 US C%sT 2007_10_4_2 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Indiana/Tell_City": [ "-5:47:3 - LMT 1883_10_18_12_12_57 -5:47:3", @@ -851,7 +693,7 @@ data = { "-5 - EST 1969 -5", "-5 US E%sT 1971 -5", "-5 - EST 2006_3_2_2 -5", - "-6 US C%sT" + "-6 US C%sT", ], "America/Indiana/Vevay": [ "-5:40:16 - LMT 1883_10_18_12_19_44 -5:40:16", @@ -859,7 +701,7 @@ data = { "-5 - EST 1969 -5", "-5 US E%sT 1973 -5", "-5 - EST 2006 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Indiana/Vincennes": [ "-5:50:7 - LMT 1883_10_18_12_9_53 -5:50:7", @@ -869,7 +711,7 @@ data = { "-5 US E%sT 1971 -5", "-5 - EST 2006_3_2_2 -5", "-6 US C%sT 2007_10_4_2 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Indiana/Winamac": [ "-5:46:25 - LMT 1883_10_18_12_13_35 -5:46:25", @@ -879,26 +721,26 @@ data = { "-5 US E%sT 1971 -5", "-5 - EST 2006_3_2_2 -5", "-6 US C%sT 2007_2_11_2 -6", - "-5 US E%sT" + "-5 US E%sT", ], "America/Inuvik": [ "0 - zzz 1953", "-8 NT_YK P%sT 1979_3_29_2 -8", "-7 NT_YK M%sT 1980 -7", - "-7 Canada M%sT" + "-7 Canada M%sT", ], "America/Iqaluit": [ "0 - zzz 1942_7", "-5 NT_YK E%sT 1999_9_31_2 -4", "-6 Canada C%sT 2000_9_29_2 -5", - "-5 Canada E%sT" + "-5 Canada E%sT", ], "America/Jamaica": [ "-5:7:12 - LMT 1890 -5:7:12", "-5:7:12 - KMT 1912_1 -5:7:12", "-5 - EST 1974_3_28_2 -5", "-5 US E%sT 1984 -5", - "-5 - EST" + "-5 - EST", ], "America/Juneau": [ "15:2:19 - LMT 1867_9_18 15:2:19", @@ -910,7 +752,7 @@ data = { "-9 US Y%sT 1980_9_26_2 -8", "-8 US P%sT 1983_9_30_2 -7", "-9 US Y%sT 1983_10_30 -9", - "-9 US AK%sT" + "-9 US AK%sT", ], "America/Kentucky/Louisville": [ "-5:43:2 - LMT 1883_10_18_12_16_58 -5:43:2", @@ -921,31 +763,31 @@ data = { "-5 - EST 1968 -5", "-5 US E%sT 1974_0_6_2 -5", "-5 - CDT 1974_9_27_2 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Kentucky/Monticello": [ "-5:39:24 - LMT 1883_10_18_12_20_36 -5:39:24", "-6 US C%sT 1946 -6", "-6 - CST 1968 -6", "-6 US C%sT 2000_9_29_2 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/La_Paz": [ "-4:32:36 - LMT 1890 -4:32:36", "-4:32:36 - CMT 1931_9_15 -4:32:36", "-3:32:36 - BOST 1932_2_21 -3:32:36", - "-4 - BOT" + "-4 - BOT", ], "America/Lima": [ "-5:8:12 - LMT 1890 -5:8:12", "-5:8:36 - LMT 1908_6_28 -5:8:36", - "-5 Peru PE%sT" + "-5 Peru PE%sT", ], "America/Los_Angeles": [ "-7:52:58 - LMT 1883_10_18_12_7_2 -7:52:58", "-8 US P%sT 1946 -8", "-8 CA P%sT 1967 -8", - "-8 US P%sT" + "-8 US P%sT", ], "America/Maceio": [ "-2:22:52 - LMT 1914 -2:22:52", @@ -956,7 +798,7 @@ data = { "-3 Brazil BR%sT 2000_9_22 -2", "-3 - BRT 2001_8_13 -3", "-3 Brazil BR%sT 2002_9_1 -3", - "-3 - BRT" + "-3 - BRT", ], "America/Managua": [ "-5:45:8 - LMT 1890 -5:45:8", @@ -967,28 +809,28 @@ data = { "-5 - EST 1992_8_24 -5", "-6 - CST 1993 -6", "-5 - EST 1997 -5", - "-6 Nic C%sT" + "-6 Nic C%sT", ], "America/Manaus": [ "-4:0:4 - LMT 1914 -4:0:4", "-4 Brazil AM%sT 1988_8_12 -4", "-4 - AMT 1993_8_28 -4", "-4 Brazil AM%sT 1994_8_22 -4", - "-4 - AMT" + "-4 - AMT", ], "America/Martinique": [ "-4:4:20 - LMT 1890 -4:4:20", "-4:4:20 - FFMT 1911_4 -4:4:20", "-4 - AST 1980_3_6 -4", "-3 - ADT 1980_8_28 -3", - "-4 - AST" + "-4 - AST", ], "America/Matamoros": [ "-6:40 - LMT 1921_11_31_23_20 -6:40", "-6 - CST 1988 -6", "-6 US C%sT 1989 -6", "-6 Mexico C%sT 2010 -6", - "-6 US C%sT" + "-6 US C%sT", ], "America/Mazatlan": [ "-7:5:40 - LMT 1921_11_31_23_54_20 -7:5:40", @@ -1000,20 +842,20 @@ data = { "-6 - CST 1942_3_24 -6", "-7 - MST 1949_0_14 -7", "-8 - PST 1970 -8", - "-7 Mexico M%sT" + "-7 Mexico M%sT", ], "America/Menominee": [ "-5:50:27 - LMT 1885_8_18_12 -5:50:27", "-6 US C%sT 1946 -6", "-6 Menominee C%sT 1969_3_27_2 -6", "-5 - EST 1973_3_29_2 -5", - "-6 US C%sT" + "-6 US C%sT", ], "America/Merida": [ "-5:58:28 - LMT 1922_0_1_0_1_32 -5:58:28", "-6 - CST 1981_11_23 -6", "-5 - EST 1982_11_2 -5", - "-6 Mexico C%sT" + "-6 Mexico C%sT", ], "America/Metlakatla": [ "15:13:42 - LMT 1867_9_18 15:13:42", @@ -1022,7 +864,7 @@ data = { "-8 US P%sT 1946 -8", "-8 - PST 1969 -8", "-8 US P%sT 1983_9_30_2 -7", - "-8 - MeST" + "-8 - MeST", ], "America/Mexico_City": [ "-6:36:36 - LMT 1922_0_1_0_23_24 -6:36:36", @@ -1033,13 +875,13 @@ data = { "-7 - MST 1932_3_1 -7", "-6 Mexico C%sT 2001_8_30_02 -5", "-6 - CST 2002_1_20 -6", - "-6 Mexico C%sT" + "-6 Mexico C%sT", ], "America/Miquelon": [ "-3:44:40 - LMT 1911_4_15 -3:44:40", "-4 - AST 1980_4 -4", "-3 - PMST 1987 -3", - "-3 Canada PM%sT" + "-3 Canada PM%sT", ], "America/Moncton": [ "-4:19:8 - LMT 1883_11_9 -4:19:8", @@ -1050,19 +892,19 @@ data = { "-4 Moncton A%sT 1973 -4", "-4 Canada A%sT 1993 -4", "-4 Moncton A%sT 2007 -4", - "-4 Canada A%sT" + "-4 Canada A%sT", ], "America/Monterrey": [ "-6:41:16 - LMT 1921_11_31_23_18_44 -6:41:16", "-6 - CST 1988 -6", "-6 US C%sT 1989 -6", - "-6 Mexico C%sT" + "-6 Mexico C%sT", ], "America/Montevideo": [ "-3:44:44 - LMT 1898_5_28 -3:44:44", "-3:44:44 - MMT 1920_4_1 -3:44:44", "-3:30 Uruguay UY%sT 1942_11_14 -3:30", - "-3 Uruguay UY%sT" + "-3 Uruguay UY%sT", ], "America/Montreal": [ "-4:54:16 - LMT 1884 -4:54:16", @@ -1071,30 +913,23 @@ data = { "-5 Mont E%sT 1942_1_9_2 -5", "-5 Canada E%sT 1946 -5", "-5 Mont E%sT 1974 -5", - "-5 Canada E%sT" - ], - "America/Montserrat": [ - "-4:8:52 - LMT 1911_6_1_0_1 -4:8:52", - "-4 - AST" - ], - "America/Nassau": [ - "-5:9:30 - LMT 1912_2_2 -5:9:30", - "-5 Bahamas E%sT 1976 -5", - "-5 US E%sT" + "-5 Canada E%sT", ], + "America/Montserrat": ["-4:8:52 - LMT 1911_6_1_0_1 -4:8:52", "-4 - AST"], + "America/Nassau": ["-5:9:30 - LMT 1912_2_2 -5:9:30", "-5 Bahamas E%sT 1976 -5", "-5 US E%sT"], "America/New_York": [ "-4:56:2 - LMT 1883_10_18_12_3_58 -4:56:2", "-5 US E%sT 1920 -5", "-5 NYC E%sT 1942 -5", "-5 US E%sT 1946 -5", "-5 NYC E%sT 1967 -5", - "-5 US E%sT" + "-5 US E%sT", ], "America/Nipigon": [ "-5:53:4 - LMT 1895 -5:53:4", "-5 Canada E%sT 1940_8_29 -5", "-4 - EDT 1942_1_9_2 -5", - "-5 Canada E%sT" + "-5 Canada E%sT", ], "America/Nome": [ "12:58:21 - LMT 1867_9_18 12:58:21", @@ -1105,7 +940,7 @@ data = { "-11 - BST 1969 -11", "-11 US B%sT 1983_9_30_2 -10", "-9 US Y%sT 1983_10_30 -9", - "-9 US AK%sT" + "-9 US AK%sT", ], "America/Noronha": [ "-2:9:40 - LMT 1914 -2:9:40", @@ -1114,22 +949,22 @@ data = { "-2 Brazil FN%sT 2000_9_15 -1", "-2 - FNT 2001_8_13 -2", "-2 Brazil FN%sT 2002_9_1 -2", - "-2 - FNT" + "-2 - FNT", ], "America/North_Dakota/Beulah": [ "-6:47:7 - LMT 1883_10_18_12_12_53 -6:47:7", "-7 US M%sT 2010_10_7_2 -6", - "-6 US C%sT" + "-6 US C%sT", ], "America/North_Dakota/Center": [ "-6:45:12 - LMT 1883_10_18_12_14_48 -6:45:12", "-7 US M%sT 1992_9_25_02 -6", - "-6 US C%sT" + "-6 US C%sT", ], "America/North_Dakota/New_Salem": [ "-6:45:39 - LMT 1883_10_18_12_14_21 -6:45:39", "-7 US M%sT 2003_9_26_02 -6", - "-6 US C%sT" + "-6 US C%sT", ], "America/Ojinaga": [ "-6:57:40 - LMT 1922_0_1_0_2_20 -6:57:40", @@ -1142,19 +977,19 @@ data = { "-6 Mexico C%sT 1998 -6", "-6 - CST 1998_3_5_3 -6", "-7 Mexico M%sT 2010 -7", - "-7 US M%sT" + "-7 US M%sT", ], "America/Panama": [ "-5:18:8 - LMT 1890 -5:18:8", "-5:19:36 - CMT 1908_3_22 -5:19:36", - "-5 - EST" + "-5 - EST", ], "America/Pangnirtung": [ "0 - zzz 1921", "-4 NT_YK A%sT 1995_3_2_2 -4", "-5 Canada E%sT 1999_9_31_2 -4", "-6 Canada C%sT 2000_9_29_2 -5", - "-5 Canada E%sT" + "-5 Canada E%sT", ], "America/Paramaribo": [ "-3:40:40 - LMT 1911 -3:40:40", @@ -1162,7 +997,7 @@ data = { "-3:40:36 - PMT 1945_9 -3:40:36", "-3:30 - NEGT 1975_10_20 -3:30", "-3:30 - SRT 1984_9 -3:30", - "-3 - SRT" + "-3 - SRT", ], "America/Phoenix": [ "-7:28:18 - LMT 1883_10_18_11_31_42 -7:28:18", @@ -1171,39 +1006,36 @@ data = { "-7 US M%sT 1944_9_1_00_1 -6", "-7 - MST 1967 -7", "-7 US M%sT 1968_2_21 -7", - "-7 - MST" + "-7 - MST", ], "America/Port-au-Prince": [ "-4:49:20 - LMT 1890 -4:49:20", "-4:49 - PPMT 1917_0_24_12 -4:49", - "-5 Haiti E%sT" - ], - "America/Port_of_Spain": [ - "-4:6:4 - LMT 1912_2_2 -4:6:4", - "-4 - AST" + "-5 Haiti E%sT", ], + "America/Port_of_Spain": ["-4:6:4 - LMT 1912_2_2 -4:6:4", "-4 - AST"], "America/Porto_Velho": [ "-4:15:36 - LMT 1914 -4:15:36", "-4 Brazil AM%sT 1988_8_12 -4", - "-4 - AMT" + "-4 - AMT", ], "America/Puerto_Rico": [ "-4:24:25 - LMT 1899_2_28_12 -4:24:25", "-4 - AST 1942_4_3 -4", "-4 US A%sT 1946 -4", - "-4 - AST" + "-4 - AST", ], "America/Rainy_River": [ "-6:18:16 - LMT 1895 -6:18:16", "-6 Canada C%sT 1940_8_29 -6", "-5 - CDT 1942_1_9_2 -6", - "-6 Canada C%sT" + "-6 Canada C%sT", ], "America/Rankin_Inlet": [ "0 - zzz 1957", "-6 NT_YK C%sT 2000_9_29_2 -5", "-5 - EST 2001_3_1_3 -5", - "-6 Canada C%sT" + "-6 Canada C%sT", ], "America/Recife": [ "-2:19:36 - LMT 1914 -2:19:36", @@ -1212,12 +1044,12 @@ data = { "-3 Brazil BR%sT 2000_9_15 -2", "-3 - BRT 2001_8_13 -3", "-3 Brazil BR%sT 2002_9_1 -3", - "-3 - BRT" + "-3 - BRT", ], "America/Regina": [ "-6:58:36 - LMT 1905_8 -6:58:36", "-7 Regina M%sT 1960_3_24_2 -7", - "-6 - CST" + "-6 - CST", ], "America/Resolute": [ "0 - zzz 1947_7_31", @@ -1225,13 +1057,13 @@ data = { "-5 - EST 2001_3_1_3 -5", "-6 Canada C%sT 2006_9_29_2 -5", "-5 - EST 2007_2_11_3 -5", - "-6 Canada C%sT" + "-6 Canada C%sT", ], "America/Rio_Branco": [ "-4:31:12 - LMT 1914 -4:31:12", "-5 Brazil AC%sT 1988_8_12 -5", "-5 - ACT 2008_5_24_00 -5", - "-4 - AMT" + "-4 - AMT", ], "America/Santa_Isabel": [ "-7:39:28 - LMT 1922_0_1_0_20_32 -7:39:28", @@ -1251,13 +1083,13 @@ data = { "-8 US P%sT 1996 -8", "-8 Mexico P%sT 2001 -8", "-8 US P%sT 2002_1_20 -8", - "-8 Mexico P%sT" + "-8 Mexico P%sT", ], "America/Santarem": [ "-3:38:48 - LMT 1914 -3:38:48", "-4 Brazil AM%sT 1988_8_12 -4", "-4 - AMT 2008_5_24_00 -4", - "-3 - BRT" + "-3 - BRT", ], "America/Santiago": [ "-4:42:46 - LMT 1890 -4:42:46", @@ -1267,7 +1099,7 @@ data = { "-4 - CLT 1919_6_1 -4", "-4:42:46 - SMT 1927_8_1 -4:42:46", "-5 Chile CL%sT 1947_4_22 -5", - "-4 Chile CL%sT" + "-4 Chile CL%sT", ], "America/Santo_Domingo": [ "-4:39:36 - LMT 1890 -4:39:36", @@ -1275,19 +1107,19 @@ data = { "-5 DR E%sT 1974_9_27 -5", "-4 - AST 2000_9_29_02 -4", "-5 US E%sT 2000_11_3_01 -5", - "-4 - AST" + "-4 - AST", ], "America/Sao_Paulo": [ "-3:6:28 - LMT 1914 -3:6:28", "-3 Brazil BR%sT 1963_9_23_00 -3", "-2 - BRST 1964 -2", - "-3 Brazil BR%sT" + "-3 Brazil BR%sT", ], "America/Scoresbysund": [ "-1:27:52 - LMT 1916_6_28 -1:27:52", "-2 - CGT 1980_3_6_2 -2", "-2 C-Eur CG%sT 1981_2_29 -2", - "-1 EU EG%sT" + "-1 EU EG%sT", ], "America/Sitka": [ "14:58:47 - LMT 1867_9_18 14:58:47", @@ -1297,7 +1129,7 @@ data = { "-8 - PST 1969 -8", "-8 US P%sT 1983_9_30_2 -7", "-9 US Y%sT 1983_10_30 -9", - "-9 US AK%sT" + "-9 US AK%sT", ], "America/St_Johns": [ "-3:30:52 - LMT 1884 -3:30:52", @@ -1307,41 +1139,21 @@ data = { "-3:30 StJohns N%sT 1942_4_11 -3:30", "-3:30 Canada N%sT 1946 -3:30", "-3:30 StJohns N%sT 2011_10 -2:30", - "-3:30 Canada N%sT" - ], - "America/St_Kitts": [ - "-4:10:52 - LMT 1912_2_2 -4:10:52", - "-4 - AST" - ], - "America/St_Lucia": [ - "-4:4 - LMT 1890 -4:4", - "-4:4 - CMT 1912 -4:4", - "-4 - AST" - ], - "America/St_Thomas": [ - "-4:19:44 - LMT 1911_6 -4:19:44", - "-4 - AST" - ], - "America/St_Vincent": [ - "-4:4:56 - LMT 1890 -4:4:56", - "-4:4:56 - KMT 1912 -4:4:56", - "-4 - AST" + "-3:30 Canada N%sT", ], + "America/St_Kitts": ["-4:10:52 - LMT 1912_2_2 -4:10:52", "-4 - AST"], + "America/St_Lucia": ["-4:4 - LMT 1890 -4:4", "-4:4 - CMT 1912 -4:4", "-4 - AST"], + "America/St_Thomas": ["-4:19:44 - LMT 1911_6 -4:19:44", "-4 - AST"], + "America/St_Vincent": ["-4:4:56 - LMT 1890 -4:4:56", "-4:4:56 - KMT 1912 -4:4:56", "-4 - AST"], "America/Swift_Current": [ "-7:11:20 - LMT 1905_8 -7:11:20", "-7 Canada M%sT 1946_3_28_2 -7", "-7 Regina M%sT 1950 -7", "-7 Swift M%sT 1972_3_30_2 -7", - "-6 - CST" - ], - "America/Tegucigalpa": [ - "-5:48:52 - LMT 1921_3 -5:48:52", - "-6 Hond C%sT" - ], - "America/Thule": [ - "-4:35:8 - LMT 1916_6_28 -4:35:8", - "-4 Thule A%sT" + "-6 - CST", ], + "America/Tegucigalpa": ["-5:48:52 - LMT 1921_3 -5:48:52", "-6 Hond C%sT"], + "America/Thule": ["-4:35:8 - LMT 1916_6_28 -4:35:8", "-4 Thule A%sT"], "America/Thunder_Bay": [ "-5:57 - LMT 1895 -5:57", "-6 - CST 1910 -6", @@ -1349,7 +1161,7 @@ data = { "-5 Canada E%sT 1970 -5", "-5 Mont E%sT 1973 -5", "-5 - EST 1974 -5", - "-5 Canada E%sT" + "-5 Canada E%sT", ], "America/Tijuana": [ "-7:48:4 - LMT 1922_0_1_0_11_56 -7:48:4", @@ -1370,7 +1182,7 @@ data = { "-8 Mexico P%sT 2001 -8", "-8 US P%sT 2002_1_20 -8", "-8 Mexico P%sT 2010 -8", - "-8 US P%sT" + "-8 US P%sT", ], "America/Toronto": [ "-5:17:32 - LMT 1895 -5:17:32", @@ -1378,27 +1190,20 @@ data = { "-5 Toronto E%sT 1942_1_9_2 -5", "-5 Canada E%sT 1946 -5", "-5 Toronto E%sT 1974 -5", - "-5 Canada E%sT" - ], - "America/Tortola": [ - "-4:18:28 - LMT 1911_6 -4:18:28", - "-4 - AST" - ], - "America/Vancouver": [ - "-8:12:28 - LMT 1884 -8:12:28", - "-8 Vanc P%sT 1987 -8", - "-8 Canada P%sT" + "-5 Canada E%sT", ], + "America/Tortola": ["-4:18:28 - LMT 1911_6 -4:18:28", "-4 - AST"], + "America/Vancouver": ["-8:12:28 - LMT 1884 -8:12:28", "-8 Vanc P%sT 1987 -8", "-8 Canada P%sT"], "America/Whitehorse": [ "-9:0:12 - LMT 1900_7_20 -9:0:12", "-9 NT_YK Y%sT 1966_6_1_2 -9", "-8 NT_YK P%sT 1980 -8", - "-8 Canada P%sT" + "-8 Canada P%sT", ], "America/Winnipeg": [ "-6:28:36 - LMT 1887_6_16 -6:28:36", "-6 Winn C%sT 2006 -6", - "-6 Canada C%sT" + "-6 Canada C%sT", ], "America/Yakutat": [ "14:41:5 - LMT 1867_9_18 14:41:5", @@ -1407,20 +1212,16 @@ data = { "-9 US Y%sT 1946 -9", "-9 - YST 1969 -9", "-9 US Y%sT 1983_10_30 -9", - "-9 US AK%sT" - ], - "America/Yellowknife": [ - "0 - zzz 1935", - "-7 NT_YK M%sT 1980 -7", - "-7 Canada M%sT" + "-9 US AK%sT", ], + "America/Yellowknife": ["0 - zzz 1935", "-7 NT_YK M%sT 1980 -7", "-7 Canada M%sT"], "Antarctica/Casey": [ "0 - zzz 1969", "8 - WST 2009_9_18_2 8", "11 - CAST 2010_2_5_2 11", "8 - WST 2011_9_28_2 8", "11 - CAST 2012_1_21_17", - "8 - WST" + "8 - WST", ], "Antarctica/Davis": [ "0 - zzz 1957_0_13", @@ -1430,13 +1231,13 @@ data = { "5 - DAVT 2010_2_10_20", "7 - DAVT 2011_9_28_2 7", "5 - DAVT 2012_1_21_20", - "7 - DAVT" + "7 - DAVT", ], "Antarctica/DumontDUrville": [ "0 - zzz 1947", "10 - PMT 1952_0_14 10", "0 - zzz 1956_10", - "10 - DDUT" + "10 - DDUT", ], "Antarctica/Macquarie": [ "0 - zzz 1899_10", @@ -1446,58 +1247,36 @@ data = { "0 - zzz 1948_2_25", "10 Aus EST 1967 10", "10 AT EST 2010_3_4_3 11", - "11 - MIST" - ], - "Antarctica/Mawson": [ - "0 - zzz 1954_1_13", - "6 - MAWT 2009_9_18_2 6", - "5 - MAWT" - ], - "Antarctica/McMurdo": [ - "0 - zzz 1956", - "12 NZAQ NZ%sT" + "11 - MIST", ], + "Antarctica/Mawson": ["0 - zzz 1954_1_13", "6 - MAWT 2009_9_18_2 6", "5 - MAWT"], + "Antarctica/McMurdo": ["0 - zzz 1956", "12 NZAQ NZ%sT"], "Antarctica/Palmer": [ "0 - zzz 1965", "-4 ArgAQ AR%sT 1969_9_5 -4", "-3 ArgAQ AR%sT 1982_4 -3", - "-4 ChileAQ CL%sT" - ], - "Antarctica/Rothera": [ - "0 - zzz 1976_11_1", - "-3 - ROTT" - ], - "Antarctica/Syowa": [ - "0 - zzz 1957_0_29", - "3 - SYOT" - ], - "Antarctica/Vostok": [ - "0 - zzz 1957_11_16", - "6 - VOST" + "-4 ChileAQ CL%sT", ], + "Antarctica/Rothera": ["0 - zzz 1976_11_1", "-3 - ROTT"], + "Antarctica/Syowa": ["0 - zzz 1957_0_29", "3 - SYOT"], + "Antarctica/Vostok": ["0 - zzz 1957_11_16", "6 - VOST"], "Europe/Oslo": [ "0:43 - LMT 1895_0_1 0:43", "1 Norway CE%sT 1940_7_10_23 1", "1 C-Eur CE%sT 1945_3_2_2 1", "1 Norway CE%sT 1980 1", - "1 EU CE%sT" - ], - "Asia/Aden": [ - "2:59:54 - LMT 1950 2:59:54", - "3 - AST" + "1 EU CE%sT", ], + "Asia/Aden": ["2:59:54 - LMT 1950 2:59:54", "3 - AST"], "Asia/Almaty": [ "5:7:48 - LMT 1924_4_2 5:7:48", "5 - ALMT 1930_5_21 5", "6 RussiaAsia ALM%sT 1991 6", "6 - ALMT 1992 6", "6 RussiaAsia ALM%sT 2005_2_15 6", - "6 - ALMT" - ], - "Asia/Amman": [ - "2:23:44 - LMT 1931 2:23:44", - "2 Jordan EE%sT" + "6 - ALMT", ], + "Asia/Amman": ["2:23:44 - LMT 1931 2:23:44", "2 Jordan EE%sT"], "Asia/Anadyr": [ "11:49:56 - LMT 1924_4_2 11:49:56", "12 - ANAT 1930_5_21 12", @@ -1506,7 +1285,7 @@ data = { "11 Russia ANA%sT 1992_0_19_2 11", "12 Russia ANA%sT 2010_2_28_2 12", "11 Russia ANA%sT 2011_2_27_2 11", - "12 - ANAT" + "12 - ANAT", ], "Asia/Aqtau": [ "3:21:4 - LMT 1924_4_2 3:21:4", @@ -1518,7 +1297,7 @@ data = { "5 - SHET 1991_11_16 5", "5 RussiaAsia AQT%sT 1995_2_26_2 5", "4 RussiaAsia AQT%sT 2005_2_15 4", - "5 - AQTT" + "5 - AQTT", ], "Asia/Aqtobe": [ "3:48:40 - LMT 1924_4_2 3:48:40", @@ -1529,7 +1308,7 @@ data = { "5 RussiaAsia AKT%sT 1991 5", "5 - AKTT 1991_11_16 5", "5 RussiaAsia AQT%sT 2005_2_15 5", - "5 - AQTT" + "5 - AQTT", ], "Asia/Ashgabat": [ "3:53:32 - LMT 1924_4_2 3:53:32", @@ -1537,19 +1316,15 @@ data = { "5 RussiaAsia ASH%sT 1991_2_31_2 5", "4 RussiaAsia ASH%sT 1991_9_27 4", "4 RussiaAsia TM%sT 1992_0_19_2 4", - "5 - TMT" + "5 - TMT", ], "Asia/Baghdad": [ "2:57:40 - LMT 1890 2:57:40", "2:57:36 - BMT 1918 2:57:36", "3 - AST 1982_4 3", - "3 Iraq A%sT" - ], - "Asia/Bahrain": [ - "3:22:20 - LMT 1920 3:22:20", - "4 - GST 1972_5 4", - "3 - AST" + "3 Iraq A%sT", ], + "Asia/Bahrain": ["3:22:20 - LMT 1920 3:22:20", "4 - GST 1972_5 4", "3 - AST"], "Asia/Baku": [ "3:19:24 - LMT 1924_4_2 3:19:24", "3 - BAKT 1957_2 3", @@ -1558,42 +1333,27 @@ data = { "3 RussiaAsia AZ%sT 1992_8_26_23 4", "4 - AZT 1996 4", "4 EUAsia AZ%sT 1997 4", - "4 Azer AZ%sT" - ], - "Asia/Bangkok": [ - "6:42:4 - LMT 1880 6:42:4", - "6:42:4 - BMT 1920_3 6:42:4", - "7 - ICT" - ], - "Asia/Beirut": [ - "2:22 - LMT 1880 2:22", - "2 Lebanon EE%sT" + "4 Azer AZ%sT", ], + "Asia/Bangkok": ["6:42:4 - LMT 1880 6:42:4", "6:42:4 - BMT 1920_3 6:42:4", "7 - ICT"], + "Asia/Beirut": ["2:22 - LMT 1880 2:22", "2 Lebanon EE%sT"], "Asia/Bishkek": [ "4:58:24 - LMT 1924_4_2 4:58:24", "5 - FRUT 1930_5_21 5", "6 RussiaAsia FRU%sT 1991_2_31_2 6", "6 - FRUST 1991_7_31_2 6", "5 Kyrgyz KG%sT 2005_7_12 6", - "6 - KGT" - ], - "Asia/Brunei": [ - "7:39:40 - LMT 1926_2 7:39:40", - "7:30 - BNT 1933 7:30", - "8 - BNT" + "6 - KGT", ], + "Asia/Brunei": ["7:39:40 - LMT 1926_2 7:39:40", "7:30 - BNT 1933 7:30", "8 - BNT"], "Asia/Choibalsan": [ "7:38 - LMT 1905_7 7:38", "7 - ULAT 1978 7", "8 - ULAT 1983_3 8", "9 Mongol CHO%sT 2008_2_31 9", - "8 Mongol CHO%sT" - ], - "Asia/Chongqing": [ - "7:6:20 - LMT 1928 7:6:20", - "7 - LONT 1980_4 7", - "8 PRC C%sT" + "8 Mongol CHO%sT", ], + "Asia/Chongqing": ["7:6:20 - LMT 1928 7:6:20", "7 - LONT 1980_4 7", "8 PRC C%sT"], "Asia/Colombo": [ "5:19:24 - LMT 1880 5:19:24", "5:19:32 - MMT 1906 5:19:32", @@ -1603,12 +1363,9 @@ data = { "5:30 - IST 1996_4_25_0 5:30", "6:30 - LKT 1996_9_26_0_30 6:30", "6 - LKT 2006_3_15_0_30 6", - "5:30 - IST" - ], - "Asia/Damascus": [ - "2:25:12 - LMT 1920 2:25:12", - "2 Syria EE%sT" + "5:30 - IST", ], + "Asia/Damascus": ["2:25:12 - LMT 1920 2:25:12", "2 Syria EE%sT"], "Asia/Dhaka": [ "6:1:40 - LMT 1890 6:1:40", "5:53:20 - HMT 1941_9 5:53:20", @@ -1617,7 +1374,7 @@ data = { "6:30 - BURT 1951_8_30 6:30", "6 - DACT 1971_2_26 6", "6 - BDT 2009 6", - "6 Dhaka BD%sT" + "6 Dhaka BD%sT", ], "Asia/Dili": [ "8:22:20 - LMT 1912 8:22:20", @@ -1625,18 +1382,15 @@ data = { "9 - JST 1945_8_23 9", "9 - TLT 1976_4_3 9", "8 - CIT 2000_8_17_00 8", - "9 - TLT" - ], - "Asia/Dubai": [ - "3:41:12 - LMT 1920 3:41:12", - "4 - GST" + "9 - TLT", ], + "Asia/Dubai": ["3:41:12 - LMT 1920 3:41:12", "4 - GST"], "Asia/Dushanbe": [ "4:35:12 - LMT 1924_4_2 4:35:12", "5 - DUST 1930_5_21 5", "6 RussiaAsia DUS%sT 1991_2_31_2 6", "6 - DUSST 1991_8_9_2 5", - "5 - TJT" + "5 - TJT", ], "Asia/Gaza": [ "2:17:52 - LMT 1900_9 2:17:52", @@ -1650,7 +1404,7 @@ data = { "2 - EET 2010_2_27_0_1 2", "2 Palestine EE%sT 2011_7_1 3", "2 - EET 2012 2", - "2 Palestine EE%sT" + "2 Palestine EE%sT", ], "Asia/Harbin": [ "8:26:44 - LMT 1928 8:26:44", @@ -1658,7 +1412,7 @@ data = { "8 - CST 1940 8", "9 - CHAT 1966_4 9", "8:30 - CHAT 1980_4 8:30", - "8 PRC C%sT" + "8 PRC C%sT", ], "Asia/Hebron": [ "2:20:23 - LMT 1900_9 2:20:23", @@ -1666,26 +1420,22 @@ data = { "2 EgyptAsia EE%sT 1967_5_5 3", "2 Zion I%sT 1996 2", "2 Jordan EE%sT 1999 2", - "2 Palestine EE%sT" + "2 Palestine EE%sT", ], "Asia/Ho_Chi_Minh": [ "7:6:40 - LMT 1906_5_9 7:6:40", "7:6:20 - SMT 1911_2_11_0_1 7:6:20", "7 - ICT 1912_4 7", "8 - ICT 1931_4 8", - "7 - ICT" + "7 - ICT", ], "Asia/Hong_Kong": [ "7:36:42 - LMT 1904_9_30 7:36:42", "8 HK HK%sT 1941_11_25 8", "9 - JST 1945_8_15 9", - "8 HK HK%sT" - ], - "Asia/Hovd": [ - "6:6:36 - LMT 1905_7 6:6:36", - "6 - HOVT 1978 6", - "7 Mongol HOV%sT" + "8 HK HK%sT", ], + "Asia/Hovd": ["6:6:36 - LMT 1905_7 6:6:36", "6 - HOVT 1978 6", "7 Mongol HOV%sT"], "Asia/Irkutsk": [ "6:57:20 - LMT 1880 6:57:20", "6:57:20 - IMT 1920_0_25 6:57:20", @@ -1693,7 +1443,7 @@ data = { "8 Russia IRK%sT 1991_2_31_2 8", "7 Russia IRK%sT 1992_0_19_2 7", "8 Russia IRK%sT 2011_2_27_2 8", - "9 - IRKT" + "9 - IRKT", ], "Asia/Jakarta": [ "7:7:12 - LMT 1867_7_10 7:7:12", @@ -1704,24 +1454,16 @@ data = { "7:30 - WIT 1948_4 7:30", "8 - WIT 1950_4 8", "7:30 - WIT 1964 7:30", - "7 - WIT" + "7 - WIT", ], "Asia/Jayapura": [ "9:22:48 - LMT 1932_10 9:22:48", "9 - EIT 1944_8_1 9", "9:30 - CST 1964 9:30", - "9 - EIT" - ], - "Asia/Jerusalem": [ - "2:20:56 - LMT 1880 2:20:56", - "2:20:40 - JMT 1918 2:20:40", - "2 Zion I%sT" - ], - "Asia/Kabul": [ - "4:36:48 - LMT 1890 4:36:48", - "4 - AFT 1945 4", - "4:30 - AFT" + "9 - EIT", ], + "Asia/Jerusalem": ["2:20:56 - LMT 1880 2:20:56", "2:20:40 - JMT 1918 2:20:40", "2 Zion I%sT"], + "Asia/Kabul": ["4:36:48 - LMT 1890 4:36:48", "4 - AFT 1945 4", "4:30 - AFT"], "Asia/Kamchatka": [ "10:34:36 - LMT 1922_10_10 10:34:36", "11 - PETT 1930_5_21 11", @@ -1729,7 +1471,7 @@ data = { "11 Russia PET%sT 1992_0_19_2 11", "12 Russia PET%sT 2010_2_28_2 12", "11 Russia PET%sT 2011_2_27_2 11", - "12 - PETT" + "12 - PETT", ], "Asia/Karachi": [ "4:28:12 - LMT 1907 4:28:12", @@ -1737,19 +1479,15 @@ data = { "6:30 - IST 1945_9_15 6:30", "5:30 - IST 1951_8_30 5:30", "5 - KART 1971_2_26 5", - "5 Pakistan PK%sT" + "5 Pakistan PK%sT", ], "Asia/Kashgar": [ "5:3:56 - LMT 1928 5:3:56", "5:30 - KAST 1940 5:30", "5 - KAST 1980_4 5", - "8 PRC C%sT" - ], - "Asia/Kathmandu": [ - "5:41:16 - LMT 1920 5:41:16", - "5:30 - IST 1986 5:30", - "5:45 - NPT" + "8 PRC C%sT", ], + "Asia/Kathmandu": ["5:41:16 - LMT 1920 5:41:16", "5:30 - IST 1986 5:30", "5:45 - NPT"], "Asia/Khandyga": [ "9:2:13 - LMT 1919_11_15 9:2:13", "8 - YAKT 1930_5_21 8", @@ -1758,7 +1496,7 @@ data = { "9 Russia YAK%sT 2004 9", "10 Russia VLA%sT 2011_2_27_2 10", "11 - VLAT 2011_8_13_0 11", - "10 - YAKT" + "10 - YAKT", ], "Asia/Kolkata": [ "5:53:28 - LMT 1880 5:53:28", @@ -1766,7 +1504,7 @@ data = { "6:30 - BURT 1942_4_15 6:30", "5:30 - IST 1942_8 5:30", "6:30 - IST 1945_9_15 6:30", - "5:30 - IST" + "5:30 - IST", ], "Asia/Krasnoyarsk": [ "6:11:20 - LMT 1920_0_6 6:11:20", @@ -1774,7 +1512,7 @@ data = { "7 Russia KRA%sT 1991_2_31_2 7", "6 Russia KRA%sT 1992_0_19_2 6", "7 Russia KRA%sT 2011_2_27_2 7", - "8 - KRAT" + "8 - KRAT", ], "Asia/Kuala_Lumpur": [ "6:46:46 - LMT 1901_0_1 6:46:46", @@ -1785,7 +1523,7 @@ data = { "7:30 - MALT 1942_1_16 7:30", "9 - JST 1945_8_12 9", "7:30 - MALT 1982_0_1 7:30", - "8 - MYT" + "8 - MYT", ], "Asia/Kuching": [ "7:21:20 - LMT 1926_2 7:21:20", @@ -1793,47 +1531,37 @@ data = { "8 NBorneo BOR%sT 1942_1_16 8", "9 - JST 1945_8_12 9", "8 - BORT 1982_0_1 8", - "8 - MYT" - ], - "Asia/Kuwait": [ - "3:11:56 - LMT 1950 3:11:56", - "3 - AST" - ], - "Asia/Macau": [ - "7:34:20 - LMT 1912 7:34:20", - "8 Macau MO%sT 1999_11_20 8", - "8 PRC C%sT" + "8 - MYT", ], + "Asia/Kuwait": ["3:11:56 - LMT 1950 3:11:56", "3 - AST"], + "Asia/Macau": ["7:34:20 - LMT 1912 7:34:20", "8 Macau MO%sT 1999_11_20 8", "8 PRC C%sT"], "Asia/Magadan": [ "10:3:12 - LMT 1924_4_2 10:3:12", "10 - MAGT 1930_5_21 10", "11 Russia MAG%sT 1991_2_31_2 11", "10 Russia MAG%sT 1992_0_19_2 10", "11 Russia MAG%sT 2011_2_27_2 11", - "12 - MAGT" + "12 - MAGT", ], "Asia/Makassar": [ "7:57:36 - LMT 1920 7:57:36", "7:57:36 - MMT 1932_10 7:57:36", "8 - CIT 1942_1_9 8", "9 - JST 1945_8_23 9", - "8 - CIT" + "8 - CIT", ], "Asia/Manila": [ "-15:56 - LMT 1844_11_31 -15:56", "8:4 - LMT 1899_4_11 8:4", "8 Phil PH%sT 1942_4 8", "9 - JST 1944_10 9", - "8 Phil PH%sT" - ], - "Asia/Muscat": [ - "3:54:24 - LMT 1920 3:54:24", - "4 - GST" + "8 Phil PH%sT", ], + "Asia/Muscat": ["3:54:24 - LMT 1920 3:54:24", "4 - GST"], "Asia/Nicosia": [ "2:13:28 - LMT 1921_10_14 2:13:28", "2 Cyprus EE%sT 1998_8 3", - "2 EUAsia EE%sT" + "2 EUAsia EE%sT", ], "Asia/Novokuznetsk": [ "5:48:48 - NMT 1920_0_6 5:48:48", @@ -1842,7 +1570,7 @@ data = { "6 Russia KRA%sT 1992_0_19_2 6", "7 Russia KRA%sT 2010_2_28_2 7", "6 Russia NOV%sT 2011_2_27_2 6", - "7 - NOVT" + "7 - NOVT", ], "Asia/Novosibirsk": [ "5:31:40 - LMT 1919_11_14_6 5:31:40", @@ -1851,7 +1579,7 @@ data = { "6 Russia NOV%sT 1992_0_19_2 6", "7 Russia NOV%sT 1993_4_23 8", "6 Russia NOV%sT 2011_2_27_2 6", - "7 - NOVT" + "7 - NOVT", ], "Asia/Omsk": [ "4:53:36 - LMT 1919_10_14 4:53:36", @@ -1859,7 +1587,7 @@ data = { "6 Russia OMS%sT 1991_2_31_2 6", "5 Russia OMS%sT 1992_0_19_2 5", "6 Russia OMS%sT 2011_2_27_2 6", - "7 - OMST" + "7 - OMST", ], "Asia/Oral": [ "3:25:24 - LMT 1924_4_2 3:25:24", @@ -1871,14 +1599,14 @@ data = { "4 RussiaAsia URA%sT 1991 4", "4 - URAT 1991_11_16 4", "4 RussiaAsia ORA%sT 2005_2_15 4", - "5 - ORAT" + "5 - ORAT", ], "Asia/Phnom_Penh": [ "6:59:40 - LMT 1906_5_9 6:59:40", "7:6:20 - SMT 1911_2_11_0_1 7:6:20", "7 - ICT 1912_4 7", "8 - ICT 1931_4 8", - "7 - ICT" + "7 - ICT", ], "Asia/Pontianak": [ "7:17:20 - LMT 1908_4 7:17:20", @@ -1889,7 +1617,7 @@ data = { "8 - WIT 1950_4 8", "7:30 - WIT 1964 7:30", "8 - CIT 1988_0_1 8", - "7 - WIT" + "7 - WIT", ], "Asia/Pyongyang": [ "8:23 - LMT 1890 8:23", @@ -1898,13 +1626,9 @@ data = { "8:30 - KST 1932 8:30", "9 - KST 1954_2_21 9", "8 - KST 1961_7_10 8", - "9 - KST" - ], - "Asia/Qatar": [ - "3:26:8 - LMT 1920 3:26:8", - "4 - GST 1972_5 4", - "3 - AST" + "9 - KST", ], + "Asia/Qatar": ["3:26:8 - LMT 1920 3:26:8", "4 - GST 1972_5 4", "3 - AST"], "Asia/Qyzylorda": [ "4:21:52 - LMT 1924_4_2 4:21:52", "4 - KIZT 1930_5_21 4", @@ -1915,19 +1639,16 @@ data = { "5 - KIZT 1991_11_16 5", "5 - QYZT 1992_0_19_2 5", "6 RussiaAsia QYZ%sT 2005_2_15 6", - "6 - QYZT" + "6 - QYZT", ], "Asia/Rangoon": [ "6:24:40 - LMT 1880 6:24:40", "6:24:40 - RMT 1920 6:24:40", "6:30 - BURT 1942_4 6:30", "9 - JST 1945_4_3 9", - "6:30 - MMT" - ], - "Asia/Riyadh": [ - "3:6:52 - LMT 1950 3:6:52", - "3 - AST" + "6:30 - MMT", ], + "Asia/Riyadh": ["3:6:52 - LMT 1950 3:6:52", "3 - AST"], "Asia/Sakhalin": [ "9:30:48 - LMT 1905_7_23 9:30:48", "9 - CJT 1938 9", @@ -1936,7 +1657,7 @@ data = { "10 Russia SAK%sT 1992_0_19_2 10", "11 Russia SAK%sT 1997_2_30_2 11", "10 Russia SAK%sT 2011_2_27_2 10", - "11 - SAKT" + "11 - SAKT", ], "Asia/Samarkand": [ "4:27:12 - LMT 1924_4_2 4:27:12", @@ -1946,7 +1667,7 @@ data = { "6 - TAST 1982_3_1 6", "5 RussiaAsia SAM%sT 1991_8_1 6", "5 RussiaAsia UZ%sT 1992 5", - "5 - UZT" + "5 - UZT", ], "Asia/Seoul": [ "8:27:52 - LMT 1890 8:27:52", @@ -1956,13 +1677,9 @@ data = { "9 - KST 1954_2_21 9", "8 ROK K%sT 1961_7_10 8", "8:30 - KST 1968_9 8:30", - "9 ROK K%sT" - ], - "Asia/Shanghai": [ - "8:5:57 - LMT 1928 8:5:57", - "8 Shang C%sT 1949 8", - "8 PRC C%sT" + "9 ROK K%sT", ], + "Asia/Shanghai": ["8:5:57 - LMT 1928 8:5:57", "8 Shang C%sT 1949 8", "8 PRC C%sT"], "Asia/Singapore": [ "6:55:25 - LMT 1901_0_1 6:55:25", "6:55:25 - SMT 1905_5_1 6:55:25", @@ -1973,19 +1690,16 @@ data = { "9 - JST 1945_8_12 9", "7:30 - MALT 1965_7_9 7:30", "7:30 - SGT 1982_0_1 7:30", - "8 - SGT" - ], - "Asia/Taipei": [ - "8:6 - LMT 1896 8:6", - "8 Taiwan C%sT" + "8 - SGT", ], + "Asia/Taipei": ["8:6 - LMT 1896 8:6", "8 Taiwan C%sT"], "Asia/Tashkent": [ "4:37:12 - LMT 1924_4_2 4:37:12", "5 - TAST 1930_5_21 5", "6 RussiaAsia TAS%sT 1991_2_31_2 6", "5 RussiaAsia TAS%sT 1991_8_1 6", "5 RussiaAsia UZ%sT 1992 5", - "5 - UZT" + "5 - UZT", ], "Asia/Tbilisi": [ "2:59:16 - LMT 1880 2:59:16", @@ -1999,36 +1713,24 @@ data = { "5 - GEST 1997_2_30 5", "4 E-EurAsia GE%sT 2004_5_27 5", "3 RussiaAsia GE%sT 2005_2_27_2 3", - "4 - GET" + "4 - GET", ], "Asia/Tehran": [ "3:25:44 - LMT 1916 3:25:44", "3:25:44 - TMT 1946 3:25:44", "3:30 - IRST 1977_10 3:30", "4 Iran IR%sT 1979 4", - "3:30 Iran IR%sT" - ], - "Asia/Thimphu": [ - "5:58:36 - LMT 1947_7_15 5:58:36", - "5:30 - IST 1987_9 5:30", - "6 - BTT" + "3:30 Iran IR%sT", ], + "Asia/Thimphu": ["5:58:36 - LMT 1947_7_15 5:58:36", "5:30 - IST 1987_9 5:30", "6 - BTT"], "Asia/Tokyo": [ "9:18:59 - LMT 1887_11_31_15", "9 - JST 1896 9", "9 - CJT 1938 9", - "9 Japan J%sT" - ], - "Asia/Ulaanbaatar": [ - "7:7:32 - LMT 1905_7 7:7:32", - "7 - ULAT 1978 7", - "8 Mongol ULA%sT" - ], - "Asia/Urumqi": [ - "5:50:20 - LMT 1928 5:50:20", - "6 - URUT 1980_4 6", - "8 PRC C%sT" + "9 Japan J%sT", ], + "Asia/Ulaanbaatar": ["7:7:32 - LMT 1905_7 7:7:32", "7 - ULAT 1978 7", "8 Mongol ULA%sT"], + "Asia/Urumqi": ["5:50:20 - LMT 1928 5:50:20", "6 - URUT 1980_4 6", "8 PRC C%sT"], "Asia/Ust-Nera": [ "9:32:54 - LMT 1919_11_15 9:32:54", "8 - YAKT 1930_5_21 8", @@ -2037,14 +1739,14 @@ data = { "10 Russia MAG%sT 1992_0_19_2 10", "11 Russia MAG%sT 2011_2_27_2 11", "12 - MAGT 2011_8_13_0 12", - "11 - VLAT" + "11 - VLAT", ], "Asia/Vientiane": [ "6:50:24 - LMT 1906_5_9 6:50:24", "7:6:20 - SMT 1911_2_11_0_1 7:6:20", "7 - ICT 1912_4 7", "8 - ICT 1931_4 8", - "7 - ICT" + "7 - ICT", ], "Asia/Vladivostok": [ "8:47:44 - LMT 1922_10_15 8:47:44", @@ -2052,7 +1754,7 @@ data = { "10 Russia VLA%sT 1991_2_31_2 10", "9 Russia VLA%sST 1992_0_19_2 9", "10 Russia VLA%sT 2011_2_27_2 10", - "11 - VLAT" + "11 - VLAT", ], "Asia/Yakutsk": [ "8:38:40 - LMT 1919_11_15 8:38:40", @@ -2060,7 +1762,7 @@ data = { "9 Russia YAK%sT 1991_2_31_2 9", "8 Russia YAK%sT 1992_0_19_2 8", "9 Russia YAK%sT 2011_2_27_2 9", - "10 - YAKT" + "10 - YAKT", ], "Asia/Yekaterinburg": [ "4:2:24 - LMT 1919_6_15_4 4:2:24", @@ -2068,7 +1770,7 @@ data = { "5 Russia SVE%sT 1991_2_31_2 5", "4 Russia SVE%sT 1992_0_19_2 4", "5 Russia YEK%sT 2011_2_27_2 5", - "6 - YEKT" + "6 - YEKT", ], "Asia/Yerevan": [ "2:58 - LMT 1924_4_2 2:58", @@ -2078,7 +1780,7 @@ data = { "3 RussiaAsia AM%sT 1995_8_24_2 3", "4 - AMT 1997 4", "4 RussiaAsia AM%sT 2012_2_25_2 4", - "4 - AMT" + "4 - AMT", ], "Atlantic/Azores": [ "-1:42:40 - LMT 1884 -1:42:40", @@ -2087,54 +1789,47 @@ data = { "-1 Port AZO%sT 1983_8_25_1 -1", "-1 W-Eur AZO%sT 1992_8_27_1 -1", "0 EU WE%sT 1993_2_28_1", - "-1 EU AZO%sT" + "-1 EU AZO%sT", ], "Atlantic/Bermuda": [ "-4:19:18 - LMT 1930_0_1_2 -4:19:18", "-4 - AST 1974_3_28_2 -4", "-4 Bahamas A%sT 1976 -4", - "-4 US A%sT" + "-4 US A%sT", ], "Atlantic/Canary": [ "-1:1:36 - LMT 1922_2 -1:1:36", "-1 - CANT 1946_8_30_1 -1", "0 - WET 1980_3_6_0", "1 - WEST 1980_8_28_0", - "0 EU WE%sT" + "0 EU WE%sT", ], "Atlantic/Cape_Verde": [ "-1:34:4 - LMT 1907 -1:34:4", "-2 - CVT 1942_8 -2", "-1 - CVST 1945_9_15 -1", "-2 - CVT 1975_10_25_2 -2", - "-1 - CVT" - ], - "Atlantic/Faroe": [ - "-0:27:4 - LMT 1908_0_11 -0:27:4", - "0 - WET 1981", - "0 EU WE%sT" + "-1 - CVT", ], + "Atlantic/Faroe": ["-0:27:4 - LMT 1908_0_11 -0:27:4", "0 - WET 1981", "0 EU WE%sT"], "Atlantic/Madeira": [ "-1:7:36 - LMT 1884 -1:7:36", "-1:7:36 - FMT 1911_4_24 -1:7:36", "-1 Port MAD%sT 1966_3_3_2 -1", "0 Port WE%sT 1983_8_25_1", - "0 EU WE%sT" + "0 EU WE%sT", ], "Atlantic/Reykjavik": [ "-1:27:24 - LMT 1837 -1:27:24", "-1:27:48 - RMT 1908 -1:27:48", "-1 Iceland IS%sT 1968_3_7_1 -1", - "0 - GMT" - ], - "Atlantic/South_Georgia": [ - "-2:26:8 - LMT 1890 -2:26:8", - "-2 - GST" + "0 - GMT", ], + "Atlantic/South_Georgia": ["-2:26:8 - LMT 1890 -2:26:8", "-2 - GST"], "Atlantic/St_Helena": [ "-0:22:48 - LMT 1890 -0:22:48", "-0:22:48 - JMT 1951 -0:22:48", - "0 - GMT" + "0 - GMT", ], "Atlantic/Stanley": [ "-3:51:24 - LMT 1890 -3:51:24", @@ -2142,210 +1837,106 @@ data = { "-4 Falk FK%sT 1983_4 -4", "-3 Falk FK%sT 1985_8_15 -3", "-4 Falk FK%sT 2010_8_5_02 -4", - "-3 - FKST" + "-3 - FKST", ], "Australia/Adelaide": [ "9:14:20 - LMT 1895_1 9:14:20", "9 - CST 1899_4 9", "9:30 Aus CST 1971 9:30", - "9:30 AS CST" - ], - "Australia/Brisbane": [ - "10:12:8 - LMT 1895 10:12:8", - "10 Aus EST 1971 10", - "10 AQ EST" + "9:30 AS CST", ], + "Australia/Brisbane": ["10:12:8 - LMT 1895 10:12:8", "10 Aus EST 1971 10", "10 AQ EST"], "Australia/Broken_Hill": [ "9:25:48 - LMT 1895_1 9:25:48", "10 - EST 1896_7_23 10", "9 - CST 1899_4 9", "9:30 Aus CST 1971 9:30", "9:30 AN CST 2000 10:30", - "9:30 AS CST" + "9:30 AS CST", ], "Australia/Currie": [ "9:35:28 - LMT 1895_8 9:35:28", "10 - EST 1916_9_1_2 10", "11 - EST 1917_1 11", "10 Aus EST 1971_6 10", - "10 AT EST" - ], - "Australia/Darwin": [ - "8:43:20 - LMT 1895_1 8:43:20", - "9 - CST 1899_4 9", - "9:30 Aus CST" + "10 AT EST", ], + "Australia/Darwin": ["8:43:20 - LMT 1895_1 8:43:20", "9 - CST 1899_4 9", "9:30 Aus CST"], "Australia/Eucla": [ "8:35:28 - LMT 1895_11 8:35:28", "8:45 Aus CWST 1943_6 8:45", - "8:45 AW CWST" + "8:45 AW CWST", ], "Australia/Hobart": [ "9:49:16 - LMT 1895_8 9:49:16", "10 - EST 1916_9_1_2 10", "11 - EST 1917_1 11", "10 Aus EST 1967 10", - "10 AT EST" + "10 AT EST", ], "Australia/Lindeman": [ "9:55:56 - LMT 1895 9:55:56", "10 Aus EST 1971 10", "10 AQ EST 1992_6 10", - "10 Holiday EST" - ], - "Australia/Lord_Howe": [ - "10:36:20 - LMT 1895_1 10:36:20", - "10 - EST 1981_2 10", - "10:30 LH LHST" - ], - "Australia/Melbourne": [ - "9:39:52 - LMT 1895_1 9:39:52", - "10 Aus EST 1971 10", - "10 AV EST" - ], - "Australia/Perth": [ - "7:43:24 - LMT 1895_11 7:43:24", - "8 Aus WST 1943_6 8", - "8 AW WST" - ], - "Australia/Sydney": [ - "10:4:52 - LMT 1895_1 10:4:52", - "10 Aus EST 1971 10", - "10 AN EST" - ], - "CET": [ - "1 C-Eur CE%sT" - ], - "CST6CDT": [ - "-6 US C%sT" - ], - "EET": [ - "2 EU EE%sT" - ], - "EST": [ - "-5 - EST" - ], - "EST5EDT": [ - "-5 US E%sT" - ], - "HST": [ - "-10 - HST" - ], - "MET": [ - "1 C-Eur ME%sT" - ], - "MST": [ - "-7 - MST" - ], - "MST7MDT": [ - "-7 US M%sT" - ], - "PST8PDT": [ - "-8 US P%sT" - ], - "WET": [ - "0 EU WE%sT" - ], - "Etc/GMT": [ - "0 - GMT" - ], - "Etc/GMT+1": [ - "-1 - GMT+1" - ], - "Etc/GMT+10": [ - "-10 - GMT+10" - ], - "Etc/GMT+11": [ - "-11 - GMT+11" - ], - "Etc/GMT+12": [ - "-12 - GMT+12" - ], - "Etc/GMT+2": [ - "-2 - GMT+2" - ], - "Etc/GMT+3": [ - "-3 - GMT+3" - ], - "Etc/GMT+4": [ - "-4 - GMT+4" - ], - "Etc/GMT+5": [ - "-5 - GMT+5" - ], - "Etc/GMT+6": [ - "-6 - GMT+6" - ], - "Etc/GMT+7": [ - "-7 - GMT+7" - ], - "Etc/GMT+8": [ - "-8 - GMT+8" - ], - "Etc/GMT+9": [ - "-9 - GMT+9" - ], - "Etc/GMT-1": [ - "1 - GMT-1" - ], - "Etc/GMT-10": [ - "10 - GMT-10" - ], - "Etc/GMT-11": [ - "11 - GMT-11" - ], - "Etc/GMT-12": [ - "12 - GMT-12" - ], - "Etc/GMT-13": [ - "13 - GMT-13" - ], - "Etc/GMT-14": [ - "14 - GMT-14" - ], - "Etc/GMT-2": [ - "2 - GMT-2" - ], - "Etc/GMT-3": [ - "3 - GMT-3" - ], - "Etc/GMT-4": [ - "4 - GMT-4" - ], - "Etc/GMT-5": [ - "5 - GMT-5" - ], - "Etc/GMT-6": [ - "6 - GMT-6" - ], - "Etc/GMT-7": [ - "7 - GMT-7" - ], - "Etc/GMT-8": [ - "8 - GMT-8" - ], - "Etc/GMT-9": [ - "9 - GMT-9" - ], - "Etc/UCT": [ - "0 - UCT" - ], - "Etc/UTC": [ - "0 - UTC" - ], + "10 Holiday EST", + ], + "Australia/Lord_Howe": ["10:36:20 - LMT 1895_1 10:36:20", "10 - EST 1981_2 10", "10:30 LH LHST"], + "Australia/Melbourne": ["9:39:52 - LMT 1895_1 9:39:52", "10 Aus EST 1971 10", "10 AV EST"], + "Australia/Perth": ["7:43:24 - LMT 1895_11 7:43:24", "8 Aus WST 1943_6 8", "8 AW WST"], + "Australia/Sydney": ["10:4:52 - LMT 1895_1 10:4:52", "10 Aus EST 1971 10", "10 AN EST"], + "CET": ["1 C-Eur CE%sT"], + "CST6CDT": ["-6 US C%sT"], + "EET": ["2 EU EE%sT"], + "EST": ["-5 - EST"], + "EST5EDT": ["-5 US E%sT"], + "HST": ["-10 - HST"], + "MET": ["1 C-Eur ME%sT"], + "MST": ["-7 - MST"], + "MST7MDT": ["-7 US M%sT"], + "PST8PDT": ["-8 US P%sT"], + "WET": ["0 EU WE%sT"], + "Etc/GMT": ["0 - GMT"], + "Etc/GMT+1": ["-1 - GMT+1"], + "Etc/GMT+10": ["-10 - GMT+10"], + "Etc/GMT+11": ["-11 - GMT+11"], + "Etc/GMT+12": ["-12 - GMT+12"], + "Etc/GMT+2": ["-2 - GMT+2"], + "Etc/GMT+3": ["-3 - GMT+3"], + "Etc/GMT+4": ["-4 - GMT+4"], + "Etc/GMT+5": ["-5 - GMT+5"], + "Etc/GMT+6": ["-6 - GMT+6"], + "Etc/GMT+7": ["-7 - GMT+7"], + "Etc/GMT+8": ["-8 - GMT+8"], + "Etc/GMT+9": ["-9 - GMT+9"], + "Etc/GMT-1": ["1 - GMT-1"], + "Etc/GMT-10": ["10 - GMT-10"], + "Etc/GMT-11": ["11 - GMT-11"], + "Etc/GMT-12": ["12 - GMT-12"], + "Etc/GMT-13": ["13 - GMT-13"], + "Etc/GMT-14": ["14 - GMT-14"], + "Etc/GMT-2": ["2 - GMT-2"], + "Etc/GMT-3": ["3 - GMT-3"], + "Etc/GMT-4": ["4 - GMT-4"], + "Etc/GMT-5": ["5 - GMT-5"], + "Etc/GMT-6": ["6 - GMT-6"], + "Etc/GMT-7": ["7 - GMT-7"], + "Etc/GMT-8": ["8 - GMT-8"], + "Etc/GMT-9": ["9 - GMT-9"], + "Etc/UCT": ["0 - UCT"], + "Etc/UTC": ["0 - UTC"], "Europe/Amsterdam": [ "0:19:32 - LMT 1835 0:19:32", "0:19:32 Neth %s 1937_6_1 1:19:32", "0:20 Neth NE%sT 1940_4_16_0 0:20", "1 C-Eur CE%sT 1945_3_2_2 1", "1 Neth CE%sT 1977 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Andorra": [ "0:6:4 - LMT 1901 0:6:4", "0 - WET 1946_8_30", "1 - CET 1985_2_31_2 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Athens": [ "1:34:52 - LMT 1895_8_14 1:34:52", @@ -2353,7 +1944,7 @@ data = { "2 Greece EE%sT 1941_3_30 3", "1 Greece CE%sT 1944_3_4 1", "2 Greece EE%sT 1981 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Belgrade": [ "1:22 - LMT 1884 1:22", @@ -2362,21 +1953,21 @@ data = { "1 - CET 1945_4_8_2 1", "2 - CEST 1945_8_16_2 1", "1 - CET 1982_10_27 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Berlin": [ "0:53:28 - LMT 1893_3 0:53:28", "1 C-Eur CE%sT 1945_4_24_2 2", "1 SovietZone CE%sT 1946 1", "1 Germany CE%sT 1980 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Prague": [ "0:57:44 - LMT 1850 0:57:44", "0:57:44 - PMT 1891_9 0:57:44", "1 C-Eur CE%sT 1944_8_17_2 1", "1 Czech CE%sT 1979 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Brussels": [ "0:17:30 - LMT 1880 0:17:30", @@ -2387,7 +1978,7 @@ data = { "0 Belgium WE%sT 1940_4_20_2", "1 C-Eur CE%sT 1944_8_3 2", "1 Belgium CE%sT 1977 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Bucharest": [ "1:44:24 - LMT 1891_9 1:44:24", @@ -2396,7 +1987,7 @@ data = { "2 C-Eur EE%sT 1991 2", "2 Romania EE%sT 1994 2", "2 E-Eur EE%sT 1997 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Budapest": [ "1:16:20 - LMT 1890_9 1:16:20", @@ -2404,13 +1995,13 @@ data = { "1 Hungary CE%sT 1941_3_6_2 1", "1 C-Eur CE%sT 1945 1", "1 Hungary CE%sT 1980_8_28_2 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Zurich": [ "0:34:8 - LMT 1848_8_12 0:34:8", "0:29:44 - BMT 1894_5 0:29:44", "1 Swiss CE%sT 1981 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Chisinau": [ "1:55:20 - LMT 1880 1:55:20", @@ -2424,7 +2015,7 @@ data = { "2 - EET 1991 2", "2 Russia EE%sT 1992 2", "2 E-Eur EE%sT 1997 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Copenhagen": [ "0:50:20 - LMT 1890 0:50:20", @@ -2432,7 +2023,7 @@ data = { "1 Denmark CE%sT 1942_10_2_2 1", "1 C-Eur CE%sT 1945_3_2_2 1", "1 Denmark CE%sT 1980 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Dublin": [ "-0:25 - LMT 1880_7_2 -0:25", @@ -2447,26 +2038,26 @@ data = { "0 GB-Eire GMT/IST 1968_9_27 1", "1 - IST 1971_9_31_2", "0 GB-Eire GMT/IST 1996", - "0 EU GMT/IST" + "0 EU GMT/IST", ], "Europe/Gibraltar": [ "-0:21:24 - LMT 1880_7_2_0 -0:21:24", "0 GB-Eire %s 1957_3_14_2", "1 - CET 1982 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/London": [ "-0:1:15 - LMT 1847_11_1_0 -0:1:15", "0 GB-Eire %s 1968_9_27 1", "1 - BST 1971_9_31_2", "0 GB-Eire %s 1996", - "0 EU GMT/BST" + "0 EU GMT/BST", ], "Europe/Helsinki": [ "1:39:52 - LMT 1878_4_31 1:39:52", "1:39:52 - HMT 1921_4 1:39:52", "2 Finland EE%sT 1983 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Istanbul": [ "1:55:52 - LMT 1880 1:55:52", @@ -2476,7 +2067,7 @@ data = { "2 Turkey EE%sT 2007 2", "2 EU EE%sT 2011_2_27_1", "2 - EET 2011_2_28_1", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Kaliningrad": [ "1:22 - LMT 1893_3 1:22", @@ -2484,7 +2075,7 @@ data = { "2 Poland CE%sT 1946 2", "3 Russia MSK/MSD 1991_2_31_2 3", "2 Russia EE%sT 2011_2_27_2 2", - "3 - FET" + "3 - FET", ], "Europe/Kiev": [ "2:2:4 - LMT 1880 2:2:4", @@ -2496,7 +2087,7 @@ data = { "3 - MSK 1990_6_1_2 3", "2 - EET 1992 2", "2 E-Eur EE%sT 1995 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Lisbon": [ "-0:36:32 - LMT 1884 -0:36:32", @@ -2506,7 +2097,7 @@ data = { "0 Port WE%sT 1983_8_25_1", "0 W-Eur WE%sT 1992_8_27_1", "1 EU CE%sT 1996_2_31_1", - "0 EU WE%sT" + "0 EU WE%sT", ], "Europe/Luxembourg": [ "0:24:36 - LMT 1904_5 0:24:36", @@ -2515,13 +2106,13 @@ data = { "0 Belgium WE%sT 1940_4_14_3 1", "1 C-Eur WE%sT 1944_8_18_3 2", "1 Belgium CE%sT 1977 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Madrid": [ "-0:14:44 - LMT 1901_0_1_0 -0:14:44", "0 Spain WE%sT 1946_8_30 2", "1 Spain CE%sT 1979 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Malta": [ "0:58:4 - LMT 1893_10_2_0 0:58:4", @@ -2529,7 +2120,7 @@ data = { "1 C-Eur CE%sT 1945_3_2_2 1", "1 Italy CE%sT 1973_2_31 1", "1 Malta CE%sT 1981 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Minsk": [ "1:50:16 - LMT 1880 1:50:16", @@ -2543,14 +2134,14 @@ data = { "2 - EET 1992_2_29_0 2", "3 - EEST 1992_8_27_0 2", "2 Russia EE%sT 2011_2_27_2 2", - "3 - FET" + "3 - FET", ], "Europe/Monaco": [ "0:29:32 - LMT 1891_2_15 0:29:32", "0:9:21 - PMT 1911_2_11 0:9:21", "0 France WE%sT 1945_8_16_3 2", "1 France CE%sT 1977 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Moscow": [ "2:30:20 - LMT 1880 2:30:20", @@ -2561,7 +2152,7 @@ data = { "3 Russia MSK/MSD 1991_2_31_2 3", "2 Russia EE%sT 1992_0_19_2 2", "3 Russia MSK/MSD 2011_2_27_2 3", - "4 - MSK" + "4 - MSK", ], "Europe/Paris": [ "0:9:21 - LMT 1891_2_15_0_1 0:9:21", @@ -2570,7 +2161,7 @@ data = { "1 C-Eur CE%sT 1944_7_25 2", "0 France WE%sT 1945_8_16_3 2", "1 France CE%sT 1977 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Riga": [ "1:36:24 - LMT 1880 1:36:24", @@ -2587,7 +2178,7 @@ data = { "2 Latvia EE%sT 1997_0_21 2", "2 EU EE%sT 2000_1_29 2", "2 - EET 2001_0_2 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Rome": [ "0:49:56 - LMT 1866_8_22 0:49:56", @@ -2595,7 +2186,7 @@ data = { "1 Italy CE%sT 1942_10_2_2 1", "1 C-Eur CE%sT 1944_6 2", "1 Italy CE%sT 1980 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Samara": [ "3:20:36 - LMT 1919_6_1_2 3:20:36", @@ -2607,7 +2198,7 @@ data = { "3 - KUYT 1991_9_20_3 3", "4 Russia SAM%sT 2010_2_28_2 4", "3 Russia SAM%sT 2011_2_27_2 3", - "4 - SAMT" + "4 - SAMT", ], "Europe/Simferopol": [ "2:16:24 - LMT 1880 2:16:24", @@ -2623,7 +2214,7 @@ data = { "4 - MSD 1996_9_27_3 3", "3 Russia MSK/MSD 1997 3", "3 - MSK 1997_2_30_1", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Sofia": [ "1:33:16 - LMT 1880 1:33:16", @@ -2635,7 +2226,7 @@ data = { "2 Bulg EE%sT 1982_8_26_2 3", "2 C-Eur EE%sT 1991 2", "2 E-Eur EE%sT 1997 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Stockholm": [ "1:12:12 - LMT 1879_0_1 1:12:12", @@ -2643,7 +2234,7 @@ data = { "1 - CET 1916_4_14_23 1", "2 - CEST 1916_9_1_01 2", "1 - CET 1980 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Tallinn": [ "1:39 - LMT 1880 1:39", @@ -2658,13 +2249,13 @@ data = { "2 C-Eur EE%sT 1998_8_22 3", "2 EU EE%sT 1999_10_1 3", "2 - EET 2002_1_21 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Tirane": [ "1:19:20 - LMT 1914 1:19:20", "1 - CET 1940_5_16 1", "1 Albania CE%sT 1984_6 2", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Uzhgorod": [ "1:29:12 - LMT 1890_9 1:29:12", @@ -2677,13 +2268,9 @@ data = { "1 - CET 1991_2_31_3 1", "2 - EET 1992 2", "2 E-Eur EE%sT 1995 2", - "2 EU EE%sT" - ], - "Europe/Vaduz": [ - "0:38:4 - LMT 1894_5 0:38:4", - "1 - CET 1981 1", - "1 EU CE%sT" + "2 EU EE%sT", ], + "Europe/Vaduz": ["0:38:4 - LMT 1894_5 0:38:4", "1 - CET 1981 1", "1 EU CE%sT"], "Europe/Vienna": [ "1:5:21 - LMT 1893_3 1:5:21", "1 C-Eur CE%sT 1920 1", @@ -2692,7 +2279,7 @@ data = { "2 - CEST 1945_3_12_2 1", "1 - CET 1946 1", "1 Austria CE%sT 1981 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Vilnius": [ "1:41:16 - LMT 1880 1:41:16", @@ -2709,7 +2296,7 @@ data = { "2 - EET 1998_2_29_1", "1 EU CE%sT 1999_9_31_1", "2 - EET 2003_0_1 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Europe/Volgograd": [ "2:57:40 - LMT 1920_0_3 2:57:40", @@ -2720,7 +2307,7 @@ data = { "3 Russia VOL%sT 1991_2_31_2 3", "4 - VOLT 1992_2_29_2 4", "3 Russia VOL%sT 2011_2_27_2 3", - "4 - VOLT" + "4 - VOLT", ], "Europe/Warsaw": [ "1:24 - LMT 1880 1:24", @@ -2731,7 +2318,7 @@ data = { "1 C-Eur CE%sT 1944_9 2", "1 Poland CE%sT 1977 1", "1 W-Eur CE%sT 1988 1", - "1 EU CE%sT" + "1 EU CE%sT", ], "Europe/Zaporozhye": [ "2:20:40 - LMT 1880 2:20:40", @@ -2741,56 +2328,24 @@ data = { "1 C-Eur CE%sT 1943_9_25 1", "3 Russia MSK/MSD 1991_2_31_2 3", "2 E-Eur EE%sT 1995 2", - "2 EU EE%sT" + "2 EU EE%sT", ], "Indian/Antananarivo": [ "3:10:4 - LMT 1911_6 3:10:4", "3 - EAT 1954_1_27_23 3", "4 - EAST 1954_4_29_23 3", - "3 - EAT" - ], - "Indian/Chagos": [ - "4:49:40 - LMT 1907 4:49:40", - "5 - IOT 1996 5", - "6 - IOT" - ], - "Indian/Christmas": [ - "7:2:52 - LMT 1895_1 7:2:52", - "7 - CXT" - ], - "Indian/Cocos": [ - "6:27:40 - LMT 1900 6:27:40", - "6:30 - CCT" - ], - "Indian/Comoro": [ - "2:53:4 - LMT 1911_6 2:53:4", - "3 - EAT" - ], - "Indian/Kerguelen": [ - "0 - zzz 1950", - "5 - TFT" - ], - "Indian/Mahe": [ - "3:41:48 - LMT 1906_5 3:41:48", - "4 - SCT" - ], - "Indian/Maldives": [ - "4:54 - LMT 1880 4:54", - "4:54 - MMT 1960 4:54", - "5 - MVT" - ], - "Indian/Mauritius": [ - "3:50 - LMT 1907 3:50", - "4 Mauritius MU%sT" - ], - "Indian/Mayotte": [ - "3:0:56 - LMT 1911_6 3:0:56", - "3 - EAT" - ], - "Indian/Reunion": [ - "3:41:52 - LMT 1911_5 3:41:52", - "4 - RET" - ], + "3 - EAT", + ], + "Indian/Chagos": ["4:49:40 - LMT 1907 4:49:40", "5 - IOT 1996 5", "6 - IOT"], + "Indian/Christmas": ["7:2:52 - LMT 1895_1 7:2:52", "7 - CXT"], + "Indian/Cocos": ["6:27:40 - LMT 1900 6:27:40", "6:30 - CCT"], + "Indian/Comoro": ["2:53:4 - LMT 1911_6 2:53:4", "3 - EAT"], + "Indian/Kerguelen": ["0 - zzz 1950", "5 - TFT"], + "Indian/Mahe": ["3:41:48 - LMT 1906_5 3:41:48", "4 - SCT"], + "Indian/Maldives": ["4:54 - LMT 1880 4:54", "4:54 - MMT 1960 4:54", "5 - MVT"], + "Indian/Mauritius": ["3:50 - LMT 1907 3:50", "4 Mauritius MU%sT"], + "Indian/Mayotte": ["3:0:56 - LMT 1911_6 3:0:56", "3 - EAT"], + "Indian/Reunion": ["3:41:52 - LMT 1911_5 3:41:52", "4 - RET"], "Pacific/Apia": [ "12:33:4 - LMT 1879_6_5 12:33:4", "-11:26:56 - LMT 1911 -11:26:56", @@ -2800,68 +2355,39 @@ data = { "-11 - WST 2011_8_24_3 -11", "-10 - WSDT 2011_11_30 -10", "14 - WSDT 2012_3_1_4 14", - "13 WS WS%sT" + "13 WS WS%sT", ], "Pacific/Auckland": [ "11:39:4 - LMT 1868_10_2 11:39:4", "11:30 NZ NZ%sT 1946_0_1 12", - "12 NZ NZ%sT" - ], - "Pacific/Chatham": [ - "12:13:48 - LMT 1957_0_1 12:13:48", - "12:45 Chatham CHA%sT" - ], - "Pacific/Chuuk": [ - "10:7:8 - LMT 1901 10:7:8", - "10 - CHUT" + "12 NZ NZ%sT", ], + "Pacific/Chatham": ["12:13:48 - LMT 1957_0_1 12:13:48", "12:45 Chatham CHA%sT"], + "Pacific/Chuuk": ["10:7:8 - LMT 1901 10:7:8", "10 - CHUT"], "Pacific/Easter": [ "-7:17:44 - LMT 1890 -7:17:44", "-7:17:28 - EMT 1932_8 -7:17:28", "-7 Chile EAS%sT 1982_2_13_21 -6", - "-6 Chile EAS%sT" - ], - "Pacific/Efate": [ - "11:13:16 - LMT 1912_0_13 11:13:16", - "11 Vanuatu VU%sT" + "-6 Chile EAS%sT", ], + "Pacific/Efate": ["11:13:16 - LMT 1912_0_13 11:13:16", "11 Vanuatu VU%sT"], "Pacific/Enderbury": [ "-11:24:20 - LMT 1901 -11:24:20", "-12 - PHOT 1979_9 -12", "-11 - PHOT 1995 -11", - "13 - PHOT" - ], - "Pacific/Fakaofo": [ - "-11:24:56 - LMT 1901 -11:24:56", - "-11 - TKT 2011_11_30 -11", - "13 - TKT" - ], - "Pacific/Fiji": [ - "11:55:44 - LMT 1915_9_26 11:55:44", - "12 Fiji FJ%sT" - ], - "Pacific/Funafuti": [ - "11:56:52 - LMT 1901 11:56:52", - "12 - TVT" - ], - "Pacific/Galapagos": [ - "-5:58:24 - LMT 1931 -5:58:24", - "-5 - ECT 1986 -5", - "-6 - GALT" - ], - "Pacific/Gambier": [ - "-8:59:48 - LMT 1912_9 -8:59:48", - "-9 - GAMT" - ], - "Pacific/Guadalcanal": [ - "10:39:48 - LMT 1912_9 10:39:48", - "11 - SBT" - ], + "13 - PHOT", + ], + "Pacific/Fakaofo": ["-11:24:56 - LMT 1901 -11:24:56", "-11 - TKT 2011_11_30 -11", "13 - TKT"], + "Pacific/Fiji": ["11:55:44 - LMT 1915_9_26 11:55:44", "12 Fiji FJ%sT"], + "Pacific/Funafuti": ["11:56:52 - LMT 1901 11:56:52", "12 - TVT"], + "Pacific/Galapagos": ["-5:58:24 - LMT 1931 -5:58:24", "-5 - ECT 1986 -5", "-6 - GALT"], + "Pacific/Gambier": ["-8:59:48 - LMT 1912_9 -8:59:48", "-9 - GAMT"], + "Pacific/Guadalcanal": ["10:39:48 - LMT 1912_9 10:39:48", "11 - SBT"], "Pacific/Guam": [ "-14:21 - LMT 1844_11_31 -14:21", "9:39 - LMT 1901 9:39", "10 - GST 2000_11_23 10", - "10 - ChST" + "10 - ChST", ], "Pacific/Honolulu": [ "-10:31:26 - LMT 1896_0_13_12 -10:31:26", @@ -2870,134 +2396,97 @@ data = { "-10:30 - HST 1942_1_09_2 -10:30", "-9:30 - HDT 1945_8_30_2 -9:30", "-10:30 - HST 1947_5_8_2 -10:30", - "-10 - HST" - ], - "Pacific/Johnston": [ - "-10 - HST" + "-10 - HST", ], + "Pacific/Johnston": ["-10 - HST"], "Pacific/Kiritimati": [ "-10:29:20 - LMT 1901 -10:29:20", "-10:40 - LINT 1979_9 -10:40", "-10 - LINT 1995 -10", - "14 - LINT" + "14 - LINT", ], "Pacific/Kosrae": [ "10:51:56 - LMT 1901 10:51:56", "11 - KOST 1969_9 11", "12 - KOST 1999 12", - "11 - KOST" + "11 - KOST", ], "Pacific/Kwajalein": [ "11:9:20 - LMT 1901 11:9:20", "11 - MHT 1969_9 11", "-12 - KWAT 1993_7_20 -12", - "12 - MHT" - ], - "Pacific/Majuro": [ - "11:24:48 - LMT 1901 11:24:48", - "11 - MHT 1969_9 11", - "12 - MHT" - ], - "Pacific/Marquesas": [ - "-9:18 - LMT 1912_9 -9:18", - "-9:30 - MART" + "12 - MHT", ], + "Pacific/Majuro": ["11:24:48 - LMT 1901 11:24:48", "11 - MHT 1969_9 11", "12 - MHT"], + "Pacific/Marquesas": ["-9:18 - LMT 1912_9 -9:18", "-9:30 - MART"], "Pacific/Midway": [ "-11:49:28 - LMT 1901 -11:49:28", "-11 - NST 1956_5_3 -11", "-10 - NDT 1956_8_2 -10", "-11 - NST 1967_3 -11", "-11 - BST 1983_10_30 -11", - "-11 - SST" + "-11 - SST", ], "Pacific/Nauru": [ "11:7:40 - LMT 1921_0_15 11:7:40", "11:30 - NRT 1942_2_15 11:30", "9 - JST 1944_7_15 9", "11:30 - NRT 1979_4 11:30", - "12 - NRT" + "12 - NRT", ], "Pacific/Niue": [ "-11:19:40 - LMT 1901 -11:19:40", "-11:20 - NUT 1951 -11:20", "-11:30 - NUT 1978_9_1 -11:30", - "-11 - NUT" - ], - "Pacific/Norfolk": [ - "11:11:52 - LMT 1901 11:11:52", - "11:12 - NMT 1951 11:12", - "11:30 - NFT" - ], - "Pacific/Noumea": [ - "11:5:48 - LMT 1912_0_13 11:5:48", - "11 NC NC%sT" + "-11 - NUT", ], + "Pacific/Norfolk": ["11:11:52 - LMT 1901 11:11:52", "11:12 - NMT 1951 11:12", "11:30 - NFT"], + "Pacific/Noumea": ["11:5:48 - LMT 1912_0_13 11:5:48", "11 NC NC%sT"], "Pacific/Pago_Pago": [ "12:37:12 - LMT 1879_6_5 12:37:12", "-11:22:48 - LMT 1911 -11:22:48", "-11:30 - SAMT 1950 -11:30", "-11 - NST 1967_3 -11", "-11 - BST 1983_10_30 -11", - "-11 - SST" - ], - "Pacific/Palau": [ - "8:57:56 - LMT 1901 8:57:56", - "9 - PWT" + "-11 - SST", ], + "Pacific/Palau": ["8:57:56 - LMT 1901 8:57:56", "9 - PWT"], "Pacific/Pitcairn": [ "-8:40:20 - LMT 1901 -8:40:20", "-8:30 - PNT 1998_3_27_00 -8:30", - "-8 - PST" - ], - "Pacific/Pohnpei": [ - "10:32:52 - LMT 1901 10:32:52", - "11 - PONT" + "-8 - PST", ], + "Pacific/Pohnpei": ["10:32:52 - LMT 1901 10:32:52", "11 - PONT"], "Pacific/Port_Moresby": [ "9:48:40 - LMT 1880 9:48:40", "9:48:32 - PMMT 1895 9:48:32", - "10 - PGT" + "10 - PGT", ], "Pacific/Rarotonga": [ "-10:39:4 - LMT 1901 -10:39:4", "-10:30 - CKT 1978_10_12 -10:30", - "-10 Cook CK%sT" + "-10 Cook CK%sT", ], "Pacific/Saipan": [ "-14:17 - LMT 1844_11_31 -14:17", "9:43 - LMT 1901 9:43", "9 - MPT 1969_9 9", "10 - MPT 2000_11_23 10", - "10 - ChST" - ], - "Pacific/Tahiti": [ - "-9:58:16 - LMT 1912_9 -9:58:16", - "-10 - TAHT" - ], - "Pacific/Tarawa": [ - "11:32:4 - LMT 1901 11:32:4", - "12 - GILT" + "10 - ChST", ], + "Pacific/Tahiti": ["-9:58:16 - LMT 1912_9 -9:58:16", "-10 - TAHT"], + "Pacific/Tarawa": ["11:32:4 - LMT 1901 11:32:4", "12 - GILT"], "Pacific/Tongatapu": [ "12:19:20 - LMT 1901 12:19:20", "12:20 - TOT 1941 12:20", "13 - TOT 1999 13", - "13 Tonga TO%sT" - ], - "Pacific/Wake": [ - "11:6:28 - LMT 1901 11:6:28", - "12 - WAKT" + "13 Tonga TO%sT", ], - "Pacific/Wallis": [ - "12:15:20 - LMT 1901 12:15:20", - "12 - WFT" - ] + "Pacific/Wake": ["11:6:28 - LMT 1901 11:6:28", "12 - WAKT"], + "Pacific/Wallis": ["12:15:20 - LMT 1901 12:15:20", "12 - WFT"], }, "rules": { - "Ghana": [ - "1936 1942 8 1 7 0 0 0:20 GHST", - "1936 1942 11 31 7 0 0 0 GMT" - ], + "Ghana": ["1936 1942 8 1 7 0 0 0:20 GHST", "1936 1942 11 31 7 0 0 0 GMT"], "Algeria": [ "1916 1916 5 14 7 23 2 1 S", "1916 1919 9 1 0 23 2 0", @@ -3020,7 +2509,7 @@ data = { "1978 1978 2 24 7 1 0 1 S", "1978 1978 8 22 7 3 0 0", "1980 1980 3 25 7 0 0 1 S", - "1980 1980 9 31 7 2 0 0" + "1980 1980 9 31 7 2 0 0", ], "Egypt": [ "1940 1940 6 15 7 0 0 1 S", @@ -3050,7 +2539,7 @@ data = { "2009 2009 7 20 7 23 2 0", "2010 2010 7 11 7 0 0 0", "2010 2010 8 10 7 0 0 1 S", - "2010 2010 8 4 8 23 2 0" + "2010 2010 8 4 8 23 2 0", ], "Morocco": [ "1939 1939 8 12 7 0 0 1 S", @@ -3097,7 +2586,7 @@ data = { "2020 2020 4 24 7 2 0 1 S", "2021 2021 4 13 7 2 0 1 S", "2022 2022 4 3 7 2 0 1 S", - "2023 9999 3 0 8 2 0 1 S" + "2023 9999 3 0 8 2 0 1 S", ], "Spain": [ "1917 1917 4 5 7 23 2 1 S", @@ -3130,7 +2619,7 @@ data = { "1976 1976 2 27 7 23 0 1 S", "1976 1977 8 0 8 1 0 0", "1977 1978 3 2 7 23 0 1 S", - "1978 1978 9 1 7 1 0 0" + "1978 1978 9 1 7 1 0 0", ], "SpainAfrica": [ "1967 1967 5 3 7 12 0 1 S", @@ -3141,7 +2630,7 @@ data = { "1976 1976 7 1 7 0 0 0", "1977 1977 8 28 7 0 0 0", "1978 1978 5 1 7 0 0 1 S", - "1978 1978 7 4 7 0 0 0" + "1978 1978 7 4 7 0 0 0", ], "EU": [ "1977 1980 3 1 0 1 1 1 S", @@ -3149,23 +2638,20 @@ data = { "1978 1978 9 1 7 1 1 0", "1979 1995 8 0 8 1 1 0", "1981 9999 2 0 8 1 1 1 S", - "1996 9999 9 0 8 1 1 0" + "1996 9999 9 0 8 1 1 0", ], "SL": [ "1935 1942 5 1 7 0 0 0:40 SLST", "1935 1942 9 1 7 0 0 0 WAT", "1957 1962 5 1 7 0 0 1 SLST", - "1957 1962 8 1 7 0 0 0 GMT" - ], - "SA": [ - "1942 1943 8 15 0 2 0 1", - "1943 1944 2 15 0 2 0 0" + "1957 1962 8 1 7 0 0 0 GMT", ], + "SA": ["1942 1943 8 15 0 2 0 1", "1943 1944 2 15 0 2 0 0"], "Sudan": [ "1970 1970 4 1 7 0 0 1 S", "1970 1985 9 15 7 0 0 0", "1971 1971 3 30 7 0 0 1 S", - "1972 1985 3 0 8 0 0 1 S" + "1972 1985 3 0 8 0 0 1 S", ], "Libya": [ "1951 1951 9 14 7 2 0 1 S", @@ -3184,7 +2670,7 @@ data = { "1997 1997 3 4 7 0 0 1 S", "1997 1997 9 4 7 0 0 0", "2013 9999 2 5 8 1 0 1 S", - "2013 9999 9 5 8 2 0 0" + "2013 9999 9 5 8 2 0 0", ], "Tunisia": [ "1939 1939 3 15 7 23 2 1 S", @@ -3211,12 +2697,9 @@ data = { "2005 2005 4 1 7 0 2 1 S", "2005 2005 8 30 7 1 2 0", "2006 2008 2 0 8 2 2 1 S", - "2006 2008 9 0 8 2 2 0" - ], - "Namibia": [ - "1994 9999 8 1 0 2 0 1 S", - "1995 9999 3 1 0 2 0 0" + "2006 2008 9 0 8 2 2 0", ], + "Namibia": ["1994 9999 8 1 0 2 0 1 S", "1995 9999 3 1 0 2 0 0"], "US": [ "1918 1919 2 0 8 2 0 1 D", "1918 1919 9 0 8 2 0 0 S", @@ -3230,7 +2713,7 @@ data = { "1976 1986 3 0 8 2 0 1 D", "1987 2006 3 1 0 2 0 1 D", "2007 9999 2 8 0 2 0 1 D", - "2007 9999 10 1 0 2 0 0 S" + "2007 9999 10 1 0 2 0 0 S", ], "Brazil": [ "1931 1931 9 3 7 11 0 1 S", @@ -3296,7 +2779,7 @@ data = { "2034 2034 1 22 0 0 0 0", "2035 2036 1 15 0 0 0 0", "2037 2037 1 22 0 0 0 0", - "2038 9999 1 15 0 0 0 0" + "2038 9999 1 15 0 0 0 0", ], "Arg": [ "1930 1930 11 1 7 0 0 1 S", @@ -3327,12 +2810,9 @@ data = { "2000 2000 2 3 7 0 0 0", "2007 2007 11 30 7 0 0 1 S", "2008 2009 2 15 0 0 0 0", - "2008 2008 9 15 0 0 0 1 S" - ], - "SanLuis": [ - "2008 2009 2 8 0 0 0 0", - "2007 2009 9 8 0 0 0 1 S" + "2008 2008 9 15 0 0 0 1 S", ], + "SanLuis": ["2008 2009 2 8 0 0 0 0", "2007 2009 9 8 0 0 0 1 S"], "Para": [ "1975 1988 9 1 7 0 0 1 S", "1975 1978 2 1 7 0 0 0", @@ -3355,7 +2835,7 @@ data = { "2005 2009 2 8 0 0 0 0", "2010 9999 9 1 0 0 0 1 S", "2010 2012 3 8 0 0 0 0", - "2013 9999 2 22 0 0 0 0" + "2013 9999 2 22 0 0 0 0", ], "Canada": [ "1918 1918 3 14 7 2 0 1 D", @@ -3367,7 +2847,7 @@ data = { "1974 2006 9 0 8 2 0 0 S", "1987 2006 3 1 0 2 0 1 D", "2007 9999 2 8 0 2 0 1 D", - "2007 9999 10 1 0 2 0 0 S" + "2007 9999 10 1 0 2 0 0 S", ], "Mexico": [ "1939 1939 1 5 7 0 0 1 D", @@ -3383,14 +2863,14 @@ data = { "2001 2001 4 1 0 2 0 1 D", "2001 2001 8 0 8 2 0 0 S", "2002 9999 3 1 0 2 0 1 D", - "2002 9999 9 0 8 2 0 0 S" + "2002 9999 9 0 8 2 0 0 S", ], "Barb": [ "1977 1977 5 12 7 2 0 1 D", "1977 1978 9 1 0 2 0 0 S", "1978 1980 3 15 0 2 0 1 D", "1979 1979 8 30 7 2 0 0 S", - "1980 1980 8 25 7 2 0 0 S" + "1980 1980 8 25 7 2 0 0 S", ], "Belize": [ "1918 1942 9 2 0 0 0 0:30 HD", @@ -3398,12 +2878,9 @@ data = { "1973 1973 11 5 7 0 0 1 D", "1974 1974 1 9 7 0 0 0 S", "1982 1982 11 18 7 0 0 1 D", - "1983 1983 1 12 7 0 0 0 S" - ], - "CO": [ - "1992 1992 4 3 7 0 0 1 S", - "1993 1993 3 4 7 0 0 0" + "1983 1983 1 12 7 0 0 0 S", ], + "CO": ["1992 1992 4 3 7 0 0 1 S", "1993 1993 3 4 7 0 0 0"], "NT_YK": [ "1918 1918 3 14 7 2 0 1 D", "1918 1918 9 27 7 2 0 0 S", @@ -3416,7 +2893,7 @@ data = { "1965 1965 9 0 8 2 0 0 S", "1980 1986 3 0 8 2 0 1 D", "1980 2006 9 0 8 2 0 0 S", - "1987 2006 3 1 0 2 0 1 D" + "1987 2006 3 1 0 2 0 1 D", ], "Chicago": [ "1920 1920 5 13 7 2 0 1 D", @@ -3424,14 +2901,14 @@ data = { "1921 1921 2 0 8 2 0 1 D", "1922 1966 3 0 8 2 0 1 D", "1922 1954 8 0 8 2 0 0 S", - "1955 1966 9 0 8 2 0 0 S" + "1955 1966 9 0 8 2 0 0 S", ], "CR": [ "1979 1980 1 0 8 0 0 1 D", "1979 1980 5 1 0 0 0 0 S", "1991 1992 0 15 6 0 0 1 D", "1991 1991 6 1 7 0 0 0 S", - "1992 1992 2 15 7 0 0 0 S" + "1992 1992 2 15 7 0 0 0 S", ], "Vanc": [ "1918 1918 3 14 7 2 0 1 D", @@ -3442,20 +2919,20 @@ data = { "1946 1986 3 0 8 2 0 1 D", "1946 1946 9 13 7 2 0 0 S", "1947 1961 8 0 8 2 0 0 S", - "1962 2006 9 0 8 2 0 0 S" + "1962 2006 9 0 8 2 0 0 S", ], "Denver": [ "1920 1921 2 0 8 2 0 1 D", "1920 1920 9 0 8 2 0 0 S", "1921 1921 4 22 7 2 0 0 S", "1965 1966 3 0 8 2 0 1 D", - "1965 1966 9 0 8 2 0 0 S" + "1965 1966 9 0 8 2 0 0 S", ], "Detroit": [ "1948 1948 3 0 8 2 0 1 D", "1948 1948 8 0 8 2 0 0 S", "1967 1967 5 14 7 2 0 1 D", - "1967 1967 9 0 8 2 0 0 S" + "1967 1967 9 0 8 2 0 0 S", ], "Edm": [ "1918 1919 3 8 0 2 0 1 D", @@ -3474,12 +2951,9 @@ data = { "1969 1969 3 0 8 2 0 1 D", "1969 1969 9 0 8 2 0 0 S", "1972 1986 3 0 8 2 0 1 D", - "1972 2006 9 0 8 2 0 0 S" - ], - "Salv": [ - "1987 1988 4 1 0 0 0 1 D", - "1987 1988 8 0 8 0 0 0 S" + "1972 2006 9 0 8 2 0 0 S", ], + "Salv": ["1987 1988 4 1 0 0 0 1 D", "1987 1988 8 0 8 0 0 0 S"], "Halifax": [ "1916 1916 3 1 7 0 0 1 D", "1916 1916 9 1 7 0 0 0 S", @@ -3521,7 +2995,7 @@ data = { "1956 1959 3 0 8 2 0 1 D", "1956 1959 8 0 8 2 0 0 S", "1962 1973 3 0 8 2 0 1 D", - "1962 1973 9 0 8 2 0 0 S" + "1962 1973 9 0 8 2 0 0 S", ], "StJohns": [ "1917 1917 3 8 7 2 0 1 D", @@ -3542,14 +3016,14 @@ data = { "1988 1988 3 1 0 0:1 0 2 DD", "1989 2006 3 1 0 0:1 0 1 D", "2007 2011 2 8 0 0:1 0 1 D", - "2007 2010 10 1 0 0:1 0 0 S" + "2007 2010 10 1 0 0:1 0 0 S", ], "TC": [ "1979 1986 3 0 8 2 0 1 D", "1979 2006 9 0 8 2 0 0 S", "1987 2006 3 1 0 2 0 1 D", "2007 9999 2 8 0 2 0 1 D", - "2007 9999 10 1 0 2 0 0 S" + "2007 9999 10 1 0 2 0 0 S", ], "Guat": [ "1973 1973 10 25 7 0 0 1 D", @@ -3559,7 +3033,7 @@ data = { "1991 1991 2 23 7 0 0 1 D", "1991 1991 8 7 7 0 0 0 S", "2006 2006 3 30 7 0 0 1 D", - "2006 2006 9 1 7 0 0 0 S" + "2006 2006 9 1 7 0 0 0 S", ], "Cuba": [ "1928 1928 5 10 7 0 0 1 D", @@ -3599,31 +3073,31 @@ data = { "2011 2011 10 13 7 0 2 0 S", "2012 2012 3 1 7 0 2 1 D", "2012 9999 10 1 0 0 2 0 S", - "2013 9999 2 8 0 0 2 1 D" + "2013 9999 2 8 0 0 2 1 D", ], "Indianapolis": [ "1941 1941 5 22 7 2 0 1 D", "1941 1954 8 0 8 2 0 0 S", - "1946 1954 3 0 8 2 0 1 D" + "1946 1954 3 0 8 2 0 1 D", ], "Starke": [ "1947 1961 3 0 8 2 0 1 D", "1947 1954 8 0 8 2 0 0 S", "1955 1956 9 0 8 2 0 0 S", "1957 1958 8 0 8 2 0 0 S", - "1959 1961 9 0 8 2 0 0 S" + "1959 1961 9 0 8 2 0 0 S", ], "Marengo": [ "1951 1951 3 0 8 2 0 1 D", "1951 1951 8 0 8 2 0 0 S", "1954 1960 3 0 8 2 0 1 D", - "1954 1960 8 0 8 2 0 0 S" + "1954 1960 8 0 8 2 0 0 S", ], "Pike": [ "1955 1955 4 1 7 0 0 1 D", "1955 1960 8 0 8 2 0 0 S", "1956 1964 3 0 8 2 0 1 D", - "1961 1964 9 0 8 2 0 0 S" + "1961 1964 9 0 8 2 0 0 S", ], "Perry": [ "1946 1946 3 0 8 2 0 1 D", @@ -3634,7 +3108,7 @@ data = { "1956 1963 3 0 8 2 0 1 D", "1960 1960 9 0 8 2 0 0 S", "1961 1961 8 0 8 2 0 0 S", - "1962 1963 9 0 8 2 0 0 S" + "1962 1963 9 0 8 2 0 0 S", ], "Vincennes": [ "1946 1946 3 0 8 2 0 1 D", @@ -3645,13 +3119,13 @@ data = { "1956 1963 3 0 8 2 0 1 D", "1960 1960 9 0 8 2 0 0 S", "1961 1961 8 0 8 2 0 0 S", - "1962 1963 9 0 8 2 0 0 S" + "1962 1963 9 0 8 2 0 0 S", ], "Pulaski": [ "1946 1960 3 0 8 2 0 1 D", "1946 1954 8 0 8 2 0 0 S", "1955 1956 9 0 8 2 0 0 S", - "1957 1960 8 0 8 2 0 0 S" + "1957 1960 8 0 8 2 0 0 S", ], "Louisville": [ "1921 1921 4 1 7 2 0 1 D", @@ -3660,7 +3134,7 @@ data = { "1941 1941 8 0 8 2 0 0 S", "1946 1946 5 2 7 2 0 0 S", "1950 1955 8 0 8 2 0 0 S", - "1956 1960 9 0 8 2 0 0 S" + "1956 1960 9 0 8 2 0 0 S", ], "Peru": [ "1938 1938 0 1 7 0 0 1 S", @@ -3672,14 +3146,14 @@ data = { "1990 1990 0 1 7 0 0 1 S", "1990 1990 3 1 7 0 0 0", "1994 1994 0 1 7 0 0 1 S", - "1994 1994 3 1 7 0 0 0" + "1994 1994 3 1 7 0 0 0", ], "CA": [ "1948 1948 2 14 7 2 0 1 D", "1949 1949 0 1 7 2 0 0 S", "1950 1966 3 0 8 2 0 1 D", "1950 1961 8 0 8 2 0 0 S", - "1962 1966 9 0 8 2 0 0 S" + "1962 1966 9 0 8 2 0 0 S", ], "Nic": [ "1979 1980 2 16 0 0 0 1 D", @@ -3687,13 +3161,13 @@ data = { "2005 2005 3 10 7 0 0 1 D", "2005 2005 9 1 0 0 0 0 S", "2006 2006 3 30 7 2 0 1 D", - "2006 2006 9 1 0 1 0 0 S" + "2006 2006 9 1 0 1 0 0 S", ], "Menominee": [ "1946 1946 3 0 8 2 0 1 D", "1946 1946 8 0 8 2 0 0 S", "1966 1966 3 0 8 2 0 1 D", - "1966 1966 9 0 8 2 0 0 S" + "1966 1966 9 0 8 2 0 0 S", ], "Moncton": [ "1933 1935 5 8 0 1 0 1 D", @@ -3708,7 +3182,7 @@ data = { "1946 1956 8 0 8 2 0 0 S", "1957 1972 9 0 8 2 0 0 S", "1993 2006 3 1 0 0:1 0 1 D", - "1993 2006 9 0 8 0:1 0 0 S" + "1993 2006 9 0 8 0:1 0 0 S", ], "Uruguay": [ "1923 1923 9 2 7 0 0 0:30 HS", @@ -3755,7 +3229,7 @@ data = { "2005 2005 9 9 7 2 0 1 S", "2006 2006 2 12 7 2 0 0", "2006 9999 9 1 0 2 0 1 S", - "2007 9999 2 8 0 2 0 0" + "2007 9999 2 8 0 2 0 0", ], "Mont": [ "1917 1917 2 25 7 2 0 1 D", @@ -3780,18 +3254,15 @@ data = { "1945 1948 8 0 8 2 0 0 S", "1949 1950 9 0 8 2 0 0 S", "1951 1956 8 0 8 2 0 0 S", - "1957 1973 9 0 8 2 0 0 S" - ], - "Bahamas": [ - "1964 1975 9 0 8 2 0 0 S", - "1964 1975 3 0 8 2 0 1 D" + "1957 1973 9 0 8 2 0 0 S", ], + "Bahamas": ["1964 1975 9 0 8 2 0 0 S", "1964 1975 3 0 8 2 0 1 D"], "NYC": [ "1920 1920 2 0 8 2 0 1 D", "1920 1920 9 0 8 2 0 0 S", "1921 1966 3 0 8 2 0 1 D", "1921 1954 8 0 8 2 0 0 S", - "1955 1966 9 0 8 2 0 0 S" + "1955 1966 9 0 8 2 0 0 S", ], "Haiti": [ "1983 1983 4 8 7 0 0 1 D", @@ -3802,7 +3273,7 @@ data = { "2005 2006 3 1 0 0 0 1 D", "2005 2006 9 0 8 0 0 0 S", "2012 9999 2 8 0 2 0 1 D", - "2012 9999 10 1 0 2 0 0 S" + "2012 9999 10 1 0 2 0 0 S", ], "Regina": [ "1918 1918 3 14 7 2 0 1 D", @@ -3821,7 +3292,7 @@ data = { "1947 1957 3 0 8 2 0 1 D", "1947 1957 8 0 8 2 0 0 S", "1959 1959 3 0 8 2 0 1 D", - "1959 1959 9 0 8 2 0 0 S" + "1959 1959 9 0 8 2 0 0 S", ], "Chile": [ "1927 1932 8 1 7 0 0 1 S", @@ -3860,7 +3331,7 @@ data = { "2011 2011 4 2 0 3 1 0", "2011 2011 7 16 0 4 1 1 S", "2012 9999 3 23 0 3 1 0", - "2012 9999 8 2 0 4 1 1 S" + "2012 9999 8 2 0 4 1 1 S", ], "DR": [ "1966 1966 9 30 7 0 0 1 D", @@ -3868,7 +3339,7 @@ data = { "1969 1973 9 0 8 0 0 0:30 HD", "1970 1970 1 21 7 0 0 0 S", "1971 1971 0 20 7 0 0 0 S", - "1972 1974 0 21 7 0 0 0 S" + "1972 1974 0 21 7 0 0 0 S", ], "C-Eur": [ "1916 1916 3 30 7 23 0 1 S", @@ -3887,20 +3358,20 @@ data = { "1978 1978 9 1 7 2 2 0", "1979 1995 8 0 8 2 2 0", "1981 9999 2 0 8 2 2 1 S", - "1996 9999 9 0 8 2 2 0" + "1996 9999 9 0 8 2 2 0", ], "Swift": [ "1957 1957 3 0 8 2 0 1 D", "1957 1957 9 0 8 2 0 0 S", "1959 1961 3 0 8 2 0 1 D", "1959 1959 9 0 8 2 0 0 S", - "1960 1961 8 0 8 2 0 0 S" + "1960 1961 8 0 8 2 0 0 S", ], "Hond": [ "1987 1988 4 1 0 0 0 1 D", "1987 1988 8 0 8 0 0 0 S", "2006 2006 4 1 0 0 0 1 D", - "2006 2006 7 1 1 0 0 0 S" + "2006 2006 7 1 1 0 0 0 S", ], "Thule": [ "1991 1992 2 0 8 2 0 1 D", @@ -3908,7 +3379,7 @@ data = { "1993 2006 3 1 0 2 0 1 D", "1993 2006 9 0 8 2 0 0 S", "2007 9999 2 8 0 2 0 1 D", - "2007 9999 10 1 0 2 0 0 S" + "2007 9999 10 1 0 2 0 0 S", ], "Toronto": [ "1919 1919 2 30 7 23:30 0 1 D", @@ -3934,7 +3405,7 @@ data = { "1950 1973 3 0 8 2 0 1 D", "1950 1950 10 0 8 2 0 0 S", "1951 1956 8 0 8 2 0 0 S", - "1957 1973 9 0 8 2 0 0 S" + "1957 1973 9 0 8 2 0 0 S", ], "Winn": [ "1916 1916 3 23 7 0 0 1 D", @@ -3960,7 +3431,7 @@ data = { "1963 1963 8 22 7 2 0 0 S", "1966 1986 3 0 8 2 2 1 D", "1966 2005 9 0 8 2 2 0 S", - "1987 2005 3 1 0 2 2 1 D" + "1987 2005 3 1 0 2 2 1 D", ], "Aus": [ "1917 1917 0 1 7 0:1 0 1", @@ -3969,7 +3440,7 @@ data = { "1942 1942 2 29 7 2 0 0", "1942 1942 8 27 7 2 0 1", "1943 1944 2 0 8 2 0 0", - "1943 1943 9 3 7 2 0 1" + "1943 1943 9 3 7 2 0 1", ], "AT": [ "1967 1967 9 1 0 2 2 1", @@ -3990,7 +3461,7 @@ data = { "2001 9999 9 1 0 2 2 1", "2006 2006 3 1 0 2 2 0", "2007 2007 2 0 8 2 2 0", - "2008 9999 3 1 0 2 2 0" + "2008 9999 3 1 0 2 2 0", ], "NZAQ": [ "1974 1974 10 3 7 2 2 1 D", @@ -4001,7 +3472,7 @@ data = { "1976 1989 2 1 0 2 2 0 S", "1990 2007 2 15 0 2 2 0 S", "2007 9999 8 0 8 2 2 1 D", - "2008 9999 3 1 0 2 2 0 S" + "2008 9999 3 1 0 2 2 0 S", ], "ArgAQ": [ "1964 1966 2 1 7 0 0 0", @@ -4010,7 +3481,7 @@ data = { "1967 1968 9 1 0 0 0 1 S", "1968 1969 3 1 0 0 0 0", "1974 1974 0 23 7 0 0 1 S", - "1974 1974 4 1 7 0 0 0" + "1974 1974 4 1 7 0 0 0", ], "ChileAQ": [ "1972 1986 2 9 0 3 1 0", @@ -4035,7 +3506,7 @@ data = { "2011 2011 4 2 0 3 1 0", "2011 2011 7 16 0 4 1 1 S", "2012 9999 3 23 0 3 1 0", - "2012 9999 8 2 0 4 1 1 S" + "2012 9999 8 2 0 4 1 1 S", ], "Norway": [ "1916 1916 4 22 7 1 0 1 S", @@ -4044,7 +3515,7 @@ data = { "1945 1945 9 1 7 2 2 0", "1959 1964 2 15 0 2 2 1 S", "1959 1965 8 15 0 2 2 0", - "1965 1965 3 25 7 2 2 1 S" + "1965 1965 3 25 7 2 2 1 S", ], "RussiaAsia": [ "1981 1984 3 1 7 0 0 1 S", @@ -4055,7 +3526,7 @@ data = { "1992 1992 8 6 8 23 0 0", "1993 9999 2 0 8 2 2 1 S", "1993 1995 8 0 8 2 2 0", - "1996 9999 9 0 8 2 2 0" + "1996 9999 9 0 8 2 2 0", ], "Jordan": [ "1973 1973 5 6 7 0 0 1 S", @@ -4086,7 +3557,7 @@ data = { "2004 2004 9 15 7 0 2 0", "2005 2005 8 5 8 0 2 0", "2006 2011 9 5 8 0 2 0", - "2013 9999 9 5 8 0 2 0" + "2013 9999 9 5 8 0 2 0", ], "Russia": [ "1917 1917 6 1 7 23 0 1 MST", @@ -4108,7 +3579,7 @@ data = { "1992 1992 8 6 8 23 0 0", "1993 2010 2 0 8 2 2 1 S", "1993 1995 8 0 8 2 2 0", - "1996 2010 9 0 8 2 2 0" + "1996 2010 9 0 8 2 2 0", ], "Iraq": [ "1982 1982 4 1 7 0 0 1 D", @@ -4118,17 +3589,10 @@ data = { "1985 1990 8 0 8 1 2 0 S", "1986 1990 2 0 8 1 2 1 D", "1991 2007 3 1 7 3 2 1 D", - "1991 2007 9 1 7 3 2 0 S" - ], - "EUAsia": [ - "1981 9999 2 0 8 1 1 1 S", - "1979 1995 8 0 8 1 1 0", - "1996 9999 9 0 8 1 1 0" - ], - "Azer": [ - "1997 9999 2 0 8 4 0 1 S", - "1997 9999 9 0 8 5 0 0" + "1991 2007 9 1 7 3 2 0 S", ], + "EUAsia": ["1981 9999 2 0 8 1 1 1 S", "1979 1995 8 0 8 1 1 0", "1996 9999 9 0 8 1 1 0"], + "Azer": ["1997 9999 2 0 8 4 0 1 S", "1997 9999 9 0 8 5 0 0"], "Lebanon": [ "1920 1920 2 28 7 0 0 1 S", "1920 1920 9 25 7 0 0 0", @@ -4153,13 +3617,13 @@ data = { "1992 1992 9 4 7 0 0 0", "1993 9999 2 0 8 0 0 1 S", "1993 1998 8 0 8 0 0 0", - "1999 9999 9 0 8 0 0 0" + "1999 9999 9 0 8 0 0 0", ], "Kyrgyz": [ "1992 1996 3 7 0 0 2 1 S", "1992 1996 8 0 8 0 0 0", "1997 2005 2 0 8 2:30 0 1 S", - "1997 2004 9 0 8 2:30 0 0" + "1997 2004 9 0 8 2:30 0 0", ], "Mongol": [ "1983 1984 3 1 7 0 0 1 S", @@ -4168,13 +3632,9 @@ data = { "1984 1998 8 0 8 0 0 0", "2001 2001 3 6 8 2 0 1 S", "2001 2006 8 6 8 2 0 0", - "2002 2006 2 6 8 2 0 1 S" - ], - "PRC": [ - "1986 1986 4 4 7 0 0 1 D", - "1986 1991 8 11 0 0 0 0 S", - "1987 1991 3 10 0 0 0 1 D" + "2002 2006 2 6 8 2 0 1 S", ], + "PRC": ["1986 1986 4 4 7 0 0 1 D", "1986 1991 8 11 0 0 0 0 S", "1987 1991 3 10 0 0 0 1 D"], "Syria": [ "1920 1923 3 15 0 2 0 1 S", "1920 1923 9 1 0 2 0 0", @@ -4216,12 +3676,9 @@ data = { "2009 2009 2 5 8 0 0 1 S", "2010 2011 3 1 5 0 0 1 S", "2012 9999 2 5 8 0 0 1 S", - "2009 9999 9 5 8 0 0 0" - ], - "Dhaka": [ - "2009 2009 5 19 7 23 0 1 S", - "2009 2009 11 31 7 23:59 0 0" + "2009 9999 9 5 8 0 0 0", ], + "Dhaka": ["2009 2009 5 19 7 23 0 1 S", "2009 2009 11 31 7 23:59 0 0"], "Zion": [ "1940 1940 5 1 7 0 0 1 D", "1942 1944 10 1 7 0 0 0 S", @@ -4310,7 +3767,7 @@ data = { "2013 9999 2 23 5 2 0 1 D", "2013 2026 9 2 0 2 0 0 S", "2027 2027 9 3 1 2 0 0 S", - "2028 9999 9 2 0 2 0 0 S" + "2028 9999 9 2 0 2 0 0 S", ], "EgyptAsia": [ "1957 1957 4 10 7 0 0 1 S", @@ -4318,7 +3775,7 @@ data = { "1958 1958 4 1 7 0 0 1 S", "1959 1967 4 1 7 1 0 1 S", "1959 1965 8 30 7 3 0 0", - "1966 1966 9 1 7 3 0 0" + "1966 1966 9 1 7 3 0 0", ], "Palestine": [ "1999 2005 3 15 5 0 0 1 S", @@ -4338,7 +3795,7 @@ data = { "2011 2011 7 30 7 0 0 1 S", "2011 2011 8 30 7 0 0 0", "2012 9999 2 4 8 24 0 1 S", - "2012 9999 8 21 5 1 0 0" + "2012 9999 8 21 5 1 0 0", ], "HK": [ "1941 1941 3 1 7 3:30 0 1 S", @@ -4359,7 +3816,7 @@ data = { "1965 1976 9 16 0 3:30 0 0", "1973 1973 11 30 7 3:30 0 1 S", "1979 1979 4 8 0 3:30 0 1 S", - "1979 1979 9 16 0 3:30 0 0" + "1979 1979 9 16 0 3:30 0 0", ], "Pakistan": [ "2002 2002 3 2 0 0:1 0 1 S", @@ -4367,12 +3824,9 @@ data = { "2008 2008 5 1 7 0 0 1 S", "2008 2008 10 1 7 0 0 0", "2009 2009 3 15 7 0 0 1 S", - "2009 2009 10 1 7 0 0 0" - ], - "NBorneo": [ - "1935 1941 8 14 7 0 0 0:20 TS", - "1935 1941 11 14 7 0 0 0" + "2009 2009 10 1 7 0 0 0", ], + "NBorneo": ["1935 1941 8 14 7 0 0 0:20 TS", "1935 1941 11 14 7 0 0 0"], "Macau": [ "1961 1962 2 16 0 3:30 0 1 S", "1961 1964 10 1 0 3:30 0 0", @@ -4387,7 +3841,7 @@ data = { "1974 1977 9 15 0 3:30 0 0", "1975 1977 3 15 0 3:30 0 1 S", "1978 1980 3 15 0 0 0 1 S", - "1978 1980 9 15 0 0 0 0" + "1978 1980 9 15 0 0 0 0", ], "Phil": [ "1936 1936 10 1 7 0 0 1 S", @@ -4395,7 +3849,7 @@ data = { "1954 1954 3 12 7 0 0 1 S", "1954 1954 6 1 7 0 0 0", "1978 1978 2 22 7 0 0 1 S", - "1978 1978 8 21 7 0 0 0" + "1978 1978 8 21 7 0 0 0", ], "Cyprus": [ "1975 1975 3 13 7 0 0 1 S", @@ -4406,19 +3860,15 @@ data = { "1977 1977 8 25 7 0 0 0", "1978 1978 9 2 7 0 0 0", "1979 1997 8 0 8 0 0 0", - "1981 1998 2 0 8 0 0 1 S" + "1981 1998 2 0 8 0 0 1 S", ], "ROK": [ "1960 1960 4 15 7 0 0 1 D", "1960 1960 8 13 7 0 0 0 S", "1987 1988 4 8 0 0 0 1 D", - "1987 1988 9 8 0 0 0 0 S" - ], - "Shang": [ - "1940 1940 5 3 7 0 0 1 D", - "1940 1941 9 1 7 0 0 0 S", - "1941 1941 2 16 7 0 0 1 D" + "1987 1988 9 8 0 0 0 0 S", ], + "Shang": ["1940 1940 5 3 7 0 0 1 D", "1940 1941 9 1 7 0 0 0 S", "1941 1941 2 16 7 0 0 1 D"], "Taiwan": [ "1945 1951 4 1 7 0 0 1 D", "1945 1951 9 1 7 0 0 0 S", @@ -4430,13 +3880,9 @@ data = { "1974 1975 3 1 7 0 0 1 D", "1974 1975 9 1 7 0 0 0 S", "1979 1979 5 30 7 0 0 1 D", - "1979 1979 8 30 7 0 0 0 S" - ], - "E-EurAsia": [ - "1981 9999 2 0 8 0 0 1 S", - "1979 1995 8 0 8 0 0 0", - "1996 9999 9 0 8 0 0 0" + "1979 1979 8 30 7 0 0 0 S", ], + "E-EurAsia": ["1981 9999 2 0 8 0 0 1 S", "1979 1995 8 0 8 0 0 0", "1996 9999 9 0 8 0 0 0"], "Iran": [ "1978 1980 2 21 7 0 0 1 D", "1978 1978 9 21 7 0 0 0 S", @@ -4486,13 +3932,13 @@ data = { "2034 2035 2 22 7 0 0 1 D", "2034 2035 8 22 7 0 0 0 S", "2036 2037 2 21 7 0 0 1 D", - "2036 2037 8 21 7 0 0 0 S" + "2036 2037 8 21 7 0 0 0 S", ], "Japan": [ "1948 1948 4 1 0 2 0 1 D", "1948 1951 8 8 6 2 0 0 S", "1949 1949 3 1 0 2 0 1 D", - "1950 1951 4 1 0 2 0 1 D" + "1950 1951 4 1 0 2 0 1 D", ], "Port": [ "1916 1916 5 17 7 23 0 1 S", @@ -4544,7 +3990,7 @@ data = { "1979 1982 8 0 8 1 2 0", "1980 1980 2 0 8 0 2 1 S", "1981 1982 2 0 8 1 2 1 S", - "1983 1983 2 0 8 2 2 1 S" + "1983 1983 2 0 8 2 2 1 S", ], "W-Eur": [ "1977 1980 3 1 0 1 2 1 S", @@ -4552,7 +3998,7 @@ data = { "1978 1978 9 1 7 1 2 0", "1979 1995 8 0 8 1 2 0", "1981 9999 2 0 8 1 2 1 S", - "1996 9999 9 0 8 1 2 0" + "1996 9999 9 0 8 1 2 0", ], "Iceland": [ "1917 1918 1 19 7 23 0 1 S", @@ -4571,7 +4017,7 @@ data = { "1947 1967 3 1 0 1 2 1 S", "1949 1949 9 30 7 1 2 0", "1950 1966 9 22 0 1 2 0", - "1967 1967 9 29 7 1 2 0" + "1967 1967 9 29 7 1 2 0", ], "Falk": [ "1937 1938 8 0 8 0 0 1 S", @@ -4585,7 +4031,7 @@ data = { "1985 2000 8 9 0 0 0 1 S", "1986 2000 3 16 0 0 0 0", "2001 2010 3 15 0 2 0 0", - "2001 2010 8 1 0 2 0 1 S" + "2001 2010 8 1 0 2 0 1 S", ], "AS": [ "1971 1985 9 0 8 2 2 1", @@ -4602,13 +4048,13 @@ data = { "2006 2006 3 2 7 2 2 0", "2007 2007 2 0 8 2 2 0", "2008 9999 3 1 0 2 2 0", - "2008 9999 9 1 0 2 2 1" + "2008 9999 9 1 0 2 2 1", ], "AQ": [ "1971 1971 9 0 8 2 2 1", "1972 1972 1 0 8 2 2 0", "1989 1991 9 0 8 2 2 1", - "1990 1992 2 1 0 2 2 0" + "1990 1992 2 1 0 2 2 0", ], "AN": [ "1971 1985 9 0 8 2 2 1", @@ -4626,7 +4072,7 @@ data = { "2006 2006 3 1 0 2 2 0", "2007 2007 2 0 8 2 2 0", "2008 9999 3 1 0 2 2 0", - "2008 9999 9 1 0 2 2 1" + "2008 9999 9 1 0 2 2 1", ], "AW": [ "1974 1974 9 0 8 2 2 1", @@ -4637,12 +4083,9 @@ data = { "1992 1992 2 1 0 2 2 0", "2006 2006 11 3 7 2 2 1", "2007 2009 2 0 8 2 2 0", - "2007 2008 9 0 8 2 2 1" - ], - "Holiday": [ - "1992 1993 9 0 8 2 2 1", - "1993 1994 2 1 0 2 2 0" + "2007 2008 9 0 8 2 2 1", ], + "Holiday": ["1992 1993 9 0 8 2 2 1", "1993 1994 2 1 0 2 2 0"], "LH": [ "1981 1984 9 0 8 2 0 1", "1982 1985 2 1 0 2 0 0", @@ -4657,7 +4100,7 @@ data = { "2006 2006 3 1 0 2 0 0", "2007 2007 2 0 8 2 0 0", "2008 9999 3 1 0 2 0 0", - "2008 9999 9 1 0 2 0 0:30" + "2008 9999 9 1 0 2 0 0:30", ], "AV": [ "1971 1985 9 0 8 2 2 1", @@ -4673,7 +4116,7 @@ data = { "2006 2006 3 1 0 2 2 0", "2007 2007 2 0 8 2 2 0", "2008 9999 3 1 0 2 2 0", - "2008 9999 9 1 0 2 2 1" + "2008 9999 9 1 0 2 2 1", ], "Neth": [ "1916 1916 4 1 7 0 0 1 NST", @@ -4695,7 +4138,7 @@ data = { "1937 1939 9 2 0 2 2 0", "1938 1939 4 15 7 2 2 1 S", "1945 1945 3 2 7 2 2 1 S", - "1945 1945 8 16 7 2 2 0" + "1945 1945 8 16 7 2 2 0", ], "Greece": [ "1932 1932 6 7 7 0 0 1 S", @@ -4716,12 +4159,12 @@ data = { "1979 1979 3 1 7 9 0 1 S", "1979 1979 8 29 7 2 0 0", "1980 1980 3 1 7 0 0 1 S", - "1980 1980 8 28 7 0 0 0" + "1980 1980 8 28 7 0 0 0", ], "SovietZone": [ "1945 1945 4 24 7 2 0 2 M", "1945 1945 8 24 7 3 0 1 S", - "1945 1945 10 18 7 2 2 0" + "1945 1945 10 18 7 2 2 0", ], "Germany": [ "1946 1946 3 14 7 2 2 1 S", @@ -4731,7 +4174,7 @@ data = { "1947 1947 4 11 7 2 2 2 M", "1947 1947 5 29 7 3 0 1 S", "1948 1948 3 18 7 2 2 1 S", - "1949 1949 3 10 7 2 2 1 S" + "1949 1949 3 10 7 2 2 1 S", ], "Czech": [ "1945 1945 3 8 7 2 2 1 S", @@ -4740,7 +4183,7 @@ data = { "1946 1949 9 1 0 2 2 0", "1947 1947 3 20 7 2 2 1 S", "1948 1948 3 18 7 2 2 1 S", - "1949 1949 3 9 7 2 2 1 S" + "1949 1949 3 9 7 2 2 1 S", ], "Belgium": [ "1918 1918 2 9 7 0 2 1 S", @@ -4776,7 +4219,7 @@ data = { "1945 1945 3 2 7 2 2 1 S", "1945 1945 8 16 7 2 2 0", "1946 1946 4 19 7 2 2 1 S", - "1946 1946 9 7 7 2 2 0" + "1946 1946 9 7 7 2 2 0", ], "Romania": [ "1932 1932 4 21 7 0 2 1 S", @@ -4787,7 +4230,7 @@ data = { "1980 1980 3 5 7 23 0 1 S", "1980 1980 8 0 8 1 0 0", "1991 1993 2 0 8 0 2 1 S", - "1991 1993 8 0 8 0 2 0" + "1991 1993 8 0 8 0 2 0", ], "E-Eur": [ "1977 1980 3 1 0 0 0 1 S", @@ -4795,7 +4238,7 @@ data = { "1978 1978 9 1 7 0 0 0", "1979 1995 8 0 8 0 0 0", "1981 9999 2 0 8 0 0 1 S", - "1996 9999 9 0 8 0 0 0" + "1996 9999 9 0 8 0 0 0", ], "Hungary": [ "1918 1918 3 1 7 3 0 1 S", @@ -4817,12 +4260,9 @@ data = { "1956 1956 8 0 8 0 0 0", "1957 1957 5 1 0 1 0 1 S", "1957 1957 8 0 8 3 0 0", - "1980 1980 3 6 7 1 0 1 S" - ], - "Swiss": [ - "1941 1942 4 1 1 1 0 1 S", - "1941 1942 9 1 1 2 0 0" + "1980 1980 3 6 7 1 0 1 S", ], + "Swiss": ["1941 1942 4 1 1 1 0 1 S", "1941 1942 9 1 1 2 0 0"], "Denmark": [ "1916 1916 4 14 7 23 0 1 S", "1916 1916 8 30 7 23 0 0", @@ -4834,7 +4274,7 @@ data = { "1947 1947 4 4 7 2 2 1 S", "1947 1947 7 10 7 2 2 0", "1948 1948 4 9 7 2 2 1 S", - "1948 1948 7 8 7 2 2 0" + "1948 1948 7 8 7 2 2 0", ], "GB-Eire": [ "1916 1916 4 21 7 2 2 1 BST", @@ -4901,13 +4341,13 @@ data = { "1972 1980 9 23 0 2 2 0 GMT", "1981 1995 2 0 8 1 1 1 BST", "1981 1989 9 23 0 1 1 0 GMT", - "1990 1995 9 22 0 1 1 0 GMT" + "1990 1995 9 22 0 1 1 0 GMT", ], "Finland": [ "1942 1942 3 3 7 0 0 1 S", "1942 1942 9 3 7 0 0 0", "1981 1982 2 0 8 2 0 1 S", - "1981 1982 8 0 8 3 0 0" + "1981 1982 8 0 8 3 0 0", ], "Turkey": [ "1916 1916 4 1 7 0 0 1 S", @@ -4963,7 +4403,7 @@ data = { "1986 1990 8 0 8 2 2 0", "1991 2006 2 0 8 1 2 1 S", "1991 1995 8 0 8 1 2 0", - "1996 2006 9 0 8 1 2 0" + "1996 2006 9 0 8 1 2 0", ], "Poland": [ "1918 1919 8 16 7 2 2 0", @@ -4985,7 +4425,7 @@ data = { "1959 1961 9 1 0 1 2 0", "1960 1960 3 3 7 1 2 1 S", "1961 1964 4 0 8 1 2 1 S", - "1962 1964 8 0 8 1 2 0" + "1962 1964 8 0 8 1 2 0", ], "Lux": [ "1916 1916 4 14 7 23 0 1 S", @@ -5010,7 +4450,7 @@ data = { "1926 1926 3 17 7 23 0 1 S", "1927 1927 3 9 7 23 0 1 S", "1928 1928 3 14 7 23 0 1 S", - "1929 1929 3 20 7 23 0 1 S" + "1929 1929 3 20 7 23 0 1 S", ], "Italy": [ "1916 1916 5 3 7 0 2 1 S", @@ -5048,7 +4488,7 @@ data = { "1976 1976 4 30 7 0 2 1 S", "1977 1979 4 22 0 0 2 1 S", "1978 1978 9 1 7 0 2 0", - "1979 1979 8 30 7 0 2 0" + "1979 1979 8 30 7 0 2 0", ], "Malta": [ "1973 1973 2 31 7 0 2 1 S", @@ -5057,7 +4497,7 @@ data = { "1974 1974 8 16 7 0 2 0", "1975 1979 3 15 0 2 0 1 S", "1975 1980 8 15 0 2 0 0", - "1980 1980 2 31 7 2 0 1 S" + "1980 1980 2 31 7 2 0 1 S", ], "France": [ "1916 1916 5 14 7 23 2 1 S", @@ -5101,18 +4541,15 @@ data = { "1945 1945 3 2 7 2 0 2 M", "1945 1945 8 16 7 3 0 0", "1976 1976 2 28 7 1 0 1 S", - "1976 1976 8 26 7 1 0 0" - ], - "Latvia": [ - "1989 1996 2 0 8 2 2 1 S", - "1989 1996 8 0 8 2 2 0" + "1976 1976 8 26 7 1 0 0", ], + "Latvia": ["1989 1996 2 0 8 2 2 1 S", "1989 1996 8 0 8 2 2 0"], "Bulg": [ "1979 1979 2 31 7 23 0 1 S", "1979 1979 9 1 7 1 0 0", "1980 1982 3 1 6 23 0 1 S", "1980 1980 8 29 7 1 0 0", - "1981 1981 8 27 7 2 0 0" + "1981 1981 8 27 7 2 0 0", ], "Albania": [ "1940 1940 5 16 7 0 0 1 S", @@ -5139,7 +4576,7 @@ data = { "1982 1982 9 3 7 0 0 0", "1983 1983 3 18 7 0 0 1 S", "1983 1983 9 1 7 0 0 0", - "1984 1984 3 1 7 0 0 1 S" + "1984 1984 3 1 7 0 0 1 S", ], "Austria": [ "1920 1920 3 5 7 2 2 1 S", @@ -5149,18 +4586,15 @@ data = { "1947 1947 3 6 7 2 2 1 S", "1948 1948 3 18 7 2 2 1 S", "1980 1980 3 6 7 0 0 1 S", - "1980 1980 8 28 7 0 0 0" + "1980 1980 8 28 7 0 0 0", ], "Mauritius": [ "1982 1982 9 10 7 0 0 1 S", "1983 1983 2 21 7 0 0 0", "2008 2008 9 0 8 2 0 1 S", - "2009 2009 2 0 8 2 0 0" - ], - "WS": [ - "2012 9999 8 0 8 3 0 1 D", - "2012 9999 3 1 0 4 0 0" + "2009 2009 2 0 8 2 0 0", ], + "WS": ["2012 9999 8 0 8 3 0 1 D", "2012 9999 3 1 0 4 0 0"], "NZ": [ "1927 1927 10 6 7 2 0 1 S", "1928 1928 2 4 7 2 0 0 M", @@ -5177,7 +4611,7 @@ data = { "1990 2006 9 1 0 2 2 1 D", "1990 2007 2 15 0 2 2 0 S", "2007 9999 8 0 8 2 2 1 D", - "2008 9999 3 1 0 2 2 0 S" + "2008 9999 3 1 0 2 2 0 S", ], "Chatham": [ "1974 1974 10 1 0 2:45 2 1 D", @@ -5188,7 +4622,7 @@ data = { "1990 2006 9 1 0 2:45 2 1 D", "1990 2007 2 15 0 2:45 2 0 S", "2007 9999 8 0 8 2:45 2 1 D", - "2008 9999 3 1 0 2:45 2 0 S" + "2008 9999 3 1 0 2:45 2 0 S", ], "Vanuatu": [ "1983 1983 8 25 7 0 0 1 S", @@ -5196,7 +4630,7 @@ data = { "1984 1984 9 23 7 0 0 1 S", "1985 1991 8 23 0 0 0 1 S", "1992 1993 0 23 0 0 0 0", - "1992 1992 9 23 0 0 0 1 S" + "1992 1992 9 23 0 0 0 1 S", ], "Fiji": [ "1998 1999 10 1 0 2 0 1 S", @@ -5205,25 +4639,25 @@ data = { "2010 2010 2 0 8 3 0 0", "2010 9999 9 18 0 2 0 1 S", "2011 2011 2 1 0 3 0 0", - "2012 9999 0 18 0 3 0 0" + "2012 9999 0 18 0 3 0 0", ], "NC": [ "1977 1978 11 1 0 0 0 1 S", "1978 1979 1 27 7 0 0 0", "1996 1996 11 1 7 2 2 1 S", - "1997 1997 2 2 7 2 2 0" + "1997 1997 2 2 7 2 2 0", ], "Cook": [ "1978 1978 10 12 7 0 0 0:30 HS", "1979 1991 2 1 0 0 0 0", - "1979 1990 9 0 8 0 0 0:30 HS" + "1979 1990 9 0 8 0 0 0:30 HS", ], "Tonga": [ "1999 1999 9 7 7 2 2 1 S", "2000 2000 2 19 7 2 2 0", "2000 2001 10 1 0 2 0 1 S", - "2001 2002 0 0 8 2 0 0" - ] + "2001 2002 0 0 8 2 0 0", + ], }, "links": { "America/Kralendijk": "America/Curacao", @@ -5246,7 +4680,6 @@ data = { "Europe/Skopje": "Europe/Belgrade", "Europe/Vatican": "Europe/Rome", "Europe/Zagreb": "Europe/Belgrade", - # backward compatibility for deprecated timezones as per iana database "America/Virgin": "America/Port_of_Spain", "America/Buenos_Aires": "America/Argentina/Buenos_Aires", @@ -5357,6 +4790,6 @@ data = { "Egypt": "Africa/Cairo", "Singapore": "Asia/Singapore", "Brazil/East": "America/Sao_Paulo", - "Brazil/Acre": "America/Rio_Branco" - } + "Brazil/Acre": "America/Rio_Branco", + }, } diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index a1fe04ccb6..57dd004c28 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -20,39 +20,51 @@ from frappe.query_builder.functions import Coalesce, Max from frappe.query_builder.utils import DocType -class NestedSetRecursionError(frappe.ValidationError): pass -class NestedSetMultipleRootsError(frappe.ValidationError): pass -class NestedSetChildExistsError(frappe.ValidationError): pass -class NestedSetInvalidMergeError(frappe.ValidationError): pass +class NestedSetRecursionError(frappe.ValidationError): + pass + + +class NestedSetMultipleRootsError(frappe.ValidationError): + pass + + +class NestedSetChildExistsError(frappe.ValidationError): + pass + + +class NestedSetInvalidMergeError(frappe.ValidationError): + pass + # called in the on_update method def update_nsm(doc): # get fields, data from the DocType - old_parent_field = 'old_parent' + old_parent_field = "old_parent" parent_field = "parent_" + frappe.scrub(doc.doctype) - if hasattr(doc,'nsm_parent_field'): + if hasattr(doc, "nsm_parent_field"): parent_field = doc.nsm_parent_field - if hasattr(doc,'nsm_oldparent_field'): + if hasattr(doc, "nsm_oldparent_field"): old_parent_field = doc.nsm_oldparent_field parent, old_parent = doc.get(parent_field) or None, doc.get(old_parent_field) or None # has parent changed (?) or parent is None (root) if not doc.lft and not doc.rgt: - update_add_node(doc, parent or '', parent_field) + update_add_node(doc, parent or "", parent_field) elif old_parent != parent: update_move_node(doc, parent_field) # set old parent doc.set(old_parent_field, parent) - frappe.db.set_value(doc.doctype, doc.name, old_parent_field, parent or '', update_modified=False) + frappe.db.set_value(doc.doctype, doc.name, old_parent_field, parent or "", update_modified=False) doc.reload() + def update_add_node(doc, parent, parent_field): """ - insert a new node + insert a new node """ doctype = doc.doctype name = doc.name @@ -62,10 +74,13 @@ def update_add_node(doc, parent, parent_field): if parent: left, right = frappe.db.get_value(doctype, {"name": parent}, ["lft", "rgt"], for_update=True) validate_loop(doc.doctype, doc.name, left, right) - else: # root - right = frappe.qb.from_(Table).select( - Coalesce(Max(Table.rgt), 0) + 1 - ).where(Coalesce(Table[parent_field], "") == "").run(pluck=True)[0] + else: # root + right = ( + frappe.qb.from_(Table) + .select(Coalesce(Max(Table.rgt), 0) + 1) + .where(Coalesce(Table[parent_field], "") == "") + .run(pluck=True)[0] + ) right = right or 1 @@ -73,11 +88,15 @@ def update_add_node(doc, parent, parent_field): frappe.qb.update(Table).set(Table.rgt, Table.rgt + 2).where(Table.rgt >= right).run() frappe.qb.update(Table).set(Table.lft, Table.lft + 2).where(Table.lft >= right).run() - if frappe.qb.from_(Table).select("*").where((Table.lft == right) | (Table.rgt == right + 1)).run(): + if ( + frappe.qb.from_(Table).select("*").where((Table.lft == right) | (Table.rgt == right + 1)).run() + ): frappe.throw(_("Nested set error. Please contact the Administrator.")) # update index of new node - frappe.qb.update(Table).set(Table.lft, right).set(Table.rgt, right + 1).where(Table.name == name).run() + frappe.qb.update(Table).set(Table.lft, right).set(Table.rgt, right + 1).where( + Table.name == name + ).run() return right @@ -86,14 +105,18 @@ def update_move_node(doc: Document, parent_field: str): Table = DocType(doc.doctype) if parent: - new_parent = frappe.qb.from_(Table).select( - Table.lft, Table.rgt - ).where(Table.name == parent).for_update().run(as_dict=True)[0] + new_parent = ( + frappe.qb.from_(Table) + .select(Table.lft, Table.rgt) + .where(Table.name == parent) + .for_update() + .run(as_dict=True)[0] + ) validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt) # move to dark side - frappe.qb.update(Table).set(Table.lft, - Table.lft).set(Table.rgt, - Table.rgt).where( + frappe.qb.update(Table).set(Table.lft, -Table.lft).set(Table.rgt, -Table.rgt).where( (Table.lft >= doc.lft) & (Table.rgt <= doc.rgt) ).run() @@ -110,9 +133,13 @@ def update_move_node(doc: Document, parent_field: str): if parent: # re-query value due to computation above - new_parent = frappe.qb.from_(Table).select( - Table.lft, Table.rgt - ).where(Table.name == parent).for_update().run(as_dict=True)[0] + new_parent = ( + frappe.qb.from_(Table) + .select(Table.lft, Table.rgt) + .where(Table.name == parent) + .for_update() + .run(as_dict=True)[0] + ) # set parent lft, rgt frappe.qb.update(Table).set(Table.rgt, Table.rgt + diff).where(Table.name == parent).run() @@ -134,9 +161,7 @@ def update_move_node(doc: Document, parent_field: str): new_diff = max_rgt + 1 - doc.lft # bring back from dark side - frappe.qb.update(Table).set( - Table.lft, -Table.lft + new_diff - ).set( + frappe.qb.update(Table).set(Table.lft, -Table.lft + new_diff).set( Table.rgt, -Table.rgt + new_diff ).where(Table.lft < 0).run() @@ -144,17 +169,19 @@ def update_move_node(doc: Document, parent_field: str): @frappe.whitelist() def rebuild_tree(doctype, parent_field): """ - call rebuild_node for all root nodes + call rebuild_node for all root nodes """ # Check for perm if called from client-side - if frappe.request and frappe.local.form_dict.cmd == 'rebuild_tree': - frappe.only_for('System Manager') + if frappe.request and frappe.local.form_dict.cmd == "rebuild_tree": + frappe.only_for("System Manager") meta = frappe.get_meta(doctype) if not meta.has_field("lft") or not meta.has_field("rgt"): - frappe.throw(_("Rebuilding of tree is not supported for {}").format(frappe.bold(doctype)), - title=_("Invalid Action")) + frappe.throw( + _("Rebuilding of tree is not supported for {}").format(frappe.bold(doctype)), + title=_("Invalid Action"), + ) # get all roots right = 1 @@ -162,9 +189,7 @@ def rebuild_tree(doctype, parent_field): column = getattr(table, parent_field) result = ( frappe.qb.from_(table) - .where( - (column == "") | (column.isnull()) - ) + .where((column == "") | (column.isnull())) .orderby(table.name, order=Order.asc) .select(table.name) ).run() @@ -176,35 +201,38 @@ def rebuild_tree(doctype, parent_field): frappe.db.auto_commit_on_many_writes = 0 + def rebuild_node(doctype, parent, left, parent_field): """ - reset lft, rgt and recursive call for all children + reset lft, rgt and recursive call for all children """ # the right value of this node is the left value + 1 - right = left+1 + right = left + 1 # get all children of this node table = DocType(doctype) column = getattr(table, parent_field) - result = ( - frappe.qb.from_(table).where(column == parent).select(table.name) - ).run() + result = (frappe.qb.from_(table).where(column == parent).select(table.name)).run() for r in result: right = rebuild_node(doctype, r[0], right, parent_field) # we've got the left value, and now that we've processed # the children of this node we also know the right value - frappe.db.set_value(doctype, parent, {"lft": left, "rgt": right}, for_update=False, update_modified=False) + frappe.db.set_value( + doctype, parent, {"lft": left, "rgt": right}, for_update=False, update_modified=False + ) - #return the right value of this node + 1 - return right+1 + # return the right value of this node + 1 + return right + 1 def validate_loop(doctype, name, lft, rgt): """check if item not an ancestor (loop)""" - if name in frappe.get_all(doctype, filters={"lft": ["<=", lft], "rgt": [">=", rgt]}, pluck="name"): + if name in frappe.get_all( + doctype, filters={"lft": ["<=", lft], "rgt": [">=", rgt]}, pluck="name" + ): frappe.throw(_("Item cannot be added to its own descendents"), NestedSetRecursionError) @@ -218,7 +246,7 @@ class NestedSet(Document): self.validate_ledger() def on_trash(self, allow_root_deletion=False): - if not getattr(self, 'nsm_parent_field', None): + if not getattr(self, "nsm_parent_field", None): self.nsm_parent_field = frappe.scrub(self.doctype) + "_parent" parent = self.get(self.nsm_parent_field) @@ -241,13 +269,18 @@ class NestedSet(Document): def validate_if_child_exists(self): has_children = frappe.db.count(self.doctype, filters={self.nsm_parent_field: self.name}) if has_children: - frappe.throw(_("Cannot delete {0} as it has child nodes").format(self.name), NestedSetChildExistsError) + frappe.throw( + _("Cannot delete {0} as it has child nodes").format(self.name), NestedSetChildExistsError + ) def before_rename(self, olddn, newdn, merge=False, group_fname="is_group"): if merge and hasattr(self, group_fname): is_group = frappe.db.get_value(self.doctype, newdn, group_fname) if self.get(group_fname) != is_group: - frappe.throw(_("Merging is only possible between Group-to-Group or Leaf Node-to-Leaf Node"), NestedSetInvalidMergeError) + frappe.throw( + _("Merging is only possible between Group-to-Group or Leaf Node-to-Leaf Node"), + NestedSetInvalidMergeError, + ) def after_rename(self, olddn, newdn, merge=False): if not self.nsm_parent_field: @@ -256,7 +289,13 @@ class NestedSet(Document): parent_field = self.nsm_parent_field # set old_parent for children - frappe.db.set_value(self.doctype, {"old_parent": newdn}, {parent_field: newdn}, update_modified=False, for_update=False) + frappe.db.set_value( + self.doctype, + {"old_parent": newdn}, + {parent_field: newdn}, + update_modified=False, + for_update=False, + ) if merge: rebuild_tree(self.doctype, parent_field) @@ -267,14 +306,14 @@ class NestedSet(Document): frappe.throw(_("""Multiple root nodes not allowed."""), NestedSetMultipleRootsError) def get_root_node_count(self): - return frappe.db.count(self.doctype, { - self.nsm_parent_field: '' - }) + return frappe.db.count(self.doctype, {self.nsm_parent_field: ""}) def validate_ledger(self, group_identifier="is_group"): if hasattr(self, group_identifier) and not bool(self.get(group_identifier)): if frappe.get_all(self.doctype, {self.nsm_parent_field: self.name, "docstatus": ("!=", 2)}): - frappe.throw(_("{0} {1} cannot be a leaf node as it has children").format(_(self.doctype), self.name)) + frappe.throw( + _("{0} {1} cannot be a leaf node as it has children").format(_(self.doctype), self.name) + ) def get_ancestors(self): return get_ancestors_of(self.doctype, self.name) @@ -287,7 +326,9 @@ class NestedSet(Document): def get_children(self) -> Iterator["NestedSet"]: """Return a generator that yields child Documents.""" - child_names = frappe.get_list(self.doctype, filters={self.nsm_parent_field: self.name}, pluck="name") + child_names = frappe.get_list( + self.doctype, filters={self.nsm_parent_field: self.name}, pluck="name" + ) for name in child_names: yield frappe.get_doc(self.doctype, name) @@ -301,30 +342,44 @@ def get_root_of(doctype): t1 = Table.as_("t1") t2 = Table.as_("t2") - subq = frappe.qb.from_(t2).select(Count("*")).where( - (t2.lft < t1.lft) & (t2.rgt > t1.rgt) - ) - result = frappe.qb.from_(t1).select(t1.name).where( - (subqry(subq) == 0) & (t1.rgt > t1.lft) - ).run() + subq = frappe.qb.from_(t2).select(Count("*")).where((t2.lft < t1.lft) & (t2.rgt > t1.rgt)) + result = frappe.qb.from_(t1).select(t1.name).where((subqry(subq) == 0) & (t1.rgt > t1.lft)).run() return result[0][0] if result else None + def get_ancestors_of(doctype, name, order_by="lft desc", limit=None): """Get ancestor elements of a DocType with a tree structure""" lft, rgt = frappe.db.get_value(doctype, name, ["lft", "rgt"]) - result = [d["name"] for d in frappe.db.get_all(doctype, {"lft": ["<", lft], "rgt": [">", rgt]}, - "name", order_by=order_by, limit_page_length=limit)] + result = [ + d["name"] + for d in frappe.db.get_all( + doctype, + {"lft": ["<", lft], "rgt": [">", rgt]}, + "name", + order_by=order_by, + limit_page_length=limit, + ) + ] return result or [] -def get_descendants_of(doctype, name, order_by="lft desc", limit=None, - ignore_permissions=False): - '''Return descendants of the current record''' - lft, rgt = frappe.db.get_value(doctype, name, ['lft', 'rgt']) - result = [d["name"] for d in frappe.db.get_list(doctype, {"lft": [">", lft], "rgt": ["<", rgt]}, - "name", order_by=order_by, limit_page_length=limit, ignore_permissions=ignore_permissions)] +def get_descendants_of(doctype, name, order_by="lft desc", limit=None, ignore_permissions=False): + """Return descendants of the current record""" + lft, rgt = frappe.db.get_value(doctype, name, ["lft", "rgt"]) + + result = [ + d["name"] + for d in frappe.db.get_list( + doctype, + {"lft": [">", lft], "rgt": ["<", rgt]}, + "name", + order_by=order_by, + limit_page_length=limit, + ignore_permissions=ignore_permissions, + ) + ] return result or [] diff --git a/frappe/utils/oauth.py b/frappe/utils/oauth.py index df2f5dca62..9e7138cbbd 100644 --- a/frappe/utils/oauth.py +++ b/frappe/utils/oauth.py @@ -1,14 +1,20 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import base64 +import json + +import jwt + import frappe import frappe.utils -import json, jwt -import base64 from frappe import _ from frappe.utils.password import get_decrypted_password -class SignupDisabledError(frappe.PermissionError): pass + +class SignupDisabledError(frappe.PermissionError): + pass + def get_oauth2_providers(): out = {} @@ -23,7 +29,7 @@ def get_oauth2_providers(): "name": provider.name, "authorize_url": authorize_url, "access_token_url": access_token_url, - "base_url": provider.base_url + "base_url": provider.base_url, }, "redirect_uri": provider.redirect_url, "api_endpoint": provider.api_endpoint, @@ -36,6 +42,7 @@ def get_oauth2_providers(): return out + def get_oauth_keys(provider): """get client_id and client_secret from database or conf""" @@ -44,28 +51,29 @@ def get_oauth_keys(provider): if not keys: # try database - client_id, client_secret = frappe.get_value("Social Login Key", provider, ["client_id", "client_secret"]) + client_id, client_secret = frappe.get_value( + "Social Login Key", provider, ["client_id", "client_secret"] + ) client_secret = get_decrypted_password("Social Login Key", provider, "client_secret") - keys = { - "client_id": client_id, - "client_secret": client_secret - } + keys = {"client_id": client_id, "client_secret": client_secret} return keys else: - return { - "client_id": keys["client_id"], - "client_secret": keys["client_secret"] - } + return {"client_id": keys["client_id"], "client_secret": keys["client_secret"]} + def get_oauth2_authorize_url(provider, redirect_to): flow = get_oauth2_flow(provider) - state = { "site": frappe.utils.get_url(), "token": frappe.generate_hash(), "redirect_to": redirect_to } + state = { + "site": frappe.utils.get_url(), + "token": frappe.generate_hash(), + "redirect_to": redirect_to, + } # relative to absolute url data = { "redirect_uri": get_redirect_uri(provider), - "state": base64.b64encode(bytes(json.dumps(state).encode("utf-8"))) + "state": base64.b64encode(bytes(json.dumps(state).encode("utf-8"))), } oauth2_providers = get_oauth2_providers() @@ -75,6 +83,7 @@ def get_oauth2_authorize_url(provider, redirect_to): return flow.get_authorize_url(**data) + def get_oauth2_flow(provider): from rauth import OAuth2Service @@ -89,6 +98,7 @@ def get_oauth2_flow(provider): # and we have setup the communication lines return OAuth2Service(**params) + def get_redirect_uri(provider): keys = frappe.conf.get("{provider}_login".format(provider=provider)) @@ -104,14 +114,17 @@ def get_redirect_uri(provider): # this uses the site's url + the relative redirect uri return frappe.utils.get_url(redirect_uri) + def login_via_oauth2(provider, code, state, decoder=None): info = get_info_via_oauth(provider, code, decoder) login_oauth_user(info, provider=provider, state=state) + def login_via_oauth2_id_token(provider, code, state, decoder=None): info = get_info_via_oauth(provider, code, decoder, id_token=True) login_oauth_user(info, provider=provider, state=state) + def get_info_via_oauth(provider, code, decoder=None, id_token=False): flow = get_oauth2_flow(provider) oauth2_providers = get_oauth2_providers() @@ -120,7 +133,7 @@ def get_info_via_oauth(provider, code, decoder=None, id_token=False): "data": { "code": code, "redirect_uri": get_redirect_uri(provider), - "grant_type": "authorization_code" + "grant_type": "authorization_code", } } @@ -132,7 +145,7 @@ def get_info_via_oauth(provider, code, decoder=None, id_token=False): if id_token: parsed_access = json.loads(session.access_token_response.text) - token = parsed_access['id_token'] + token = parsed_access["id_token"] info = jwt.decode(token, flow.client_secret, options={"verify_signature": False}) else: @@ -151,11 +164,14 @@ def get_info_via_oauth(provider, code, decoder=None, id_token=False): return info -def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=None, generate_login_token=False): + +def login_oauth_user( + data=None, provider=None, state=None, email_id=None, key=None, generate_login_token=False +): # NOTE: This could lead to security issue as the signed in user can type any email address in complete_signup # if email_id and key: # data = json.loads(frappe.db.get_temp(key)) - # # What if data is missing because of an invalid key + # # What if data is missing because of an invalid key # data["email"] = email_id # # elif not (data.get("email") and get_first_name(data)) and not frappe.db.exists("User", data.get("email")): @@ -181,7 +197,9 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No user = get_email(data) if not user: - frappe.respond_as_web_page(_("Invalid Request"), _("Please ensure that your profile has an email address")) + frappe.respond_as_web_page( + _("Invalid Request"), _("Please ensure that your profile has an email address") + ) return try: @@ -189,8 +207,12 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No return except SignupDisabledError: - return frappe.respond_as_web_page("Signup is Disabled", "Sorry. Signup from Website is disabled.", - success=False, http_status_code=403) + return frappe.respond_as_web_page( + "Signup is Disabled", + "Sorry. Signup from Website is disabled.", + success=False, + http_status_code=403, + ) frappe.local.login_manager.user = user frappe.local.login_manager.post_login() @@ -200,19 +222,21 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No if frappe.utils.cint(generate_login_token): login_token = frappe.generate_hash(length=32) - frappe.cache().set_value("login_token:{0}".format(login_token), frappe.local.session.sid, expires_in_sec=120) + frappe.cache().set_value( + "login_token:{0}".format(login_token), frappe.local.session.sid, expires_in_sec=120 + ) frappe.response["login_token"] = login_token - else: redirect_to = state.get("redirect_to") redirect_post_login( - desk_user=frappe.local.response.get('message') == 'Logged In', + desk_user=frappe.local.response.get("message") == "Logged In", redirect_to=redirect_to, - provider=provider + provider=provider, ) + def update_oauth_user(user, data, provider): if isinstance(data.get("location"), dict): data["location"] = data.get("location").get("name") @@ -234,49 +258,49 @@ def update_oauth_user(user, data, provider): doc = frappe.new_doc("Gender", {"gender": gender}) doc.insert(ignore_permissions=True) - user.update({ - "doctype": "User", - "first_name": get_first_name(data), - "last_name": get_last_name(data), - "email": get_email(data), - "gender": gender, - "enabled": 1, - "new_password": frappe.generate_hash(get_email(data)), - "location": data.get("location"), - "user_type": "Website User", - "user_image": data.get("picture") or data.get("avatar_url") - }) + user.update( + { + "doctype": "User", + "first_name": get_first_name(data), + "last_name": get_last_name(data), + "email": get_email(data), + "gender": gender, + "enabled": 1, + "new_password": frappe.generate_hash(get_email(data)), + "location": data.get("location"), + "user_type": "Website User", + "user_image": data.get("picture") or data.get("avatar_url"), + } + ) else: user = frappe.get_doc("User", user) if not user.enabled: - frappe.respond_as_web_page(_('Not Allowed'), _('User {0} is disabled').format(user.email)) + frappe.respond_as_web_page(_("Not Allowed"), _("User {0} is disabled").format(user.email)) return False - if provider=="facebook" and not user.get_social_login_userid(provider): + if provider == "facebook" and not user.get_social_login_userid(provider): save = True user.set_social_login_userid(provider, userid=data["id"], username=data.get("username")) - user.update({ - "user_image": "https://graph.facebook.com/{id}/picture".format(id=data["id"]) - }) + user.update({"user_image": "https://graph.facebook.com/{id}/picture".format(id=data["id"])}) - elif provider=="google" and not user.get_social_login_userid(provider): + elif provider == "google" and not user.get_social_login_userid(provider): save = True user.set_social_login_userid(provider, userid=data["id"]) - elif provider=="github" and not user.get_social_login_userid(provider): + elif provider == "github" and not user.get_social_login_userid(provider): save = True user.set_social_login_userid(provider, userid=data["id"], username=data.get("login")) - elif provider=="frappe" and not user.get_social_login_userid(provider): + elif provider == "frappe" and not user.get_social_login_userid(provider): save = True user.set_social_login_userid(provider, userid=data["sub"]) - elif provider=="office_365" and not user.get_social_login_userid(provider): + elif provider == "office_365" and not user.get_social_login_userid(provider): save = True user.set_social_login_userid(provider, userid=data["sub"]) - elif provider=="salesforce" and not user.get_social_login_userid(provider): + elif provider == "salesforce" and not user.get_social_login_userid(provider): save = True user.set_social_login_userid(provider, userid="/".join(data["sub"].split("/")[-2:])) @@ -296,22 +320,26 @@ def update_oauth_user(user, data, provider): user.save() + def get_first_name(data): return data.get("first_name") or data.get("given_name") or data.get("name") + def get_last_name(data): return data.get("last_name") or data.get("family_name") + def get_email(data): return data.get("email") or data.get("upn") or data.get("unique_name") + def redirect_post_login(desk_user, redirect_to=None, provider=None): # redirect! frappe.local.response["type"] = "redirect" if not redirect_to: # the #desktop is added to prevent a facebook redirect bug - desk_uri = "/app/workspace" if provider == 'facebook' else '/app' + desk_uri = "/app/workspace" if provider == "facebook" else "/app" redirect_to = desk_uri if desk_user else "/me" redirect_to = frappe.utils.get_url(redirect_to) diff --git a/frappe/utils/password.py b/frappe/utils/password.py index b5218806c8..f2c4b9685a 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -2,15 +2,17 @@ # License: MIT. See LICENSE import string + +from cryptography.fernet import Fernet, InvalidToken +from passlib.context import CryptContext +from passlib.hash import mysql41, pbkdf2_sha256 +from passlib.registry import register_crypt_handler +from pypika.terms import Values + import frappe from frappe import _ from frappe.query_builder import Table from frappe.utils import cstr, encode -from cryptography.fernet import Fernet, InvalidToken -from passlib.hash import pbkdf2_sha256, mysql41 -from passlib.registry import register_crypt_handler -from passlib.context import CryptContext -from pypika.terms import Values Auth = Table("__Auth") @@ -23,8 +25,10 @@ class LegacyPassword(pbkdf2_sha256): # check if this is a mysql hash # it is possible that we will generate a false positive if the users password happens to be 40 hex chars proceeded # by an * char, but this seems highly unlikely - if not (secret[0] == "*" and len(secret) == 41 and all(c in string.hexdigits for c in secret[1:])): - secret = mysql41.hash(secret + self.salt.decode('utf-8')) + if not ( + secret[0] == "*" and len(secret) == 41 and all(c in string.hexdigits for c in secret[1:]) + ): + secret = mysql41.hash(secret + self.salt.decode("utf-8")) return super(LegacyPassword, self)._calc_checksum(secret) @@ -72,9 +76,7 @@ def set_encrypted_password(doctype, name, pwd, fieldname="password"): if frappe.db.db_type == "mariadb": query = query.on_duplicate_key_update(Auth.password, Values(Auth.password)) elif frappe.db.db_type == "postgres": - query = ( - query.on_conflict(Auth.doctype, Auth.name, Auth.fieldname).do_update(Auth.password) - ) + query = query.on_conflict(Auth.doctype, Auth.name, Auth.fieldname).do_update(Auth.password) try: query.run() @@ -85,12 +87,9 @@ def set_encrypted_password(doctype, name, pwd, fieldname="password"): raise e -def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.delete("__Auth", { - "doctype": doctype, - "name": name, - "fieldname": fieldname - }) +def remove_encrypted_password(doctype, name, fieldname="password"): + frappe.db.delete("__Auth", {"doctype": doctype, "name": name, "fieldname": fieldname}) + def check_password(user, pwd, doctype="User", fieldname="password", delete_tracker_cache=True): """Checks if user and password are correct, else raises frappe.AuthenticationError""" @@ -131,16 +130,16 @@ def delete_login_failed_cache(user): frappe.cache().hdel("locked_account_time", user) -def update_password(user, pwd, doctype='User', fieldname='password', logout_all_sessions=False): - ''' - Update the password for the User +def update_password(user, pwd, doctype="User", fieldname="password", logout_all_sessions=False): + """ + Update the password for the User - :param user: username - :param pwd: new password - :param doctype: doctype name (for encryption) - :param fieldname: fieldname (in given doctype) (for encryption) - :param logout_all_session: delete all other session - ''' + :param user: username + :param pwd: new password + :param doctype: doctype name (for encryption) + :param fieldname: fieldname (in given doctype) (for encryption) + :param logout_all_session: delete all other session + """ hashPwd = passlibctx.hash(pwd) query = ( @@ -151,9 +150,8 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ # TODO: Simplify this via aliasing methods in `frappe.qb` if frappe.db.db_type == "mariadb": - query = ( - query.on_duplicate_key_update(Auth.password, hashPwd) - .on_duplicate_key_update(Auth.encrypted, 0) + query = query.on_duplicate_key_update(Auth.password, hashPwd).on_duplicate_key_update( + Auth.encrypted, 0 ) elif frappe.db.db_type == "postgres": query = ( @@ -167,15 +165,13 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ # clear all the sessions except current if logout_all_sessions: from frappe.sessions import clear_sessions + clear_sessions(user=user, keep_current=True, force=True) def delete_all_passwords_for(doctype, name): try: - frappe.db.delete("__Auth", { - "doctype": doctype, - "name": name - }) + frappe.db.delete("__Auth", {"doctype": doctype, "name": name}) except Exception as e: if not frappe.db.is_missing_column(e): raise @@ -184,15 +180,13 @@ def delete_all_passwords_for(doctype, name): def rename_password(doctype, old_name, new_name): # NOTE: fieldname is not considered, since the document is renamed frappe.qb.update(Auth).set(Auth.name, new_name).where( - (Auth.doctype == doctype) - & (Auth.name == old_name) + (Auth.doctype == doctype) & (Auth.name == old_name) ).run() def rename_password_field(doctype, old_fieldname, new_fieldname): frappe.qb.update(Auth).set(Auth.fieldname, new_fieldname).where( - (Auth.doctype == doctype) - & (Auth.fieldname == old_fieldname) + (Auth.doctype == doctype) & (Auth.fieldname == old_fieldname) ).run() @@ -208,7 +202,7 @@ def encrypt(txt, encryption_key=None): cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) except Exception: # encryption_key is not in 32 url-safe base64-encoded format - frappe.throw(_('Encryption key is in invalid format!')) + frappe.throw(_("Encryption key is in invalid format!")) cipher_text = cstr(cipher_suite.encrypt(encode(txt))) return cipher_text diff --git a/frappe/utils/password_strength.py b/frappe/utils/password_strength.py index fdfa2054ba..1f7a171ce9 100644 --- a/frappe/utils/password_strength.py +++ b/frappe/utils/password_strength.py @@ -11,13 +11,12 @@ from frappe import _ def test_password_strength(password, user_inputs=None): - '''Wrapper around zxcvbn.password_strength''' + """Wrapper around zxcvbn.password_strength""" result = zxcvbn(password, user_inputs) - result.update({ - "feedback": get_feedback(result.get('score'), result.get('sequence')) - }) + result.update({"feedback": get_feedback(result.get("score"), result.get("sequence"))}) return result + # NOTE: code modified for frappe translations # ------------------------------------------- # feedback functionality code from https://github.com/sans-serif/python-zxcvbn/blob/master/zxcvbn/feedback.py @@ -25,6 +24,7 @@ def test_password_strength(password, user_inputs=None): # Used for regex matching capitalization import re + # Used to get the regex patterns for capitalization # (Used the same way in the original zxcvbn) from zxcvbn import scoring @@ -32,7 +32,7 @@ from zxcvbn import scoring # Default feedback value default_feedback = { "warning": "", - "suggestions":[ + "suggestions": [ _("Use a few words, avoid common phrases."), _("No need for symbols, digits, or uppercase letters."), ], @@ -44,7 +44,9 @@ def get_feedback(score, sequence): Returns the feedback dictionary consisting of ("warning","suggestions") for the given sequences. """ global default_feedback - minimum_password_score = int(frappe.db.get_single_value("System Settings", "minimum_password_score") or 2) + minimum_password_score = int( + frappe.db.get_single_value("System Settings", "minimum_password_score") or 2 + ) # Starting feedback if len(sequence) == 0: @@ -55,7 +57,7 @@ def get_feedback(score, sequence): return dict({"warning": "", "suggestions": []}) # Tie feedback to the longest match for longer sequences - longest_match = max(sequence, key=lambda seq: len(seq.get('token', ''))) + longest_match = max(sequence, key=lambda seq: len(seq.get("token", ""))) # Get feedback for this match feedback = get_match_feedback(longest_match, len(sequence) == 1) @@ -64,9 +66,7 @@ def get_feedback(score, sequence): if not feedback: feedback = { "warning": "", - "suggestions":[ - _("Better add a few more letters or another word") - ], + "suggestions": [_("Better add a few more letters or another word")], } return feedback @@ -75,6 +75,7 @@ def get_match_feedback(match, is_sole_match): """ Returns feedback as a dictionary for a certain match """ + def fun_bruteforce(): # Define a number of functions that are used in a look up dictionary return None @@ -85,18 +86,14 @@ def get_match_feedback(match, is_sole_match): def fun_spatial(): feedback = { - "warning": _('Short keyboard patterns are easy to guess'), - "suggestions": [ - _("Make use of longer keyboard patterns") - ], + "warning": _("Short keyboard patterns are easy to guess"), + "suggestions": [_("Make use of longer keyboard patterns")], } if match.get("turns") == 1: feedback = { - "warning": _('Straight rows of keys are easy to guess'), - "suggestions": [ - _("Try to use a longer keyboard pattern with more turns") - ], + "warning": _("Straight rows of keys are easy to guess"), + "suggestions": [_("Try to use a longer keyboard pattern with more turns")], } return feedback @@ -104,42 +101,31 @@ def get_match_feedback(match, is_sole_match): def fun_repeat(): feedback = { "warning": _('Repeats like "abcabcabc" are only slightly harder to guess than "abc"'), - "suggestions": [ - _("Try to avoid repeated words and characters") - ], + "suggestions": [_("Try to avoid repeated words and characters")], } if match.get("repeated_char") and len(match.get("repeated_char")) == 1: feedback = { "warning": _('Repeats like "aaa" are easy to guess'), - "suggestions": [ - _("Let's avoid repeated words and characters") - ], + "suggestions": [_("Let's avoid repeated words and characters")], } return feedback def fun_sequence(): return { - "suggestions": [ - _("Avoid sequences like abc or 6543 as they are easy to guess") - ], + "suggestions": [_("Avoid sequences like abc or 6543 as they are easy to guess")], } def fun_regex(): if match["regex_name"] == "recent_year": return { "warning": _("Recent years are easy to guess."), - "suggestions": [ - _("Avoid recent years."), - _("Avoid years that are associated with you.") - ], + "suggestions": [_("Avoid recent years."), _("Avoid years that are associated with you.")], } def fun_date(): return { "warning": _("Dates are often easy to guess."), - "suggestions": [ - _("Avoid dates and years that are associated with you.") - ], + "suggestions": [_("Avoid dates and years that are associated with you.")], } # Dictionary that maps pattern names to funtions that return feedback @@ -151,11 +137,12 @@ def get_match_feedback(match, is_sole_match): "sequence": fun_sequence, "regex": fun_regex, "date": fun_date, - "year": fun_date + "year": fun_date, } - pattern_fn = patterns.get(match['pattern']) + pattern_fn = patterns.get(match["pattern"]) if pattern_fn: - return (pattern_fn()) + return pattern_fn() + def get_dictionary_match_feedback(match, is_sole_match): """ @@ -199,7 +186,4 @@ def get_dictionary_match_feedback(match, is_sole_match): if match.get("l33t_entropy"): suggestions.append(_("Predictable substitutions like '@' instead of 'a' don't help very much.")) - return { - "warning": warning, - "suggestions": suggestions - } + return {"warning": warning, "suggestions": suggestions} diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index b8e684869e..952717434c 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -3,8 +3,8 @@ import io import os import re -from distutils.version import LooseVersion import subprocess +from distutils.version import LooseVersion import pdfkit from bs4 import BeautifulSoup @@ -15,21 +15,22 @@ from frappe import _ from frappe.utils import scrub_urls from frappe.utils.jinja_globals import bundled_asset, is_rtl -PDF_CONTENT_ERRORS = ["ContentNotFoundError", "ContentOperationNotPermittedError", - "UnknownContentError", "RemoteHostClosedError"] +PDF_CONTENT_ERRORS = [ + "ContentNotFoundError", + "ContentOperationNotPermittedError", + "UnknownContentError", + "RemoteHostClosedError", +] def get_pdf(html, options=None, output=None): html = scrub_urls(html) html, options = prepare_options(html, options) - options.update({ - "disable-javascript": "", - "disable-local-file-access": "" - }) + options.update({"disable-javascript": "", "disable-local-file-access": ""}) - filedata = '' - if LooseVersion(get_wkhtmltopdf_version()) > LooseVersion('0.12.3'): + filedata = "" + if LooseVersion(get_wkhtmltopdf_version()) > LooseVersion("0.12.3"): options.update({"disable-smart-shrinking": ""}) try: @@ -88,21 +89,23 @@ def prepare_options(html, options): if not options: options = {} - options.update({ - 'print-media-type': None, - 'background': None, - 'images': None, - 'quiet': None, - # 'no-outline': None, - 'encoding': "UTF-8", - # 'load-error-handling': 'ignore' - }) + options.update( + { + "print-media-type": None, + "background": None, + "images": None, + "quiet": None, + # 'no-outline': None, + "encoding": "UTF-8", + # 'load-error-handling': 'ignore' + } + ) if not options.get("margin-right"): - options['margin-right'] = '15mm' + options["margin-right"] = "15mm" if not options.get("margin-left"): - options['margin-left'] = '15mm' + options["margin-left"] = "15mm" html, html_options = read_options_from_html(html) options.update(html_options or {}) @@ -112,9 +115,7 @@ def prepare_options(html, options): # page size pdf_page_size = ( - options.get("page-size") - or frappe.db.get_single_value("Print Settings", "pdf_page_size") - or "A4" + options.get("page-size") or frappe.db.get_single_value("Print Settings", "pdf_page_size") or "A4" ) if pdf_page_size == "Custom": @@ -142,10 +143,11 @@ def get_cookie_options(): with open(cookiejar, "w") as f: f.write("sid={}; Domain={};\n".format(frappe.session.sid, domain)) - options['cookie-jar'] = cookiejar + options["cookie-jar"] = cookiejar return options + def read_options_from_html(html): options = {} soup = BeautifulSoup(html, "html5lib") @@ -155,7 +157,17 @@ def read_options_from_html(html): toggle_visible_pdf(soup) # use regex instead of soup-parser - for attr in ("margin-top", "margin-bottom", "margin-left", "margin-right", "page-size", "header-spacing", "orientation", "page-width", "page-height"): + for attr in ( + "margin-top", + "margin-bottom", + "margin-left", + "margin-right", + "page-size", + "header-spacing", + "orientation", + "page-width", + "page-height", + ): try: pattern = re.compile(r"(\.print-format)([\S|\s][^}]*?)(" + str(attr) + r":)(.+)(mm;)") match = pattern.findall(html) @@ -173,7 +185,7 @@ def prepare_header_footer(soup): head = soup.find("head").contents styles = soup.find_all("style") - print_css = bundled_asset('print.bundle.css').lstrip('/') + print_css = bundled_asset("print.bundle.css").lstrip("/") css = frappe.read_file(os.path.join(frappe.local.sites_path, print_css)) # extract header and footer @@ -185,15 +197,18 @@ def prepare_header_footer(soup): tag.extract() toggle_visible_pdf(content) - html = frappe.render_template("templates/print_formats/pdf_header_footer.html", { - "head": head, - "content": content, - "styles": styles, - "html_id": html_id, - "css": css, - "lang": frappe.local.lang, - "layout_direction": "rtl" if is_rtl() else "ltr" - }) + html = frappe.render_template( + "templates/print_formats/pdf_header_footer.html", + { + "head": head, + "content": content, + "styles": styles, + "html_id": html_id, + "css": css, + "lang": frappe.local.lang, + "layout_direction": "rtl" if is_rtl() else "ltr", + }, + ) # create temp file fname = os.path.join("/tmp", "frappe-pdf-{0}.html".format(frappe.generate_hash())) @@ -216,24 +231,26 @@ def cleanup(options): if options.get(key) and os.path.exists(options[key]): os.remove(options[key]) + def toggle_visible_pdf(soup): for tag in soup.find_all(attrs={"class": "visible-pdf"}): # remove visible-pdf class to unhide - tag.attrs['class'].remove('visible-pdf') + tag.attrs["class"].remove("visible-pdf") for tag in soup.find_all(attrs={"class": "hidden-pdf"}): # remove tag from html tag.extract() + def get_wkhtmltopdf_version(): wkhtmltopdf_version = frappe.cache().hget("wkhtmltopdf_version", None) if not wkhtmltopdf_version: try: res = subprocess.check_output(["wkhtmltopdf", "--version"]) - wkhtmltopdf_version = res.decode('utf-8').split(" ")[1] + wkhtmltopdf_version = res.decode("utf-8").split(" ")[1] frappe.cache().hset("wkhtmltopdf_version", None, wkhtmltopdf_version) except Exception: pass - return (wkhtmltopdf_version or '0') + return wkhtmltopdf_version or "0" diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index 06f15ced27..35842217d1 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -1,15 +1,18 @@ -import frappe, os -from frappe import _ +import os -from frappe.utils.pdf import get_pdf,cleanup -from frappe.core.doctype.access_log.access_log import make_access_log from PyPDF2 import PdfFileWriter +import frappe +from frappe import _ +from frappe.core.doctype.access_log.access_log import make_access_log +from frappe.utils.pdf import cleanup, get_pdf + no_cache = 1 base_template_path = "www/printview.html" standard_format = "templates/print_formats/standard.html" + @frappe.whitelist() def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options=None): """ @@ -19,19 +22,19 @@ def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options= can be from a single DocType or multiple DocTypes Note: The design may seem a little weird, but it exists exists to - ensure backward compatibility. The correct way to use this function is to - pass a dict to doctype as described below + ensure backward compatibility. The correct way to use this function is to + pass a dict to doctype as described below NEW FUNCTIONALITY ================= Parameters: doctype (dict): - key (string): DocType name - value (list): of strings of doc names which need to be concatenated and printed + key (string): DocType name + value (list): of strings of doc names which need to be concatenated and printed name (string): - name of the pdf which is generated + name of the pdf which is generated format: - Print Format to be used + Print Format to be used Returns: PDF: A PDF generated by the concatenation of the mentioned input docs @@ -40,18 +43,19 @@ def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options= ========================================= Parameters: doctype (string): - name of the DocType to which the docs belong which need to be printed + name of the DocType to which the docs belong which need to be printed name (string or list): - If string the name of the doc which needs to be printed - If list the list of strings of doc names which needs to be printed + If string the name of the doc which needs to be printed + If list the list of strings of doc names which needs to be printed format: - Print Format to be used + Print Format to be used Returns: PDF: A PDF generated by the concatenation of the mentioned input docs """ import json + output = PdfFileWriter() if isinstance(options, str): @@ -62,13 +66,31 @@ def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options= # Concatenating pdf files for i, ss in enumerate(result): - output = frappe.get_print(doctype, ss, format, as_pdf=True, output=output, no_letterhead=no_letterhead, pdf_options=options) - frappe.local.response.filename = "{doctype}.pdf".format(doctype=doctype.replace(" ", "-").replace("/", "-")) + output = frappe.get_print( + doctype, + ss, + format, + as_pdf=True, + output=output, + no_letterhead=no_letterhead, + pdf_options=options, + ) + frappe.local.response.filename = "{doctype}.pdf".format( + doctype=doctype.replace(" ", "-").replace("/", "-") + ) else: for doctype_name in doctype: for doc_name in doctype[doctype_name]: try: - output = frappe.get_print(doctype_name, doc_name, format, as_pdf=True, output=output, no_letterhead=no_letterhead, pdf_options=options) + output = frappe.get_print( + doctype_name, + doc_name, + format, + as_pdf=True, + output=output, + no_letterhead=no_letterhead, + pdf_options=options, + ) except Exception: frappe.log_error("Permission Error on doc {} of doctype {}".format(doc_name, doctype_name)) frappe.local.response.filename = "{}.pdf".format(name) @@ -76,32 +98,40 @@ def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options= frappe.local.response.filecontent = read_multi_pdf(output) frappe.local.response.type = "download" + def read_multi_pdf(output): # Get the content of the merged pdf files fname = os.path.join("/tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash())) - output.write(open(fname,"wb")) + output.write(open(fname, "wb")) with open(fname, "rb") as fileobj: filedata = fileobj.read() return filedata + @frappe.whitelist() def download_pdf(doctype, name, format=None, doc=None, no_letterhead=0): html = frappe.get_print(doctype, name, format, doc=doc, no_letterhead=no_letterhead) - frappe.local.response.filename = "{name}.pdf".format(name=name.replace(" ", "-").replace("/", "-")) + frappe.local.response.filename = "{name}.pdf".format( + name=name.replace(" ", "-").replace("/", "-") + ) frappe.local.response.filecontent = get_pdf(html) frappe.local.response.type = "pdf" + @frappe.whitelist() def report_to_pdf(html, orientation="Landscape"): - make_access_log(file_type='PDF', method='PDF', page=html) + make_access_log(file_type="PDF", method="PDF", page=html) frappe.local.response.filename = "report.pdf" frappe.local.response.filecontent = get_pdf(html, {"orientation": orientation}) frappe.local.response.type = "pdf" + @frappe.whitelist() -def print_by_server(doctype, name, printer_setting, print_format=None, doc=None, no_letterhead=0, file_path=None): +def print_by_server( + doctype, name, printer_setting, print_format=None, doc=None, no_letterhead=0, file_path=None +): print_settings = frappe.get_doc("Network Printer Settings", printer_setting) try: import cups @@ -113,16 +143,20 @@ def print_by_server(doctype, name, printer_setting, print_format=None, doc=None, cups.setPort(print_settings.port) conn = cups.Connection() output = PdfFileWriter() - output = frappe.get_print(doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf = True, output = output) + output = frappe.get_print( + doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf=True, output=output + ) if not file_path: file_path = os.path.join("/", "tmp", "frappe-pdf-{0}.pdf".format(frappe.generate_hash())) - output.write(open(file_path,"wb")) - conn.printFile(print_settings.printer_name,file_path , name, {}) + output.write(open(file_path, "wb")) + conn.printFile(print_settings.printer_name, file_path, name, {}) except IOError as e: - if ("ContentNotFoundError" in e.message + if ( + "ContentNotFoundError" in e.message or "ContentOperationNotPermittedError" in e.message or "UnknownContentError" in e.message - or "RemoteHostClosedError" in e.message): + or "RemoteHostClosedError" in e.message + ): frappe.throw(_("PDF generation failed")) except cups.IPPError: frappe.throw(_("Printing failed")) diff --git a/frappe/utils/redis_queue.py b/frappe/utils/redis_queue.py index b344b0caa5..89f8fe16f3 100644 --- a/frappe/utils/redis_queue.py +++ b/frappe/utils/redis_queue.py @@ -3,13 +3,13 @@ import redis import frappe from frappe.utils import get_bench_id, random_string + class RedisQueue: def __init__(self, conn): self.conn = conn def add_user(self, username, password=None): - """Create or update the user. - """ + """Create or update the user.""" password = password or self.conn.acl_genpass() user_settings = self.get_new_user_settings(username, password) is_created = self.conn.acl_setuser(**user_settings) @@ -25,14 +25,14 @@ class RedisQueue: return conn @classmethod - def new(cls, username='default', password=None): + def new(cls, username="default", password=None): return cls(cls.get_connection(username, password)) @classmethod def set_admin_password(cls, cur_password=None, new_password=None, reset_passwords=False): - username = 'default' + username = "default" conn = cls.get_connection(username, cur_password) - password = '+'+(new_password or conn.acl_genpass()) + password = "+" + (new_password or conn.acl_genpass()) conn.acl_setuser( username=username, enabled=True, reset_passwords=reset_passwords, passwords=password ) @@ -41,26 +41,25 @@ class RedisQueue: @classmethod def get_new_user_settings(cls, username, password): d = {} - d['username'] = username - d['passwords'] = '+'+password - d['reset_keys'] = True - d['enabled'] = True - d['keys'] = cls.get_acl_key_rules() - d['commands'] = cls.get_acl_command_rules() + d["username"] = username + d["passwords"] = "+" + password + d["reset_keys"] = True + d["enabled"] = True + d["keys"] = cls.get_acl_key_rules() + d["commands"] = cls.get_acl_command_rules() return d @classmethod def get_acl_key_rules(cls, include_key_prefix=False): - """FIXME: Find better way - """ - rules = ['rq:[^q]*', 'rq:queues', f'rq:queue:{get_bench_id()}:*'] + """FIXME: Find better way""" + rules = ["rq:[^q]*", "rq:queues", f"rq:queue:{get_bench_id()}:*"] if include_key_prefix: - return ['~'+pattern for pattern in rules] + return ["~" + pattern for pattern in rules] return rules @classmethod def get_acl_command_rules(cls): - return ['+@all', '-@admin'] + return ["+@all", "-@admin"] @classmethod def gen_acl_list(cls, set_admin_password=False): @@ -70,14 +69,17 @@ class RedisQueue: """ bench_username = get_bench_id() bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() - bench_user_rule_str = ' '.join(bench_user_rules).strip() + bench_user_rule_str = " ".join(bench_user_rules).strip() bench_user_password = random_string(20) - default_username = 'default' - _default_user_password = random_string(20) if set_admin_password else '' - default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' + default_username = "default" + _default_user_password = random_string(20) if set_admin_password else "" + default_user_password = ">" + _default_user_password if _default_user_password else "nopass" return [ - f'user {default_username} on {default_user_password} ~* &* +@all', - f'user {bench_username} on >{bench_user_password} {bench_user_rule_str}' - ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} + f"user {default_username} on {default_user_password} ~* &* +@all", + f"user {bench_username} on >{bench_user_password} {bench_user_rule_str}", + ], { + "bench": (bench_username, bench_user_password), + "default": (default_username, _default_user_password), + } diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index fbf272e906..0101355174 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -11,6 +11,7 @@ from frappe.utils import cstr class RedisWrapper(redis.Redis): """Redis client that will automatically prefix conf.db_name""" + def connected(self): try: self.ping() @@ -27,7 +28,7 @@ class RedisWrapper(redis.Redis): key = "user:{0}:{1}".format(user, key) - return "{0}|{1}".format(frappe.conf.db_name, key).encode('utf-8') + return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8") def set_value(self, key, val, user=None, expires_in_sec=None, shared=False): """Sets cache value. @@ -53,7 +54,7 @@ class RedisWrapper(redis.Redis): def get_value(self, key, generator=None, user=None, expires=False, shared=False): """Returns cache value. If not found and generator function is - given, it will call the generator. + given, it will call the generator. :param key: Cache key. :param generator: Function to be called to generate a value if `None` is returned. @@ -115,7 +116,7 @@ class RedisWrapper(redis.Redis): def delete_value(self, keys, user=None, make_keys=True, shared=False): """Delete value, list of values.""" if not isinstance(keys, (list, tuple)): - keys = (keys, ) + keys = (keys,) for key in keys: if make_keys: @@ -163,23 +164,21 @@ class RedisWrapper(redis.Redis): # set in redis try: - super(RedisWrapper, self).hset(_name, - key, pickle.dumps(value)) + super(RedisWrapper, self).hset(_name, key, pickle.dumps(value)) except redis.exceptions.ConnectionError: pass def hgetall(self, name): value = super(RedisWrapper, self).hgetall(self.make_key(name)) - return { - key: pickle.loads(value) for key, value in value.items() - } + return {key: pickle.loads(value) for key, value in value.items()} def hget(self, name, key, generator=None, shared=False): _name = self.make_key(name, shared=shared) if _name not in frappe.local.cache: frappe.local.cache[_name] = {} - if not key: return None + if not key: + return None if key in frappe.local.cache[_name]: return frappe.local.cache[_name][key] @@ -247,4 +246,3 @@ class RedisWrapper(redis.Redis): def smembers(self, name): """Return all members of the set""" return super(RedisWrapper, self).smembers(self.make_key(name)) - diff --git a/frappe/utils/response.py b/frappe/utils/response.py index a852c584c6..c537460713 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -1,31 +1,38 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import json import datetime import decimal +import json import mimetypes import os -import frappe -from frappe import _ -import frappe.model.document -import frappe.utils -import frappe.sessions +from urllib.parse import quote + import werkzeug.utils +from werkzeug.exceptions import Forbidden, NotFound from werkzeug.local import LocalProxy -from werkzeug.wsgi import wrap_file from werkzeug.wrappers import Response -from werkzeug.exceptions import NotFound, Forbidden -from frappe.utils import cint, format_timedelta -from urllib.parse import quote +from werkzeug.wsgi import wrap_file + +import frappe +import frappe.model.document +import frappe.sessions +import frappe.utils +from frappe import _ from frappe.core.doctype.access_log.access_log import make_access_log +from frappe.utils import cint, format_timedelta def report_error(status_code): - '''Build error. Show traceback in developer mode''' - allow_traceback = cint(frappe.db.get_system_setting('allow_error_traceback')) if frappe.db else True - if (allow_traceback and (status_code!=404 or frappe.conf.logging) - and not frappe.local.flags.disable_traceback): + """Build error. Show traceback in developer mode""" + allow_traceback = ( + cint(frappe.db.get_system_setting("allow_error_traceback")) if frappe.db else True + ) + if ( + allow_traceback + and (status_code != 404 or frappe.conf.logging) + and not frappe.local.flags.disable_traceback + ): traceback = frappe.utils.get_traceback() if traceback: frappe.errprint(traceback) @@ -35,90 +42,115 @@ def report_error(status_code): response.status_code = status_code return response + def build_response(response_type=None): if "docs" in frappe.local.response and not frappe.local.response.docs: del frappe.local.response["docs"] response_type_map = { - 'csv': as_csv, - 'txt': as_txt, - 'download': as_raw, - 'json': as_json, - 'pdf': as_pdf, - 'page': as_page, - 'redirect': redirect, - 'binary': as_binary + "csv": as_csv, + "txt": as_txt, + "download": as_raw, + "json": as_json, + "pdf": as_pdf, + "page": as_page, + "redirect": redirect, + "binary": as_binary, } - return response_type_map[frappe.response.get('type') or response_type]() + return response_type_map[frappe.response.get("type") or response_type]() + def as_csv(): response = Response() - response.mimetype = 'text/csv' - response.charset = 'utf-8' - response.headers["Content-Disposition"] = ("attachment; filename=\"%s.csv\"" % frappe.response['doctype'].replace(' ', '_')).encode("utf-8") - response.data = frappe.response['result'] + response.mimetype = "text/csv" + response.charset = "utf-8" + response.headers["Content-Disposition"] = ( + 'attachment; filename="%s.csv"' % frappe.response["doctype"].replace(" ", "_") + ).encode("utf-8") + response.data = frappe.response["result"] return response + def as_txt(): response = Response() - response.mimetype = 'text' - response.charset = 'utf-8' - response.headers["Content-Disposition"] = ("attachment; filename=\"%s.txt\"" % frappe.response['doctype'].replace(' ', '_')).encode("utf-8") - response.data = frappe.response['result'] + response.mimetype = "text" + response.charset = "utf-8" + response.headers["Content-Disposition"] = ( + 'attachment; filename="%s.txt"' % frappe.response["doctype"].replace(" ", "_") + ).encode("utf-8") + response.data = frappe.response["result"] return response + def as_raw(): response = Response() - response.mimetype = frappe.response.get("content_type") or mimetypes.guess_type(frappe.response['filename'])[0] or "application/unknown" - response.headers["Content-Disposition"] = (f'{frappe.response.get("display_content_as","attachment")}; filename="{frappe.response["filename"].replace(" ", "_")}"').encode("utf-8") - response.data = frappe.response['filecontent'] + response.mimetype = ( + frappe.response.get("content_type") + or mimetypes.guess_type(frappe.response["filename"])[0] + or "application/unknown" + ) + response.headers["Content-Disposition"] = ( + f'{frappe.response.get("display_content_as","attachment")}; filename="{frappe.response["filename"].replace(" ", "_")}"' + ).encode("utf-8") + response.data = frappe.response["filecontent"] return response + def as_json(): make_logs() response = Response() if frappe.local.response.http_status_code: - response.status_code = frappe.local.response['http_status_code'] - del frappe.local.response['http_status_code'] + response.status_code = frappe.local.response["http_status_code"] + del frappe.local.response["http_status_code"] - response.mimetype = 'application/json' - response.charset = 'utf-8' - response.data = json.dumps(frappe.local.response, default=json_handler, separators=(',',':')) + response.mimetype = "application/json" + response.charset = "utf-8" + response.data = json.dumps(frappe.local.response, default=json_handler, separators=(",", ":")) return response + def as_pdf(): response = Response() response.mimetype = "application/pdf" - encoded_filename = quote(frappe.response['filename'].replace(' ', '_')) - response.headers["Content-Disposition"] = ("filename=\"%s\"" % frappe.response['filename'].replace(' ', '_') + ";filename*=utf-8''%s" % encoded_filename).encode("utf-8") - response.data = frappe.response['filecontent'] + encoded_filename = quote(frappe.response["filename"].replace(" ", "_")) + response.headers["Content-Disposition"] = ( + 'filename="%s"' % frappe.response["filename"].replace(" ", "_") + + ";filename*=utf-8''%s" % encoded_filename + ).encode("utf-8") + response.data = frappe.response["filecontent"] return response + def as_binary(): response = Response() - response.mimetype = 'application/octet-stream' - response.headers["Content-Disposition"] = ("filename=\"%s\"" % frappe.response['filename'].replace(' ', '_')).encode("utf-8") - response.data = frappe.response['filecontent'] + response.mimetype = "application/octet-stream" + response.headers["Content-Disposition"] = ( + 'filename="%s"' % frappe.response["filename"].replace(" ", "_") + ).encode("utf-8") + response.data = frappe.response["filecontent"] return response -def make_logs(response = None): + +def make_logs(response=None): """make strings for msgprint and errprint""" if not response: response = frappe.local.response if frappe.error_log: - response['exc'] = json.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log]) + response["exc"] = json.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log]) if frappe.local.message_log: - response['_server_messages'] = json.dumps([frappe.utils.cstr(d) for - d in frappe.local.message_log]) + response["_server_messages"] = json.dumps( + [frappe.utils.cstr(d) for d in frappe.local.message_log] + ) if frappe.debug_log and frappe.conf.get("logging") or False: - response['_debug_messages'] = json.dumps(frappe.local.debug_log) + response["_debug_messages"] = json.dumps(frappe.local.debug_log) if frappe.flags.error_message: - response['_error_message'] = frappe.flags.error_message + response["_error_message"] = frappe.flags.error_message + def json_handler(obj): """serialize non-serializable data for json""" @@ -143,34 +175,44 @@ def json_handler(obj): elif isinstance(obj, Iterable): return list(obj) - elif type(obj)==type or isinstance(obj, Exception): + elif type(obj) == type or isinstance(obj, Exception): return repr(obj) else: - raise TypeError("""Object of type %s with value of %s is not JSON serializable""" % \ - (type(obj), repr(obj))) + raise TypeError( + """Object of type %s with value of %s is not JSON serializable""" % (type(obj), repr(obj)) + ) + def as_page(): """print web page""" from frappe.website.serve import get_response - return get_response(frappe.response['route'], http_status_code=frappe.response.get("http_status_code")) + + return get_response( + frappe.response["route"], http_status_code=frappe.response.get("http_status_code") + ) + def redirect(): return werkzeug.utils.redirect(frappe.response.location) + def download_backup(path): try: frappe.only_for(("System Manager", "Administrator")) - make_access_log(report_name='Backup') + make_access_log(report_name="Backup") except frappe.PermissionError: - raise Forbidden(_("You need to be logged in and have System Manager Role to be able to access backups.")) + raise Forbidden( + _("You need to be logged in and have System Manager Role to be able to access backups.") + ) return send_private_file(path) + def download_private_file(path): """Checks permissions and sends back private file""" - files = frappe.db.get_all('File', {'file_url': path}) + files = frappe.db.get_all("File", {"file_url": path}) can_access = False # this file might be attached to multiple documents # if the file is accessible from any one of those documents @@ -179,7 +221,7 @@ def download_private_file(path): _file = frappe.get_doc("File", f) can_access = _file.is_downloadable() if can_access: - make_access_log(doctype='File', document=_file.name, file_type=os.path.splitext(path)[-1][1:]) + make_access_log(doctype="File", document=_file.name, file_type=os.path.splitext(path)[-1][1:]) break if not can_access: @@ -189,18 +231,18 @@ def download_private_file(path): def send_private_file(path): - path = os.path.join(frappe.local.conf.get('private_path', 'private'), path.strip("/")) + path = os.path.join(frappe.local.conf.get("private_path", "private"), path.strip("/")) filename = os.path.basename(path) - if frappe.local.request.headers.get('X-Use-X-Accel-Redirect'): - path = '/protected/' + path + if frappe.local.request.headers.get("X-Use-X-Accel-Redirect"): + path = "/protected/" + path response = Response() - response.headers['X-Accel-Redirect'] = quote(frappe.utils.encode(path)) + response.headers["X-Accel-Redirect"] = quote(frappe.utils.encode(path)) else: filepath = frappe.utils.get_site_path(path) try: - f = open(filepath, 'rb') + f = open(filepath, "rb") except IOError: raise NotFound @@ -210,18 +252,25 @@ def send_private_file(path): # Except for those that can be injected with scripts. extension = os.path.splitext(path)[1] - blacklist = ['.svg', '.html', '.htm', '.xml'] + blacklist = [".svg", ".html", ".htm", ".xml"] if extension.lower() in blacklist: - response.headers.add('Content-Disposition', 'attachment', filename=filename.encode("utf-8")) + response.headers.add("Content-Disposition", "attachment", filename=filename.encode("utf-8")) - response.mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + response.mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream" return response + def handle_session_stopped(): from frappe.website.serve import get_response - frappe.respond_as_web_page(_("Updating"), + + frappe.respond_as_web_page( + _("Updating"), _("The system is being updated. Please refresh again after a few moments."), - http_status_code=503, indicator_color='orange', fullpage = True, primary_action=None) + http_status_code=503, + indicator_color="orange", + fullpage=True, + primary_action=None, + ) return get_response("message", http_status_code=503) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 3614711b55..0a775fafec 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -1,4 +1,3 @@ - import copy import inspect import json @@ -14,12 +13,13 @@ import frappe.integrations.utils import frappe.utils import frappe.utils.data from frappe import _ -from frappe.handler import execute_cmd from frappe.frappeclient import FrappeClient +from frappe.handler import execute_cmd from frappe.modules import scrub +from frappe.utils.background_jobs import enqueue, get_jobs from frappe.website.utils import get_next_link, get_shade, get_toc from frappe.www.printview import get_visible_columns -from frappe.utils.background_jobs import enqueue, get_jobs + class ServerScriptNotEnabled(frappe.PermissionError): pass @@ -27,11 +27,14 @@ class ServerScriptNotEnabled(frappe.PermissionError): class NamespaceDict(frappe._dict): """Raise AttributeError if function not found in namespace""" + def __getattr__(self, key): ret = self.get(key) if (not ret and key.startswith("__")) or (key not in self): + def default_function(*args, **kwargs): raise AttributeError(f"module has no attribute '{key}'") + return default_function return ret @@ -39,13 +42,13 @@ class NamespaceDict(frappe._dict): def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=False): # server scripts can be disabled via site_config.json # they are enabled by default - if 'server_script_enabled' in frappe.conf: + if "server_script_enabled" in frappe.conf: enabled = frappe.conf.server_script_enabled else: enabled = True if not enabled: - frappe.throw(_('Please Enable Server Scripts'), ServerScriptNotEnabled) + frappe.throw(_("Please Enable Server Scripts"), ServerScriptNotEnabled) # build globals exec_globals = get_safe_globals() @@ -53,16 +56,17 @@ def safe_exec(script, _globals=None, _locals=None, restrict_commit_rollback=Fals exec_globals.update(_globals) if restrict_commit_rollback: - exec_globals.frappe.db.pop('commit', None) - exec_globals.frappe.db.pop('rollback', None) + exec_globals.frappe.db.pop("commit", None) + exec_globals.frappe.db.pop("rollback", None) # execute script compiled by RestrictedPython frappe.flags.in_safe_exec = True - exec(compile_restricted(script), exec_globals, _locals) # pylint: disable=exec-used + exec(compile_restricted(script), exec_globals, _locals) # pylint: disable=exec-used frappe.flags.in_safe_exec = False return exec_globals, _locals + def get_safe_globals(): datautils = frappe._dict() @@ -75,7 +79,7 @@ def get_safe_globals(): add_data_utils(datautils) - form_dict = getattr(frappe.local, 'form_dict', frappe._dict()) + form_dict = getattr(frappe.local, "form_dict", frappe._dict()) if "_" in form_dict: del frappe.local.form_dict["_"] @@ -84,10 +88,7 @@ def get_safe_globals(): out = NamespaceDict( # make available limited methods of frappe - json=NamespaceDict( - loads=json.loads, - dumps=json.dumps - ), + json=NamespaceDict(loads=json.loads, dumps=json.dumps), as_json=frappe.as_json, dict=dict, log=frappe.log, @@ -106,7 +107,6 @@ def get_safe_globals(): copy_doc=frappe.copy_doc, errprint=frappe.errprint, qb=frappe.qb, - get_meta=frappe.get_meta, get_doc=frappe.get_doc, get_cached_doc=frappe.get_cached_doc, @@ -114,7 +114,6 @@ def get_safe_globals(): get_all=frappe.get_all, get_system_settings=frappe.get_system_settings, rename_doc=frappe.rename_doc, - utils=datautils, get_url=frappe.utils.get_url, render_template=frappe.render_template, @@ -123,15 +122,18 @@ def get_safe_globals(): sendmail=frappe.sendmail, get_print=frappe.get_print, attach_print=frappe.attach_print, - user=user, get_fullname=frappe.utils.get_fullname, get_gravatar=frappe.utils.get_gravatar_url, - full_name=frappe.local.session.data.full_name if getattr(frappe.local, "session", None) else "Guest", - request=getattr(frappe.local, 'request', {}), + full_name=frappe.local.session.data.full_name + if getattr(frappe.local, "session", None) + else "Guest", + request=getattr(frappe.local, "request", {}), session=frappe._dict( user=user, - csrf_token=frappe.local.session.data.csrf_token if getattr(frappe.local, "session", None) else '' + csrf_token=frappe.local.session.data.csrf_token + if getattr(frappe.local, "session", None) + else "", ), make_get_request=frappe.integrations.utils.make_get_request, make_post_request=frappe.integrations.utils.make_post_request, @@ -140,7 +142,7 @@ def get_safe_globals(): enqueue=safe_enqueue, sanitize_html=frappe.utils.sanitize_html, log_error=frappe.log_error, - db = NamespaceDict( + db=NamespaceDict( get_list=frappe.get_list, get_all=frappe.get_all, get_value=frappe.db.get_value, @@ -156,9 +158,7 @@ def get_safe_globals(): ), ), FrappeClient=FrappeClient, - style=frappe._dict( - border_color='#d1d8dd' - ), + style=frappe._dict(border_color="#d1d8dd"), get_toc=get_toc, get_next_link=get_next_link, _=frappe._, @@ -172,7 +172,9 @@ def get_safe_globals(): get_visible_columns=get_visible_columns, ) - add_module_properties(frappe.exceptions, out.frappe, lambda obj: inspect.isclass(obj) and issubclass(obj, Exception)) + add_module_properties( + frappe.exceptions, out.frappe, lambda obj: inspect.isclass(obj) and issubclass(obj, Exception) + ) if frappe.response: out.frappe.response = frappe.response @@ -193,47 +195,47 @@ def get_safe_globals(): return out + def is_job_queued(job_name, queue="default"): - ''' + """ :param job_name: used to identify a queued job, usually dotted path to function :param queue: should be either long, default or short - ''' + """ site = frappe.local.site - queued_jobs = get_jobs(site=site, queue=queue, key='job_name').get(site) + queued_jobs = get_jobs(site=site, queue=queue, key="job_name").get(site) return queued_jobs and job_name in queued_jobs + def safe_enqueue(function, **kwargs): - ''' - Enqueue function to be executed using a background worker - Accepts frappe.enqueue params like job_name, queue, timeout, etc. - in addition to params to be passed to function - - :param function: whitelised function or API Method set in Server Script - ''' - - return enqueue( - 'frappe.utils.safe_exec.call_whitelisted_function', - function=function, - **kwargs - ) + """ + Enqueue function to be executed using a background worker + Accepts frappe.enqueue params like job_name, queue, timeout, etc. + in addition to params to be passed to function + + :param function: whitelised function or API Method set in Server Script + """ + + return enqueue("frappe.utils.safe_exec.call_whitelisted_function", function=function, **kwargs) + def call_whitelisted_function(function, **kwargs): - '''Executes a whitelisted function or Server Script of type API''' + """Executes a whitelisted function or Server Script of type API""" return call_with_form_dict(lambda: execute_cmd(function), kwargs) + def run_script(script, **kwargs): - '''run another server script''' + """run another server script""" return call_with_form_dict( - lambda: frappe.get_doc('Server Script', script).execute_method(), - kwargs + lambda: frappe.get_doc("Server Script", script).execute_method(), kwargs ) + def call_with_form_dict(function, kwargs): # temporarily update form_dict, to use inside below call - form_dict = getattr(frappe.local, 'form_dict', frappe._dict()) + form_dict = getattr(frappe.local, "form_dict", frappe._dict()) if kwargs: frappe.local.form_dict = form_dict.copy().update(kwargs) @@ -242,32 +244,35 @@ def call_with_form_dict(function, kwargs): finally: frappe.local.form_dict = form_dict + def get_python_builtins(): return { - 'abs': abs, - 'all': all, - 'any': any, - 'bool': bool, - 'dict': dict, - 'enumerate': enumerate, - 'isinstance': isinstance, - 'issubclass': issubclass, - 'list': list, - 'max': max, - 'min': min, - 'range': range, - 'set': set, - 'sorted': sorted, - 'sum': sum, - 'tuple': tuple, + "abs": abs, + "all": all, + "any": any, + "bool": bool, + "dict": dict, + "enumerate": enumerate, + "isinstance": isinstance, + "issubclass": issubclass, + "list": list, + "max": max, + "min": min, + "range": range, + "set": set, + "sorted": sorted, + "sum": sum, + "tuple": tuple, } + def get_hooks(hook=None, default=None, app_name=None): hooks = frappe.get_hooks(hook=hook, default=default, app_name=app_name) return copy.deepcopy(hooks) + def read_sql(query, *args, **kwargs): - '''a wrapper for frappe.db.sql to allow reads''' + """a wrapper for frappe.db.sql to allow reads""" query = str(query) if frappe.flags.in_safe_exec: check_safe_sql_query(query) @@ -275,23 +280,27 @@ def read_sql(query, *args, **kwargs): def check_safe_sql_query(query: str, throw: bool = True) -> bool: - """ Check if SQL query is safe for running in restricted context. + """Check if SQL query is safe for running in restricted context. - Safe queries: - 1. Read only 'select' or 'explain' queries - 2. CTE on mariadb where writes are not allowed. + Safe queries: + 1. Read only 'select' or 'explain' queries + 2. CTE on mariadb where writes are not allowed. """ query = query.strip().lower() whitelisted_statements = ("select", "explain") - if (query.startswith(whitelisted_statements) - or (query.startswith("with") and frappe.db.db_type == "mariadb")): + if query.startswith(whitelisted_statements) or ( + query.startswith("with") and frappe.db.db_type == "mariadb" + ): return True if throw: - frappe.throw(_("Query must be of SELECT or read-only WITH type."), - title=_("Unsafe SQL query"), exc=frappe.PermissionError) + frappe.throw( + _("Query must be of SELECT or read-only WITH type."), + title=_("Unsafe SQL query"), + exc=frappe.PermissionError, + ) return False @@ -299,10 +308,11 @@ def check_safe_sql_query(query: str, throw: bool = True) -> bool: def _getitem(obj, key): # guard function for RestrictedPython # allow any key to be accessed as long as it does not start with underscore - if isinstance(key, str) and key.startswith('_'): - raise SyntaxError('Key starts with _') + if isinstance(key, str) and key.startswith("_"): + raise SyntaxError("Key starts with _") return obj[key] + def _getattr(object, name, default=None): # guard function for RestrictedPython # allow any key to be accessed as long as @@ -311,29 +321,37 @@ def _getattr(object, name, default=None): UNSAFE_ATTRIBUTES = { # Generator Attributes - "gi_frame", "gi_code", + "gi_frame", + "gi_code", # Coroutine Attributes - "cr_frame", "cr_code", "cr_origin", + "cr_frame", + "cr_code", + "cr_origin", # Async Generator Attributes - "ag_code", "ag_frame", + "ag_code", + "ag_frame", # Traceback Attributes - "tb_frame", "tb_next", + "tb_frame", + "tb_next", } if isinstance(name, str) and (name in UNSAFE_ATTRIBUTES): raise SyntaxError("{name} is an unsafe attribute".format(name=name)) return RestrictedPython.Guards.safer_getattr(object, name, default=default) + def _write(obj): # guard function for RestrictedPython # allow writing to any object return obj + def add_data_utils(data): for key, obj in frappe.utils.data.__dict__.items(): if key in VALID_UTILS: data[key] = obj + def add_module_properties(module, data, filter_method): for key, obj in module.__dict__.items(): if key.startswith("_"): @@ -344,112 +362,113 @@ def add_module_properties(module, data, filter_method): # only allow functions data[key] = obj + VALID_UTILS = ( -"DATE_FORMAT", -"TIME_FORMAT", -"DATETIME_FORMAT", -"is_invalid_date_string", -"getdate", -"get_datetime", -"to_timedelta", -"get_timedelta", -"add_to_date", -"add_days", -"add_months", -"add_years", -"date_diff", -"month_diff", -"time_diff", -"time_diff_in_seconds", -"time_diff_in_hours", -"now_datetime", -"get_timestamp", -"get_eta", -"get_time_zone", -"convert_utc_to_user_timezone", -"now", -"nowdate", -"today", -"nowtime", -"get_first_day", -"get_quarter_start", -"get_first_day_of_week", -"get_year_start", -"get_last_day_of_week", -"get_last_day", -"get_time", -"get_datetime_in_timezone", -"get_datetime_str", -"get_date_str", -"get_time_str", -"get_user_date_format", -"get_user_time_format", -"format_date", -"format_time", -"format_datetime", -"format_duration", -"get_weekdays", -"get_weekday", -"get_timespan_date_range", -"global_date_format", -"has_common", -"flt", -"cint", -"floor", -"ceil", -"cstr", -"rounded", -"remainder", -"safe_div", -"round_based_on_smallest_currency_fraction", -"encode", -"parse_val", -"fmt_money", -"get_number_format_info", -"money_in_words", -"in_words", -"is_html", -"is_image", -"get_thumbnail_base64_for_image", -"image_to_base64", -"pdf_to_base64", -"strip_html", -"escape_html", -"pretty_date", -"comma_or", -"comma_and", -"comma_sep", -"new_line_sep", -"filter_strip_join", -"get_url", -"get_host_name_from_request", -"url_contains_port", -"get_host_name", -"get_link_to_form", -"get_link_to_report", -"get_absolute_url", -"get_url_to_form", -"get_url_to_list", -"get_url_to_report", -"get_url_to_report_with_filters", -"evaluate_filters", -"compare", -"get_filter", -"make_filter_tuple", -"make_filter_dict", -"sanitize_column", -"scrub_urls", -"expand_relative_urls", -"quoted", -"quote_urls", -"unique", -"strip", -"to_markdown", -"md_to_html", -"markdown", -"is_subset", -"generate_hash", -"formatdate", -"get_user_info_for_avatar", -"get_abbr" + "DATE_FORMAT", + "TIME_FORMAT", + "DATETIME_FORMAT", + "is_invalid_date_string", + "getdate", + "get_datetime", + "to_timedelta", + "get_timedelta", + "add_to_date", + "add_days", + "add_months", + "add_years", + "date_diff", + "month_diff", + "time_diff", + "time_diff_in_seconds", + "time_diff_in_hours", + "now_datetime", + "get_timestamp", + "get_eta", + "get_time_zone", + "convert_utc_to_user_timezone", + "now", + "nowdate", + "today", + "nowtime", + "get_first_day", + "get_quarter_start", + "get_first_day_of_week", + "get_year_start", + "get_last_day_of_week", + "get_last_day", + "get_time", + "get_datetime_in_timezone", + "get_datetime_str", + "get_date_str", + "get_time_str", + "get_user_date_format", + "get_user_time_format", + "format_date", + "format_time", + "format_datetime", + "format_duration", + "get_weekdays", + "get_weekday", + "get_timespan_date_range", + "global_date_format", + "has_common", + "flt", + "cint", + "floor", + "ceil", + "cstr", + "rounded", + "remainder", + "safe_div", + "round_based_on_smallest_currency_fraction", + "encode", + "parse_val", + "fmt_money", + "get_number_format_info", + "money_in_words", + "in_words", + "is_html", + "is_image", + "get_thumbnail_base64_for_image", + "image_to_base64", + "pdf_to_base64", + "strip_html", + "escape_html", + "pretty_date", + "comma_or", + "comma_and", + "comma_sep", + "new_line_sep", + "filter_strip_join", + "get_url", + "get_host_name_from_request", + "url_contains_port", + "get_host_name", + "get_link_to_form", + "get_link_to_report", + "get_absolute_url", + "get_url_to_form", + "get_url_to_list", + "get_url_to_report", + "get_url_to_report_with_filters", + "evaluate_filters", + "compare", + "get_filter", + "make_filter_tuple", + "make_filter_dict", + "sanitize_column", + "scrub_urls", + "expand_relative_urls", + "quoted", + "quote_urls", + "unique", + "strip", + "to_markdown", + "md_to_html", + "markdown", + "is_subset", + "generate_hash", + "formatdate", + "get_user_info_for_avatar", + "get_abbr", ) diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 8ebb4b2937..d1cda3d0fc 100755 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -21,23 +21,26 @@ from frappe.installer import update_site_config from frappe.utils import get_sites, now_datetime from frappe.utils.background_jobs import get_jobs +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" -DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' def start_scheduler(): - '''Run enqueue_events_for_all_sites every 2 minutes (default). - Specify scheduler_interval in seconds in common_site_config.json''' + """Run enqueue_events_for_all_sites every 2 minutes (default). + Specify scheduler_interval in seconds in common_site_config.json""" - schedule.every(frappe.get_conf().scheduler_tick_interval or 60).seconds.do(enqueue_events_for_all_sites) + schedule.every(frappe.get_conf().scheduler_tick_interval or 60).seconds.do( + enqueue_events_for_all_sites + ) while True: schedule.run_pending() time.sleep(1) + def enqueue_events_for_all_sites(): - '''Loop through sites and enqueue events that are not already queued''' + """Loop through sites and enqueue events that are not already queued""" - if os.path.exists(os.path.join('.', '.restarting')): + if os.path.exists(os.path.join(".", ".restarting")): # Don't add task to queue if webserver is in restart mode return @@ -48,11 +51,14 @@ def enqueue_events_for_all_sites(): try: enqueue_events_for_site(site=site) except Exception as e: - print(e.__class__, 'Failed to enqueue events for site: {}'.format(site)) + print(e.__class__, "Failed to enqueue events for site: {}".format(site)) + def enqueue_events_for_site(site): def log_and_raise(): - error_message = 'Exception in Enqueue Events for Site {0}\n{1}'.format(site, frappe.get_traceback()) + error_message = "Exception in Enqueue Events for Site {0}\n{1}".format( + site, frappe.get_traceback() + ) frappe.logger("scheduler").error(error_message) try: @@ -63,10 +69,10 @@ def enqueue_events_for_site(site): enqueue_events(site=site) - frappe.logger("scheduler").debug('Queued events for site {0}'.format(site)) + frappe.logger("scheduler").debug("Queued events for site {0}".format(site)) except frappe.db.OperationalError as e: if frappe.db.is_access_denied(e): - frappe.logger("scheduler").debug('Access denied for site {0}'.format(site)) + frappe.logger("scheduler").debug("Access denied for site {0}".format(site)) else: log_and_raise() except: @@ -75,14 +81,16 @@ def enqueue_events_for_site(site): finally: frappe.destroy() + def enqueue_events(site): if schedule_jobs_based_on_activity(): frappe.flags.enqueued_jobs = [] - queued_jobs = get_jobs(site=site, key='job_type').get(site) or [] - for job_type in frappe.get_all('Scheduled Job Type', ('name', 'method'), dict(stopped=0)): + queued_jobs = get_jobs(site=site, key="job_type").get(site) or [] + for job_type in frappe.get_all("Scheduled Job Type", ("name", "method"), dict(stopped=0)): if not job_type.method in queued_jobs: # don't add it to queue if still pending - frappe.get_doc('Scheduled Job Type', job_type.name).enqueue() + frappe.get_doc("Scheduled Job Type", job_type.name).enqueue() + def is_scheduler_inactive(): if frappe.local.conf.maintenance_mode: @@ -96,31 +104,36 @@ def is_scheduler_inactive(): return False + def is_scheduler_disabled(): if frappe.conf.disable_scheduler: return True return not frappe.utils.cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) + def toggle_scheduler(enable): frappe.db.set_value("System Settings", None, "enable_scheduler", 1 if enable else 0) + def enable_scheduler(): toggle_scheduler(True) + def disable_scheduler(): toggle_scheduler(False) + def schedule_jobs_based_on_activity(check_time=None): - '''Returns True for active sites defined by Activity Log - Returns True for inactive sites once in 24 hours''' + """Returns True for active sites defined by Activity Log + Returns True for inactive sites once in 24 hours""" if is_dormant(check_time=check_time): # ensure last job is one day old - last_job_timestamp = frappe.db.get_last_created('Scheduled Job Log') + last_job_timestamp = frappe.db.get_last_created("Scheduled Job Log") if not last_job_timestamp: return True else: - if ((check_time or now_datetime()) - last_job_timestamp).total_seconds() >= 86400: + if ((check_time or now_datetime()) - last_job_timestamp).total_seconds() >= 86400: # one day is passed since jobs are run, so lets do this return True else: @@ -130,9 +143,10 @@ def schedule_jobs_based_on_activity(check_time=None): # site active, lets run the jobs return True + def is_dormant(check_time=None): - last_activity_log_timestamp = frappe.db.get_last_created('Activity Log') - since = (frappe.get_system_settings('dormant_days') or 4) * 86400 + last_activity_log_timestamp = frappe.db.get_last_created("Activity Log") + since = (frappe.get_system_settings("dormant_days") or 4) * 86400 if not last_activity_log_timestamp: return True if ((check_time or now_datetime()) - last_activity_log_timestamp).total_seconds() >= since: @@ -145,4 +159,4 @@ def activate_scheduler(): if is_scheduler_disabled(): enable_scheduler() if frappe.conf.pause_scheduler: - update_site_config('pause_scheduler', 0) + update_site_config("pause_scheduler", 0) diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index 6fec393eb2..95455ab594 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -2,14 +2,18 @@ # License: MIT. See LICENSE import frappe -def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): - frappe.get_doc({ - "doctype": "Custom Field", - "dt": doctype, - "fieldname": fieldname, - "fieldtype": fieldtype, - "options": options - }).insert() + +def add_custom_field(doctype, fieldname, fieldtype="Data", options=None): + frappe.get_doc( + { + "doctype": "Custom Field", + "dt": doctype, + "fieldname": fieldname, + "fieldtype": fieldtype, + "options": options, + } + ).insert() + def clear_custom_fields(doctype): frappe.db.delete("Custom Field", {"dt": doctype}) diff --git a/frappe/utils/user.py b/frappe/utils/user.py index 43d9d26ab8..308ab85f05 100644 --- a/frappe/utils/user.py +++ b/frappe/utils/user.py @@ -2,7 +2,7 @@ # License: MIT. See LICENSE from email.utils import formataddr -from typing import Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, List, Optional import frappe import frappe.share @@ -10,9 +10,8 @@ from frappe import _dict from frappe.boot import get_allowed_reports from frappe.core.doctype.domain_settings.domain_settings import get_active_modules from frappe.permissions import get_roles, get_valid_perms -from frappe.query_builder import DocType +from frappe.query_builder import DocType, Order from frappe.query_builder.functions import Concat_ws -from frappe.query_builder import Order if TYPE_CHECKING: from frappe.core.doctype.user.user import User @@ -22,9 +21,10 @@ class UserPermissions: """ A user permission object can be accessed as `frappe.get_user()` """ - def __init__(self, name=''): + + def __init__(self, name=""): self.defaults = None - self.name = name or frappe.session.get('user') + self.name = name or frappe.session.get("user") self.roles = [] self.all_read = [] @@ -54,7 +54,8 @@ class UserPermissions: pass except Exception as e: # install boo-boo - if not frappe.db.is_table_missing(e): raise + if not frappe.db.is_table_missing(e): + raise return user @@ -74,7 +75,18 @@ class UserPermissions: self.doctype_map = {} active_domains = frappe.get_active_domains() - all_doctypes = frappe.get_all("DocType", fields=["name", "in_create", "module", "istable", "issingle", "read_only", "restrict_to_domain"]) + all_doctypes = frappe.get_all( + "DocType", + fields=[ + "name", + "in_create", + "module", + "istable", + "issingle", + "read_only", + "restrict_to_domain", + ], + ) for dt in all_doctypes: if not dt.restrict_to_domain or (dt.restrict_to_domain in active_domains): @@ -84,7 +96,7 @@ class UserPermissions: """build map of permissions at level 0""" self.perm_map = {} for r in get_valid_perms(): - dt = r['parent'] + dt = r["parent"] if dt not in self.perm_map: self.perm_map[dt] = {} @@ -96,8 +108,8 @@ class UserPermissions: def build_permissions(self): """build lists of what the user can read / write / create quirks: - read_only => Not in Search - in_create => Not in create + read_only => Not in Search + in_create => Not in create """ self.build_doctype_map() self.build_perm_map() @@ -112,52 +124,54 @@ class UserPermissions: if not p.get("read") and (dt in user_shared): p["read"] = 1 - if p.get('select'): + if p.get("select"): self.can_select.append(dt) - if not dtp.get('istable'): - if p.get('create') and not dtp.get('issingle'): - if dtp.get('in_create'): + if not dtp.get("istable"): + if p.get("create") and not dtp.get("issingle"): + if dtp.get("in_create"): self.in_create.append(dt) else: self.can_create.append(dt) - elif p.get('write'): + elif p.get("write"): self.can_write.append(dt) - elif p.get('read'): - if dtp.get('read_only'): + elif p.get("read"): + if dtp.get("read_only"): # read_only = "User Cannot Search" self.all_read.append(dt) no_list_view_link.append(dt) else: self.can_read.append(dt) - if p.get('cancel'): + if p.get("cancel"): self.can_cancel.append(dt) - if p.get('delete'): + if p.get("delete"): self.can_delete.append(dt) - if (p.get('read') or p.get('write') or p.get('create')): - if p.get('report'): + if p.get("read") or p.get("write") or p.get("create"): + if p.get("report"): self.can_get_report.append(dt) for key in ("import", "export", "print", "email", "set_user_permissions"): if p.get(key): getattr(self, "can_" + key).append(dt) - if not dtp.get('istable'): - if not dtp.get('issingle') and not dtp.get('read_only'): + if not dtp.get("istable"): + if not dtp.get("issingle") and not dtp.get("read_only"): self.can_search.append(dt) - if dtp.get('module') not in self.allow_modules: - if active_modules and dtp.get('module') not in active_modules: + if dtp.get("module") not in self.allow_modules: + if active_modules and dtp.get("module") not in active_modules: pass else: - self.allow_modules.append(dtp.get('module')) + self.allow_modules.append(dtp.get("module")) self.can_write += self.can_create self.can_write += self.in_create self.can_read += self.can_write - self.shared = frappe.get_all("DocShare", {"user": self.name, "read": 1}, distinct=True, pluck="share_doctype") + self.shared = frappe.get_all( + "DocShare", {"user": self.name, "read": 1}, distinct=True, pluck="share_doctype" + ) self.can_read = list(set(self.can_read + self.shared)) self.all_read += self.can_read @@ -166,7 +180,7 @@ class UserPermissions: self.can_read.remove(dt) if "System Manager" in self.get_roles(): - self.can_import += frappe.get_all("DocType", {'allow_import': 1}, pluck="name") + self.can_import += frappe.get_all("DocType", {"allow_import": 1}, pluck="name") self.can_import += frappe.get_all( "Property Setter", pluck="doc_type", @@ -177,6 +191,7 @@ class UserPermissions: def get_defaults(self): import frappe.defaults + self.defaults = frappe.defaults.get_defaults(self.name) return self.defaults @@ -217,10 +232,24 @@ class UserPermissions: d.name = self.name d.roles = self.get_roles() d.defaults = self.get_defaults() - for key in ("can_select", "can_create", "can_write", "can_read", "can_cancel", - "can_delete", "can_get_report", "allow_modules", "all_read", "can_search", - "in_create", "can_export", "can_import", "can_print", "can_email", - "can_set_user_permissions"): + for key in ( + "can_select", + "can_create", + "can_write", + "can_read", + "can_cancel", + "can_delete", + "can_get_report", + "allow_modules", + "all_read", + "can_search", + "in_create", + "can_export", + "can_import", + "can_print", + "can_email", + "can_set_user_permissions", + ): d[key] = list(set(getattr(self, key))) d.all_reports = self.get_all_reports() @@ -343,10 +372,7 @@ def get_enabled_system_users() -> List[Dict]: def is_website_user(username: Optional[str] = None) -> Optional[str]: - return ( - frappe.db.get_value("User", username or frappe.session.user, "user_type") - == "Website User" - ) + return frappe.db.get_value("User", username or frappe.session.user, "user_type") == "Website User" def is_system_user(username: Optional[str] = None) -> Optional[str]: diff --git a/frappe/utils/verified_command.py b/frappe/utils/verified_command.py index 582e0226aa..e342ef1810 100644 --- a/frappe/utils/verified_command.py +++ b/frappe/utils/verified_command.py @@ -1,11 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import hmac, hashlib +import hashlib +import hmac from urllib.parse import urlencode -from frappe import _ import frappe import frappe.utils +from frappe import _ + def get_signed_params(params): """Sign a url by appending `&_signature=xxxxx` to given params (string or dict). @@ -18,37 +20,46 @@ def get_signed_params(params): signature.update(get_secret().encode()) return params + "&_signature=" + signature.hexdigest() + def get_secret(): - return frappe.local.conf.get("secret") or str(frappe.db.get_value("User", "Administrator", "creation")) + return frappe.local.conf.get("secret") or str( + frappe.db.get_value("User", "Administrator", "creation") + ) + def verify_request(): """Verify if the incoming signed request if it is correct.""" - query_string = frappe.safe_decode(frappe.local.flags.signed_query_string or \ - getattr(frappe.request, 'query_string', None)) + query_string = frappe.safe_decode( + frappe.local.flags.signed_query_string or getattr(frappe.request, "query_string", None) + ) valid = False - signature_string = '&_signature=' + signature_string = "&_signature=" if signature_string in query_string: params, signature = query_string.split(signature_string) - given_signature = hmac.new(params.encode('utf-8'), digestmod=hashlib.md5) + given_signature = hmac.new(params.encode("utf-8"), digestmod=hashlib.md5) given_signature.update(get_secret().encode()) valid = signature == given_signature.hexdigest() if not valid: - frappe.respond_as_web_page(_("Invalid Link"), - _("This link is invalid or expired. Please make sure you have pasted correctly.")) + frappe.respond_as_web_page( + _("Invalid Link"), + _("This link is invalid or expired. Please make sure you have pasted correctly."), + ) return valid + def get_url(cmd, params, nonce=None, secret=None): if not nonce: nonce = params signature = get_signature(params, nonce, secret) - params['signature'] = signature - return frappe.utils.get_url("".join(['api/method/', cmd, '?', urlencode(params)])) + params["signature"] = signature + return frappe.utils.get_url("".join(["api/method/", cmd, "?", urlencode(params)])) + def get_signature(params, nonce, secret=None): params = "".join((frappe.utils.cstr(p) for p in params.values())) @@ -60,10 +71,12 @@ def get_signature(params, nonce, secret=None): signature.update(params) return signature.hexdigest() + def verify_using_doc(doc, signature, cmd): params = doc.get_signature_params() return signature == get_signature(params, doc.get_nonce()) + def get_url_using_doc(doc, cmd): params = doc.get_signature_params() return get_url(cmd, params, doc.get_nonce()) diff --git a/frappe/utils/weasyprint.py b/frappe/utils/weasyprint.py index 607d7ea49c..ceb064a955 100644 --- a/frappe/utils/weasyprint.py +++ b/frappe/utils/weasyprint.py @@ -5,6 +5,7 @@ import click import frappe + @frappe.whitelist() def download_pdf(doctype, name, print_format, letterhead=None): doc = frappe.get_doc(doctype, name) @@ -37,11 +38,11 @@ class PrintFormatGenerator: Parameters ---------- print_format: str - Name of the Print Format + Name of the Print Format doc: str - Document to print + Document to print letterhead: str - Letter Head to apply (optional) + Letter Head to apply (optional) """ self.base_url = frappe.utils.get_url() self.print_format = frappe.get_doc("Print Format", print_format) @@ -55,9 +56,7 @@ class PrintFormatGenerator: self.print_settings = frappe.get_doc("Print Settings") page_width_map = {"A4": 210, "Letter": 216} page_width = page_width_map.get(self.print_settings.pdf_page_size) or 210 - body_width = ( - page_width - self.print_format.margin_left - self.print_format.margin_right - ) + body_width = page_width - self.print_format.margin_left - self.print_format.margin_right print_style = ( frappe.get_doc("Print Style", self.print_settings.print_style) if self.print_settings.print_style @@ -86,20 +85,14 @@ class PrintFormatGenerator: self.context.css = frappe.render_template( "templates/print_format/print_format.css", self.context ) - return frappe.render_template( - "templates/print_format/print_format.html", self.context - ) + return frappe.render_template("templates/print_format/print_format.html", self.context) def get_header_footer_html(self): header_html = footer_html = None if self.letterhead: - header_html = frappe.render_template( - "templates/print_format/print_header.html", self.context - ) + header_html = frappe.render_template("templates/print_format/print_header.html", self.context) if self.letterhead: - footer_html = frappe.render_template( - "templates/print_format/print_footer.html", self.context - ) + footer_html = frappe.render_template("templates/print_format/print_footer.html", self.context) return header_html, footer_html def render_pdf(self): @@ -107,15 +100,13 @@ class PrintFormatGenerator: Returns ------- pdf: a bytes sequence - The rendered PDF. + The rendered PDF. """ HTML, CSS = import_weasyprint() self._make_header_footer() - self.context.update( - {"header_height": self.header_height, "footer_height": self.footer_height} - ) + self.context.update({"header_height": self.header_height, "footer_height": self.footer_height}) main_html = self.get_main_html() html = HTML(string=main_html, base_url=self.base_url) @@ -132,29 +123,26 @@ class PrintFormatGenerator: Parameters ---------- element: str - Either 'header' or 'footer' + Either 'header' or 'footer' Returns ------- element_body: BlockBox - A Weasyprint pre-rendered representation of an html element + A Weasyprint pre-rendered representation of an html element element_height: float - The height of this element, which will be then translated in a html height + The height of this element, which will be then translated in a html height """ HTML, CSS = import_weasyprint() - html = HTML(string=getattr(self, f"{element}_html"), base_url=self.base_url,) - element_doc = html.render( - stylesheets=[CSS(string="@page {size: A4 portrait; margin: 0;}")] + html = HTML( + string=getattr(self, f"{element}_html"), + base_url=self.base_url, ) + element_doc = html.render(stylesheets=[CSS(string="@page {size: A4 portrait; margin: 0;}")]) element_page = element_doc.pages[0] - element_body = PrintFormatGenerator.get_element( - element_page._page_box.all_children(), "body" - ) + element_body = PrintFormatGenerator.get_element(element_page._page_box.all_children(), "body") element_body = element_body.copy_with_children(element_body.all_children()) - element_html = PrintFormatGenerator.get_element( - element_page._page_box.all_children(), element - ) + element_html = PrintFormatGenerator.get_element(element_page._page_box.all_children(), element) if element == "header": element_height = element_html.height @@ -170,11 +158,11 @@ class PrintFormatGenerator: Parameters ---------- main_doc: Document - The top level representation for a PDF page in Weasyprint. + The top level representation for a PDF page in Weasyprint. header_body: BlockBox - A representation for an html element in Weasyprint. + A representation for an html element in Weasyprint. footer_body: BlockBox - A representation for an html element in Weasyprint. + A representation for an html element in Weasyprint. """ for page in main_doc.pages: page_body = PrintFormatGenerator.get_element(page._page_box.all_children(), "body") @@ -250,16 +238,16 @@ class PrintFormatGenerator: def import_weasyprint(): try: - from weasyprint import HTML, CSS + from weasyprint import CSS, HTML + return HTML, CSS except OSError: - message = "\n".join([ - "WeasyPrint depdends on additional system dependencies.", - "Follow instructions specific to your operating system:", - "https://doc.courtbouillon.org/weasyprint/stable/first_steps.html" - ]) - click.secho( - message, - fg="yellow" + message = "\n".join( + [ + "WeasyPrint depdends on additional system dependencies.", + "Follow instructions specific to your operating system:", + "https://doc.courtbouillon.org/weasyprint/stable/first_steps.html", + ] ) + click.secho(message, fg="yellow") frappe.throw(message) diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 38a076212a..ad02cd8327 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -12,7 +12,7 @@ from openpyxl.utils import get_column_letter import frappe from frappe.utils.html_utils import unescape_html -ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]') +ILLEGAL_CHARACTERS_RE = re.compile(r"[\000-\010]|[\013-\014]|[\016-\037]") # return xlsx file object @@ -28,19 +28,19 @@ def make_xlsx(data, sheet_name, wb=None, column_widths=None): ws.column_dimensions[get_column_letter(i + 1)].width = column_width row1 = ws.row_dimensions[1] - row1.font = Font(name='Calibri', bold=True) + row1.font = Font(name="Calibri", bold=True) for row in data: clean_row = [] for item in row: - if isinstance(item, str) and (sheet_name not in ['Data Import Template', 'Data Export']): + if isinstance(item, str) and (sheet_name not in ["Data Import Template", "Data Export"]): value = handle_html(item) else: value = item if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None): # Remove illegal characters from the string - value = re.sub(ILLEGAL_CHARACTERS_RE, '', value) + value = re.sub(ILLEGAL_CHARACTERS_RE, "", value) clean_row.append(value) @@ -57,7 +57,7 @@ def handle_html(data): # return if no html tags found data = frappe.as_unicode(data) - if '<' not in data or '>' not in data: + if "<" not in data or ">" not in data: return data h = unescape_html(data or "") @@ -72,9 +72,9 @@ def handle_html(data): # unable to parse html, send it raw return data - value = ", ".join(value.split(' \n')) - value = " ".join(value.split('\n')) - value = ", ".join(value.split('# ')) + value = ", ".join(value.split(" \n")) + value = " ".join(value.split("\n")) + value = ", ".join(value.split("# ")) return value @@ -114,6 +114,6 @@ def read_xls_file_from_attached_file(content): def build_xlsx_response(data, filename): xlsx_file = make_xlsx(data, filename) # write out response as a xlsx type - frappe.response['filename'] = filename + '.xlsx' - frappe.response['filecontent'] = xlsx_file.getvalue() - frappe.response['type'] = 'binary' + frappe.response["filename"] = filename + ".xlsx" + frappe.response["filecontent"] = xlsx_file.getvalue() + frappe.response["type"] = "binary" diff --git a/frappe/website/dashboard_fixtures.py b/frappe/website/dashboard_fixtures.py index 1ac7ca60ec..0d641cb547 100644 --- a/frappe/website/dashboard_fixtures.py +++ b/frappe/website/dashboard_fixtures.py @@ -1,36 +1,43 @@ import frappe + def get_data(): - return frappe._dict({ - "dashboards": get_dashboards(), - "charts": get_charts(), - "number_cards": None, - }) + return frappe._dict( + { + "dashboards": get_dashboards(), + "charts": get_charts(), + "number_cards": None, + } + ) + def get_dashboards(): - return [{ - "name": "Website", - "dashboard_name": "Website", - "charts": [ - { "chart": "Website Analytics", "width": "Full" } - ] - }] + return [ + { + "name": "Website", + "dashboard_name": "Website", + "charts": [{"chart": "Website Analytics", "width": "Full"}], + } + ] + def get_charts(): - return [{ - "chart_name": "Website Analytics", - "chart_type": "Report", - "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"tooltipOptions\": {}}", - "doctype": "Dashboard Chart", - "filters_json": "{}", - "group_by_type": "Count", - "is_custom": 1, - "is_public": 1, - "name": "Website Analytics", - "number_of_groups": 0, - "report_name": "Website Analytics", - "time_interval": "Yearly", - "timeseries": 0, - "timespan": "Last Year", - "type": "Line" - }] \ No newline at end of file + return [ + { + "chart_name": "Website Analytics", + "chart_type": "Report", + "custom_options": '{"type": "line", "lineOptions": {"regionFill": 1}, "axisOptions": {"shortenYAxisNumbers": 1}, "tooltipOptions": {}}', + "doctype": "Dashboard Chart", + "filters_json": "{}", + "group_by_type": "Count", + "is_custom": 1, + "is_public": 1, + "name": "Website Analytics", + "number_of_groups": 0, + "report_name": "Website Analytics", + "time_interval": "Yearly", + "timeseries": 0, + "timespan": "Last Year", + "type": "Line", + } + ] diff --git a/frappe/website/doctype/about_us_settings/about_us_settings.py b/frappe/website/doctype/about_us_settings/about_us_settings.py index 0b9458323f..4989f96d00 100644 --- a/frappe/website/doctype/about_us_settings/about_us_settings.py +++ b/frappe/website/doctype/about_us_settings/about_us_settings.py @@ -4,17 +4,16 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document -class AboutUsSettings(Document): +class AboutUsSettings(Document): def on_update(self): from frappe.website.utils import clear_cache + clear_cache("about") + def get_args(): obj = frappe.get_doc("About Us Settings") - return { - "obj": obj - } \ No newline at end of file + return {"obj": obj} diff --git a/frappe/website/doctype/about_us_settings/test_about_us_settings.py b/frappe/website/doctype/about_us_settings/test_about_us_settings.py index 17dbb2c9a8..a4b2da7718 100644 --- a/frappe/website/doctype/about_us_settings/test_about_us_settings.py +++ b/frappe/website/doctype/about_us_settings/test_about_us_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestAboutUsSettings(unittest.TestCase): pass diff --git a/frappe/website/doctype/about_us_team_member/about_us_team_member.py b/frappe/website/doctype/about_us_team_member/about_us_team_member.py index b9a003073a..93a867faa1 100644 --- a/frappe/website/doctype/about_us_team_member/about_us_team_member.py +++ b/frappe/website/doctype/about_us_team_member/about_us_team_member.py @@ -4,8 +4,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class AboutUsTeamMember(Document): - pass \ No newline at end of file + pass diff --git a/frappe/website/doctype/blog_category/blog_category.py b/frappe/website/doctype/blog_category/blog_category.py index 3f8cbad85e..31212ac61b 100644 --- a/frappe/website/doctype/blog_category/blog_category.py +++ b/frappe/website/doctype/blog_category/blog_category.py @@ -1,8 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from frappe.website.website_generator import WebsiteGenerator from frappe.website.utils import clear_cache +from frappe.website.website_generator import WebsiteGenerator + class BlogCategory(WebsiteGenerator): def autoname(self): @@ -14,4 +15,4 @@ class BlogCategory(WebsiteGenerator): def set_route(self): # Override blog route since it has to been templated - self.route = 'blog/' + self.name + self.route = "blog/" + self.name diff --git a/frappe/website/doctype/blog_category/test_blog_category.py b/frappe/website/doctype/blog_category/test_blog_category.py index 495473856a..c62e22dfe0 100644 --- a/frappe/website/doctype/blog_category/test_blog_category.py +++ b/frappe/website/doctype/blog_category/test_blog_category.py @@ -1,7 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + + class TestBlogCategory(unittest.TestCase): pass diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index 9ac51133fa..2f5d8e4ace 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -1,21 +1,37 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from math import ceil + import frappe from frappe import _ +from frappe.utils import ( + cint, + get_fullname, + global_date_format, + markdown, + sanitize_html, + strip_html_tags, + today, +) +from frappe.website.utils import ( + clear_cache, + find_first_image, + get_comment_list, + get_html_content_based_on_type, +) from frappe.website.website_generator import WebsiteGenerator -from frappe.website.utils import clear_cache -from frappe.utils import today, cint, global_date_format, get_fullname, strip_html_tags, markdown, sanitize_html -from math import ceil -from frappe.website.utils import (find_first_image, get_html_content_based_on_type, - get_comment_list) + class BlogPost(WebsiteGenerator): @frappe.whitelist() def make_route(self): if not self.route: - return frappe.db.get_value('Blog Category', self.blog_category, - 'route') + '/' + self.scrub(self.title) + return ( + frappe.db.get_value("Blog Category", self.blog_category, "route") + + "/" + + self.scrub(self.title) + ) def get_feed(self): return self.title @@ -24,7 +40,7 @@ class BlogPost(WebsiteGenerator): super(BlogPost, self).validate() if not self.blog_intro: - content = get_html_content_based_on_type(self, 'content', self.content_type) + content = get_html_content_based_on_type(self, "content", self.content_type) self.blog_intro = content[:200] self.blog_intro = strip_html_tags(self.blog_intro) @@ -75,35 +91,41 @@ class BlogPost(WebsiteGenerator): context.updated = global_date_format(self.published_on) context.social_links = self.fetch_social_links_info() context.cta = self.fetch_cta() - context.enable_cta = not self.hide_cta and frappe.db.get_single_value("Blog Settings", "show_cta_in_blog", cache=True) + context.enable_cta = not self.hide_cta and frappe.db.get_single_value( + "Blog Settings", "show_cta_in_blog", cache=True + ) if self.blogger: context.blogger_info = frappe.get_doc("Blogger", self.blogger).as_dict() context.author = self.blogger + context.content = get_html_content_based_on_type(self, "content", self.content_type) - context.content = get_html_content_based_on_type(self, 'content', self.content_type) - - #if meta description is not present, then blog intro or first 140 characters of the blog will be set as description - context.description = self.meta_description or self.blog_intro or strip_html_tags(context.content[:140]) + # if meta description is not present, then blog intro or first 140 characters of the blog will be set as description + context.description = ( + self.meta_description or self.blog_intro or strip_html_tags(context.content[:140]) + ) context.metatags = { "name": self.meta_title, "description": context.description, } - #if meta image is not present, then first image inside the blog will be set as the meta image + # if meta image is not present, then first image inside the blog will be set as the meta image image = find_first_image(context.content) context.metatags["image"] = self.meta_image or image or None self.load_comments(context) self.load_feedback(context) - context.category = frappe.db.get_value("Blog Category", - context.doc.blog_category, ["title", "route"], as_dict=1) - context.parents = [{"name": _("Home"), "route":"/"}, + context.category = frappe.db.get_value( + "Blog Category", context.doc.blog_category, ["title", "route"], as_dict=1 + ) + context.parents = [ + {"name": _("Home"), "route": "/"}, {"name": "Blog", "route": "/blog"}, - {"label": context.category.title, "route":context.category.route}] + {"label": context.category.title, "route": context.category.route}, + ] context.guest_allowed = frappe.db.get_single_value("Blog Settings", "allow_guest_to_comment") def fetch_cta(self): @@ -115,7 +137,7 @@ class BlogPost(WebsiteGenerator): "title": blog_settings.title, "subtitle": blog_settings.subtitle, "cta_label": blog_settings.cta_label, - "cta_url": blog_settings.cta_url + "cta_url": blog_settings.cta_url, } return {} @@ -124,13 +146,16 @@ class BlogPost(WebsiteGenerator): if not frappe.db.get_single_value("Blog Settings", "enable_social_sharing", cache=True): return [] - url = frappe.local.site + "/" +self.route + url = frappe.local.site + "/" + self.route social_links = [ - { "icon": "twitter", "link": "https://twitter.com/intent/tweet?text=" + self.title + "&url=" + url }, - { "icon": "facebook", "link": "https://www.facebook.com/sharer.php?u=" + url }, - { "icon": "linkedin", "link": "https://www.linkedin.com/sharing/share-offsite/?url=" + url }, - { "icon": "envelope", "link": "mailto:?subject=" + self.title + "&body=" + url } + { + "icon": "twitter", + "link": "https://twitter.com/intent/tweet?text=" + self.title + "&url=" + url, + }, + {"icon": "facebook", "link": "https://www.facebook.com/sharer.php?u=" + url}, + {"icon": "linkedin", "link": "https://www.linkedin.com/sharing/share-offsite/?url=" + url}, + {"icon": "envelope", "link": "mailto:?subject=" + self.title + "&body=" + url}, ] return social_links @@ -146,48 +171,48 @@ class BlogPost(WebsiteGenerator): def load_feedback(self, context): user = frappe.session.user - feedback = frappe.get_all('Feedback', - fields=['like'], + feedback = frappe.get_all( + "Feedback", + fields=["like"], filters=dict( reference_doctype=self.doctype, reference_name=self.name, ip_address=frappe.local.request_ip, - owner=user - ) + owner=user, + ), ) like_count = 0 - if frappe.db.count('Feedback'): - like_count = frappe.db.count('Feedback', - filters = dict( - reference_doctype = self.doctype, - reference_name = self.name, - like = True - ) + if frappe.db.count("Feedback"): + like_count = frappe.db.count( + "Feedback", filters=dict(reference_doctype=self.doctype, reference_name=self.name, like=True) ) - context.user_feedback = feedback[0] if feedback else '' + context.user_feedback = feedback[0] if feedback else "" context.like_count = like_count def set_read_time(self): - content = self.content or self.content_html or '' + content = self.content or self.content_html or "" if self.content_type == "Markdown": content = markdown(self.content_md) total_words = len(strip_html_tags(content).split()) - self.read_time = ceil(total_words/250) + self.read_time = ceil(total_words / 250) + def get_list_context(context=None): list_context = frappe._dict( - get_list = get_blog_list, - no_breadcrumbs = True, - hide_filters = True, + get_list=get_blog_list, + no_breadcrumbs=True, + hide_filters=True, # show_search = True, - title = _('Blog') + title=_("Blog"), ) - category = frappe.utils.escape_html(frappe.local.form_dict.blog_category or frappe.local.form_dict.category) + category = frappe.utils.escape_html( + frappe.local.form_dict.blog_category or frappe.local.form_dict.category + ) if category: category_title = get_blog_category(category) list_context.sub_title = _("Posts filed under {0}").format(category_title) @@ -202,8 +227,7 @@ def get_list_context(context=None): list_context.sub_title = _('Filtered by "{0}"').format(sanitize_html(frappe.local.form_dict.txt)) if list_context.sub_title: - list_context.parents = [{"name": _("Home"), "route": "/"}, - {"name": "Blog", "route": "/blog"}] + list_context.parents = [{"name": _("Home"), "route": "/"}, {"name": "Blog", "route": "/blog"}] else: list_context.parents = [{"name": _("Home"), "route": "/"}] @@ -236,27 +260,38 @@ def get_blog_categories(): .run(as_dict=1) ) + def clear_blog_cache(): - for blog in frappe.db.sql_list("""select route from - `tabBlog Post` where ifnull(published,0)=1"""): + for blog in frappe.db.sql_list( + """select route from + `tabBlog Post` where ifnull(published,0)=1""" + ): clear_cache(blog) clear_cache("writers") + def get_blog_category(route): return frappe.db.get_value("Blog Category", {"name": route}, "title") or route -def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_length=20, order_by=None): + +def get_blog_list( + doctype, txt=None, filters=None, limit_start=0, limit_page_length=20, order_by=None +): conditions = [] - category = filters.blog_category or frappe.utils.escape_html(frappe.local.form_dict.blog_category or frappe.local.form_dict.category) + category = filters.blog_category or frappe.utils.escape_html( + frappe.local.form_dict.blog_category or frappe.local.form_dict.category + ) if filters: if filters.blogger: - conditions.append('t1.blogger=%s' % frappe.db.escape(filters.blogger)) + conditions.append("t1.blogger=%s" % frappe.db.escape(filters.blogger)) if category: - conditions.append('t1.blog_category=%s' % frappe.db.escape(category)) + conditions.append("t1.blog_category=%s" % frappe.db.escape(category)) if txt: - conditions.append('(t1.content like {0} or t1.title like {0}")'.format(frappe.db.escape('%' + txt + '%'))) + conditions.append( + '(t1.content like {0} or t1.title like {0}")'.format(frappe.db.escape("%" + txt + "%")) + ) if conditions: frappe.local.no_cache = 1 @@ -285,31 +320,37 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len %(condition)s order by featured desc, published_on desc, name asc limit %(page_len)s OFFSET %(start)s""" % { - "start": limit_start, "page_len": limit_page_length, - "condition": (" and " + " and ".join(conditions)) if conditions else "" - } + "start": limit_start, + "page_len": limit_page_length, + "condition": (" and " + " and ".join(conditions)) if conditions else "", + } posts = frappe.db.sql(query, as_dict=1) for post in posts: - post.content = get_html_content_based_on_type(post, 'content', post.content_type) + post.content = get_html_content_based_on_type(post, "content", post.content_type) if not post.cover_image: post.cover_image = find_first_image(post.content) post.published = global_date_format(post.creation) post.content = strip_html_tags(post.content) if not post.comments: - post.comment_text = _('No comments yet') - elif post.comments==1: - post.comment_text = _('1 comment') + post.comment_text = _("No comments yet") + elif post.comments == 1: + post.comment_text = _("1 comment") else: - post.comment_text = _('{0} comments').format(str(post.comments)) + post.comment_text = _("{0} comments").format(str(post.comments)) post.avatar = post.avatar or "" - post.category = frappe.db.get_value('Blog Category', post.blog_category, - ['name', 'route', 'title'], as_dict=True) + post.category = frappe.db.get_value( + "Blog Category", post.blog_category, ["name", "route", "title"], as_dict=True + ) - if post.avatar and (not "http:" in post.avatar and not "https:" in post.avatar) and not post.avatar.startswith("/"): + if ( + post.avatar + and (not "http:" in post.avatar and not "https:" in post.avatar) + and not post.avatar.startswith("/") + ): post.avatar = "/" + post.avatar return posts diff --git a/frappe/website/doctype/blog_post/test_blog_post.py b/frappe/website/doctype/blog_post/test_blog_post.py index 575b6c0fc0..0eddad4bfe 100644 --- a/frappe/website/doctype/blog_post/test_blog_post.py +++ b/frappe/website/doctype/blog_post/test_blog_post.py @@ -1,27 +1,29 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe +import re import unittest + from bs4 import BeautifulSoup -import re -from frappe.utils import set_request -from frappe.website.serve import get_response -from frappe.utils import random_string +import frappe +from frappe.custom.doctype.customize_form.customize_form import reset_customization +from frappe.utils import random_string, set_request from frappe.website.doctype.blog_post.blog_post import get_blog_list +from frappe.website.serve import get_response from frappe.website.utils import clear_website_cache from frappe.website.website_generator import WebsiteGenerator -from frappe.custom.doctype.customize_form.customize_form import reset_customization -test_dependencies = ['Blog Post'] +test_dependencies = ["Blog Post"] + class TestBlogPost(unittest.TestCase): def setUp(self): - reset_customization('Blog Post') + reset_customization("Blog Post") def test_generator_view(self): - pages = frappe.get_all('Blog Post', fields=['name', 'route'], - filters={'published': 1, 'route': ('!=', '')}, limit =1) + pages = frappe.get_all( + "Blog Post", fields=["name", "route"], filters={"published": 1, "route": ("!=", "")}, limit=1 + ) set_request(path=pages[0].route) response = get_response() @@ -29,15 +31,16 @@ class TestBlogPost(unittest.TestCase): self.assertTrue(response.status_code, 200) html = response.get_data().decode() - self.assertTrue('
' in html) + self.assertTrue( + '
' in html + ) def test_generator_not_found(self): - pages = frappe.get_all('Blog Post', fields=['name', 'route'], - filters={'published': 0}, limit =1) + pages = frappe.get_all("Blog Post", fields=["name", "route"], filters={"published": 0}, limit=1) - route = f'test-route-{frappe.generate_hash(length=5)}' + route = f"test-route-{frappe.generate_hash(length=5)}" - frappe.db.set_value('Blog Post', pages[0].name, 'route', route) + frappe.db.set_value("Blog Post", pages[0].name, "route", route) set_request(path=route) response = get_response() @@ -46,7 +49,7 @@ class TestBlogPost(unittest.TestCase): def test_category_link(self): # Make a temporary Blog Post (and a Blog Category) - blog = make_test_blog('Test Category Link') + blog = make_test_blog("Test Category Link") # Visit the blog post page set_request(path=blog.route) @@ -55,11 +58,11 @@ class TestBlogPost(unittest.TestCase): # On blog post page find link to the category page soup = BeautifulSoup(blog_page_html, "lxml") - category_page_link = list(soup.find_all('a', href=re.compile(blog.blog_category)))[0] + category_page_link = list(soup.find_all("a", href=re.compile(blog.blog_category)))[0] category_page_url = category_page_link["href"] - cached_value = frappe.db.value_cache[('DocType', 'Blog Post', 'name')] - frappe.db.value_cache[('DocType', 'Blog Post', 'name')] = (('Blog Post',),) + cached_value = frappe.db.value_cache[("DocType", "Blog Post", "name")] + frappe.db.value_cache[("DocType", "Blog Post", "name")] = (("Blog Post",),) # Visit the category page (by following the link found in above stage) set_request(path=category_page_url) @@ -69,7 +72,7 @@ class TestBlogPost(unittest.TestCase): self.assertIn(blog.title, category_page_html) # Cleanup - frappe.db.value_cache[('DocType', 'Blog Post', 'name')] = cached_value + frappe.db.value_cache[("DocType", "Blog Post", "name")] = cached_value frappe.delete_doc("Blog Post", blog.name) frappe.delete_doc("Blog Category", blog.blog_category) @@ -101,8 +104,12 @@ class TestBlogPost(unittest.TestCase): clear_website_cache() # first response no-cache - pages = frappe.get_all('Blog Post', fields=['name', 'route'], - filters={'published': 1, 'title': "_Test Blog Post"}, limit=1) + pages = frappe.get_all( + "Blog Post", + fields=["name", "route"], + filters={"published": 1, "title": "_Test Blog Post"}, + limit=1, + ) route = pages[0].route set_request(path=route) @@ -113,13 +120,13 @@ class TestBlogPost(unittest.TestCase): set_request(path=route) response = get_response() - self.assertIn(('X-From-Cache', 'True'), list(response.headers)) + self.assertIn(("X-From-Cache", "True"), list(response.headers)) frappe.flags.force_website_cache = True def test_spam_comments(self): # Make a temporary Blog Post (and a Blog Category) - blog = make_test_blog('Test Spam Comment') + blog = make_test_blog("Test Spam Comment") # Create a spam comment frappe.get_doc( @@ -127,10 +134,10 @@ class TestBlogPost(unittest.TestCase): comment_type="Comment", reference_doctype="Blog Post", reference_name=blog.name, - comment_email="spam", - comment_by="spam", + comment_email='spam', + comment_by='spam', published=1, - content="More spam content. spam with link.", + content='More spam content. spam with link.', ).insert() # Visit the blog post page @@ -145,30 +152,30 @@ class TestBlogPost(unittest.TestCase): frappe.delete_doc("Blog Post", blog.name) frappe.delete_doc("Blog Category", blog.blog_category) + def scrub(text): return WebsiteGenerator.scrub(None, text) + def make_test_blog(category_title="Test Blog Category"): category_name = scrub(category_title) - if not frappe.db.exists('Blog Category', category_name): - frappe.get_doc(dict( - doctype = 'Blog Category', - title=category_title)).insert() - if not frappe.db.exists('Blogger', 'test-blogger'): - frappe.get_doc(dict( - doctype = 'Blogger', - short_name='test-blogger', - full_name='Test Blogger')).insert() - - test_blog = frappe.get_doc(dict( - doctype = 'Blog Post', - blog_category = category_name, - blogger = 'test-blogger', - title = random_string(20), - route = random_string(20), - content = random_string(20), - published = 1 - )).insert() + if not frappe.db.exists("Blog Category", category_name): + frappe.get_doc(dict(doctype="Blog Category", title=category_title)).insert() + if not frappe.db.exists("Blogger", "test-blogger"): + frappe.get_doc( + dict(doctype="Blogger", short_name="test-blogger", full_name="Test Blogger") + ).insert() - return test_blog + test_blog = frappe.get_doc( + dict( + doctype="Blog Post", + blog_category=category_name, + blogger="test-blogger", + title=random_string(20), + route=random_string(20), + content=random_string(20), + published=1, + ) + ).insert() + return test_blog diff --git a/frappe/website/doctype/blog_settings/blog_settings.py b/frappe/website/doctype/blog_settings/blog_settings.py index a7fca30054..ed22f64fd7 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.py +++ b/frappe/website/doctype/blog_settings/blog_settings.py @@ -4,18 +4,20 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document -class BlogSettings(Document): +class BlogSettings(Document): def on_update(self): from frappe.website.utils import clear_cache + clear_cache("blog") clear_cache("writers") + def get_feedback_limit(): return frappe.db.get_single_value("Blog Settings", "feedback_limit") or 5 + def get_comment_limit(): - return frappe.db.get_single_value("Blog Settings", "comment_limit") or 5 \ No newline at end of file + return frappe.db.get_single_value("Blog Settings", "comment_limit") or 5 diff --git a/frappe/website/doctype/blog_settings/test_blog_settings.py b/frappe/website/doctype/blog_settings/test_blog_settings.py index b7659d58a4..23607812fb 100644 --- a/frappe/website/doctype/blog_settings/test_blog_settings.py +++ b/frappe/website/doctype/blog_settings/test_blog_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestBlogSettings(unittest.TestCase): pass diff --git a/frappe/website/doctype/blogger/blogger.py b/frappe/website/doctype/blogger/blogger.py index fb1c3e2831..9f348a2ea1 100644 --- a/frappe/website/doctype/blogger/blogger.py +++ b/frappe/website/doctype/blogger/blogger.py @@ -5,28 +5,30 @@ import frappe from frappe import _ - from frappe.model.document import Document + class Blogger(Document): def validate(self): if self.user and not frappe.db.exists("User", self.user): # for data import - frappe.get_doc({ - "doctype":"User", - "email": self.user, - "first_name": self.user.split("@")[0] - }).insert() + frappe.get_doc( + {"doctype": "User", "email": self.user, "first_name": self.user.split("@")[0]} + ).insert() def on_update(self): "if user is set, then update all older blogs" from frappe.website.doctype.blog_post.blog_post import clear_blog_cache + clear_blog_cache() if self.user: - for blog in frappe.db.sql_list("""select name from `tabBlog Post` where owner=%s - and ifnull(blogger,'')=''""", self.user): + for blog in frappe.db.sql_list( + """select name from `tabBlog Post` where owner=%s + and ifnull(blogger,'')=''""", + self.user, + ): b = frappe.get_doc("Blog Post", blog) b.blogger = self.name b.save() diff --git a/frappe/website/doctype/blogger/test_blogger.py b/frappe/website/doctype/blogger/test_blogger.py index 68d7311906..44a1e2e2a0 100644 --- a/frappe/website/doctype/blogger/test_blogger.py +++ b/frappe/website/doctype/blogger/test_blogger.py @@ -2,4 +2,5 @@ # License: MIT. See LICENSE import frappe -test_records = frappe.get_test_records('Blogger') \ No newline at end of file + +test_records = frappe.get_test_records("Blogger") diff --git a/frappe/website/doctype/color/color.py b/frappe/website/doctype/color/color.py index 888ad5dd69..4d822d6b69 100644 --- a/frappe/website/doctype/color/color.py +++ b/frappe/website/doctype/color/color.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class Color(Document): pass diff --git a/frappe/website/doctype/color/test_color.py b/frappe/website/doctype/color/test_color.py index 300fce61a9..2fd1cf1ba8 100644 --- a/frappe/website/doctype/color/test_color.py +++ b/frappe/website/doctype/color/test_color.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestColor(unittest.TestCase): pass diff --git a/frappe/website/doctype/company_history/company_history.py b/frappe/website/doctype/company_history/company_history.py index f161952dba..366c43ce03 100644 --- a/frappe/website/doctype/company_history/company_history.py +++ b/frappe/website/doctype/company_history/company_history.py @@ -4,8 +4,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class CompanyHistory(Document): - pass \ No newline at end of file + pass diff --git a/frappe/website/doctype/contact_us_settings/contact_us_settings.py b/frappe/website/doctype/contact_us_settings/contact_us_settings.py index 1fce8377d4..36bbedb57c 100644 --- a/frappe/website/doctype/contact_us_settings/contact_us_settings.py +++ b/frappe/website/doctype/contact_us_settings/contact_us_settings.py @@ -4,11 +4,11 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document -class ContactUsSettings(Document): +class ContactUsSettings(Document): def on_update(self): from frappe.website.utils import clear_cache - clear_cache("contact") \ No newline at end of file + + clear_cache("contact") diff --git a/frappe/website/doctype/discussion_reply/discussion_reply.py b/frappe/website/doctype/discussion_reply/discussion_reply.py index efb18b4da7..1ac62d3b7d 100644 --- a/frappe/website/doctype/discussion_reply/discussion_reply.py +++ b/frappe/website/doctype/discussion_reply/discussion_reply.py @@ -4,60 +4,58 @@ import frappe from frappe.model.document import Document -class DiscussionReply(Document): +class DiscussionReply(Document): def on_update(self): frappe.publish_realtime( event="update_message", - message = { - "reply": frappe.utils.md_to_html(self.reply), - "reply_name": self.name - }, - after_commit=True) + message={"reply": frappe.utils.md_to_html(self.reply), "reply_name": self.name}, + after_commit=True, + ) def after_insert(self): replies = frappe.db.count("Discussion Reply", {"topic": self.topic}) - topic_info = frappe.get_all("Discussion Topic", + topic_info = frappe.get_all( + "Discussion Topic", {"name": self.topic}, - ["reference_doctype", "reference_docname", "name", "title", "owner", "creation"]) - - template = frappe.render_template("frappe/templates/discussions/reply_card.html", { - "reply": self, - "topic": { - "name": self.topic + ["reference_doctype", "reference_docname", "name", "title", "owner", "creation"], + ) + + template = frappe.render_template( + "frappe/templates/discussions/reply_card.html", + { + "reply": self, + "topic": {"name": self.topic}, + "loop": {"index": replies}, + "single_thread": True if not topic_info[0].title else False, }, - "loop": { - "index": replies - }, - "single_thread": True if not topic_info[0].title else False - }) + ) - sidebar = frappe.render_template("frappe/templates/discussions/sidebar.html", { - "topic": topic_info[0] - }) + sidebar = frappe.render_template( + "frappe/templates/discussions/sidebar.html", {"topic": topic_info[0]} + ) - new_topic_template = frappe.render_template("frappe/templates/discussions/reply_section.html", { - "topic": topic_info[0] - }) + new_topic_template = frappe.render_template( + "frappe/templates/discussions/reply_section.html", {"topic": topic_info[0]} + ) frappe.publish_realtime( event="publish_message", - message = { + message={ "template": template, "topic_info": topic_info[0], "sidebar": sidebar, "new_topic_template": new_topic_template, - "reply_owner": self.owner + "reply_owner": self.owner, }, - after_commit=True) + after_commit=True, + ) def after_delete(self): frappe.publish_realtime( - event="delete_message", - message = { - "reply_name": self.name - }, - after_commit=True) + event="delete_message", message={"reply_name": self.name}, after_commit=True + ) + @frappe.whitelist() def delete_message(reply_name): diff --git a/frappe/website/doctype/discussion_reply/test_discussion_reply.py b/frappe/website/doctype/discussion_reply/test_discussion_reply.py index 454852423b..855a6801b2 100644 --- a/frappe/website/doctype/discussion_reply/test_discussion_reply.py +++ b/frappe/website/doctype/discussion_reply/test_discussion_reply.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDiscussionReply(unittest.TestCase): pass diff --git a/frappe/website/doctype/discussion_topic/discussion_topic.py b/frappe/website/doctype/discussion_topic/discussion_topic.py index f69b424279..7eb661cd02 100644 --- a/frappe/website/doctype/discussion_topic/discussion_topic.py +++ b/frappe/website/doctype/discussion_topic/discussion_topic.py @@ -4,9 +4,11 @@ import frappe from frappe.model.document import Document + class DiscussionTopic(Document): pass + @frappe.whitelist() def submit_discussion(doctype, docname, reply, title, topic_name=None, reply_name=None): @@ -20,26 +22,27 @@ def submit_discussion(doctype, docname, reply, title, topic_name=None, reply_nam save_message(reply, topic_name) return topic_name - topic = frappe.get_doc({ - "doctype": "Discussion Topic", - "title": title, - "reference_doctype": doctype, - "reference_docname": docname - }) + topic = frappe.get_doc( + { + "doctype": "Discussion Topic", + "title": title, + "reference_doctype": doctype, + "reference_docname": docname, + } + ) topic.save(ignore_permissions=True) save_message(reply, topic.name) return topic.name + def save_message(reply, topic): - frappe.get_doc({ - "doctype": "Discussion Reply", - "reply": reply, - "topic": topic - }).save(ignore_permissions=True) + frappe.get_doc({"doctype": "Discussion Reply", "reply": reply, "topic": topic}).save( + ignore_permissions=True + ) + @frappe.whitelist(allow_guest=True) def get_docname(route): if not route: route = frappe.db.get_single_value("Website Settings", "home_page") return frappe.db.get_value("Web Page", {"route": route}, ["name"]) - diff --git a/frappe/website/doctype/discussion_topic/test_discussion_topic.py b/frappe/website/doctype/discussion_topic/test_discussion_topic.py index 56eaec14ae..7197352dae 100644 --- a/frappe/website/doctype/discussion_topic/test_discussion_topic.py +++ b/frappe/website/doctype/discussion_topic/test_discussion_topic.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestDiscussionTopic(unittest.TestCase): pass diff --git a/frappe/website/doctype/help_article/help_article.py b/frappe/website/doctype/help_article/help_article.py index ba6f79f0b9..e70de07703 100644 --- a/frappe/website/doctype/help_article/help_article.py +++ b/frappe/website/doctype/help_article/help_article.py @@ -2,28 +2,33 @@ # License: MIT. See LICENSE import frappe -from frappe.website.website_generator import WebsiteGenerator -from frappe.utils import is_markdown, markdown, cint -from frappe.website.utils import get_comment_list from frappe import _ +from frappe.utils import cint, is_markdown, markdown +from frappe.website.utils import get_comment_list +from frappe.website.website_generator import WebsiteGenerator + class HelpArticle(WebsiteGenerator): def validate(self): self.set_route() def set_route(self): - '''Set route from category and title if missing''' + """Set route from category and title if missing""" if not self.route: - self.route = '/'.join([frappe.get_value('Help Category', self.category, 'route'), - self.scrub(self.title)]) + self.route = "/".join( + [frappe.get_value("Help Category", self.category, "route"), self.scrub(self.title)] + ) def on_update(self): self.update_category() clear_cache() def update_category(self): - cnt = frappe.db.sql("""select count(*) from `tabHelp Article` - where category=%s and ifnull(published,0)=1""", self.category)[0][0] + cnt = frappe.db.sql( + """select count(*) from `tabHelp Article` + where category=%s and ifnull(published,0)=1""", + self.category, + )[0][0] cat = frappe.get_doc("Help Category", self.category) cat.help_articles = cnt cat.save() @@ -32,7 +37,7 @@ class HelpArticle(WebsiteGenerator): if is_markdown(context.content): context.content = markdown(context.content) context.login_required = True - context.category = frappe.get_doc('Help Category', self.category) + context.category = frappe.get_doc("Help Category", self.category) context.level_class = get_level_class(self.level) context.comment_list = get_comment_list(self.doctype, self.name) context.show_sidebar = True @@ -40,44 +45,43 @@ class HelpArticle(WebsiteGenerator): context.parents = self.get_parents(context) def get_parents(self, context): - return [{"title": context.category.category_name, "route":context.category.route}] + return [{"title": context.category.category_name, "route": context.category.route}] + def get_list_context(context=None): filters = dict(published=1) - category = frappe.db.get_value("Help Category", { "route": frappe.local.path }) + category = frappe.db.get_value("Help Category", {"route": frappe.local.path}) if category: - filters['category'] = category + filters["category"] = category list_context = frappe._dict( - title = category or _("Knowledge Base"), - get_level_class = get_level_class, - show_sidebar = True, - sidebar_items = get_sidebar_items(), - hide_filters = True, - filters = filters, - category = frappe.local.form_dict.category, - no_breadcrumbs = True + title=category or _("Knowledge Base"), + get_level_class=get_level_class, + show_sidebar=True, + sidebar_items=get_sidebar_items(), + hide_filters=True, + filters=filters, + category=frappe.local.form_dict.category, + no_breadcrumbs=True, ) - if frappe.local.form_dict.txt: list_context.blog_subtitle = _('Filtered by "{0}"').format(frappe.local.form_dict.txt) # # list_context.update(frappe.get_doc("Blog Settings", "Blog Settings").as_dict()) return list_context + def get_level_class(level): - return { - "Beginner": "green", - "Intermediate": "orange", - "Expert": "red" - }[level] + return {"Beginner": "green", "Intermediate": "orange", "Expert": "red"}[level] + def get_sidebar_items(): def _get(): - return frappe.db.sql("""select + return frappe.db.sql( + """select concat(category_name, " (", help_articles, ")") as title, concat('/', route) as route from @@ -85,20 +89,26 @@ def get_sidebar_items(): where ifnull(published,0)=1 and help_articles > 0 order by - help_articles desc""", as_dict=True) + help_articles desc""", + as_dict=True, + ) return frappe.cache().get_value("knowledge_base:category_sidebar", _get) + def clear_cache(): clear_website_cache() from frappe.website.utils import clear_cache + clear_cache() + def clear_website_cache(path=None): frappe.cache().delete_value("knowledge_base:category_sidebar") frappe.cache().delete_value("knowledge_base:faq") + @frappe.whitelist(allow_guest=True) def add_feedback(article, helpful): field = "helpful" @@ -106,4 +116,4 @@ def add_feedback(article, helpful): field = "not_helpful" value = cint(frappe.db.get_value("Help Article", article, field)) - frappe.db.set_value("Help Article", article, field, value+1, update_modified=False) \ No newline at end of file + frappe.db.set_value("Help Article", article, field, value + 1, update_modified=False) diff --git a/frappe/website/doctype/help_article/test_help_article.py b/frappe/website/doctype/help_article/test_help_article.py index ef988544e0..e7dec4080c 100644 --- a/frappe/website/doctype/help_article/test_help_article.py +++ b/frappe/website/doctype/help_article/test_help_article.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Help Article') + class TestHelpArticle(unittest.TestCase): pass diff --git a/frappe/website/doctype/help_category/help_category.py b/frappe/website/doctype/help_category/help_category.py index 69e15029a8..741b958aa6 100644 --- a/frappe/website/doctype/help_category/help_category.py +++ b/frappe/website/doctype/help_category/help_category.py @@ -2,17 +2,15 @@ # License: MIT. See LICENSE import frappe -from frappe.website.website_generator import WebsiteGenerator from frappe.website.doctype.help_article.help_article import clear_cache +from frappe.website.website_generator import WebsiteGenerator + class HelpCategory(WebsiteGenerator): - website = frappe._dict( - condition_field = "published", - page_title_field = "category_name" - ) + website = frappe._dict(condition_field="published", page_title_field="category_name") def before_insert(self): - self.published=1 + self.published = 1 def autoname(self): self.name = self.category_name @@ -22,7 +20,7 @@ class HelpCategory(WebsiteGenerator): def set_route(self): if not self.route: - self.route = 'kb/' + self.scrub(self.category_name) + self.route = "kb/" + self.scrub(self.category_name) def on_update(self): clear_cache() diff --git a/frappe/website/doctype/help_category/test_help_category.py b/frappe/website/doctype/help_category/test_help_category.py index e7b603561a..de7d288555 100644 --- a/frappe/website/doctype/help_category/test_help_category.py +++ b/frappe/website/doctype/help_category/test_help_category.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Help Category') + class TestHelpCategory(unittest.TestCase): pass diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index e2f583fd48..45c1a5ad38 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -2,16 +2,16 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE +import json import re import frappe from frappe import _ +from frappe.core.utils import find from frappe.model.document import Document -from frappe.utils import get_fullname, time_diff_in_hours, get_datetime +from frappe.utils import get_datetime, get_fullname, time_diff_in_hours from frappe.utils.user import get_system_managers from frappe.utils.verified_command import get_signed_params, verify_request -import json -from frappe.core.utils import find class PersonalDataDeletionRequest(Document): @@ -19,9 +19,7 @@ class PersonalDataDeletionRequest(Document): super().__init__(*args, **kwargs) self.user_data_fields = frappe.get_hooks("user_data_fields") - self.full_match_privacy_docs = [ - x for x in self.user_data_fields if x.get("redact_fields") - ] + self.full_match_privacy_docs = [x for x in self.user_data_fields if x.get("redact_fields")] self.partial_privacy_docs = [ x for x in self.user_data_fields if x.get("partial") or not x.get("redact_fields") ] @@ -131,7 +129,7 @@ class PersonalDataDeletionRequest(Document): "host_name": frappe.utils.get_url(), }, header=[_("Your account has been deleted"), "green"], - now=True + now=True, ) def add_deletion_steps(self): @@ -226,20 +224,19 @@ class PersonalDataDeletionRequest(Document): if filter_by_meta and filter_by_meta.fieldtype != "Link": if self.email in doc[filter_by]: - value = re.sub( - self.full_name_regex, self.anonymization_value_map["Data"], doc[filter_by] - ) + value = re.sub(self.full_name_regex, self.anonymization_value_map["Data"], doc[filter_by]) value = re.sub(self.email_regex, self.anon, value) self.anonymize_fields_dict[filter_by] = value frappe.db.set_value( - ref["doctype"], doc["name"], self.anonymize_fields_dict, modified_by="Administrator", + ref["doctype"], + doc["name"], + self.anonymize_fields_dict, + modified_by="Administrator", ) if ref.get("rename") and doc["name"] != self.anon: - frappe.rename_doc( - ref["doctype"], doc["name"], self.anon, force=True, show_alert=False - ) + frappe.rename_doc(ref["doctype"], doc["name"], self.anon, force=True, show_alert=False) def _anonymize_data(self, email=None, anon=None, set_data=True, commit=False): email = email or self.email @@ -253,17 +250,13 @@ class PersonalDataDeletionRequest(Document): self.full_match_doctypes = ( x for x in self.full_match_privacy_docs - if filter( - lambda x: x.document_type == x and x.status == "Pending", self.deletion_steps - ) + if filter(lambda x: x.document_type == x and x.status == "Pending", self.deletion_steps) ) self.partial_match_doctypes = ( x for x in self.partial_privacy_docs - if filter( - lambda x: x.document_type == x and x.status == "Pending", self.deletion_steps - ) + if filter(lambda x: x.document_type == x and x.status == "Pending", self.deletion_steps) ) for doctype in self.full_match_doctypes: @@ -326,9 +319,7 @@ class PersonalDataDeletionRequest(Document): update_predicate = f"SET {', '.join(match_fields)}" where_predicate = ( - "" - if doctype.get("strict") - else f"WHERE `{doctype.get('filter_by', 'owner')}` = %(email)s" + "" if doctype.get("strict") else f"WHERE `{doctype.get('filter_by', 'owner')}` = %(email)s" ) frappe.db.sql( @@ -340,23 +331,28 @@ class PersonalDataDeletionRequest(Document): def put_on_hold(self): self.db_set("status", "On Hold") + def process_data_deletion_request(): auto_account_deletion = frappe.db.get_single_value("Website Settings", "auto_account_deletion") if auto_account_deletion < 1: return - requests = frappe.get_all("Personal Data Deletion Request", - filters = { - "status": "Pending Approval" - }, - pluck="name") + requests = frappe.get_all( + "Personal Data Deletion Request", filters={"status": "Pending Approval"}, pluck="name" + ) for request in requests: doc = frappe.get_doc("Personal Data Deletion Request", request) if time_diff_in_hours(get_datetime(), doc.creation) >= auto_account_deletion: - doc.add_comment("Comment", _("The User record for this request has been auto-deleted due to inactivity by system admins.")) + doc.add_comment( + "Comment", + _( + "The User record for this request has been auto-deleted due to inactivity by system admins." + ), + ) doc.trigger_data_deletion() + def remove_unverified_record(): frappe.db.sql( """ @@ -365,6 +361,7 @@ def remove_unverified_record(): AND `creation` < (NOW() - INTERVAL '7' DAY)""" ) + @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): if not verify_request(): @@ -380,8 +377,9 @@ def confirm_deletion(email, name, host_name): frappe.db.commit() frappe.respond_as_web_page( _("Confirmed"), - _("The process for deletion of {0} data associated with {1} has been initiated.") - .format(host_name, email), + _("The process for deletion of {0} data associated with {1} has been initiated.").format( + host_name, email + ), indicator_color="green", ) diff --git a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py index 675a891130..01cde6e961 100644 --- a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py @@ -1,15 +1,17 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +from datetime import datetime, timedelta + +import frappe from frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request import ( - remove_unverified_record, process_data_deletion_request + process_data_deletion_request, + remove_unverified_record, ) from frappe.website.doctype.personal_data_download_request.test_personal_data_download_request import ( - create_user_if_not_exists + create_user_if_not_exists, ) -from datetime import datetime, timedelta class TestPersonalDataDeletionRequest(unittest.TestCase): @@ -43,7 +45,7 @@ class TestPersonalDataDeletionRequest(unittest.TestCase): self.assertEqual(deleted_user.phone, self.delete_request.anonymization_value_map["Phone"]) self.assertEqual( deleted_user.birth_date, - datetime.strptime(self.delete_request.anonymization_value_map["Date"], "%Y-%m-%d").date() + datetime.strptime(self.delete_request.anonymization_value_map["Date"], "%Y-%m-%d").date(), ) self.assertEqual(self.delete_request.status, "Deleted") @@ -55,9 +57,7 @@ class TestPersonalDataDeletionRequest(unittest.TestCase): self.delete_request.db_set("status", "Pending Verification") remove_unverified_record() - self.assertFalse( - frappe.db.exists("Personal Data Deletion Request", self.delete_request.name) - ) + self.assertFalse(frappe.db.exists("Personal Data Deletion Request", self.delete_request.name)) def test_process_auto_request(self): frappe.db.set_value("Website Settings", None, "auto_account_deletion", "1") diff --git a/frappe/website/doctype/personal_data_deletion_step/personal_data_deletion_step.py b/frappe/website/doctype/personal_data_deletion_step/personal_data_deletion_step.py index 6259f70a24..222d19c27e 100644 --- a/frappe/website/doctype/personal_data_deletion_step/personal_data_deletion_step.py +++ b/frappe/website/doctype/personal_data_deletion_step/personal_data_deletion_step.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class PersonalDataDeletionStep(Document): pass diff --git a/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py b/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py index 845d28ba6e..f04234dbe5 100644 --- a/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py +++ b/frappe/website/doctype/personal_data_download_request/personal_data_download_request.py @@ -2,55 +2,69 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe import json + +import frappe from frappe import _ from frappe.model.document import Document from frappe.utils.verified_command import get_signed_params + class PersonalDataDownloadRequest(Document): def after_insert(self): personal_data = get_user_data(self.user) - frappe.enqueue_doc(self.doctype, self.name, 'generate_file_and_send_mail', - queue='short', personal_data=personal_data, now=frappe.flags.in_test) + frappe.enqueue_doc( + self.doctype, + self.name, + "generate_file_and_send_mail", + queue="short", + personal_data=personal_data, + now=frappe.flags.in_test, + ) def generate_file_and_send_mail(self, personal_data): """generate the file link for download""" - user_name = self.user_name.replace(' ','-') - f = frappe.get_doc({ - 'doctype': 'File', - 'file_name': 'Personal-Data-'+user_name+'-'+self.name+'.json', - "attached_to_doctype": 'Personal Data Download Request', - "attached_to_name": self.name, - 'content': str(personal_data), - 'is_private': 1 - }) + user_name = self.user_name.replace(" ", "-") + f = frappe.get_doc( + { + "doctype": "File", + "file_name": "Personal-Data-" + user_name + "-" + self.name + ".json", + "attached_to_doctype": "Personal Data Download Request", + "attached_to_name": self.name, + "content": str(personal_data), + "is_private": 1, + } + ) f.save(ignore_permissions=True) - file_link = frappe.utils.get_url("/api/method/frappe.utils.file_manager.download_file") +\ - "?" + get_signed_params({"file_url": f.file_url}) + file_link = ( + frappe.utils.get_url("/api/method/frappe.utils.file_manager.download_file") + + "?" + + get_signed_params({"file_url": f.file_url}) + ) host_name = frappe.local.site frappe.sendmail( recipients=self.user, subject=_("Download Your Data"), template="download_data", args={ - 'user': self.user, - 'user_name': self.user_name, - 'link': file_link, - 'host_name': host_name + "user": self.user, + "user_name": self.user_name, + "link": file_link, + "host_name": host_name, }, - header=[_("Download Your Data"), "green"] + header=[_("Download Your Data"), "green"], ) + def get_user_data(user): - """ returns user data not linked to User doctype """ + """returns user data not linked to User doctype""" hooks = frappe.get_hooks("user_data_fields") data = {} for hook in hooks: - d = data.get(hook.get("doctype"),[]) + d = data.get(hook.get("doctype"), []) d += frappe.get_all(hook.get("doctype"), {hook.get("filter_by", "owner"): user}, ["*"]) if d: - data.update({ hook.get("doctype"):d }) - return json.dumps(data, indent=2, default=str) \ No newline at end of file + data.update({hook.get("doctype"): d}) + return json.dumps(data, indent=2, default=str) diff --git a/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py b/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py index e038279437..4f115325df 100644 --- a/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py +++ b/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py @@ -1,61 +1,69 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe -import unittest import json -from frappe.website.doctype.personal_data_download_request.personal_data_download_request import get_user_data +import unittest + +import frappe from frappe.contacts.doctype.contact.contact import get_contact_name from frappe.core.doctype.user.user import create_contact +from frappe.website.doctype.personal_data_download_request.personal_data_download_request import ( + get_user_data, +) + class TestRequestPersonalData(unittest.TestCase): def setUp(self): - create_user_if_not_exists(email='test_privacy@example.com') + create_user_if_not_exists(email="test_privacy@example.com") def tearDown(self): frappe.db.delete("Personal Data Download Request") def test_user_data_creation(self): - user_data = json.loads(get_user_data('test_privacy@example.com')) - contact_name = get_contact_name('test_privacy@example.com') - expected_data = {'Contact': frappe.get_all('Contact', {"name": contact_name}, ["*"])} + user_data = json.loads(get_user_data("test_privacy@example.com")) + contact_name = get_contact_name("test_privacy@example.com") + expected_data = {"Contact": frappe.get_all("Contact", {"name": contact_name}, ["*"])} expected_data = json.loads(json.dumps(expected_data, default=str)) - self.assertEqual({'Contact': user_data['Contact']}, expected_data) + self.assertEqual({"Contact": user_data["Contact"]}, expected_data) def test_file_and_email_creation(self): - frappe.set_user('test_privacy@example.com') - download_request = frappe.get_doc({ - "doctype": 'Personal Data Download Request', - 'user': 'test_privacy@example.com' - }) + frappe.set_user("test_privacy@example.com") + download_request = frappe.get_doc( + {"doctype": "Personal Data Download Request", "user": "test_privacy@example.com"} + ) download_request.save(ignore_permissions=True) - frappe.set_user('Administrator') + frappe.set_user("Administrator") - file_count = frappe.db.count('File', { - 'attached_to_doctype':'Personal Data Download Request', - 'attached_to_name': download_request.name - }) + file_count = frappe.db.count( + "File", + { + "attached_to_doctype": "Personal Data Download Request", + "attached_to_name": download_request.name, + }, + ) self.assertEqual(file_count, 1) - email_queue = frappe.get_all('Email Queue', - fields=['message'], - order_by="creation DESC", - limit=1) + email_queue = frappe.get_all( + "Email Queue", fields=["message"], order_by="creation DESC", limit=1 + ) self.assertTrue("Subject: Download Your Data" in email_queue[0].message) frappe.db.delete("Email Queue") -def create_user_if_not_exists(email, first_name = None): + +def create_user_if_not_exists(email, first_name=None): frappe.delete_doc_if_exists("User", email) - user = frappe.get_doc({ - "doctype": "User", - "user_type": "Website User", - "email": email, - "send_welcome_email": 0, - "first_name": first_name or email.split("@")[0], - "birth_date": frappe.utils.now_datetime() - }).insert(ignore_permissions=True) + user = frappe.get_doc( + { + "doctype": "User", + "user_type": "Website User", + "email": email, + "send_welcome_email": 0, + "first_name": first_name or email.split("@")[0], + "birth_date": frappe.utils.now_datetime(), + } + ).insert(ignore_permissions=True) create_contact(user=user) diff --git a/frappe/website/doctype/portal_menu_item/portal_menu_item.py b/frappe/website/doctype/portal_menu_item/portal_menu_item.py index fadd3e276a..bb74a22249 100644 --- a/frappe/website/doctype/portal_menu_item/portal_menu_item.py +++ b/frappe/website/doctype/portal_menu_item/portal_menu_item.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class PortalMenuItem(Document): pass diff --git a/frappe/website/doctype/portal_settings/portal_settings.py b/frappe/website/doctype/portal_settings/portal_settings.py index 315c7aaf15..4749cbfb4e 100644 --- a/frappe/website/doctype/portal_settings/portal_settings.py +++ b/frappe/website/doctype/portal_settings/portal_settings.py @@ -5,31 +5,32 @@ import frappe from frappe.model.document import Document + class PortalSettings(Document): def add_item(self, item): - '''insert new portal menu item if route is not set, or role is different''' - exists = [d for d in self.get('menu', []) if d.get('route')==item.get('route')] - if exists and item.get('role'): - if exists[0].role != item.get('role'): - exists[0].role = item.get('role') + """insert new portal menu item if route is not set, or role is different""" + exists = [d for d in self.get("menu", []) if d.get("route") == item.get("route")] + if exists and item.get("role"): + if exists[0].role != item.get("role"): + exists[0].role = item.get("role") return True elif not exists: - item['enabled'] = 1 - self.append('menu', item) + item["enabled"] = 1 + self.append("menu", item) return True @frappe.whitelist() def reset(self): - '''Restore defaults''' + """Restore defaults""" self.menu = [] self.sync_menu() def sync_menu(self): - '''Sync portal menu items''' + """Sync portal menu items""" dirty = False - for item in frappe.get_hooks('standard_portal_menu_items'): - if item.get('role') and not frappe.db.exists("Role", item.get('role')): - frappe.get_doc({"doctype": "Role", "role_name": item.get('role'), "desk_access": 0}).insert() + for item in frappe.get_hooks("standard_portal_menu_items"): + if item.get("role") and not frappe.db.exists("Role", item.get("role")): + frappe.get_doc({"doctype": "Role", "role_name": item.get("role"), "desk_access": 0}).insert() if self.add_item(item): dirty = True @@ -42,11 +43,11 @@ class PortalSettings(Document): def clear_cache(self): # make js and css # clear web cache (for menus!) - frappe.clear_cache(user='Guest') + frappe.clear_cache(user="Guest") from frappe.website.utils import clear_cache + clear_cache() # clears role based home pages frappe.clear_cache() - diff --git a/frappe/website/doctype/portal_settings/test_portal_settings.py b/frappe/website/doctype/portal_settings/test_portal_settings.py index 3945ac04ec..c59e66d419 100644 --- a/frappe/website/doctype/portal_settings/test_portal_settings.py +++ b/frappe/website/doctype/portal_settings/test_portal_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestPortalSettings(unittest.TestCase): pass diff --git a/frappe/website/doctype/social_link_settings/social_link_settings.py b/frappe/website/doctype/social_link_settings/social_link_settings.py index 11777d3eba..1de408efed 100644 --- a/frappe/website/doctype/social_link_settings/social_link_settings.py +++ b/frappe/website/doctype/social_link_settings/social_link_settings.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class SocialLinkSettings(Document): pass diff --git a/frappe/website/doctype/top_bar_item/top_bar_item.py b/frappe/website/doctype/top_bar_item/top_bar_item.py index 8896aa404b..5730dc7962 100644 --- a/frappe/website/doctype/top_bar_item/top_bar_item.py +++ b/frappe/website/doctype/top_bar_item/top_bar_item.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class TopBarItem(Document): - pass \ No newline at end of file + pass diff --git a/frappe/website/doctype/web_form/test_web_form.py b/frappe/website/doctype/web_form/test_web_form.py index 3e05b221d8..f94a2a19e7 100644 --- a/frappe/website/doctype/web_form/test_web_form.py +++ b/frappe/website/doctype/web_form/test_web_form.py @@ -1,12 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe -import unittest, json +import json +import unittest -from frappe.website.serve import get_response_content +import frappe from frappe.website.doctype.web_form.web_form import accept +from frappe.website.serve import get_response_content + +test_dependencies = ["Web Form"] -test_dependencies = ['Web Form'] class TestWebForm(unittest.TestCase): def setUp(self): @@ -25,48 +27,49 @@ class TestWebForm(unittest.TestCase): frappe.set_user("Administrator") doc = { - 'doctype': 'Event', - 'subject': '_Test Event Web Form', - 'description': '_Test Event Description', - 'starts_on': '2014-09-09' + "doctype": "Event", + "subject": "_Test Event Web Form", + "description": "_Test Event Description", + "starts_on": "2014-09-09", } frappe.form_dict.web_form = "manage-events" frappe.form_dict.data = json.dumps(doc) - frappe.local.request_ip = '127.0.0.1' + frappe.local.request_ip = "127.0.0.1" - accept(web_form='manage-events', data=json.dumps(doc)) + accept(web_form="manage-events", data=json.dumps(doc)) - self.event_name = frappe.db.get_value("Event", - {"subject": "_Test Event Web Form"}) + self.event_name = frappe.db.get_value("Event", {"subject": "_Test Event Web Form"}) self.assertTrue(self.event_name) def test_edit(self): self.test_accept() - doc={ - 'doctype': 'Event', - 'subject': '_Test Event Web Form', - 'description': '_Test Event Description 1', - 'starts_on': '2014-09-09', - 'name': self.event_name + doc = { + "doctype": "Event", + "subject": "_Test Event Web Form", + "description": "_Test Event Description 1", + "starts_on": "2014-09-09", + "name": self.event_name, } - self.assertNotEqual(frappe.db.get_value("Event", - self.event_name, "description"), doc.get('description')) + self.assertNotEqual( + frappe.db.get_value("Event", self.event_name, "description"), doc.get("description") + ) - frappe.form_dict.web_form = 'manage-events' + frappe.form_dict.web_form = "manage-events" frappe.form_dict.docname = self.event_name frappe.form_dict.data = json.dumps(doc) - accept(web_form='manage-events', docname=self.event_name, data=json.dumps(doc)) + accept(web_form="manage-events", docname=self.event_name, data=json.dumps(doc)) - self.assertEqual(frappe.db.get_value("Event", - self.event_name, "description"), doc.get('description')) + self.assertEqual( + frappe.db.get_value("Event", self.event_name, "description"), doc.get("description") + ) def test_webform_render(self): - content = get_response_content('request-data') - self.assertIn('

Request Data

', content) + content = get_response_content("request-data") + self.assertIn("

Request Data

", content) self.assertIn('data-doctype="Web Form"', content) self.assertIn('data-path="request-data"', content) self.assertIn('source-type="Generator"', content) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index d891ceb205..5740533ecc 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -3,6 +3,7 @@ import json import os + import frappe from frappe import _, scrub from frappe.core.doctype.file.file import get_max_file_size, remove_file_by_url @@ -10,15 +11,14 @@ from frappe.custom.doctype.customize_form.customize_form import docfield_propert from frappe.desk.form.meta import get_code_files_via_hooks from frappe.integrations.utils import get_payment_gateway_controller from frappe.modules.utils import export_module_json, get_doc_module +from frappe.rate_limiter import rate_limit from frappe.utils import cstr from frappe.website.utils import get_comment_list from frappe.website.website_generator import WebsiteGenerator -from frappe.rate_limiter import rate_limit + class WebForm(WebsiteGenerator): - website = frappe._dict( - no_cache = 1 - ) + website = frappe._dict(no_cache=1) def onload(self): super(WebForm, self).onload() @@ -29,10 +29,18 @@ class WebForm(WebsiteGenerator): super(WebForm, self).validate() if not self.module: - self.module = frappe.db.get_value('DocType', self.doc_type, 'module') - - if (not (frappe.flags.in_install or frappe.flags.in_patch or frappe.flags.in_test or frappe.flags.in_fixtures) - and self.is_standard and not frappe.conf.developer_mode): + self.module = frappe.db.get_value("DocType", self.doc_type, "module") + + if ( + not ( + frappe.flags.in_install + or frappe.flags.in_patch + or frappe.flags.in_test + or frappe.flags.in_fixtures + ) + and self.is_standard + and not frappe.conf.developer_mode + ): frappe.throw(_("You need to be in developer mode to edit a Standard Web Form")) if not frappe.flags.in_import: @@ -42,8 +50,9 @@ class WebForm(WebsiteGenerator): self.validate_payment_amount() def validate_fields(self): - '''Validate all fields are present''' + """Validate all fields are present""" from frappe.model import no_value_fields + missing = [] meta = frappe.get_meta(self.doc_type) for df in self.web_form_fields: @@ -51,7 +60,7 @@ class WebForm(WebsiteGenerator): missing.append(df.fieldname) if missing: - frappe.throw(_('Following fields are missing:') + '
' + '
'.join(missing)) + frappe.throw(_("Following fields are missing:") + "
" + "
".join(missing)) def validate_payment_amount(self): if self.amount_based_on_field and not self.amount_field: @@ -59,14 +68,13 @@ class WebForm(WebsiteGenerator): elif not self.amount_based_on_field and not self.amount > 0: frappe.throw(_("Amount must be greater than 0.")) - def reset_field_parent(self): - '''Convert link fields to select with names as options''' + """Convert link fields to select with names as options""" for df in self.web_form_fields: df.parent = self.doc_type def use_meta_fields(self): - '''Override default properties for standard web forms''' + """Override default properties for standard web forms""" meta = frappe.get_meta(self.doc_type) for df in self.web_form_fields: @@ -76,42 +84,52 @@ class WebForm(WebsiteGenerator): continue for prop in docfield_properties: - if df.fieldtype==meta_df.fieldtype and prop not in ("idx", - "reqd", "default", "description", "options", - "hidden", "read_only", "label"): + if df.fieldtype == meta_df.fieldtype and prop not in ( + "idx", + "reqd", + "default", + "description", + "options", + "hidden", + "read_only", + "label", + ): df.set(prop, meta_df.get(prop)) - # TODO translate options of Select fields like Country # export def on_update(self): """ - Writes the .txt for this page and if write_content is checked, - it will write out a .html file + Writes the .txt for this page and if write_content is checked, + it will write out a .html file """ path = export_module_json(self, self.is_standard, self.module) if path: # js - if not os.path.exists(path + '.js'): - with open(path + '.js', 'w') as f: - f.write("""frappe.ready(function() { + if not os.path.exists(path + ".js"): + with open(path + ".js", "w") as f: + f.write( + """frappe.ready(function() { // bind events here -})""") +})""" + ) # py - if not os.path.exists(path + '.py'): - with open(path + '.py', 'w') as f: - f.write("""import frappe + if not os.path.exists(path + ".py"): + with open(path + ".py", "w") as f: + f.write( + """import frappe def get_context(context): # do your magic here pass -""") +""" + ) def get_context(self, context): - '''Build context to render the `web_form.html` template''' + """Build context to render the `web_form.html` template""" self.set_web_form_module() doc, delimeter = make_route_string(frappe.form_dict) @@ -120,10 +138,16 @@ def get_context(context): # check permissions if frappe.session.user == "Guest" and frappe.form_dict.name: - frappe.throw(_("You need to be logged in to access this {0}.").format(self.doc_type), frappe.PermissionError) + frappe.throw( + _("You need to be logged in to access this {0}.").format(self.doc_type), frappe.PermissionError + ) - if frappe.form_dict.name and not self.has_web_form_permission(self.doc_type, frappe.form_dict.name): - frappe.throw(_("You don't have the permissions to access this document"), frappe.PermissionError) + if frappe.form_dict.name and not self.has_web_form_permission( + self.doc_type, frappe.form_dict.name + ): + frappe.throw( + _("You don't have the permissions to access this document"), frappe.PermissionError + ) self.reset_field_parent() @@ -137,8 +161,10 @@ def get_context(context): # list data is queried via JS context.is_list = True else: - if frappe.session.user != 'Guest' and not frappe.form_dict.name: - frappe.form_dict.name = frappe.db.get_value(self.doc_type, {"owner": frappe.session.user}, "name") + if frappe.session.user != "Guest" and not frappe.form_dict.name: + frappe.form_dict.name = frappe.db.get_value( + self.doc_type, {"owner": frappe.session.user}, "name" + ) if not frappe.form_dict.name: # only a single doc allowed and no existing doc, hence new @@ -155,14 +181,16 @@ def get_context(context): context.parents = self.get_parents(context) if self.breadcrumbs: - context.parents = frappe.safe_eval(self.breadcrumbs, { "_": _ }) + context.parents = frappe.safe_eval(self.breadcrumbs, {"_": _}) - context.has_header = ((frappe.form_dict.name or frappe.form_dict.new) - and (frappe.session.user!="Guest" or not self.login_required)) + context.has_header = (frappe.form_dict.name or frappe.form_dict.new) and ( + frappe.session.user != "Guest" or not self.login_required + ) if context.success_message: - context.success_message = frappe.db.escape(context.success_message.replace("\n", - "
")).strip("'") + context.success_message = frappe.db.escape(context.success_message.replace("\n", "
")).strip( + "'" + ) self.add_custom_context_and_script(context) if not context.max_attachment_size: @@ -172,16 +200,16 @@ def get_context(context): self.load_translations(context) def load_translations(self, context): - translated_messages = frappe.translate.get_dict('doctype', self.doc_type) + translated_messages = frappe.translate.get_dict("doctype", self.doc_type) # Sr is not added by default, had to be added manually - translated_messages['Sr'] = _('Sr') + translated_messages["Sr"] = _("Sr") context.translated_messages = frappe.as_json(translated_messages) def load_document(self, context): - '''Load document `doc` and `layout` properties for template''' + """Load document `doc` and `layout` properties for template""" if frappe.form_dict.name or frappe.form_dict.new: context.layout = self.get_layout() - context.parents = [{"route": self.route, "label": _(self.title) }] + context.parents = [{"route": self.route, "label": _(self.title)}] if frappe.form_dict.name: context.doc = frappe.get_doc(self.doc_type, frappe.form_dict.name) @@ -192,12 +220,18 @@ def get_context(context): context.reference_name = context.doc.name if self.show_attachments: - context.attachments = frappe.get_all('File', filters= {"attached_to_name": context.reference_name, "attached_to_doctype": context.reference_doctype, "is_private": 0}, - fields=['file_name','file_url', 'file_size']) + context.attachments = frappe.get_all( + "File", + filters={ + "attached_to_name": context.reference_name, + "attached_to_doctype": context.reference_doctype, + "is_private": 0, + }, + fields=["file_name", "file_url", "file_size"], + ) if self.allow_comments: - context.comment_list = get_comment_list(context.doc.doctype, - context.doc.name) + context.comment_list = get_comment_list(context.doc.doctype, context.doc.name) def get_payment_gateway_url(self, doc): if self.accept_payment: @@ -209,6 +243,7 @@ def get_context(context): amount = doc.get(self.amount_field) from decimal import Decimal + if amount is None or Decimal(amount) <= 0: return frappe.utils.get_url(self.success_url or self.route) @@ -222,55 +257,58 @@ def get_context(context): "payer_name": frappe.utils.get_fullname(frappe.session.user), "order_id": doc.name, "currency": self.currency, - "redirect_to": frappe.utils.get_url(self.success_url or self.route) + "redirect_to": frappe.utils.get_url(self.success_url or self.route), } # Redirect the user to this url return controller.get_payment_url(**payment_details) def add_custom_context_and_script(self, context): - '''Update context from module if standard and append script''' + """Update context from module if standard and append script""" if self.web_form_module: new_context = self.web_form_module.get_context(context) if new_context: context.update(new_context) - js_path = os.path.join(os.path.dirname(self.web_form_module.__file__), scrub(self.name) + '.js') + js_path = os.path.join(os.path.dirname(self.web_form_module.__file__), scrub(self.name) + ".js") if os.path.exists(js_path): - script = frappe.render_template(open(js_path, 'r').read(), context) + script = frappe.render_template(open(js_path, "r").read(), context) for path in get_code_files_via_hooks("webform_include_js", context.doc_type): - custom_js = frappe.render_template(open(path, 'r').read(), context) + custom_js = frappe.render_template(open(path, "r").read(), context) script = "\n\n".join([script, custom_js]) context.script = script - css_path = os.path.join(os.path.dirname(self.web_form_module.__file__), scrub(self.name) + '.css') + css_path = os.path.join( + os.path.dirname(self.web_form_module.__file__), scrub(self.name) + ".css" + ) if os.path.exists(css_path): - style = open(css_path, 'r').read() + style = open(css_path, "r").read() for path in get_code_files_via_hooks("webform_include_css", context.doc_type): - custom_css = open(path, 'r').read() + custom_css = open(path, "r").read() style = "\n\n".join([style, custom_css]) context.style = style def get_layout(self): layout = [] + def add_page(df=None): - new_page = {'sections': []} + new_page = {"sections": []} layout.append(new_page) - if df and df.fieldtype=='Page Break': + if df and df.fieldtype == "Page Break": new_page.update(df.as_dict()) return new_page def add_section(df=None): - new_section = {'columns': []} + new_section = {"columns": []} if layout: - layout[-1]['sections'].append(new_section) - if df and df.fieldtype=='Section Break': + layout[-1]["sections"].append(new_section) + if df and df.fieldtype == "Section Break": new_section.update(df.as_dict()) return new_section @@ -278,7 +316,7 @@ def get_context(context): def add_column(df=None): new_col = [] if layout: - layout[-1]['sections'][-1]['columns'].append(new_col) + layout[-1]["sections"][-1]["columns"].append(new_col) return new_col @@ -286,19 +324,19 @@ def get_context(context): for df in self.web_form_fields: # breaks - if df.fieldtype=='Page Break': + if df.fieldtype == "Page Break": page = add_page(df) section, column = None, None - if df.fieldtype=='Section Break': + if df.fieldtype == "Section Break": section = add_section(df) column = None - if df.fieldtype=='Column Break': + if df.fieldtype == "Column Break": column = add_column(df) # input - if df.fieldtype not in ('Section Break', 'Column Break', 'Page Break'): + if df.fieldtype not in ("Section Break", "Column Break", "Page Break"): if not page: page = add_page() section, column = None, None @@ -322,7 +360,7 @@ def get_context(context): return parents def set_web_form_module(self): - '''Get custom web form module if exists''' + """Get custom web form module if exists""" self.web_form_module = self.get_web_form_module() def get_web_form_module(self): @@ -330,28 +368,31 @@ def get_context(context): return get_doc_module(self.module, self.doctype, self.name) def validate_mandatory(self, doc): - '''Validate mandatory web form fields''' + """Validate mandatory web form fields""" missing = [] for f in self.web_form_fields: - if f.reqd and doc.get(f.fieldname) in (None, [], ''): + if f.reqd and doc.get(f.fieldname) in (None, [], ""): missing.append(f) if missing: - frappe.throw(_('Mandatory Information missing:') + '

' - + '
'.join('{0} ({1})'.format(d.label, d.fieldtype) for d in missing)) + frappe.throw( + _("Mandatory Information missing:") + + "

" + + "
".join("{0} ({1})".format(d.label, d.fieldtype) for d in missing) + ) def allow_website_search_indexing(self): return False - def has_web_form_permission(self, doctype, name, ptype='read'): - if frappe.session.user=="Guest": + def has_web_form_permission(self, doctype, name, ptype="read"): + if frappe.session.user == "Guest": return False if self.apply_document_permissions: return frappe.get_doc(doctype, name).has_permission() # owner matches - elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user: + elif frappe.db.get_value(doctype, name, "owner") == frappe.session.user: return True elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype): @@ -365,9 +406,9 @@ def get_context(context): @frappe.whitelist(allow_guest=True) -@rate_limit(key='web_form', limit=5, seconds=60, methods=['POST']) +@rate_limit(key="web_form", limit=5, seconds=60, methods=["POST"]) def accept(web_form, data, docname=None, for_payment=False): - '''Save the web form''' + """Save the web form""" data = frappe._dict(json.loads(data)) for_payment = frappe.parse_json(for_payment) @@ -395,11 +436,11 @@ def accept(web_form, data, docname=None, for_payment=False): df = meta.get_field(fieldname) value = data.get(fieldname, None) - if df and df.fieldtype in ('Attach', 'Attach Image'): - if value and 'data:' and 'base64' in value: + if df and df.fieldtype in ("Attach", "Attach Image"): + if value and "data:" and "base64" in value: files.append((fieldname, value)) if not doc.name: - doc.set(fieldname, '') + doc.set(fieldname, "") continue elif not value and doc.get(fieldname): @@ -409,7 +450,7 @@ def accept(web_form, data, docname=None, for_payment=False): if for_payment: web_form.validate_mandatory(doc) - doc.run_method('validate_payment') + doc.run_method("validate_payment") if doc.name: if web_form.has_web_form_permission(doc.doctype, doc.name, "write"): @@ -420,12 +461,12 @@ def accept(web_form, data, docname=None, for_payment=False): else: # insert - if web_form.login_required and frappe.session.user=="Guest": + if web_form.login_required and frappe.session.user == "Guest": frappe.throw(_("You must login to submit this form")) ignore_mandatory = True if files else False - doc.insert(ignore_permissions = True, ignore_mandatory = ignore_mandatory) + doc.insert(ignore_permissions=True, ignore_mandatory=ignore_mandatory) # add files if files: @@ -437,27 +478,29 @@ def accept(web_form, data, docname=None, for_payment=False): remove_file_by_url(doc.get(fieldname), doctype=doc.doctype, name=doc.name) # save new file - filename, dataurl = filedata.split(',', 1) - _file = frappe.get_doc({ - "doctype": "File", - "file_name": filename, - "attached_to_doctype": doc.doctype, - "attached_to_name": doc.name, - "content": dataurl, - "decode": True}) + filename, dataurl = filedata.split(",", 1) + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": filename, + "attached_to_doctype": doc.doctype, + "attached_to_name": doc.name, + "content": dataurl, + "decode": True, + } + ) _file.save() # update values doc.set(fieldname, _file.file_url) - doc.save(ignore_permissions = True) + doc.save(ignore_permissions=True) if files_to_delete: for f in files_to_delete: if f: remove_file_by_url(f, doctype=doc.doctype, name=doc.name) - frappe.flags.web_form_doc = doc if for_payment: @@ -465,6 +508,7 @@ def accept(web_form, data, docname=None, for_payment=False): else: return doc + @frappe.whitelist() def delete(web_form_name, docname): web_form = frappe.get_doc("Web Form", web_form_name) @@ -496,7 +540,9 @@ def delete_multiple(web_form_name, docnames): frappe.delete_doc(web_form.doc_type, docname, ignore_permissions=True) if restricted_docnames: - raise frappe.PermissionError("You do not have permisssion to delete " + ", ".join(restricted_docnames)) + raise frappe.PermissionError( + "You do not have permisssion to delete " + ", ".join(restricted_docnames) + ) def check_webform_perm(doctype, name): @@ -505,37 +551,40 @@ def check_webform_perm(doctype, name): if doc.has_webform_permission(): return True + @frappe.whitelist(allow_guest=True) def get_web_form_filters(web_form_name): web_form = frappe.get_doc("Web Form", web_form_name) return [field for field in web_form.web_form_fields if field.show_in_filter] + def make_route_string(parameters): route_string = "" - delimeter = '?' + delimeter = "?" if isinstance(parameters, dict): for key in parameters: if key != "web_form_name": route_string += route_string + delimeter + key + "=" + cstr(parameters[key]) - delimeter = '&' + delimeter = "&" return (route_string, delimeter) + @frappe.whitelist(allow_guest=True) def get_form_data(doctype, docname=None, web_form_name=None): - web_form = frappe.get_doc('Web Form', web_form_name) + web_form = frappe.get_doc("Web Form", web_form_name) - if web_form.login_required and frappe.session.user == 'Guest': + if web_form.login_required and frappe.session.user == "Guest": frappe.throw(_("Not Permitted"), frappe.PermissionError) out = frappe._dict() out.web_form = web_form - if frappe.session.user != 'Guest' and not docname and not web_form.allow_multiple: + if frappe.session.user != "Guest" and not docname and not web_form.allow_multiple: docname = frappe.db.get_value(doctype, {"owner": frappe.session.user}, "name") if docname: doc = frappe.get_doc(doctype, docname) - if web_form.has_web_form_permission(doctype, docname, ptype='read'): + if web_form.has_web_form_permission(doctype, docname, ptype="read"): out.doc = doc else: frappe.throw(_("Not permitted"), frappe.PermissionError) @@ -549,13 +598,12 @@ def get_form_data(doctype, docname=None, web_form_name=None): if field.fieldtype == "Link": field.fieldtype = "Autocomplete" field.options = get_link_options( - web_form_name, - field.options, - field.allow_read_on_all_link_options + web_form_name, field.options, field.allow_read_on_all_link_options ) return out + @frappe.whitelist() def get_in_list_view_fields(doctype): meta = frappe.get_meta(doctype) @@ -564,32 +612,33 @@ def get_in_list_view_fields(doctype): if meta.title_field: fields.append(meta.title_field) else: - fields.append('name') + fields.append("name") - if meta.has_field('status'): - fields.append('status') + if meta.has_field("status"): + fields.append("status") fields += [df.fieldname for df in meta.fields if df.in_list_view and df.fieldname not in fields] def get_field_df(fieldname): - if fieldname == 'name': - return { 'label': 'Name', 'fieldname': 'name', 'fieldtype': 'Data' } + if fieldname == "name": + return {"label": "Name", "fieldname": "name", "fieldtype": "Data"} return meta.get_field(fieldname).as_dict() return [get_field_df(f) for f in fields] + @frappe.whitelist(allow_guest=True) def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=False): web_form_doc = frappe.get_doc("Web Form", web_form_name) doctype_validated = False - limited_to_user = False + limited_to_user = False if web_form_doc.login_required: # check if frappe session user is not guest or admin - if frappe.session.user != 'Guest': + if frappe.session.user != "Guest": doctype_validated = True if not allow_read_on_all_link_options: - limited_to_user = True + limited_to_user = True else: for field in web_form_doc.web_form_fields: @@ -601,14 +650,16 @@ def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=Fals link_options, filters = [], {} if limited_to_user: - filters = {"owner":frappe.session.user} + filters = {"owner": frappe.session.user} - fields = ['name as value'] + fields = ["name as value"] - title_field = frappe.db.get_value('DocType', doctype, 'title_field', cache=1) - show_title_field_in_link = frappe.db.get_value('DocType', doctype, 'show_title_field_in_link', cache=1) == 1 + title_field = frappe.db.get_value("DocType", doctype, "title_field", cache=1) + show_title_field_in_link = ( + frappe.db.get_value("DocType", doctype, "show_title_field_in_link", cache=1) == 1 + ) if title_field and show_title_field_in_link: - fields.append(f'{title_field} as label') + fields.append(f"{title_field} as label") link_options = frappe.get_all(doctype, filters, fields) @@ -618,4 +669,4 @@ def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=Fals return "\n".join([doc.value for doc in link_options]) else: - raise frappe.PermissionError('Not Allowed, {0}'.format(doctype)) + raise frappe.PermissionError("Not Allowed, {0}".format(doctype)) diff --git a/frappe/website/doctype/web_form_field/web_form_field.py b/frappe/website/doctype/web_form_field/web_form_field.py index 342b329164..51e13aa3a7 100644 --- a/frappe/website/doctype/web_form_field/web_form_field.py +++ b/frappe/website/doctype/web_form_field/web_form_field.py @@ -4,5 +4,6 @@ import frappe from frappe.model.document import Document + class WebFormField(Document): pass diff --git a/frappe/website/doctype/web_page/test_web_page.py b/frappe/website/doctype/web_page/test_web_page.py index aebc6a38c1..5132800086 100644 --- a/frappe/website/doctype/web_page/test_web_page.py +++ b/frappe/website/doctype/web_page/test_web_page.py @@ -1,10 +1,11 @@ - import unittest + import frappe from frappe.website.path_resolver import PathResolver from frappe.website.serve import get_response_content -test_records = frappe.get_test_records('Web Page') +test_records = frappe.get_test_records("Web Page") + class TestWebPage(unittest.TestCase): def setUp(self): @@ -19,42 +20,47 @@ class TestWebPage(unittest.TestCase): self.assertFalse(PathResolver("test-web-page-1/test-web-page-Random").is_valid_path()) def test_content_type(self): - web_page = frappe.get_doc(dict( - doctype = 'Web Page', - title = 'Test Content Type', - published = 1, - content_type = 'Rich Text', - main_section = 'rich text', - main_section_md = '# h1\nmarkdown content', - main_section_html = '
html content
' - )).insert() - - self.assertIn('rich text', get_response_content('/test-content-type')) - - web_page.content_type = 'Markdown' + web_page = frappe.get_doc( + dict( + doctype="Web Page", + title="Test Content Type", + published=1, + content_type="Rich Text", + main_section="rich text", + main_section_md="# h1\nmarkdown content", + main_section_html="
html content
", + ) + ).insert() + + self.assertIn("rich text", get_response_content("/test-content-type")) + + web_page.content_type = "Markdown" web_page.save() - self.assertIn('markdown content', get_response_content('/test-content-type')) + self.assertIn("markdown content", get_response_content("/test-content-type")) - web_page.content_type = 'HTML' + web_page.content_type = "HTML" web_page.save() - self.assertIn('html content', get_response_content('/test-content-type')) + self.assertIn("html content", get_response_content("/test-content-type")) web_page.delete() def test_dynamic_route(self): - web_page = frappe.get_doc(dict( - doctype = 'Web Page', - title = 'Test Dynamic Route', - published = 1, - dynamic_route = 1, - route = '/doctype-view/', - content_type = 'HTML', - dynamic_template = 1, - main_section_html = '
{{ frappe.form_dict.doctype }}
' - )).insert() + web_page = frappe.get_doc( + dict( + doctype="Web Page", + title="Test Dynamic Route", + published=1, + dynamic_route=1, + route="/doctype-view/", + content_type="HTML", + dynamic_template=1, + main_section_html="
{{ frappe.form_dict.doctype }}
", + ) + ).insert() try: from frappe.utils import get_html_for_route - content = get_html_for_route('/doctype-view/DocField') - self.assertIn('
DocField
', content) + + content = get_html_for_route("/doctype-view/DocField") + self.assertIn("
DocField
", content) finally: web_page.delete() diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 3ce266dfdd..f74af1d8c7 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -11,8 +11,12 @@ from frappe.utils import get_datetime, now, quoted, strip_html from frappe.utils.jinja import render_template from frappe.utils.safe_exec import safe_exec from frappe.website.doctype.website_slideshow.website_slideshow import get_slideshow -from frappe.website.utils import (extract_title, find_first_image, - get_comment_list, get_html_content_based_on_type) +from frappe.website.utils import ( + extract_title, + find_first_image, + get_comment_list, + get_html_content_based_on_type, +) from frappe.website.website_generator import WebsiteGenerator @@ -33,14 +37,14 @@ class WebPage(WebsiteGenerator): super(WebPage, self).on_trash() def get_context(self, context): - context.main_section = get_html_content_based_on_type(self, 'main_section', self.content_type) + context.main_section = get_html_content_based_on_type(self, "main_section", self.content_type) context.source_content_type = self.content_type context.title = self.title if self.context_script: - _locals = dict(context = frappe._dict()) + _locals = dict(context=frappe._dict()) safe_exec(self.context_script, None, _locals) - context.update(_locals['context']) + context.update(_locals["context"]) self.render_dynamic(context) @@ -52,12 +56,14 @@ class WebPage(WebsiteGenerator): context.comment_list = get_comment_list(self.doctype, self.name) context.guest_allowed = True - context.update({ - "style": self.css or "", - "script": self.javascript or "", - "header": self.header, - "text_align": self.text_align, - }) + context.update( + { + "style": self.css or "", + "script": self.javascript or "", + "header": self.header, + "text_align": self.text_align, + } + ) if not self.show_title: context["no_header"] = 1 @@ -82,9 +88,9 @@ class WebPage(WebsiteGenerator): raise def set_breadcrumbs(self, context): - """Build breadcrumbs template """ + """Build breadcrumbs template""" if self.breadcrumbs: - context.parents = frappe.safe_eval(self.breadcrumbs, { "_": _ }) + context.parents = frappe.safe_eval(self.breadcrumbs, {"_": _}) if not "no_breadcrumbs" in context: if "" in context.main_section: context.no_breadcrumbs = 1 @@ -116,7 +122,7 @@ class WebPage(WebsiteGenerator): context.title = strip_html(context.header) def set_page_blocks(self, context): - if self.content_type != 'Page Builder': + if self.content_type != "Page Builder": return out = get_web_blocks_html(self.page_blocks) context.page_builder_html = out.html @@ -135,8 +141,9 @@ class WebPage(WebsiteGenerator): def check_for_redirect(self, context): if "")[0].strip() + ) raise frappe.Redirect def set_metatags(self, context): @@ -145,7 +152,7 @@ class WebPage(WebsiteGenerator): "name": self.meta_title or self.title, "description": self.meta_description, "image": self.meta_image or find_first_image(context.main_section or ""), - "og:type": "website" + "og:type": "website", } def validate_dates(self): @@ -182,18 +189,21 @@ def check_publish_status(): def get_web_blocks_html(blocks): - '''Converts a list of blocks into Raw HTML and extracts out their scripts for deduplication''' + """Converts a list of blocks into Raw HTML and extracts out their scripts for deduplication""" - out = frappe._dict(html='', scripts=[], styles=[]) + out = frappe._dict(html="", scripts=[], styles=[]) extracted_scripts = [] extracted_styles = [] for block in blocks: - web_template = frappe.get_cached_doc('Web Template', block.web_template) - rendered_html = frappe.render_template('templates/includes/web_block.html', context={ - 'web_block': block, - 'web_template_html': web_template.render(block.web_template_values), - 'web_template_type': web_template.type - }) + web_template = frappe.get_cached_doc("Web Template", block.web_template) + rendered_html = frappe.render_template( + "templates/includes/web_block.html", + context={ + "web_block": block, + "web_template_html": web_template.render(block.web_template_values), + "web_template_type": web_template.type, + }, + ) html, scripts, styles = extract_script_and_style_tags(rendered_html) out.html += html if block.web_template not in extracted_scripts: @@ -205,17 +215,19 @@ def get_web_blocks_html(blocks): return out + def extract_script_and_style_tags(html): from bs4 import BeautifulSoup + soup = BeautifulSoup(html, "html.parser") scripts = [] styles = [] - for script in soup.find_all('script'): + for script in soup.find_all("script"): scripts.append(script.string) script.extract() - for style in soup.find_all('style'): + for style in soup.find_all("style"): styles.append(style.string) style.extract() diff --git a/frappe/website/doctype/web_page_view/test_web_page_view.py b/frappe/website/doctype/web_page_view/test_web_page_view.py index e0e571a25c..3f1d86ada4 100644 --- a/frappe/website/doctype/web_page_view/test_web_page_view.py +++ b/frappe/website/doctype/web_page_view/test_web_page_view.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestWebPageView(unittest.TestCase): pass diff --git a/frappe/website/doctype/web_page_view/web_page_view.py b/frappe/website/doctype/web_page_view/web_page_view.py index cd6b4ba940..b1e64980d9 100644 --- a/frappe/website/doctype/web_page_view/web_page_view.py +++ b/frappe/website/doctype/web_page_view/web_page_view.py @@ -5,6 +5,7 @@ import frappe from frappe.model.document import Document + class WebPageView(Document): pass @@ -15,16 +16,16 @@ def make_view_log(path, referrer=None, browser=None, version=None, url=None, use return request_dict = frappe.request.__dict__ - user_agent = request_dict.get('environ', {}).get('HTTP_USER_AGENT') + user_agent = request_dict.get("environ", {}).get("HTTP_USER_AGENT") if referrer: - referrer = referrer.split('?')[0] + referrer = referrer.split("?")[0] is_unique = True if referrer.startswith(url): is_unique = False - if path != "/" and path.startswith('/'): + if path != "/" and path.startswith("/"): path = path[1:] view = frappe.new_doc("Web Page View") @@ -42,9 +43,11 @@ def make_view_log(path, referrer=None, browser=None, version=None, url=None, use if frappe.message_log: frappe.message_log.pop() + @frappe.whitelist() def get_page_view_count(path): - return frappe.db.count("Web Page View", filters={'path': path}) + return frappe.db.count("Web Page View", filters={"path": path}) + def is_tracking_enabled(): - return frappe.db.get_value("Website Settings", "Website Settings", "enable_view_tracking") \ No newline at end of file + return frappe.db.get_value("Website Settings", "Website Settings", "enable_view_tracking") diff --git a/frappe/website/doctype/web_template/test_web_template.py b/frappe/website/doctype/web_template/test_web_template.py index 9789fbd13e..2d2d7faf36 100644 --- a/frappe/website/doctype/web_template/test_web_template.py +++ b/frappe/website/doctype/web_template/test_web_template.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest + from bs4 import BeautifulSoup + +import frappe from frappe.utils import set_request from frappe.website.serve import get_response + class TestWebTemplate(unittest.TestCase): def test_render_web_template_with_values(self): doc = frappe.get_doc("Web Template", "Hero with Right Image") diff --git a/frappe/website/doctype/web_template/web_template.py b/frappe/website/doctype/web_template/web_template.py index b7cc6f3caa..93b469fe96 100644 --- a/frappe/website/doctype/web_template/web_template.py +++ b/frappe/website/doctype/web_template/web_template.py @@ -6,14 +6,10 @@ import os from shutil import rmtree import frappe +from frappe import _ from frappe.model.document import Document +from frappe.modules.export_file import get_module_path, scrub_dt_dn, write_document_file from frappe.website.utils import clear_cache -from frappe import _ -from frappe.modules.export_file import ( - write_document_file, - get_module_path, - scrub_dt_dn, -) class WebTemplate(Document): diff --git a/frappe/website/doctype/web_template_field/test_web_template_field.py b/frappe/website/doctype/web_template_field/test_web_template_field.py index 33f2d5545c..3f9160edb9 100644 --- a/frappe/website/doctype/web_template_field/test_web_template_field.py +++ b/frappe/website/doctype/web_template_field/test_web_template_field.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestWebTemplateField(unittest.TestCase): pass diff --git a/frappe/website/doctype/web_template_field/web_template_field.py b/frappe/website/doctype/web_template_field/web_template_field.py index 01b3346911..082bde41f3 100644 --- a/frappe/website/doctype/web_template_field/web_template_field.py +++ b/frappe/website/doctype/web_template_field/web_template_field.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WebTemplateField(Document): pass diff --git a/frappe/website/doctype/website_meta_tag/website_meta_tag.py b/frappe/website/doctype/website_meta_tag/website_meta_tag.py index e02da067ec..0ac84a632e 100644 --- a/frappe/website/doctype/website_meta_tag/website_meta_tag.py +++ b/frappe/website/doctype/website_meta_tag/website_meta_tag.py @@ -5,17 +5,16 @@ import frappe from frappe.model.document import Document + class WebsiteMetaTag(Document): def get_content(self): # can't have new lines in meta content - return (self.value or '').replace('\n', ' ') + return (self.value or "").replace("\n", " ") def get_meta_dict(self): - return { - self.key: self.get_content() - } + return {self.key: self.get_content()} def set_in_context(self, context): - context.setdefault('metatags', frappe._dict({})) + context.setdefault("metatags", frappe._dict({})) context.metatags[self.key] = self.get_content() return context diff --git a/frappe/website/doctype/website_route_meta/test_website_route_meta.py b/frappe/website/doctype/website_route_meta/test_website_route_meta.py index 627e7dd1cd..0a04207cea 100644 --- a/frappe/website/doctype/website_route_meta/test_website_route_meta.py +++ b/frappe/website/doctype/website_route_meta/test_website_route_meta.py @@ -1,29 +1,27 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest + +import frappe from frappe.utils import set_request from frappe.website.serve import get_response -test_dependencies = ['Blog Post'] +test_dependencies = ["Blog Post"] + + class TestWebsiteRouteMeta(unittest.TestCase): def test_meta_tag_generation(self): - blogs = frappe.get_all('Blog Post', fields=['name', 'route'], - filters={'published': 1, 'route': ('!=', '')}, limit=1) + blogs = frappe.get_all( + "Blog Post", fields=["name", "route"], filters={"published": 1, "route": ("!=", "")}, limit=1 + ) blog = blogs[0] # create meta tags for this route - doc = frappe.new_doc('Website Route Meta') - doc.append('meta_tags', { - 'key': 'type', - 'value': 'blog_post' - }) - doc.append('meta_tags', { - 'key': 'og:title', - 'value': 'My Blog' - }) + doc = frappe.new_doc("Website Route Meta") + doc.append("meta_tags", {"key": "type", "value": "blog_post"}) + doc.append("meta_tags", {"key": "og:title", "value": "My Blog"}) doc.name = blog.route doc.insert() @@ -35,8 +33,8 @@ class TestWebsiteRouteMeta(unittest.TestCase): html = response.get_data().decode() - self.assertTrue('''''' in html) - self.assertTrue('''''' in html) + self.assertTrue("""""" in html) + self.assertTrue("""""" in html) def tearDown(self): frappe.db.rollback() diff --git a/frappe/website/doctype/website_route_meta/website_route_meta.py b/frappe/website/doctype/website_route_meta/website_route_meta.py index 2800a1a02b..0c12b10583 100644 --- a/frappe/website/doctype/website_route_meta/website_route_meta.py +++ b/frappe/website/doctype/website_route_meta/website_route_meta.py @@ -4,7 +4,8 @@ from frappe.model.document import Document + class WebsiteRouteMeta(Document): def autoname(self): - if self.name and self.name.startswith('/'): + if self.name and self.name.startswith("/"): self.name = self.name[1:] diff --git a/frappe/website/doctype/website_route_redirect/website_route_redirect.py b/frappe/website/doctype/website_route_redirect/website_route_redirect.py index b285a172bc..29bcd82094 100644 --- a/frappe/website/doctype/website_route_redirect/website_route_redirect.py +++ b/frappe/website/doctype/website_route_redirect/website_route_redirect.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WebsiteRouteRedirect(Document): pass diff --git a/frappe/website/doctype/website_script/website_script.py b/frappe/website/doctype/website_script/website_script.py index 340f15bb61..f2764caf91 100644 --- a/frappe/website/doctype/website_script/website_script.py +++ b/frappe/website/doctype/website_script/website_script.py @@ -4,14 +4,14 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document -class WebsiteScript(Document): +class WebsiteScript(Document): def on_update(self): """clear cache""" - frappe.clear_cache(user = 'Guest') + frappe.clear_cache(user="Guest") from frappe.website.utils import clear_cache - clear_cache() \ No newline at end of file + + clear_cache() diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index e99152d6ae..6585642fe7 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -25,7 +25,10 @@ def authorize_access(reauthorize=None): google_settings = frappe.get_doc("Google Settings") website_settings = frappe.get_doc("Website Settings") - redirect_uri = get_request_site_address(True) + "?cmd=frappe.website.doctype.website_settings.google_indexing.google_callback" + redirect_uri = ( + get_request_site_address(True) + + "?cmd=frappe.website.doctype.website_settings.google_indexing.google_callback" + ) if not website_settings.indexing_authorization_code or reauthorize: return get_authentication_url(client_id=google_settings.client_id, redirect_uri=redirect_uri) @@ -34,14 +37,18 @@ def authorize_access(reauthorize=None): data = { "code": website_settings.indexing_authorization_code, "client_id": google_settings.client_id, - "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), + "client_secret": google_settings.get_password( + fieldname="client_secret", raise_exception=False + ), "redirect_uri": redirect_uri, - "grant_type": "authorization_code" + "grant_type": "authorization_code", } res = requests.post(get_auth_url(), data=data).json() if "refresh_token" in res: - frappe.db.set_value("Website Settings", website_settings.name, "indexing_refresh_token", res.get("refresh_token")) + frappe.db.set_value( + "Website Settings", website_settings.name, "indexing_refresh_token", res.get("refresh_token") + ) frappe.db.commit() frappe.local.response["type"] = "redirect" @@ -55,7 +62,9 @@ def authorize_access(reauthorize=None): def get_authentication_url(client_id, redirect_uri): """Return authentication url with the client id and redirect uri.""" return { - "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format(client_id, SCOPES, redirect_uri) + "url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format( + client_id, SCOPES, redirect_uri + ) } @@ -79,15 +88,12 @@ def get_google_indexing_object(): "token_uri": get_auth_url(), "client_id": google_settings.client_id, "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/indexing" + "scopes": "https://www.googleapis.com/auth/indexing", } credentials = google.oauth2.credentials.Credentials(**credentials_dict) google_indexing = build( - serviceName="indexing", - version="v3", - credentials=credentials, - static_discovery=False + serviceName="indexing", version="v3", credentials=credentials, static_discovery=False ) return google_indexing @@ -97,11 +103,8 @@ def publish_site(url, operation_type="URL_UPDATED"): """Send an update/remove url request.""" google_indexing = get_google_indexing_object() - body = { - "url": url, - "type": operation_type - } + body = {"url": url, "type": operation_type} try: - google_indexing.urlNotifications().publish(body=body, x__xgafv='2').execute() + google_indexing.urlNotifications().publish(body=body, x__xgafv="2").execute() except HttpError as e: - frappe.log_error(message=e, title='API Indexing Issue') + frappe.log_error(message=e, title="API Indexing Issue") diff --git a/frappe/website/doctype/website_settings/test_website_settings.py b/frappe/website/doctype/website_settings/test_website_settings.py index 28629716b3..c204bd7aea 100644 --- a/frappe/website/doctype/website_settings/test_website_settings.py +++ b/frappe/website/doctype/website_settings/test_website_settings.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestWebsiteSettings(unittest.TestCase): pass diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index a7a0bc5bac..e8f15290c4 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -11,6 +11,7 @@ from frappe.website.doctype.website_theme.website_theme import add_website_theme INDEXING_SCOPES = "https://www.googleapis.com/auth/indexing" + class WebsiteSettings(Document): def validate(self): self.validate_top_bar_items() @@ -22,9 +23,12 @@ class WebsiteSettings(Document): if frappe.flags.in_install: return from frappe.website.path_resolver import PathResolver + if self.home_page and not PathResolver(self.home_page).is_valid_path(): - frappe.msgprint(_("Invalid Home Page") + " (Standard pages - home, login, products, blog, about, contact)") - self.home_page = '' + frappe.msgprint( + _("Invalid Home Page") + " (Standard pages - home, login, products, blog, about, contact)" + ) + self.home_page = "" def validate_top_bar_items(self): """validate url in top bar items""" @@ -34,12 +38,17 @@ class WebsiteSettings(Document): if not parent_label_item: # invalid item - frappe.throw(_("{0} does not exist in row {1}").format(top_bar_item.parent_label, top_bar_item.idx)) + frappe.throw( + _("{0} does not exist in row {1}").format(top_bar_item.parent_label, top_bar_item.idx) + ) elif not parent_label_item[0] or parent_label_item[0].url: # parent cannot have url - frappe.throw(_("{0} in row {1} cannot have both URL and child items").format(top_bar_item.parent_label, - top_bar_item.idx)) + frappe.throw( + _("{0} in row {1} cannot have both URL and child items").format( + top_bar_item.parent_label, top_bar_item.idx + ) + ) def validate_footer_items(self): """validate url in top bar items""" @@ -49,12 +58,17 @@ class WebsiteSettings(Document): if not parent_label_item: # invalid item - frappe.throw(_("{0} does not exist in row {1}").format(footer_item.parent_label, footer_item.idx)) + frappe.throw( + _("{0} does not exist in row {1}").format(footer_item.parent_label, footer_item.idx) + ) elif not parent_label_item[0] or parent_label_item[0].url: # parent cannot have url - frappe.throw(_("{0} in row {1} cannot have both URL and child items").format(footer_item.parent_label, - footer_item.idx)) + frappe.throw( + _("{0} in row {1} cannot have both URL and child items").format( + footer_item.parent_label, footer_item.idx + ) + ) def validate_google_settings(self): if self.enable_google_indexing and not frappe.db.get_single_value("Google Settings", "enable"): @@ -66,9 +80,10 @@ class WebsiteSettings(Document): def clear_cache(self): # make js and css # clear web cache (for menus!) - frappe.clear_cache(user = 'Guest') + frappe.clear_cache(user="Guest") from frappe.website.utils import clear_cache + clear_cache() # clears role based home pages @@ -91,14 +106,18 @@ class WebsiteSettings(Document): "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), "refresh_token": self.get_password(fieldname="indexing_refresh_token", raise_exception=False), "grant_type": "refresh_token", - "scope": INDEXING_SCOPES + "scope": INDEXING_SCOPES, } try: res = requests.post(get_auth_url(), data=data).json() except requests.exceptions.HTTPError: button_label = frappe.bold(_("Allow Google Indexing Access")) - frappe.throw(_("Something went wrong during the token generation. Click on {0} to generate a new one.").format(button_label)) + frappe.throw( + _( + "Something went wrong during the token generation. Click on {0} to generate a new one." + ).format(button_label) + ) return res.get("access_token") @@ -106,30 +125,55 @@ class WebsiteSettings(Document): def get_website_settings(context=None): hooks = frappe.get_hooks() context = context or frappe._dict() - context = context.update({ - 'top_bar_items': get_items('top_bar_items'), - 'footer_items': get_items('footer_items'), - "post_login": [ - {"label": _("My Account"), "url": "/me"}, - {"label": _("Log out"), "url": "/?cmd=web_logout"} - ] - }) + context = context.update( + { + "top_bar_items": get_items("top_bar_items"), + "footer_items": get_items("footer_items"), + "post_login": [ + {"label": _("My Account"), "url": "/me"}, + {"label": _("Log out"), "url": "/?cmd=web_logout"}, + ], + } + ) settings = frappe.get_single("Website Settings") - for k in ["banner_html", "banner_image", "brand_html", "copyright", "twitter_share_via", - "facebook_share", "google_plus_one", "twitter_share", "linked_in_share", - "disable_signup", "hide_footer_signup", "head_html", "title_prefix", - "navbar_template", "footer_template", "navbar_search", "enable_view_tracking", - "footer_logo", "call_to_action", "call_to_action_url", "show_language_picker", - "footer_powered"]: + for k in [ + "banner_html", + "banner_image", + "brand_html", + "copyright", + "twitter_share_via", + "facebook_share", + "google_plus_one", + "twitter_share", + "linked_in_share", + "disable_signup", + "hide_footer_signup", + "head_html", + "title_prefix", + "navbar_template", + "footer_template", + "navbar_search", + "enable_view_tracking", + "footer_logo", + "call_to_action", + "call_to_action_url", + "show_language_picker", + "footer_powered", + ]: if hasattr(settings, k): context[k] = settings.get(k) if settings.address: context["footer_address"] = settings.address - for k in ["facebook_share", "google_plus_one", "twitter_share", "linked_in_share", - "disable_signup"]: + for k in [ + "facebook_share", + "google_plus_one", + "twitter_share", + "linked_in_share", + "disable_signup", + ]: context[k] = int(context.get(k) or 0) if frappe.request: @@ -144,8 +188,9 @@ def get_website_settings(context=None): via_hooks = frappe.get_hooks("website_context") for key in via_hooks: context[key] = via_hooks[key] - if key not in ("top_bar_items", "footer_items", "post_login") \ - and isinstance(context[key], (list, tuple)): + if key not in ("top_bar_items", "footer_items", "post_login") and isinstance( + context[key], (list, tuple) + ): context[key] = context[key][-1] add_website_theme(context) @@ -160,25 +205,31 @@ def get_website_settings(context=None): return context + def get_items(parentfield): - all_top_items = frappe.db.sql("""\ + all_top_items = frappe.db.sql( + """\ select * from `tabTop Bar Item` where parent='Website Settings' and parentfield= %s - order by idx asc""", parentfield, as_dict=1) + order by idx asc""", + parentfield, + as_dict=1, + ) top_items = all_top_items[:] # attach child items to top bar for d in all_top_items: - if d['parent_label']: + if d["parent_label"]: for t in top_items: - if t['label']==d['parent_label']: - if not 'child_items' in t: - t['child_items'] = [] - t['child_items'].append(d) + if t["label"] == d["parent_label"]: + if not "child_items" in t: + t["child_items"] = [] + t["child_items"].append(d) break return top_items + @frappe.whitelist(allow_guest=True) def get_auto_account_deletion(): return frappe.db.get_single_value("Website Settings", "auto_account_deletion") diff --git a/frappe/website/doctype/website_sidebar/test_website_sidebar.py b/frappe/website/doctype/website_sidebar/test_website_sidebar.py index a92cb04568..5e92de0f6d 100644 --- a/frappe/website/doctype/website_sidebar/test_website_sidebar.py +++ b/frappe/website/doctype/website_sidebar/test_website_sidebar.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Website Sidebar') + class TestWebsiteSidebar(unittest.TestCase): pass diff --git a/frappe/website/doctype/website_sidebar/website_sidebar.py b/frappe/website/doctype/website_sidebar/website_sidebar.py index d1f3915752..151446bac3 100644 --- a/frappe/website/doctype/website_sidebar/website_sidebar.py +++ b/frappe/website/doctype/website_sidebar/website_sidebar.py @@ -10,7 +10,7 @@ class WebsiteSidebar(Document): def get_items(self): items = frappe.get_all( "Website Sidebar Item", - filters={'parent': self.name}, + filters={"parent": self.name}, fields=["title", "route", "group"], order_by="idx asc", ) diff --git a/frappe/website/doctype/website_sidebar_item/website_sidebar_item.py b/frappe/website/doctype/website_sidebar_item/website_sidebar_item.py index 43d3a10508..e839de6d2b 100644 --- a/frappe/website/doctype/website_sidebar_item/website_sidebar_item.py +++ b/frappe/website/doctype/website_sidebar_item/website_sidebar_item.py @@ -5,5 +5,6 @@ import frappe from frappe.model.document import Document + class WebsiteSidebarItem(Document): pass diff --git a/frappe/website/doctype/website_slideshow/test_website_slideshow.py b/frappe/website/doctype/website_slideshow/test_website_slideshow.py index d8683a7cda..c18835bdc1 100644 --- a/frappe/website/doctype/website_slideshow/test_website_slideshow.py +++ b/frappe/website/doctype/website_slideshow/test_website_slideshow.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest +import frappe + # test_records = frappe.get_test_records('Website Slideshow') + class TestWebsiteSlideshow(unittest.TestCase): pass diff --git a/frappe/website/doctype/website_slideshow/website_slideshow.py b/frappe/website/doctype/website_slideshow/website_slideshow.py index 0959680872..e9571b69df 100644 --- a/frappe/website/doctype/website_slideshow/website_slideshow.py +++ b/frappe/website/doctype/website_slideshow/website_slideshow.py @@ -5,9 +5,9 @@ import frappe from frappe import _ - from frappe.model.document import Document + class WebsiteSlideshow(Document): def validate(self): self.validate_images() @@ -15,16 +15,18 @@ class WebsiteSlideshow(Document): def on_update(self): # a slide show can be in use and any change in it should get reflected from frappe.website.utils import clear_cache + clear_cache() def validate_images(self): - ''' atleast one image file should be public for slideshow ''' + """atleast one image file should be public for slideshow""" files = map(lambda row: row.image, self.slideshow_items) if files: - result = frappe.get_all("File", filters={ "file_url":("in", list(files)) }, fields="is_private") + result = frappe.get_all("File", filters={"file_url": ("in", list(files))}, fields="is_private") if any(file.is_private for file in result): frappe.throw(_("All Images attached to Website Slideshow should be public")) + def get_slideshow(doc): if not doc.slideshow: return {} @@ -32,6 +34,6 @@ def get_slideshow(doc): slideshow = frappe.get_doc("Website Slideshow", doc.slideshow) return { - "slides": slideshow.get({"doctype":"Website Slideshow Item"}), - "slideshow_header": slideshow.header or "" + "slides": slideshow.get({"doctype": "Website Slideshow Item"}), + "slideshow_header": slideshow.header or "", } diff --git a/frappe/website/doctype/website_slideshow_item/website_slideshow_item.py b/frappe/website/doctype/website_slideshow_item/website_slideshow_item.py index cc58596af9..b1ab84a294 100644 --- a/frappe/website/doctype/website_slideshow_item/website_slideshow_item.py +++ b/frappe/website/doctype/website_slideshow_item/website_slideshow_item.py @@ -4,8 +4,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class WebsiteSlideshowItem(Document): - pass \ No newline at end of file + pass diff --git a/frappe/website/doctype/website_theme/test_website_theme.py b/frappe/website/doctype/website_theme/test_website_theme.py index 926a9d3bc3..6040d70a10 100644 --- a/frappe/website/doctype/website_theme/test_website_theme.py +++ b/frappe/website/doctype/website_theme/test_website_theme.py @@ -2,37 +2,39 @@ # License: MIT. See LICENSE import os -import frappe import unittest + +import frappe + from .website_theme import get_scss_paths -class TestWebsiteTheme(unittest.TestCase): +class TestWebsiteTheme(unittest.TestCase): def test_website_theme(self): - frappe.delete_doc_if_exists('Website Theme', 'test-theme') - theme = frappe.get_doc(dict( - doctype='Website Theme', - theme='test-theme', - google_font='Inter', - custom_scss='body { font-size: 16.5px; }' # this will get minified! - )).insert() - - theme_path = frappe.get_site_path('public', theme.theme_url[1:]) + frappe.delete_doc_if_exists("Website Theme", "test-theme") + theme = frappe.get_doc( + dict( + doctype="Website Theme", + theme="test-theme", + google_font="Inter", + custom_scss="body { font-size: 16.5px; }", # this will get minified! + ) + ).insert() + + theme_path = frappe.get_site_path("public", theme.theme_url[1:]) with open(theme_path) as theme_file: css = theme_file.read() - self.assertTrue('body{font-size:16.5px}' in css) - self.assertTrue('fonts.googleapis.com' in css) + self.assertTrue("body{font-size:16.5px}" in css) + self.assertTrue("fonts.googleapis.com" in css) def test_get_scss_paths(self): - self.assertIn('frappe/public/scss/website.bundle', get_scss_paths()) + self.assertIn("frappe/public/scss/website.bundle", get_scss_paths()) def test_imports_to_ignore(self): - frappe.delete_doc_if_exists('Website Theme', 'test-theme') - theme = frappe.get_doc(dict( - doctype='Website Theme', - theme='test-theme', - ignored_apps=[{ 'app': 'frappe'}] - )).insert() + frappe.delete_doc_if_exists("Website Theme", "test-theme") + theme = frappe.get_doc( + dict(doctype="Website Theme", theme="test-theme", ignored_apps=[{"app": "frappe"}]) + ).insert() self.assertTrue('@import "frappe/public/scss/website"' not in theme.theme_scss) diff --git a/frappe/website/doctype/website_theme/website_theme.py b/frappe/website/doctype/website_theme/website_theme.py index f5b0b04487..c833430534 100644 --- a/frappe/website/doctype/website_theme/website_theme.py +++ b/frappe/website/doctype/website_theme/website_theme.py @@ -1,11 +1,16 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE +from os.path import abspath +from os.path import exists as path_exists +from os.path import join as join_path +from os.path import splitext + import frappe from frappe import _ from frappe.model.document import Document from frappe.utils import get_path -from os.path import join as join_path, exists as path_exists, abspath, splitext + class WebsiteTheme(Document): def validate(self): @@ -13,23 +18,28 @@ class WebsiteTheme(Document): self.generate_bootstrap_theme() def on_update(self): - if (not self.custom - and frappe.local.conf.get('developer_mode') - and not (frappe.flags.in_import or frappe.flags.in_test)): + if ( + not self.custom + and frappe.local.conf.get("developer_mode") + and not (frappe.flags.in_import or frappe.flags.in_test) + ): self.export_doc() self.clear_cache_if_current_theme() def is_standard_and_not_valid_user(self): - return (not self.custom - and not frappe.local.conf.get('developer_mode') - and not (frappe.flags.in_import or frappe.flags.in_test or frappe.flags.in_migrate)) + return ( + not self.custom + and not frappe.local.conf.get("developer_mode") + and not (frappe.flags.in_import or frappe.flags.in_test or frappe.flags.in_migrate) + ) def on_trash(self): if self.is_standard_and_not_valid_user(): - frappe.throw(_("You are not allowed to delete a standard Website Theme"), - frappe.PermissionError) + frappe.throw( + _("You are not allowed to delete a standard Website Theme"), frappe.PermissionError + ) def validate_if_customizable(self): if self.is_standard_and_not_valid_user(): @@ -38,22 +48,25 @@ class WebsiteTheme(Document): def export_doc(self): """Export to standard folder `[module]/website_theme/[name]/[name].json`.""" from frappe.modules.export_file import export_to_files - export_to_files(record_list=[['Website Theme', self.name]], create_init=True) + export_to_files(record_list=[["Website Theme", self.name]], create_init=True) def clear_cache_if_current_theme(self): - if frappe.flags.in_install == 'frappe': return + if frappe.flags.in_install == "frappe": + return website_settings = frappe.get_doc("Website Settings", "Website Settings") if getattr(website_settings, "website_theme", None) == self.name: website_settings.clear_cache() def generate_bootstrap_theme(self): - from subprocess import Popen, PIPE + from subprocess import PIPE, Popen - self.theme_scss = frappe.render_template('frappe/website/doctype/website_theme/website_theme_template.scss', self.as_dict()) + self.theme_scss = frappe.render_template( + "frappe/website/doctype/website_theme/website_theme_template.scss", self.as_dict() + ) # create theme file in site public files folder - folder_path = abspath(frappe.utils.get_files_path('website_theme', is_private=False)) + folder_path = abspath(frappe.utils.get_files_path("website_theme", is_private=False)) # create folder if not exist frappe.create_folder(folder_path) @@ -61,37 +74,38 @@ class WebsiteTheme(Document): self.delete_old_theme_files(folder_path) # add a random suffix - suffix = frappe.generate_hash('Website Theme', 8) if self.custom else 'style' - file_name = frappe.scrub(self.name) + '_' + suffix + '.css' + suffix = frappe.generate_hash("Website Theme", 8) if self.custom else "style" + file_name = frappe.scrub(self.name) + "_" + suffix + ".css" output_path = join_path(folder_path, file_name) self.theme_scss = content = get_scss(self) - content = content.replace('\n', '\\n') - command = ['node', 'generate_bootstrap_theme.js', output_path, content] + content = content.replace("\n", "\\n") + command = ["node", "generate_bootstrap_theme.js", output_path, content] - process = Popen(command, cwd=frappe.get_app_path('frappe', '..'), stdout=PIPE, stderr=PIPE) + process = Popen(command, cwd=frappe.get_app_path("frappe", ".."), stdout=PIPE, stderr=PIPE) stderr = process.communicate()[1] if stderr: stderr = frappe.safe_decode(stderr) - stderr = stderr.replace('\n', '
') + stderr = stderr.replace("\n", "
") frappe.throw('
{stderr}
'.format(stderr=stderr)) else: - self.theme_url = '/files/website_theme/' + file_name + self.theme_url = "/files/website_theme/" + file_name - frappe.msgprint(_('Compiled Successfully'), alert=True) + frappe.msgprint(_("Compiled Successfully"), alert=True) def delete_old_theme_files(self, folder_path): import os + for fname in os.listdir(folder_path): - if fname.startswith(frappe.scrub(self.name) + '_') and fname.endswith('.css'): + if fname.startswith(frappe.scrub(self.name) + "_") and fname.endswith(".css"): os.remove(os.path.join(folder_path, fname)) def generate_theme_if_not_exist(self): bench_path = frappe.utils.get_bench_path() if self.theme_url: - theme_path = join_path(bench_path, 'sites', self.theme_url[1:]) + theme_path = join_path(bench_path, "sites", self.theme_url[1:]) if not path_exists(theme_path): self.generate_bootstrap_theme() else: @@ -101,7 +115,7 @@ class WebsiteTheme(Document): def set_as_default(self): self.generate_bootstrap_theme() self.save() - website_settings = frappe.get_doc('Website Settings') + website_settings = frappe.get_doc("Website Settings") website_settings.website_theme = self.name website_settings.ignore_validate = True website_settings.save() @@ -109,13 +123,11 @@ class WebsiteTheme(Document): @frappe.whitelist() def get_apps(self): from frappe.utils.change_log import get_versions + apps = get_versions() out = [] for app, values in apps.items(): - out.append({ - 'name': app, - 'title': values['title'] - }) + out.append({"name": app, "title": values["title"]}) return out @@ -126,6 +138,7 @@ def add_website_theme(context): website_theme = get_active_theme() context.theme = website_theme or frappe._dict() + def get_active_theme(): website_theme = frappe.db.get_single_value("Website Settings", "website_theme") if website_theme: @@ -135,7 +148,6 @@ def get_active_theme(): pass - def get_scss(website_theme): """ Render `website_theme_template.scss` with the values defined in Website Theme. @@ -143,12 +155,14 @@ def get_scss(website_theme): params: website_theme - instance of a Website Theme """ - apps_to_ignore = tuple((d.app + '/') for d in website_theme.ignored_apps) + apps_to_ignore = tuple((d.app + "/") for d in website_theme.ignored_apps) available_imports = get_scss_paths() imports_to_include = [d for d in available_imports if not d.startswith(apps_to_ignore)] context = website_theme.as_dict() - context['website_theme_scss'] = imports_to_include - return frappe.render_template('frappe/website/doctype/website_theme/website_theme_template.scss', context) + context["website_theme_scss"] = imports_to_include + return frappe.render_template( + "frappe/website/doctype/website_theme/website_theme_template.scss", context + ) def get_scss_paths(): @@ -161,11 +175,11 @@ def get_scss_paths(): import_path_list = [] bench_path = frappe.utils.get_bench_path() - scss_files = ['public/scss/website.scss', 'public/scss/website.bundle.scss'] + scss_files = ["public/scss/website.scss", "public/scss/website.bundle.scss"] for app in frappe.get_installed_apps(): for scss_file in scss_files: relative_path = join_path(app, scss_file) - full_path = get_path('apps', app, relative_path, base=bench_path) + full_path = get_path("apps", app, relative_path, base=bench_path) if path_exists(full_path): import_path = splitext(relative_path)[0] import_path_list.append(import_path) @@ -180,10 +194,10 @@ def after_migrate(): Necessary to reflect possible changes in the imported SCSS files. Called at the end of every `bench migrate`. """ - website_theme = frappe.db.get_single_value('Website Settings', 'website_theme') - if not website_theme or website_theme == 'Standard': + website_theme = frappe.db.get_single_value("Website Settings", "website_theme") + if not website_theme or website_theme == "Standard": return - doc = frappe.get_doc('Website Theme', website_theme) + doc = frappe.get_doc("Website Theme", website_theme) doc.generate_bootstrap_theme() doc.save() diff --git a/frappe/website/doctype/website_theme_ignore_app/website_theme_ignore_app.py b/frappe/website/doctype/website_theme_ignore_app/website_theme_ignore_app.py index 99e523cf16..72652805ec 100644 --- a/frappe/website/doctype/website_theme_ignore_app/website_theme_ignore_app.py +++ b/frappe/website/doctype/website_theme_ignore_app/website_theme_ignore_app.py @@ -5,5 +5,6 @@ # import frappe from frappe.model.document import Document + class WebsiteThemeIgnoreApp(Document): pass diff --git a/frappe/website/page_renderers/base_renderer.py b/frappe/website/page_renderers/base_renderer.py index 14d4448e6c..c8b4ac7344 100644 --- a/frappe/website/page_renderers/base_renderer.py +++ b/frappe/website/page_renderers/base_renderer.py @@ -1,17 +1,18 @@ import frappe from frappe.website.utils import build_response + class BaseRenderer(object): def __init__(self, path=None, http_status_code=None): self.headers = None self.http_status_code = http_status_code or 200 if not path: path = frappe.local.request.path - self.path = path.strip('/ ') - self.basepath = '' - self.basename = '' - self.name = '' - self.route = '' + self.path = path.strip("/ ") + self.basepath = "" + self.basename = "" + self.name = "" + self.route = "" self.file_dir = None def can_render(self): @@ -21,5 +22,6 @@ class BaseRenderer(object): raise NotImplementedError def build_response(self, data, http_status_code=None, headers=None): - return build_response(self.path, data, http_status_code or self.http_status_code, headers or self.headers) - + return build_response( + self.path, data, http_status_code or self.http_status_code, headers or self.headers + ) diff --git a/frappe/website/page_renderers/base_template_page.py b/frappe/website/page_renderers/base_template_page.py index 34d51cd600..f35a7e64f7 100644 --- a/frappe/website/page_renderers/base_template_page.py +++ b/frappe/website/page_renderers/base_template_page.py @@ -7,8 +7,8 @@ from frappe.website.website_components.metatags import MetaTags class BaseTemplatePage(BaseRenderer): def __init__(self, path, http_status_code=None): super().__init__(path=path, http_status_code=http_status_code) - self.template_path = '' - self.source = '' + self.template_path = "" + self.source = "" def init_context(self): self.context = frappe._dict() @@ -18,8 +18,9 @@ class BaseTemplatePage(BaseRenderer): def add_csrf_token(self, html): if frappe.local.session: csrf_token = frappe.local.session.data.csrf_token - return html.replace("", - f'') + return html.replace( + "", f'' + ) return html @@ -40,13 +41,16 @@ class BaseTemplatePage(BaseRenderer): self.context.base_template_path = app_base[-1] if app_base else "templates/base.html" def set_title_with_prefix(self): - if (self.context.title_prefix and self.context.title - and not self.context.title.startswith(self.context.title_prefix)): - self.context.title = '{0} - {1}'.format(self.context.title_prefix, self.context.title) + if ( + self.context.title_prefix + and self.context.title + and not self.context.title.startswith(self.context.title_prefix) + ): + self.context.title = "{0} - {1}".format(self.context.title_prefix, self.context.title) def set_missing_values(self): # set using frappe.respond_as_web_page - if hasattr(frappe.local, 'response') and frappe.local.response.get('context'): + if hasattr(frappe.local, "response") and frappe.local.response.get("context"): self.context.update(frappe.local.response.context) # to be able to inspect the context dict @@ -56,15 +60,15 @@ class BaseTemplatePage(BaseRenderer): if "url_prefix" not in self.context: self.context.url_prefix = "" - if self.context.url_prefix and self.context.url_prefix[-1]!='/': - self.context.url_prefix += '/' + if self.context.url_prefix and self.context.url_prefix[-1] != "/": + self.context.url_prefix += "/" self.context.path = self.path - self.context.pathname = frappe.local.path if hasattr(frappe, 'local') else self.path + self.context.pathname = frappe.local.path if hasattr(frappe, "local") else self.path def update_website_context(self): # apply context from hooks - update_website_context = frappe.get_hooks('update_website_context') + update_website_context = frappe.get_hooks("update_website_context") for method in update_website_context: values = frappe.get_attr(method)(self.context) if values: diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 6b8d973ead..abfd72ac6f 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,16 +1,18 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage +from frappe.website.router import ( + get_doctypes_with_web_view, + get_page_info_from_web_page_with_dynamic_routes, +) from frappe.website.utils import cache_html -from frappe.website.router import (get_doctypes_with_web_view, - get_page_info_from_web_page_with_dynamic_routes) class DocumentPage(BaseTemplatePage): def can_render(self): - ''' + """ Find a document with matching `route` from all doctypes with `has_web_view`=1 - ''' + """ if self.search_in_doctypes_with_web_view(): return True @@ -29,7 +31,7 @@ class DocumentPage(BaseTemplatePage): filters[condition_field] = 1 try: - self.docname = frappe.db.get_value(doctype, filters, 'name') + self.docname = frappe.db.get_value(doctype, filters, "name") if self.docname: self.doctype = doctype return True @@ -40,7 +42,7 @@ class DocumentPage(BaseTemplatePage): def search_web_page_dynamic_routes(self): d = get_page_info_from_web_page_with_dynamic_routes(self.path) if d: - self.doctype = 'Web Page' + self.doctype = "Web Page" self.docname = d.name return True else: diff --git a/frappe/website/page_renderers/error_page.py b/frappe/website/page_renderers/error_page.py index 3501c77765..613809bfdc 100644 --- a/frappe/website/page_renderers/error_page.py +++ b/frappe/website/page_renderers/error_page.py @@ -1,10 +1,11 @@ from frappe.website.page_renderers.template_page import TemplatePage + class ErrorPage(TemplatePage): def __init__(self, path=None, http_status_code=None, exception=None): - path = 'error' + path = "error" super().__init__(path=path, http_status_code=http_status_code) - self.http_status_code = getattr(exception, 'http_status_code', None) or http_status_code or 500 + self.http_status_code = getattr(exception, "http_status_code", None) or http_status_code or 500 def can_render(self): return True diff --git a/frappe/website/page_renderers/list_page.py b/frappe/website/page_renderers/list_page.py index 61c781ea14..ad2616a03f 100644 --- a/frappe/website/page_renderers/list_page.py +++ b/frappe/website/page_renderers/list_page.py @@ -1,11 +1,12 @@ import frappe from frappe.website.page_renderers.template_page import TemplatePage + class ListPage(TemplatePage): def can_render(self): - return frappe.db.exists('DocType', self.path, True) + return frappe.db.exists("DocType", self.path, True) def render(self): frappe.local.form_dict.doctype = self.path - self.set_standard_path('list') + self.set_standard_path("list") return super().render() diff --git a/frappe/website/page_renderers/not_found_page.py b/frappe/website/page_renderers/not_found_page.py index de631f5dfe..98aeb19057 100644 --- a/frappe/website/page_renderers/not_found_page.py +++ b/frappe/website/page_renderers/not_found_page.py @@ -5,13 +5,14 @@ import frappe from frappe.website.page_renderers.template_page import TemplatePage from frappe.website.utils import can_cache -HOMEPAGE_PATHS = ('/', '/index', 'index') +HOMEPAGE_PATHS = ("/", "/index", "index") + class NotFoundPage(TemplatePage): def __init__(self, path, http_status_code=None): self.request_path = path - self.request_url = frappe.local.request.url if hasattr(frappe.local, 'request') else '' - path = '404' + self.request_url = frappe.local.request.url if hasattr(frappe.local, "request") else "" + path = "404" http_status_code = http_status_code or 404 super().__init__(path=path, http_status_code=http_status_code) @@ -20,7 +21,7 @@ class NotFoundPage(TemplatePage): def render(self): if self.can_cache_404(): - frappe.cache().hset('website_404', self.request_url, True) + frappe.cache().hset("website_404", self.request_url, True) return super().render() def can_cache_404(self): diff --git a/frappe/website/page_renderers/not_permitted_page.py b/frappe/website/page_renderers/not_permitted_page.py index e69299f5c5..bd27150617 100644 --- a/frappe/website/page_renderers/not_permitted_page.py +++ b/frappe/website/page_renderers/not_permitted_page.py @@ -1,10 +1,11 @@ import frappe from frappe import _ -from frappe.website.page_renderers.template_page import TemplatePage from frappe.utils import cstr +from frappe.website.page_renderers.template_page import TemplatePage + class NotPermittedPage(TemplatePage): - def __init__(self, path=None, http_status_code=None, exception=''): + def __init__(self, path=None, http_status_code=None, exception=""): frappe.local.message = cstr(exception) super().__init__(path=path, http_status_code=http_status_code) self.http_status_code = 403 @@ -14,11 +15,8 @@ class NotPermittedPage(TemplatePage): def render(self): frappe.local.message_title = _("Not Permitted") - frappe.local.response['context'] = dict( - indicator_color = 'red', - primary_action = '/login', - primary_label = _('Login'), - fullpage=True + frappe.local.response["context"] = dict( + indicator_color="red", primary_action="/login", primary_label=_("Login"), fullpage=True ) - self.set_standard_path('message') + self.set_standard_path("message") return super().render() diff --git a/frappe/website/page_renderers/print_page.py b/frappe/website/page_renderers/print_page.py index 05d4026e2b..e8a9d9d9a4 100644 --- a/frappe/website/page_renderers/print_page.py +++ b/frappe/website/page_renderers/print_page.py @@ -1,23 +1,24 @@ import frappe from frappe.website.page_renderers.template_page import TemplatePage + class PrintPage(TemplatePage): - ''' + """ default path returns a printable object (based on permission) /Quotation/Q-0001 - ''' + """ + def can_render(self): - parts = self.path.split('/', 1) - if len(parts)==2: - if (frappe.db.exists('DocType', parts[0], True) - and frappe.db.exists(parts[0], parts[1], True)): + parts = self.path.split("/", 1) + if len(parts) == 2: + if frappe.db.exists("DocType", parts[0], True) and frappe.db.exists(parts[0], parts[1], True): return True return False def render(self): - parts = self.path.split('/', 1) + parts = self.path.split("/", 1) frappe.form_dict.doctype = parts[0] frappe.form_dict.name = parts[1] - self.set_standard_path('printview') + self.set_standard_path("printview") return super().render() diff --git a/frappe/website/page_renderers/redirect_page.py b/frappe/website/page_renderers/redirect_page.py index 2049c375e8..b673fce3bb 100644 --- a/frappe/website/page_renderers/redirect_page.py +++ b/frappe/website/page_renderers/redirect_page.py @@ -1,6 +1,7 @@ import frappe from frappe.website.utils import build_response + class RedirectPage(object): def __init__(self, path, http_status_code=301): self.path = path @@ -10,7 +11,12 @@ class RedirectPage(object): return True def render(self): - return build_response(self.path, "", 301, { - "Location": frappe.flags.redirect_location or (frappe.local.response or {}).get('location'), - "Cache-Control": "no-store, no-cache, must-revalidate" - }) + return build_response( + self.path, + "", + 301, + { + "Location": frappe.flags.redirect_location or (frappe.local.response or {}).get("location"), + "Cache-Control": "no-store, no-cache, must-revalidate", + }, + ) diff --git a/frappe/website/page_renderers/static_page.py b/frappe/website/page_renderers/static_page.py index 48bf0f2aec..eb862d42bd 100644 --- a/frappe/website/page_renderers/static_page.py +++ b/frappe/website/page_renderers/static_page.py @@ -8,7 +8,8 @@ import frappe from frappe.website.page_renderers.base_renderer import BaseRenderer from frappe.website.utils import is_binary_file -UNSUPPORTED_STATIC_PAGE_TYPES = ('html', 'md', 'js', 'xml', 'css', 'txt', 'py', 'json') +UNSUPPORTED_STATIC_PAGE_TYPES = ("html", "md", "js", "xml", "css", "txt", "py", "json") + class StaticPage(BaseRenderer): def __init__(self, path, http_status_code=None): @@ -16,11 +17,11 @@ class StaticPage(BaseRenderer): self.set_file_path() def set_file_path(self): - self.file_path = '' + self.file_path = "" if not self.is_valid_file_path(): return for app in frappe.get_installed_apps(): - file_path = frappe.get_app_path(app, 'www') + '/' + self.path + file_path = frappe.get_app_path(app, "www") + "/" + self.path if os.path.isfile(file_path) and is_binary_file(file_path): self.file_path = file_path @@ -28,14 +29,14 @@ class StaticPage(BaseRenderer): return self.is_valid_file_path() and self.file_path def is_valid_file_path(self): - extension = self.path.rsplit('.', 1)[-1] + extension = self.path.rsplit(".", 1)[-1] if extension in UNSUPPORTED_STATIC_PAGE_TYPES: return False return True def render(self): # file descriptor to be left open, closed by middleware - f = open(self.file_path, 'rb') + f = open(self.file_path, "rb") response = Response(wrap_file(frappe.local.request.environ, f), direct_passthrough=True) - response.mimetype = mimetypes.guess_type(self.file_path)[0] or 'application/octet-stream' + response.mimetype = mimetypes.guess_type(self.file_path)[0] or "application/octet-stream" return response diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index ff3e8509bd..2ed8a62119 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -1,15 +1,29 @@ import io import os + import click import frappe -from frappe.website.router import get_page_info from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.router import get_base_template -from frappe.website.utils import (extract_comment_tag, extract_title, get_next_link, - get_toc, get_frontmatter, is_binary_file, cache_html, get_sidebar_items) - -WEBPAGE_PY_MODULE_PROPERTIES = ("base_template_path", "template", "no_cache", "sitemap", "condition_field") +from frappe.website.router import get_base_template, get_page_info +from frappe.website.utils import ( + cache_html, + extract_comment_tag, + extract_title, + get_frontmatter, + get_next_link, + get_sidebar_items, + get_toc, + is_binary_file, +) + +WEBPAGE_PY_MODULE_PROPERTIES = ( + "base_template_path", + "template", + "no_cache", + "sitemap", + "condition_field", +) COMMENT_PROPERTY_KEY_VALUE_MAP = { "no-breadcrumbs": ("no_breadcrumbs", 1), @@ -19,19 +33,20 @@ COMMENT_PROPERTY_KEY_VALUE_MAP = { "add-next-prev-links": ("add_next_prev_links", 1), "no-cache": ("no_cache", 1), "no-sitemap": ("sitemap", 0), - "sitemap": ("sitemap", 1) + "sitemap": ("sitemap", 1), } + class TemplatePage(BaseTemplatePage): def __init__(self, path, http_status_code=None): super().__init__(path=path, http_status_code=http_status_code) self.set_template_path() def set_template_path(self): - ''' + """ Searches for file matching the path in the /www and /templates/pages folders and sets path if match is found - ''' + """ folders = get_start_folders() for app in frappe.get_installed_apps(frappe_last=True): app_path = frappe.get_app_path(app) @@ -51,11 +66,13 @@ class TemplatePage(BaseTemplatePage): return def can_render(self): - return hasattr(self, 'template_path') and bool(self.template_path) + return hasattr(self, "template_path") and bool(self.template_path) @staticmethod def get_index_path_options(search_path): - return (frappe.as_unicode(f'{search_path}{d}') for d in ('', '.html', '.md', '/index.html', '/index.md')) + return ( + frappe.as_unicode(f"{search_path}{d}") for d in ("", ".html", ".md", "/index.html", "/index.md") + ) def render(self): html = self.get_html() @@ -91,7 +108,7 @@ class TemplatePage(BaseTemplatePage): if self.context.add_breadcrumbs and not self.context.parents: parent_path = os.path.dirname(self.path) - if self.path.endswith('index'): + if self.path.endswith("index"): # in case of index page move one directory up for parent path parent_path = os.path.dirname(parent_path) @@ -100,26 +117,31 @@ class TemplatePage(BaseTemplatePage): if os.path.isfile(parent_file_path): parent_page_context = get_page_info(parent_file_path, self.app, self.file_dir) if parent_page_context: - self.context.parents = [dict(route=os.path.dirname(self.path), title=parent_page_context.title)] + self.context.parents = [ + dict(route=os.path.dirname(self.path), title=parent_page_context.title) + ] break def set_pymodule(self): - ''' + """ A template may have a python module with a `get_context` method along with it in the same folder. Also the hyphens will be coverted to underscore for python module names. This method sets the pymodule_name if it exists. - ''' + """ template_basepath = os.path.splitext(self.template_path)[0] self.pymodule_name = None # replace - with _ in the internal modules names - self.pymodule_path = os.path.join(os.path.dirname(template_basepath), os.path.basename(template_basepath.replace("-", "_")) + ".py") + self.pymodule_path = os.path.join( + os.path.dirname(template_basepath), + os.path.basename(template_basepath.replace("-", "_")) + ".py", + ) if os.path.exists(os.path.join(self.app_path, self.pymodule_path)): self.pymodule_name = self.app + "." + self.pymodule_path.replace(os.path.sep, ".")[:-3] def setup_template_source(self): - '''Setup template source, frontmatter and markdown conversion''' + """Setup template source, frontmatter and markdown conversion""" self.source = self.get_raw_template() self.extract_frontmatter() self.convert_from_markdown() @@ -132,7 +154,7 @@ class TemplatePage(BaseTemplatePage): self.pymodule = frappe.get_module(self.pymodule_name) self.set_pymodule_properties() - data = self.run_pymodule_method('get_context') + data = self.run_pymodule_method("get_context") # some methods may return a "context" object if data: self.context.update(data) @@ -148,8 +170,7 @@ class TemplatePage(BaseTemplatePage): self.context[prop] = getattr(self.pymodule, prop) def set_page_properties(self): - self.context.base_template = self.context.base_template \ - or get_base_template(self.path) + self.context.base_template = self.context.base_template or get_base_template(self.path) self.context.basepath = self.basepath self.context.basename = self.basename self.context.name = self.name @@ -164,16 +185,20 @@ class TemplatePage(BaseTemplatePage): if not context.title: context.title = extract_title(self.source, self.path) - base_template = extract_comment_tag(self.source, 'base_template') + base_template = extract_comment_tag(self.source, "base_template") if base_template: context.base_template = base_template - if (context.base_template + if ( + context.base_template and "{%- extends" not in self.source and "{% extends" not in self.source - and "" not in self.source): - self.source = '''{{% extends "{0}" %}} - {{% block page_content %}}{1}{{% endblock %}}'''.format(context.base_template, self.source) + and "" not in self.source + ): + self.source = """{{% extends "{0}" %}} + {{% block page_content %}}{1}{{% endblock %}}""".format( + context.base_template, self.source + ) self.set_properties_via_comments() @@ -182,13 +207,14 @@ class TemplatePage(BaseTemplatePage): comment_tag = f"" if comment_tag in self.source: self.context[context_key] = value - click.echo(f'\n⚠️ DEPRECATION WARNING: {comment_tag} will be deprecated on 2021-12-31.') - click.echo(f'Please remove it from {self.template_path} in {self.app}') + click.echo(f"\n⚠️ DEPRECATION WARNING: {comment_tag} will be deprecated on 2021-12-31.") + click.echo(f"Please remove it from {self.template_path} in {self.app}") def run_pymodule_method(self, method_name): if hasattr(self.pymodule, method_name): try: import inspect + method = getattr(self.pymodule, method_name) if inspect.getfullargspec(method).args: return method(self.context) @@ -201,8 +227,8 @@ class TemplatePage(BaseTemplatePage): frappe.errprint(frappe.utils.get_traceback()) def render_template(self): - if self.template_path.endswith('min.js'): - html = self.source # static + if self.template_path.endswith("min.js"): + html = self.source # static else: if self.context.safe_render is not None: safe_render = self.context.safe_render @@ -214,70 +240,71 @@ class TemplatePage(BaseTemplatePage): return html def extends_template(self): - return (self.template_path.endswith(('.html', '.md')) - and ('{%- extends' in self.source - or '{% extends' in self.source)) + return self.template_path.endswith((".html", ".md")) and ( + "{%- extends" in self.source or "{% extends" in self.source + ) def get_raw_template(self): return frappe.get_jloader().get_source(frappe.get_jenv(), self.context.template)[0] def load_colocated_files(self): - '''load co-located css/js files with the same name''' - js_path = self.basename + '.js' - if os.path.exists(js_path) and '{% block script %}' not in self.source: + """load co-located css/js files with the same name""" + js_path = self.basename + ".js" + if os.path.exists(js_path) and "{% block script %}" not in self.source: self.context.colocated_js = self.get_colocated_file(js_path) - css_path = self.basename + '.css' - if os.path.exists(css_path) and '{% block style %}' not in self.source: + css_path = self.basename + ".css" + if os.path.exists(css_path) and "{% block style %}" not in self.source: self.context.colocated_css = self.get_colocated_file(css_path) def get_colocated_file(self, path): - with io.open(path, 'r', encoding = 'utf-8') as f: + with io.open(path, "r", encoding="utf-8") as f: return f.read() def extract_frontmatter(self): - if not self.template_path.endswith(('.md', '.html')): + if not self.template_path.endswith((".md", ".html")): return try: # values will be used to update self res = get_frontmatter(self.source) - if res['attributes']: - self.context.update(res['attributes']) - self.source = res['body'] + if res["attributes"]: + self.context.update(res["attributes"]) + self.source = res["body"] except Exception: pass def convert_from_markdown(self): - if self.template_path.endswith('.md'): + if self.template_path.endswith(".md"): self.source = frappe.utils.md_to_html(self.source) self.context.page_toc_html = self.source.toc_html if not self.context.show_sidebar: - self.source = '
' + self.source + '
' + self.source = '
' + self.source + "
" def update_toc(self, html): - if '{index}' in html: - html = html.replace('{index}', get_toc(self.path)) + if "{index}" in html: + html = html.replace("{index}", get_toc(self.path)) - if '{next}' in html: - html = html.replace('{next}', get_next_link(self.path)) + if "{next}" in html: + html = html.replace("{next}", get_next_link(self.path)) return html def set_standard_path(self, path): - self.app = 'frappe' - self.app_path = frappe.get_app_path('frappe') + self.app = "frappe" + self.app_path = frappe.get_app_path("frappe") self.path = path - self.template_path = 'www/{path}.html'.format(path=path) + self.template_path = "www/{path}.html".format(path=path) def set_missing_values(self): super().set_missing_values() # for backward compatibility - self.context.docs_base_url = '/docs' + self.context.docs_base_url = "/docs" def set_user_info(self): from frappe.utils.user import get_fullname_and_avatar + info = get_fullname_and_avatar(frappe.session.user) self.context["fullname"] = info.fullname self.context["user_image"] = info.avatar @@ -285,4 +312,4 @@ class TemplatePage(BaseTemplatePage): def get_start_folders(): - return frappe.local.flags.web_pages_folders or ('www', 'templates/pages') + return frappe.local.flags.web_pages_folders or ("www", "templates/pages") diff --git a/frappe/website/page_renderers/web_form.py b/frappe/website/page_renderers/web_form.py index 786aeef3d1..913252c155 100644 --- a/frappe/website/page_renderers/web_form.py +++ b/frappe/website/page_renderers/web_form.py @@ -1,10 +1,11 @@ -from frappe.website.page_renderers.document_page import DocumentPage import frappe +from frappe.website.page_renderers.document_page import DocumentPage + class WebFormPage(DocumentPage): def can_render(self): - webform_name = frappe.db.exists("Web Form", {'route': self.path}, cache=True) + webform_name = frappe.db.exists("Web Form", {"route": self.path}, cache=True) if webform_name: - self.doctype = 'Web Form' + self.doctype = "Web Form" self.docname = webform_name return bool(webform_name) diff --git a/frappe/website/path_resolver.py b/frappe/website/path_resolver.py index 0ed097416e..9015bc7566 100644 --- a/frappe/website/path_resolver.py +++ b/frappe/website/path_resolver.py @@ -1,6 +1,6 @@ import re -import click +import click from werkzeug.routing import Rule import frappe @@ -16,18 +16,18 @@ from frappe.website.router import evaluate_dynamic_routes from frappe.website.utils import can_cache, get_home_page -class PathResolver(): +class PathResolver: def __init__(self, path): - self.path = path.strip('/ ') + self.path = path.strip("/ ") def resolve(self): - '''Returns endpoint and a renderer instance that can render the endpoint''' + """Returns endpoint and a renderer instance that can render the endpoint""" request = frappe._dict() - if hasattr(frappe.local, 'request'): + if hasattr(frappe.local, "request"): request = frappe.local.request or request # check if the request url is in 404 list - if request.url and can_cache() and frappe.cache().hget('website_404', request.url): + if request.url and can_cache() and frappe.cache().hget("website_404", request.url): return self.path, NotFoundPage(self.path) try: @@ -37,7 +37,15 @@ class PathResolver(): endpoint = resolve_path(self.path) custom_renderers = self.get_custom_page_renderers() - renderers = custom_renderers + [StaticPage, WebFormPage, DocumentPage, TemplatePage, ListPage, PrintPage, NotFoundPage] + renderers = custom_renderers + [ + StaticPage, + WebFormPage, + DocumentPage, + TemplatePage, + ListPage, + PrintPage, + NotFoundPage, + ] for renderer in renderers: renderer_instance = renderer(endpoint, 200) @@ -53,64 +61,64 @@ class PathResolver(): @staticmethod def get_custom_page_renderers(): custom_renderers = [] - for renderer_path in frappe.get_hooks('page_renderer') or []: + for renderer_path in frappe.get_hooks("page_renderer") or []: try: renderer = frappe.get_attr(renderer_path) - if not hasattr(renderer, 'can_render'): - click.echo(f'{renderer.__name__} does not have can_render method') + if not hasattr(renderer, "can_render"): + click.echo(f"{renderer.__name__} does not have can_render method") continue - if not hasattr(renderer, 'render'): - click.echo(f'{renderer.__name__} does not have render method') + if not hasattr(renderer, "render"): + click.echo(f"{renderer.__name__} does not have render method") continue custom_renderers.append(renderer) except Exception: - click.echo(f'Failed to load page renderer. Import path: {renderer_path}') + click.echo(f"Failed to load page renderer. Import path: {renderer_path}") return custom_renderers - def resolve_redirect(path, query_string=None): - ''' + """ Resolve redirects from hooks Example: - website_redirect = [ - # absolute location - {"source": "/from", "target": "https://mysite/from"}, + website_redirect = [ + # absolute location + {"source": "/from", "target": "https://mysite/from"}, - # relative location - {"source": "/from", "target": "/main"}, + # relative location + {"source": "/from", "target": "/main"}, - # use regex - {"source": r"/from/(.*)", "target": r"/main/\1"} - # use r as a string prefix if you use regex groups or want to escape any string literal - ] - ''' - redirects = frappe.get_hooks('website_redirects') - redirects += frappe.db.get_all('Website Route Redirect', ['source', 'target']) + # use regex + {"source": r"/from/(.*)", "target": r"/main/\1"} + # use r as a string prefix if you use regex groups or want to escape any string literal + ] + """ + redirects = frappe.get_hooks("website_redirects") + redirects += frappe.db.get_all("Website Route Redirect", ["source", "target"]) - if not redirects: return + if not redirects: + return - redirect_to = frappe.cache().hget('website_redirects', path) + redirect_to = frappe.cache().hget("website_redirects", path) if redirect_to: frappe.flags.redirect_location = redirect_to raise frappe.Redirect for rule in redirects: - pattern = rule['source'].strip('/ ') + '$' + pattern = rule["source"].strip("/ ") + "$" path_to_match = path - if rule.get('match_with_query_string'): - path_to_match = path + '?' + frappe.safe_decode(query_string) + if rule.get("match_with_query_string"): + path_to_match = path + "?" + frappe.safe_decode(query_string) if re.match(pattern, path_to_match): - redirect_to = re.sub(pattern, rule['target'], path_to_match) + redirect_to = re.sub(pattern, rule["target"], path_to_match) frappe.flags.redirect_location = redirect_to - frappe.cache().hset('website_redirects', path_to_match, redirect_to) + frappe.cache().hset("website_redirects", path_to_match, redirect_to) raise frappe.Redirect @@ -118,7 +126,7 @@ def resolve_path(path): if not path: path = "index" - if path.endswith('.html'): + if path.endswith(".html"): path = path[:-5] if path == "index": @@ -131,20 +139,25 @@ def resolve_path(path): return path + def resolve_from_map(path): - '''transform dynamic route to a static one from hooks and route defined in doctype''' - rules = [Rule(r["from_route"], endpoint=r["to_route"], defaults=r.get("defaults")) - for r in get_website_rules()] + """transform dynamic route to a static one from hooks and route defined in doctype""" + rules = [ + Rule(r["from_route"], endpoint=r["to_route"], defaults=r.get("defaults")) + for r in get_website_rules() + ] return evaluate_dynamic_routes(rules, path) or path + def get_website_rules(): - '''Get website route rules from hooks and DocType route''' + """Get website route rules from hooks and DocType route""" + def _get(): rules = frappe.get_hooks("website_route_rules") - for d in frappe.get_all('DocType', 'name, route', dict(has_web_view=1)): + for d in frappe.get_all("DocType", "name, route", dict(has_web_view=1)): if d.route: - rules.append(dict(from_route = '/' + d.route.strip('/'), to_route=d.name)) + rules.append(dict(from_route="/" + d.route.strip("/"), to_route=d.name)) return rules @@ -152,4 +165,4 @@ def get_website_rules(): # dont cache in development return _get() - return frappe.cache().get_value('website_route_rules', _get) + return frappe.cache().get_value("website_route_rules", _get) diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index 028bbe14a7..fd9bcc01ba 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -12,6 +12,7 @@ from frappe.utils.dateutils import get_dates_from_timegrain def execute(filters=None): return WebsiteAnalytics(filters).run() + class WebsiteAnalytics(object): def __init__(self, filters=None): self.filters = frappe._dict(filters or {}) @@ -26,7 +27,7 @@ class WebsiteAnalytics(object): self.filters.range = "Daily" self.filters.to_date = frappe.utils.add_days(self.filters.to_date, 1) - self.query_filters = {'creation': ['between', [self.filters.from_date, self.filters.to_date]]} + self.query_filters = {"creation": ["between", [self.filters.from_date, self.filters.to_date]]} def run(self): columns = self.get_columns() @@ -38,24 +39,9 @@ class WebsiteAnalytics(object): def get_columns(self): return [ - { - "fieldname": "path", - "label": "Page", - "fieldtype": "Data", - "width": 300 - }, - { - "fieldname": "count", - "label": "Page Views", - "fieldtype": "Int", - "width": 150 - }, - { - "fieldname": "unique_count", - "label": "Unique Visitors", - "fieldtype": "Int", - "width": 150 - } + {"fieldname": "path", "label": "Page", "fieldtype": "Data", "width": 300}, + {"fieldname": "count", "label": "Page Views", "fieldtype": "Int", "width": 150}, + {"fieldname": "unique_count", "label": "Unique Visitors", "fieldtype": "Int", "width": 150}, ] def get_data(self): @@ -68,7 +54,7 @@ class WebsiteAnalytics(object): frappe.qb.from_(WebPageView) .select("path", count_all, count_is_unique) .where( - Coalesce(WebPageView.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date] + Coalesce(WebPageView.creation, "0001-01-01")[self.filters.from_date : self.filters.to_date] ) .groupby(WebPageView.path) .orderby("count", Order=frappe.qb.desc) @@ -76,14 +62,14 @@ class WebsiteAnalytics(object): def _get_query_for_mariadb(self): filters_range = self.filters.range - field = 'creation' - date_format = '%Y-%m-%d' + field = "creation" + date_format = "%Y-%m-%d" if filters_range == "Weekly": - field = 'ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)' + field = "ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)" elif filters_range == "Monthly": - date_format = '%Y-%m-01' + date_format = "%Y-%m-01" query = """ SELECT @@ -94,7 +80,9 @@ class WebsiteAnalytics(object): WHERE creation BETWEEN %s AND %s GROUP BY DATE_FORMAT({0}, %s) ORDER BY creation - """.format(field) + """.format( + field + ) values = (date_format, self.filters.from_date, self.filters.to_date, date_format) @@ -102,14 +90,14 @@ class WebsiteAnalytics(object): def _get_query_for_postgres(self): filters_range = self.filters.range - field = 'creation' - granularity = 'day' + field = "creation" + granularity = "day" if filters_range == "Weekly": - granularity = 'week' + granularity = "week" elif filters_range == "Monthly": - granularity = 'day' + granularity = "day" query = """ SELECT @@ -120,16 +108,18 @@ class WebsiteAnalytics(object): WHERE coalesce("tabWeb Page View".{0}, '0001-01-01') BETWEEN %s AND %s GROUP BY date_trunc(%s, {0}) ORDER BY date - """.format(field) + """.format( + field + ) values = (granularity, self.filters.from_date, self.filters.to_date, granularity) return query, values def get_chart_data(self): - current_dialect = frappe.db.db_type or 'mariadb' + current_dialect = frappe.db.db_type or "mariadb" - if current_dialect == 'mariadb': + if current_dialect == "mariadb": query, values = self._get_query_for_mariadb() else: query, values = self._get_query_for_postgres() @@ -139,7 +129,9 @@ class WebsiteAnalytics(object): return self.prepare_chart_data(self.chart_data) def prepare_chart_data(self, data): - date_range = get_dates_from_timegrain(self.filters.from_date, self.filters.to_date, self.filters.range) + date_range = get_dates_from_timegrain( + self.filters.from_date, self.filters.to_date, self.filters.range + ) if self.filters.range == "Monthly": date_range = [frappe.utils.add_days(dd, 1) for dd in date_range] @@ -152,50 +144,38 @@ class WebsiteAnalytics(object): item_date = getdate(item.get("date")) if item_date == date: return item - return {'count': 0, 'unique_count': 0} - + return {"count": 0, "unique_count": 0} for date in date_range: labels.append(date.strftime("%b %d %Y")) match = get_data_for_date(date) - total_dataset.append(match.get('count', 0)) - unique_dataset.append(match.get('unique_count', 0)) + total_dataset.append(match.get("count", 0)) + unique_dataset.append(match.get("unique_count", 0)) chart = { "data": { - 'labels': labels, - 'datasets': [ - { - 'name': "Total Views", - 'type': 'line', - 'values': total_dataset - }, - { - 'name': "Unique Visits", - 'type': 'line', - 'values': unique_dataset - } - ] + "labels": labels, + "datasets": [ + {"name": "Total Views", "type": "line", "values": total_dataset}, + {"name": "Unique Visits", "type": "line", "values": unique_dataset}, + ], }, "type": "axis-mixed", - 'lineOptions': { - 'regionFill': 1, - }, - 'axisOptions': { - 'xIsSeries': 1 + "lineOptions": { + "regionFill": 1, }, - 'colors': ['#7cd6fd', '#5e64ff'] + "axisOptions": {"xIsSeries": 1}, + "colors": ["#7cd6fd", "#5e64ff"], } return chart - def get_report_summary(self): total_count = 0 unique_count = 0 for data in self.chart_data: - unique_count += data.get('unique_count') - total_count += data.get('count') + unique_count += data.get("unique_count") + total_count += data.get("count") report_summary = [ { @@ -208,7 +188,5 @@ class WebsiteAnalytics(object): "label": "Unique Page Views", "datatype": "Int", }, - ] return report_summary - diff --git a/frappe/website/router.py b/frappe/website/router.py index eb3e77a0a2..e9f0d0f09c 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -5,36 +5,40 @@ import io import os import re +from werkzeug.routing import Map, NotFound, Rule + import frappe from frappe.website.utils import extract_title -from werkzeug.routing import Map, Rule, NotFound + def get_page_info_from_web_page_with_dynamic_routes(path): - ''' + """ Query Web Page with dynamic_route = 1 and evaluate if any of the routes match - ''' + """ rules, page_info = [], {} # build rules from all web page with `dynamic_route = 1` - for d in frappe.get_all('Web Page', fields = ['name', 'route', 'modified'], - filters = dict(published = 1, dynamic_route=1)): - rules.append(Rule('/' + d.route, endpoint = d.name)) - d.doctype = 'Web Page' + for d in frappe.get_all( + "Web Page", fields=["name", "route", "modified"], filters=dict(published=1, dynamic_route=1) + ): + rules.append(Rule("/" + d.route, endpoint=d.name)) + d.doctype = "Web Page" page_info[d.name] = d end_point = evaluate_dynamic_routes(rules, path) if end_point: return page_info[end_point] + def evaluate_dynamic_routes(rules, path): - ''' + """ Use Werkzeug routing to evaluate dynamic routes like /project/ https://werkzeug.palletsprojects.com/en/1.0.x/routing/ - ''' + """ route_map = Map(rules) endpoint = None - if hasattr(frappe.local, 'request') and frappe.local.request.environ: + if hasattr(frappe.local, "request") and frappe.local.request.environ: urls = route_map.bind_to_environ(frappe.local.request.environ) try: endpoint, args = urls.match("/" + path) @@ -49,8 +53,9 @@ def evaluate_dynamic_routes(rules, path): return endpoint + def get_pages(app=None): - '''Get all pages. Called for docs / sitemap''' + """Get all pages. Called for docs / sitemap""" def _build(app): pages = {} @@ -68,7 +73,8 @@ def get_pages(app=None): return pages - return frappe.cache().get_value('website_pages', lambda: _build(app)) + return frappe.cache().get_value("website_pages", lambda: _build(app)) + def get_pages_from_path(start, app, app_path): pages = {} @@ -76,28 +82,30 @@ def get_pages_from_path(start, app, app_path): if os.path.exists(start_path): for basepath, folders, files in os.walk(start_path): # add missing __init__.py - if not '__init__.py' in files: - open(os.path.join(basepath, '__init__.py'), 'a').close() + if not "__init__.py" in files: + open(os.path.join(basepath, "__init__.py"), "a").close() for fname in files: fname = frappe.utils.cstr(fname) - if not '.' in fname: + if not "." in fname: continue page_name, extn = fname.rsplit(".", 1) - if extn in ('js', 'css') and os.path.exists(os.path.join(basepath, page_name + '.html')): + if extn in ("js", "css") and os.path.exists(os.path.join(basepath, page_name + ".html")): # js, css is linked to html, skip continue if extn in ("html", "xml", "js", "css", "md"): - page_info = get_page_info(os.path.join(basepath, fname), - app, start, basepath, app_path, fname) + page_info = get_page_info( + os.path.join(basepath, fname), app, start, basepath, app_path, fname + ) pages[page_info.route] = page_info # print frappe.as_json(pages[-1]) return pages + def get_page_info(path, app, start, basepath=None, app_path=None, fname=None): - '''Load page info''' + """Load page info""" if fname is None: fname = os.path.basename(path) @@ -112,31 +120,32 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None): # add website route page_info = frappe._dict() - page_info.basename = page_name if extn in ('html', 'md') else fname + page_info.basename = page_name if extn in ("html", "md") else fname page_info.basepath = basepath page_info.page_or_generator = "Page" page_info.template = os.path.relpath(os.path.join(basepath, fname), app_path) - if page_info.basename == 'index': + if page_info.basename == "index": page_info.basename = "" # get route from template name - page_info.route = page_info.template.replace(start, '').strip('/') - if os.path.basename(page_info.route) in ('index.html', 'index.md'): + page_info.route = page_info.template.replace(start, "").strip("/") + if os.path.basename(page_info.route) in ("index.html", "index.md"): page_info.route = os.path.dirname(page_info.route) # remove the extension - if page_info.route.endswith('.md') or page_info.route.endswith('.html'): - page_info.route = page_info.route.rsplit('.', 1)[0] + if page_info.route.endswith(".md") or page_info.route.endswith(".html"): + page_info.route = page_info.route.rsplit(".", 1)[0] page_info.name = page_info.page_name = page_info.route # controller page_info.controller_path = os.path.join(basepath, page_name.replace("-", "_") + ".py") if os.path.exists(page_info.controller_path): - controller = app + "." + os.path.relpath(page_info.controller_path, - app_path).replace(os.path.sep, ".")[:-3] + controller = ( + app + "." + os.path.relpath(page_info.controller_path, app_path).replace(os.path.sep, ".")[:-3] + ) page_info.controller = controller @@ -151,6 +160,7 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None): return page_info + def get_frontmatter(string): """ Reference: https://github.com/jonbeebe/frontmatter @@ -159,7 +169,7 @@ def get_frontmatter(string): fmatter = "" body = "" - result = re.compile(r'^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$', re.S | re.M).search(string) + result = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M).search(string) if result: fmatter = result.group(1) @@ -170,66 +180,76 @@ def get_frontmatter(string): "body": body, } + def setup_source(page_info): - '''Get the HTML source of the template''' + """Get the HTML source of the template""" jenv = frappe.get_jenv() source = jenv.loader.get_source(jenv, page_info.template)[0] - html = '' + html = "" - if page_info.template.endswith(('.md', '.html')): + if page_info.template.endswith((".md", ".html")): # extract frontmatter block if exists try: # values will be used to update page_info res = get_frontmatter(source) - if res['attributes']: - page_info.update(res['attributes']) - source = res['body'] + if res["attributes"]: + page_info.update(res["attributes"]) + source = res["body"] except Exception: pass - if page_info.template.endswith('.md'): + if page_info.template.endswith(".md"): source = frappe.utils.md_to_html(source) page_info.page_toc_html = source.toc_html if not page_info.show_sidebar: - source = '
' + source + '
' + source = '
' + source + "
" if not page_info.base_template: page_info.base_template = get_base_template(page_info.route) - if page_info.template.endswith(('.html', '.md', )) and \ - '{%- extends' not in source and '{% extends' not in source: + if ( + page_info.template.endswith( + ( + ".html", + ".md", + ) + ) + and "{%- extends" not in source + and "{% extends" not in source + ): # set the source only if it contains raw content html = source # load css/js files - js_path = os.path.join(page_info.basepath, (page_info.basename or 'index') + '.js') - if os.path.exists(js_path) and '{% block script %}' not in html: - with io.open(js_path, 'r', encoding = 'utf-8') as f: + js_path = os.path.join(page_info.basepath, (page_info.basename or "index") + ".js") + if os.path.exists(js_path) and "{% block script %}" not in html: + with io.open(js_path, "r", encoding="utf-8") as f: js = f.read() page_info.colocated_js = js - css_path = os.path.join(page_info.basepath, (page_info.basename or 'index') + '.css') - if os.path.exists(css_path) and '{% block style %}' not in html: - with io.open(css_path, 'r', encoding='utf-8') as f: + css_path = os.path.join(page_info.basepath, (page_info.basename or "index") + ".css") + if os.path.exists(css_path) and "{% block style %}" not in html: + with io.open(css_path, "r", encoding="utf-8") as f: css = f.read() page_info.colocated_css = css if html: page_info.source = html - page_info.base_template = page_info.base_template or 'templates/web.html' + page_info.base_template = page_info.base_template or "templates/web.html" else: - page_info.source = '' + page_info.source = "" # show table of contents setup_index(page_info) + def get_base_template(path=None): - ''' + """ Returns the `base_template` for given `path`. The default `base_template` for any web route is `templates/web.html` defined in `hooks.py`. This can be overridden for certain routes in `custom_app/hooks.py` based on regex pattern. - ''' + """ if not path: path = frappe.local.request.path @@ -242,38 +262,49 @@ def get_base_template(path=None): base_template = templates[-1] return base_template + def setup_index(page_info): - '''Build page sequence from index.txt''' - if page_info.basename=='': + """Build page sequence from index.txt""" + if page_info.basename == "": # load index.txt if loading all pages - index_txt_path = os.path.join(page_info.basepath, 'index.txt') + index_txt_path = os.path.join(page_info.basepath, "index.txt") if os.path.exists(index_txt_path): - with open(index_txt_path, 'r') as f: + with open(index_txt_path, "r") as f: page_info.index = f.read().splitlines() + def load_properties_from_controller(page_info): - if not page_info.controller: return + if not page_info.controller: + return module = frappe.get_module(page_info.controller) - if not module: return + if not module: + return - for prop in ("base_template_path", "template", "no_cache", - "sitemap", "condition_field"): + for prop in ("base_template_path", "template", "no_cache", "sitemap", "condition_field"): if hasattr(module, prop): page_info[prop] = getattr(module, prop) + def get_doctypes_with_web_view(): - '''Return doctypes with Has Web View or set via hooks''' + """Return doctypes with Has Web View or set via hooks""" + def _get(): installed_apps = frappe.get_installed_apps() doctypes = frappe.get_hooks("website_generators") - doctypes_with_web_view = frappe.get_all('DocType', fields=['name', 'module'], - filters=dict(has_web_view=1)) + doctypes_with_web_view = frappe.get_all( + "DocType", fields=["name", "module"], filters=dict(has_web_view=1) + ) module_app_map = frappe.local.module_app - doctypes += [d.name for d in doctypes_with_web_view if module_app_map.get(frappe.scrub(d.module)) in installed_apps] + doctypes += [ + d.name + for d in doctypes_with_web_view + if module_app_map.get(frappe.scrub(d.module)) in installed_apps + ] return doctypes - return frappe.cache().get_value('doctypes_with_web_view', _get) + return frappe.cache().get_value("doctypes_with_web_view", _get) + def get_start_folders(): - return frappe.local.flags.web_pages_folders or ('www', 'templates/pages') + return frappe.local.flags.web_pages_folders or ("www", "templates/pages") diff --git a/frappe/website/serve.py b/frappe/website/serve.py index fe7fc77064..b30f3f1047 100644 --- a/frappe/website/serve.py +++ b/frappe/website/serve.py @@ -24,6 +24,7 @@ def get_response(path=None, http_status_code=200): return response + def get_response_content(path=None, http_status_code=200): response = get_response(path, http_status_code) - return str(response.data, 'utf-8') + return str(response.data, "utf-8") diff --git a/frappe/website/utils.py b/frappe/website/utils.py index f0a8da7736..f673a20656 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -18,7 +18,7 @@ from frappe.utils import md_to_html def delete_page_cache(path): cache = frappe.cache() - cache.delete_value('full_index') + cache.delete_value("full_index") groups = ("website_page", "page_context") if path: for name in groups: @@ -27,6 +27,7 @@ def delete_page_cache(path): for name in groups: cache.delete_key(name) + def find_first_image(html): m = re.finditer(r"""]*src\s?=\s?['"]([^'"]*)['"]""", html) try: @@ -34,6 +35,7 @@ def find_first_image(html): except StopIteration: return None + def can_cache(no_cache=False): if frappe.flags.force_website_cache: return True @@ -45,31 +47,40 @@ def can_cache(no_cache=False): def get_comment_list(doctype, name): - comments = frappe.get_all('Comment', - fields=['name', 'creation', 'owner', - 'comment_email', 'comment_by', 'content'], + comments = frappe.get_all( + "Comment", + fields=["name", "creation", "owner", "comment_email", "comment_by", "content"], filters=dict( reference_doctype=doctype, reference_name=name, - comment_type='Comment', + comment_type="Comment", ), - or_filters=[ - ['owner', '=', frappe.session.user], - ['published', '=', 1]]) - - communications = frappe.get_all("Communication", - fields=['name', 'creation', 'owner', 'owner as comment_email', - 'sender_full_name as comment_by', 'content', 'recipients'], + or_filters=[["owner", "=", frappe.session.user], ["published", "=", 1]], + ) + + communications = frappe.get_all( + "Communication", + fields=[ + "name", + "creation", + "owner", + "owner as comment_email", + "sender_full_name as comment_by", + "content", + "recipients", + ], filters=dict( reference_doctype=doctype, reference_name=name, ), or_filters=[ - ['recipients', 'like', '%{0}%'.format(frappe.session.user)], - ['cc', 'like', '%{0}%'.format(frappe.session.user)], - ['bcc', 'like', '%{0}%'.format(frappe.session.user)]]) + ["recipients", "like", "%{0}%".format(frappe.session.user)], + ["cc", "like", "%{0}%".format(frappe.session.user)], + ["bcc", "like", "%{0}%".format(frappe.session.user)], + ], + ) - return sorted((comments + communications), key=lambda comment: comment['creation'], reverse=True) + return sorted((comments + communications), key=lambda comment: comment["creation"], reverse=True) def get_home_page(): @@ -80,11 +91,12 @@ def get_home_page(): home_page = None # for user - if frappe.session.user != 'Guest': + if frappe.session.user != "Guest": # by role for role in frappe.get_roles(): - home_page = frappe.db.get_value('Role', role, 'home_page') - if home_page: break + home_page = frappe.db.get_value("Role", role, "home_page") + if home_page: + break # portal default if not home_page: @@ -99,9 +111,9 @@ def get_home_page(): home_page = frappe.db.get_single_value("Website Settings", "home_page") if not home_page: - home_page = "login" if frappe.session.user == 'Guest' else "me" + home_page = "login" if frappe.session.user == "Guest" else "me" - home_page = home_page.strip('/') + home_page = home_page.strip("/") return home_page @@ -111,14 +123,15 @@ def get_home_page(): return frappe.cache().hget("home_page", frappe.session.user, _get_home_page) + def get_home_page_via_hooks(): home_page = None - home_page_method = frappe.get_hooks('get_website_user_home_page') + home_page_method = frappe.get_hooks("get_website_user_home_page") if home_page_method: home_page = frappe.get_attr(home_page_method[-1])(frappe.session.user) - elif frappe.get_hooks('website_user_home_page'): - home_page = frappe.get_hooks('website_user_home_page')[-1] + elif frappe.get_hooks("website_user_home_page"): + home_page = frappe.get_hooks("website_user_home_page")[-1] if not home_page: role_home_page = frappe.get_hooks("role_home_page") @@ -134,61 +147,63 @@ def get_home_page_via_hooks(): home_page = home_page[-1] if home_page: - home_page = home_page.strip('/') + home_page = home_page.strip("/") return home_page def is_signup_disabled(): - return frappe.db.get_single_value('Website Settings', 'disable_signup', True) + return frappe.db.get_single_value("Website Settings", "disable_signup", True) + def cleanup_page_name(title): """make page name from title""" if not title: - return '' + return "" name = title.lower() - name = re.sub(r'[~!@#$%^&*+()<>,."\'\?]', '', name) - name = re.sub('[:/]', '-', name) - name = '-'.join(name.split()) + name = re.sub(r'[~!@#$%^&*+()<>,."\'\?]', "", name) + name = re.sub("[:/]", "-", name) + name = "-".join(name.split()) # replace repeating hyphens name = re.sub(r"(-)\1+", r"\1", name) return name[:140] def get_shade(color, percent=None): - frappe.msgprint(_('get_shade method has been deprecated.')) + frappe.msgprint(_("get_shade method has been deprecated.")) return color + def abs_url(path): """Deconstructs and Reconstructs a URL into an absolute URL or a URL relative from root '/'""" if not path: return - if path.startswith('http://') or path.startswith('https://'): + if path.startswith("http://") or path.startswith("https://"): return path - if path.startswith('tel:'): + if path.startswith("tel:"): return path - if path.startswith('data:'): + if path.startswith("data:"): return path if not path.startswith("/"): path = "/" + path return path + def get_toc(route, url_prefix=None, app=None): - '''Insert full index (table of contents) for {index} tag''' + """Insert full index (table of contents) for {index} tag""" full_index = get_full_index(app=app) - return frappe.get_template("templates/includes/full_index.html").render({ - "full_index": full_index, - "url_prefix": url_prefix or "/", - "route": route.rstrip('/') - }) + return frappe.get_template("templates/includes/full_index.html").render( + {"full_index": full_index, "url_prefix": url_prefix or "/", "route": route.rstrip("/")} + ) + def get_next_link(route, url_prefix=None, app=None): # insert next link next_item = None - route = route.rstrip('/') + route = route.rstrip("/") children_map = get_full_index(app=app) parent_route = os.path.dirname(route) children = children_map.get(parent_route, None) @@ -196,23 +211,28 @@ def get_next_link(route, url_prefix=None, app=None): if parent_route and children: for i, c in enumerate(children): if c.route == route and i < (len(children) - 1): - next_item = children[i+1] + next_item = children[i + 1] next_item.url_prefix = url_prefix or "/" if next_item: if next_item.route and next_item.title: - html = ('

' + frappe._("Next")\ - +': {title}

').format(**next_item) + html = ( + '

' + + frappe._("Next") + + ': {title}

' + ).format(**next_item) return html - return '' + return "" + def get_full_index(route=None, app=None): """Returns full index of the website for www upto the n-th level""" from frappe.website.router import get_pages if not frappe.local.flags.children_map: + def _build(): children_map = {} added = [] @@ -231,11 +251,11 @@ def get_full_index(route=None, app=None): continue page_info = pages[route] - if page_info.index or ('index' in page_info.template): + if page_info.index or ("index" in page_info.template): new_children = [] - page_info.extn = '' - for name in (page_info.index or []): - child_route = page_info.route + '/' + name + page_info.extn = "" + for name in page_info.index or []: + child_route = page_info.route + "/" + name if child_route in pages: if child_route not in added: new_children.append(pages[child_route]) @@ -254,92 +274,111 @@ def get_full_index(route=None, app=None): return children_map - children_map = frappe.cache().get_value('website_full_index', _build) + children_map = frappe.cache().get_value("website_full_index", _build) frappe.local.flags.children_map = children_map return frappe.local.flags.children_map + def extract_title(source, path): - '''Returns title from `<!-- title -->` or <h1> or path''' - title = extract_comment_tag(source, 'title') + """Returns title from `<!-- title -->` or <h1> or path""" + title = extract_comment_tag(source, "title") if not title and "

" in source: # extract title from h1 - match = re.findall('

([^<]*)', source) + match = re.findall("

([^<]*)", source) title_content = match[0].strip()[:300] - if '{{' not in title_content: + if "{{" not in title_content: title = title_content if not title: # make title from name - title = os.path.basename(path.rsplit('.', )[0].rstrip('/')).replace('_', ' ').replace('-', ' ').title() + title = ( + os.path.basename( + path.rsplit(".",)[ + 0 + ].rstrip("/") + ) + .replace("_", " ") + .replace("-", " ") + .title() + ) return title + def extract_comment_tag(source, tag): - '''Extract custom tags in comments from source. + """Extract custom tags in comments from source. :param source: raw template source in HTML :param title: tag to search, example "title" - ''' + """ if "'.format(tag), source)[0].strip() + return re.findall("".format(tag), source)[0].strip() else: return None def get_html_content_based_on_type(doc, fieldname, content_type): - ''' - Set content based on content_type - ''' - content = doc.get(fieldname) + """ + Set content based on content_type + """ + content = doc.get(fieldname) - if content_type == 'Markdown': - content = md_to_html(doc.get(fieldname + '_md')) - elif content_type == 'HTML': - content = doc.get(fieldname + '_html') + if content_type == "Markdown": + content = md_to_html(doc.get(fieldname + "_md")) + elif content_type == "HTML": + content = doc.get(fieldname + "_html") - if content is None: - content = '' + if content is None: + content = "" - return content + return content def clear_cache(path=None): - '''Clear website caches - :param path: (optional) for the given path''' - for key in ('website_generator_routes', 'website_pages', - 'website_full_index', 'sitemap_routes'): + """Clear website caches + :param path: (optional) for the given path""" + for key in ("website_generator_routes", "website_pages", "website_full_index", "sitemap_routes"): frappe.cache().delete_value(key) frappe.cache().delete_value("website_404") if path: - frappe.cache().hdel('website_redirects', path) + frappe.cache().hdel("website_redirects", path) delete_page_cache(path) else: clear_sitemap() frappe.clear_cache("Guest") - for key in ('portal_menu_items', 'home_page', 'website_route_rules', - 'doctypes_with_web_view', 'website_redirects', 'page_context', - 'website_page'): + for key in ( + "portal_menu_items", + "home_page", + "website_route_rules", + "doctypes_with_web_view", + "website_redirects", + "page_context", + "website_page", + ): frappe.cache().delete_value(key) for method in frappe.get_hooks("website_clear_cache"): frappe.get_attr(method)(path) + def clear_website_cache(path=None): clear_cache(path) + def clear_sitemap(): delete_page_cache("*") + def get_frontmatter(string): "Reference: https://github.com/jonbeebe/frontmatter" frontmatter = "" body = "" - result = re.compile(r'^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$', re.S | re.M).search(string) + result = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M).search(string) if result: frontmatter = result.group(1) body = result.group(2) @@ -349,20 +388,25 @@ def get_frontmatter(string): "body": body, } + def get_sidebar_items(parent_sidebar, basepath): import frappe.www.list + sidebar_items = [] - hooks = frappe.get_hooks('look_for_sidebar_json') + hooks = frappe.get_hooks("look_for_sidebar_json") look_for_sidebar_json = hooks[0] if hooks else frappe.flags.look_for_sidebar if basepath and look_for_sidebar_json: sidebar_items = get_sidebar_items_from_sidebar_file(basepath, look_for_sidebar_json) if not sidebar_items and parent_sidebar: - sidebar_items = frappe.get_all('Website Sidebar Item', - filters=dict(parent=parent_sidebar), fields=['title', 'route', '`group`'], - order_by='idx asc') + sidebar_items = frappe.get_all( + "Website Sidebar Item", + filters=dict(parent=parent_sidebar), + fields=["title", "route", "`group`"], + order_by="idx asc", + ) if not sidebar_items: sidebar_items = get_portal_sidebar_items() @@ -371,65 +415,68 @@ def get_sidebar_items(parent_sidebar, basepath): def get_portal_sidebar_items(): - sidebar_items = frappe.cache().hget('portal_menu_items', frappe.session.user) + sidebar_items = frappe.cache().hget("portal_menu_items", frappe.session.user) if sidebar_items is None: sidebar_items = [] roles = frappe.get_roles() - portal_settings = frappe.get_doc('Portal Settings', 'Portal Settings') + portal_settings = frappe.get_doc("Portal Settings", "Portal Settings") def add_items(sidebar_items, items): for d in items: - if d.get('enabled') and ((not d.get('role')) or d.get('role') in roles): + if d.get("enabled") and ((not d.get("role")) or d.get("role") in roles): sidebar_items.append(d.as_dict() if isinstance(d, Document) else d) if not portal_settings.hide_standard_menu: - add_items(sidebar_items, portal_settings.get('menu')) + add_items(sidebar_items, portal_settings.get("menu")) if portal_settings.custom_menu: - add_items(sidebar_items, portal_settings.get('custom_menu')) + add_items(sidebar_items, portal_settings.get("custom_menu")) - items_via_hooks = frappe.get_hooks('portal_menu_items') + items_via_hooks = frappe.get_hooks("portal_menu_items") if items_via_hooks: for i in items_via_hooks: - i['enabled'] = 1 + i["enabled"] = 1 add_items(sidebar_items, items_via_hooks) - frappe.cache().hset('portal_menu_items', frappe.session.user, sidebar_items) + frappe.cache().hset("portal_menu_items", frappe.session.user, sidebar_items) return sidebar_items + def get_sidebar_items_from_sidebar_file(basepath, look_for_sidebar_json): sidebar_items = [] sidebar_json_path = get_sidebar_json_path(basepath, look_for_sidebar_json) if not sidebar_json_path: return sidebar_items - with open(sidebar_json_path, 'r') as sidebarfile: + with open(sidebar_json_path, "r") as sidebarfile: try: sidebar_json = sidebarfile.read() sidebar_items = json.loads(sidebar_json) except json.decoder.JSONDecodeError: - frappe.throw('Invalid Sidebar JSON at ' + sidebar_json_path) + frappe.throw("Invalid Sidebar JSON at " + sidebar_json_path) return sidebar_items + def get_sidebar_json_path(path, look_for=False): - '''Get _sidebar.json path from directory path - :param path: path of the current diretory - :param look_for: if True, look for _sidebar.json going upwards from given path - :return: _sidebar.json path - ''' - if os.path.split(path)[1] == 'www' or path == '/' or not path: - return '' - - sidebar_json_path = os.path.join(path, '_sidebar.json') + """Get _sidebar.json path from directory path + :param path: path of the current diretory + :param look_for: if True, look for _sidebar.json going upwards from given path + :return: _sidebar.json path + """ + if os.path.split(path)[1] == "www" or path == "/" or not path: + return "" + + sidebar_json_path = os.path.join(path, "_sidebar.json") if os.path.exists(sidebar_json_path): return sidebar_json_path else: if look_for: return get_sidebar_json_path(os.path.split(path)[0], look_for) else: - return '' + return "" + def cache_html(func): @wraps(func) @@ -453,6 +500,7 @@ def cache_html(func): return cache_html_decorator + def build_response(path, data, http_status_code, headers: Optional[Dict] = None): # build response response = Response() @@ -468,19 +516,20 @@ def build_response(path, data, http_status_code, headers: Optional[Dict] = None) return response + def set_content_type(response, data, path): if isinstance(data, dict): - response.mimetype = 'application/json' - response.charset = 'utf-8' + response.mimetype = "application/json" + response.charset = "utf-8" data = json.dumps(data) return data - response.mimetype = 'text/html' - response.charset = 'utf-8' + response.mimetype = "text/html" + response.charset = "utf-8" # ignore paths ending with .com to avoid unnecessary download # https://bugs.python.org/issue22347 - if "." in path and not path.endswith('.com'): + if "." in path and not path.endswith(".com"): content_type, encoding = mimetypes.guess_type(path) if content_type: response.mimetype = content_type @@ -489,6 +538,7 @@ def set_content_type(response, data, path): return data + def add_preload_headers(response): from bs4 import BeautifulSoup, SoupStrainer @@ -496,10 +546,10 @@ def add_preload_headers(response): preload = [] strainer = SoupStrainer(re.compile("script|link")) soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) - for elem in soup.find_all('script', src=re.compile(".*")): + for elem in soup.find_all("script", src=re.compile(".*")): preload.append(("script", elem.get("src"))) - for elem in soup.find_all('link', rel="stylesheet"): + for elem in soup.find_all("link", rel="stylesheet"): preload.append(("style", elem.get("href"))) links = [] @@ -510,12 +560,14 @@ def add_preload_headers(response): response.headers["Link"] = ",".join(links) except Exception: import traceback + traceback.print_exc() + @lru_cache() def is_binary_file(path): # ref: https://stackoverflow.com/a/7392391/10309266 - textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f}) - with open(path, 'rb') as f: + textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7F}) + with open(path, "rb") as f: content = f.read(1024) return bool(content.translate(None, textchars)) diff --git a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.py b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.py index 75b748913a..02e3e93333 100644 --- a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.py +++ b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.py @@ -1,3 +1,3 @@ def get_context(context): # do your magic here - pass \ No newline at end of file + pass diff --git a/frappe/website/website_components/metatags.py b/frappe/website/website_components/metatags.py index e26098b773..0551f946ef 100644 --- a/frappe/website/website_components/metatags.py +++ b/frappe/website/website_components/metatags.py @@ -1,8 +1,9 @@ import frappe -METATAGS = ('title', 'description', 'image', 'author', 'published_on') +METATAGS = ("title", "description", "image", "author", "published_on") -class MetaTags(): + +class MetaTags: def __init__(self, path, context): self.path = path self.context = context @@ -18,11 +19,11 @@ class MetaTags(): if key not in self.tags and self.context.get(key): self.tags[key] = self.context[key] - if not self.tags.get('title'): - self.tags['title'] = self.context.get('name') + if not self.tags.get("title"): + self.tags["title"] = self.context.get("name") - if self.tags.get('image'): - self.tags['image'] = frappe.utils.get_url(self.tags['image']) + if self.tags.get("image"): + self.tags["image"] = frappe.utils.get_url(self.tags["image"]) self.tags["language"] = frappe.local.lang or "en" @@ -32,15 +33,15 @@ class MetaTags(): for key in METATAGS: if self.tags.get(key): - self.tags['og:' + key] = self.tags.get(key) + self.tags["og:" + key] = self.tags.get(key) def set_twitter_tags(self): for key in METATAGS: if self.tags.get(key): - self.tags['twitter:' + key] = self.tags.get(key) + self.tags["twitter:" + key] = self.tags.get(key) - if self.tags.get('image'): - self.tags['twitter:card'] = "summary_large_image" + if self.tags.get("image"): + self.tags["twitter:card"] = "summary_large_image" else: self.tags["twitter:card"] = "summary" @@ -50,21 +51,21 @@ class MetaTags(): del self.tags["published_on"] def set_metatags_from_website_route_meta(self): - ''' + """ Get meta tags from Website Route meta they can override the defaults set above - ''' + """ route = self.path - if route == '': + if route == "": # homepage - route = frappe.db.get_single_value('Website Settings', 'home_page') + route = frappe.db.get_single_value("Website Settings", "home_page") - route_exists = (route - and not route.endswith(('.js', '.css')) - and frappe.db.exists('Website Route Meta', route)) + route_exists = ( + route and not route.endswith((".js", ".css")) and frappe.db.exists("Website Route Meta", route) + ) if route_exists: - website_route_meta = frappe.get_doc('Website Route Meta', route) + website_route_meta = frappe.get_doc("Website Route Meta", route) for meta_tag in website_route_meta.meta_tags: d = meta_tag.get_meta_dict() self.tags.update(d) diff --git a/frappe/website/website_generator.py b/frappe/website/website_generator.py index e66496aa89..9d9fbd30e5 100644 --- a/frappe/website/website_generator.py +++ b/frappe/website/website_generator.py @@ -3,10 +3,10 @@ import frappe from frappe.model.document import Document -from frappe.website.utils import cleanup_page_name -from frappe.website.utils import clear_cache from frappe.modules import get_module_name -from frappe.search.website_search import update_index_for_path, remove_document_from_index +from frappe.search.website_search import remove_document_from_index, update_index_for_path +from frappe.website.utils import cleanup_page_name, clear_cache + class WebsiteGenerator(Document): website = frappe._dict() @@ -16,7 +16,7 @@ class WebsiteGenerator(Document): super(WebsiteGenerator, self).__init__(*args, **kwargs) def get_website_properties(self, key=None, default=None): - out = getattr(self, '_website', None) or getattr(self, 'website', None) or {} + out = getattr(self, "_website", None) or getattr(self, "website", None) or {} if not isinstance(out, dict): # website may be a property too, so ignore out = {} @@ -30,10 +30,9 @@ class WebsiteGenerator(Document): self.name = self.scrubbed_title() def onload(self): - self.get("__onload").update({ - "is_website_generator": True, - "published": self.is_website_published() - }) + self.get("__onload").update( + {"is_website_generator": True, "published": self.is_website_published()} + ) def validate(self): self.set_route() @@ -43,14 +42,14 @@ class WebsiteGenerator(Document): self.route = self.make_route() if self.route: - self.route = self.route.strip('/.')[:139] + self.route = self.route.strip("/.")[:139] def make_route(self): - '''Returns the default route. If `route` is specified in DocType it will be - route/title''' + """Returns the default route. If `route` is specified in DocType it will be + route/title""" from_title = self.scrubbed_title() if self.meta.route: - return self.meta.route + '/' + from_title + return self.meta.route + "/" + from_title else: return from_title @@ -58,15 +57,15 @@ class WebsiteGenerator(Document): return self.scrub(self.get(self.get_title_field())) def get_title_field(self): - '''return title field from website properties or meta.title_field''' - title_field = self.get_website_properties('page_title_field') + """return title field from website properties or meta.title_field""" + title_field = self.get_website_properties("page_title_field") if not title_field: if self.meta.title_field: title_field = self.meta.title_field - elif self.meta.has_field('title'): - title_field = 'title' + elif self.meta.has_field("title"): + title_field = "title" else: - title_field = 'name' + title_field = "name" return title_field @@ -75,10 +74,10 @@ class WebsiteGenerator(Document): clear_cache(self.route) def scrub(self, text): - return cleanup_page_name(text).replace('_', '-') + return cleanup_page_name(text).replace("_", "-") def get_parents(self, context): - '''Return breadcrumbs''' + """Return breadcrumbs""" pass def on_update(self): @@ -92,7 +91,7 @@ class WebsiteGenerator(Document): def on_trash(self): self.clear_cache() - self.send_indexing_request('URL_DELETED') + self.send_indexing_request("URL_DELETED") # On deleting the doc, remove the page from the web_routes index if self.allow_website_search_indexing(): remove_document_from_index(self.route) @@ -105,7 +104,7 @@ class WebsiteGenerator(Document): return True def get_condition_field(self): - condition_field = self.get_website_properties('condition_field') + condition_field = self.get_website_properties("condition_field") if not condition_field: if self.meta.is_published_field: condition_field = self.meta.is_published_field @@ -114,14 +113,16 @@ class WebsiteGenerator(Document): def get_page_info(self): route = frappe._dict() - route.update({ - "doc": self, - "page_or_generator": "Generator", - "ref_doctype":self.doctype, - "idx": self.idx, - "docname": self.name, - "controller": get_module_name(self.doctype, self.meta.module), - }) + route.update( + { + "doc": self, + "page_or_generator": "Generator", + "ref_doctype": self.doctype, + "idx": self.idx, + "docname": self.name, + "controller": get_module_name(self.doctype, self.meta.module), + } + ) route.update(self.get_website_properties()) @@ -132,15 +133,21 @@ class WebsiteGenerator(Document): return route - def send_indexing_request(self, operation_type='URL_UPDATED'): + def send_indexing_request(self, operation_type="URL_UPDATED"): """Send indexing request on update/trash operation.""" - if frappe.db.get_single_value('Website Settings', 'enable_google_indexing') \ - and self.is_website_published() and self.meta.allow_guest_to_view: + if ( + frappe.db.get_single_value("Website Settings", "enable_google_indexing") + and self.is_website_published() + and self.meta.allow_guest_to_view + ): url = frappe.utils.get_url(self.route) - frappe.enqueue('frappe.website.doctype.website_settings.google_indexing.publish_site', \ - url=url, operation_type=operation_type) + frappe.enqueue( + "frappe.website.doctype.website_settings.google_indexing.publish_site", + url=url, + operation_type=operation_type, + ) # Change the field value in doctype # Override this method to disable indexing @@ -159,9 +166,9 @@ class WebsiteGenerator(Document): def update_website_search_index(self): """ - Update the full test index executed on document change event. - - remove document from index if document is unpublished - - update index otherwise + Update the full test index executed on document change event. + - remove document from index if document is unpublished + - update index otherwise """ if not self.allow_website_search_indexing() or frappe.flags.in_test: return diff --git a/frappe/workflow/doctype/workflow/__init__.py b/frappe/workflow/doctype/workflow/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/workflow/doctype/workflow/__init__.py +++ b/frappe/workflow/doctype/workflow/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 14ecdfb5a1..0fb4eca232 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -1,11 +1,17 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe import unittest -from frappe.utils import random_string -from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions -from frappe.test_runner import make_test_records + +import frappe +from frappe.model.workflow import ( + WorkflowPermissionError, + WorkflowTransitionError, + apply_workflow, + get_common_transition_actions, +) from frappe.query_builder import DocType +from frappe.test_runner import make_test_records +from frappe.utils import random_string class TestWorkflow(unittest.TestCase): @@ -15,68 +21,72 @@ class TestWorkflow(unittest.TestCase): def setUp(self): self.workflow = create_todo_workflow() - frappe.set_user('Administrator') + frappe.set_user("Administrator") if self._testMethodName == "test_if_workflow_actions_were_processed_using_user": - if not frappe.db.has_column('Workflow Action', 'user'): + if not frappe.db.has_column("Workflow Action", "user"): # mariadb would raise this statement would create an implicit commit # if we do not commit before alter statement # nosemgrep frappe.db.commit() - frappe.db.multisql({ - 'mariadb': 'ALTER TABLE `tabWorkflow Action` ADD COLUMN user varchar(140)', - 'postgres': 'ALTER TABLE "tabWorkflow Action" ADD COLUMN "user" varchar(140)' - }) - frappe.cache().delete_value('table_columns') + frappe.db.multisql( + { + "mariadb": "ALTER TABLE `tabWorkflow Action` ADD COLUMN user varchar(140)", + "postgres": 'ALTER TABLE "tabWorkflow Action" ADD COLUMN "user" varchar(140)', + } + ) + frappe.cache().delete_value("table_columns") def tearDown(self): - frappe.delete_doc('Workflow', 'Test ToDo') + frappe.delete_doc("Workflow", "Test ToDo") if self._testMethodName == "test_if_workflow_actions_were_processed_using_user": - if frappe.db.has_column('Workflow Action', 'user'): + if frappe.db.has_column("Workflow Action", "user"): # mariadb would raise this statement would create an implicit commit # if we do not commit before alter statement # nosemgrep frappe.db.commit() - frappe.db.multisql({ - 'mariadb': 'ALTER TABLE `tabWorkflow Action` DROP COLUMN user', - 'postgres': 'ALTER TABLE "tabWorkflow Action" DROP COLUMN "user"' - }) - frappe.cache().delete_value('table_columns') + frappe.db.multisql( + { + "mariadb": "ALTER TABLE `tabWorkflow Action` DROP COLUMN user", + "postgres": 'ALTER TABLE "tabWorkflow Action" DROP COLUMN "user"', + } + ) + frappe.cache().delete_value("table_columns") def test_default_condition(self): - '''test default condition is set''' + """test default condition is set""" todo = create_new_todo() # default condition is set - self.assertEqual(todo.workflow_state, 'Pending') + self.assertEqual(todo.workflow_state, "Pending") return todo def test_approve(self, doc=None): - '''test simple workflow''' + """test simple workflow""" todo = doc or self.test_default_condition() - apply_workflow(todo, 'Approve') + apply_workflow(todo, "Approve") # default condition is set - self.assertEqual(todo.workflow_state, 'Approved') - self.assertEqual(todo.status, 'Closed') + self.assertEqual(todo.workflow_state, "Approved") + self.assertEqual(todo.status, "Closed") return todo def test_wrong_action(self): - '''Check illegal action (approve after reject)''' + """Check illegal action (approve after reject)""" todo = self.test_approve() - self.assertRaises(WorkflowTransitionError, apply_workflow, todo, 'Reject') + self.assertRaises(WorkflowTransitionError, apply_workflow, todo, "Reject") def test_workflow_condition(self): - '''Test condition in transition''' + """Test condition in transition""" self.workflow.transitions[0].condition = 'doc.status == "Closed"' self.workflow.save() # only approve if status is closed self.assertRaises(WorkflowTransitionError, self.test_approve) - self.workflow.transitions[0].condition = '' + self.workflow.transitions[0].condition = "" self.workflow.save() def test_get_common_transition_actions(self): @@ -85,66 +95,65 @@ class TestWorkflow(unittest.TestCase): todo3 = create_new_todo() todo4 = create_new_todo() - actions = get_common_transition_actions([todo1, todo2, todo3, todo4], 'ToDo') - self.assertSetEqual(set(actions), set(['Approve', 'Reject'])) + actions = get_common_transition_actions([todo1, todo2, todo3, todo4], "ToDo") + self.assertSetEqual(set(actions), set(["Approve", "Reject"])) - apply_workflow(todo1, 'Reject') - apply_workflow(todo2, 'Reject') - apply_workflow(todo3, 'Approve') + apply_workflow(todo1, "Reject") + apply_workflow(todo2, "Reject") + apply_workflow(todo3, "Approve") - actions = get_common_transition_actions([todo1, todo2, todo3], 'ToDo') + actions = get_common_transition_actions([todo1, todo2, todo3], "ToDo") self.assertListEqual(actions, []) - actions = get_common_transition_actions([todo1, todo2], 'ToDo') - self.assertListEqual(actions, ['Review']) + actions = get_common_transition_actions([todo1, todo2], "ToDo") + self.assertListEqual(actions, ["Review"]) def test_if_workflow_actions_were_processed_using_role(self): frappe.db.delete("Workflow Action") - user = frappe.get_doc('User', 'test2@example.com') - user.add_roles('Test Approver', 'System Manager') - frappe.set_user('test2@example.com') + user = frappe.get_doc("User", "test2@example.com") + user.add_roles("Test Approver", "System Manager") + frappe.set_user("test2@example.com") doc = self.test_default_condition() - workflow_actions = frappe.get_all('Workflow Action', fields=['*']) + workflow_actions = frappe.get_all("Workflow Action", fields=["*"]) self.assertEqual(len(workflow_actions), 1) # test if status of workflow actions are updated on approval self.test_approve(doc) - user.remove_roles('Test Approver', 'System Manager') - workflow_actions = frappe.get_all('Workflow Action', fields=['status']) + user.remove_roles("Test Approver", "System Manager") + workflow_actions = frappe.get_all("Workflow Action", fields=["status"]) self.assertEqual(len(workflow_actions), 1) - self.assertEqual(workflow_actions[0].status, 'Completed') - frappe.set_user('Administrator') + self.assertEqual(workflow_actions[0].status, "Completed") + frappe.set_user("Administrator") def test_if_workflow_actions_were_processed_using_user(self): frappe.db.delete("Workflow Action") - user = frappe.get_doc('User', 'test2@example.com') - user.add_roles('Test Approver', 'System Manager') - frappe.set_user('test2@example.com') + user = frappe.get_doc("User", "test2@example.com") + user.add_roles("Test Approver", "System Manager") + frappe.set_user("test2@example.com") doc = self.test_default_condition() - workflow_actions = frappe.get_all('Workflow Action', fields=['*']) + workflow_actions = frappe.get_all("Workflow Action", fields=["*"]) self.assertEqual(len(workflow_actions), 1) # test if status of workflow actions are updated on approval WorkflowAction = DocType("Workflow Action") WorkflowActionPermittedRole = DocType("Workflow Action Permitted Role") - frappe.qb.update(WorkflowAction).set(WorkflowAction.user, 'test2@example.com').run() - frappe.qb.update(WorkflowActionPermittedRole).set(WorkflowActionPermittedRole.role, '').run() + frappe.qb.update(WorkflowAction).set(WorkflowAction.user, "test2@example.com").run() + frappe.qb.update(WorkflowActionPermittedRole).set(WorkflowActionPermittedRole.role, "").run() self.test_approve(doc) - user.remove_roles('Test Approver', 'System Manager') - workflow_actions = frappe.get_all('Workflow Action', fields=['status']) + user.remove_roles("Test Approver", "System Manager") + workflow_actions = frappe.get_all("Workflow Action", fields=["status"]) self.assertEqual(len(workflow_actions), 1) - self.assertEqual(workflow_actions[0].status, 'Completed') - frappe.set_user('Administrator') - + self.assertEqual(workflow_actions[0].status, "Completed") + frappe.set_user("Administrator") def test_update_docstatus(self): todo = create_new_todo() - apply_workflow(todo, 'Approve') + apply_workflow(todo, "Approve") self.workflow.states[1].doc_status = 0 self.workflow.save() @@ -165,7 +174,7 @@ class TestWorkflow(unittest.TestCase): self.assertEqual(todo.docstatus, 0) todo.submit() self.assertEqual(todo.docstatus, 1) - self.assertEqual(todo.workflow_state, 'Approved') + self.assertEqual(todo.workflow_state, "Approved") self.workflow.states[1].doc_status = 0 self.workflow.save() @@ -176,48 +185,59 @@ class TestWorkflow(unittest.TestCase): with self.assertRaises(frappe.ValidationError) as se: self.workflow.save() - self.assertTrue("invalid python code" in str(se.exception).lower(), - msg="Python code validation not working") + self.assertTrue( + "invalid python code" in str(se.exception).lower(), msg="Python code validation not working" + ) def create_todo_workflow(): - if frappe.db.exists('Workflow', 'Test ToDo'): - frappe.delete_doc('Workflow', 'Test ToDo') - - if not frappe.db.exists('Role', 'Test Approver'): - frappe.get_doc(dict(doctype='Role', - role_name='Test Approver')).insert(ignore_if_duplicate=True) - workflow = frappe.new_doc('Workflow') - workflow.workflow_name = 'Test ToDo' - workflow.document_type = 'ToDo' - workflow.workflow_state_field = 'workflow_state' + if frappe.db.exists("Workflow", "Test ToDo"): + frappe.delete_doc("Workflow", "Test ToDo") + + if not frappe.db.exists("Role", "Test Approver"): + frappe.get_doc(dict(doctype="Role", role_name="Test Approver")).insert(ignore_if_duplicate=True) + workflow = frappe.new_doc("Workflow") + workflow.workflow_name = "Test ToDo" + workflow.document_type = "ToDo" + workflow.workflow_state_field = "workflow_state" workflow.is_active = 1 workflow.send_email_alert = 0 - workflow.append('states', dict( - state = 'Pending', allow_edit = 'All' - )) - workflow.append('states', dict( - state = 'Approved', allow_edit = 'Test Approver', - update_field = 'status', update_value = 'Closed' - )) - workflow.append('states', dict( - state = 'Rejected', allow_edit = 'Test Approver' - )) - workflow.append('transitions', dict( - state = 'Pending', action='Approve', next_state = 'Approved', - allowed='Test Approver', allow_self_approval= 1 - )) - workflow.append('transitions', dict( - state = 'Pending', action='Reject', next_state = 'Rejected', - allowed='Test Approver', allow_self_approval= 1 - )) - workflow.append('transitions', dict( - state = 'Rejected', action='Review', next_state = 'Pending', - allowed='All', allow_self_approval= 1 - )) + workflow.append("states", dict(state="Pending", allow_edit="All")) + workflow.append( + "states", + dict(state="Approved", allow_edit="Test Approver", update_field="status", update_value="Closed"), + ) + workflow.append("states", dict(state="Rejected", allow_edit="Test Approver")) + workflow.append( + "transitions", + dict( + state="Pending", + action="Approve", + next_state="Approved", + allowed="Test Approver", + allow_self_approval=1, + ), + ) + workflow.append( + "transitions", + dict( + state="Pending", + action="Reject", + next_state="Rejected", + allowed="Test Approver", + allow_self_approval=1, + ), + ) + workflow.append( + "transitions", + dict( + state="Rejected", action="Review", next_state="Pending", allowed="All", allow_self_approval=1 + ), + ) workflow.insert(ignore_permissions=True) return workflow + def create_new_todo(): - return frappe.get_doc(dict(doctype='ToDo', description='workflow ' + random_string(10))).insert() + return frappe.get_doc(dict(doctype="ToDo", description="workflow " + random_string(10))).insert() diff --git a/frappe/workflow/doctype/workflow/workflow.py b/frappe/workflow/doctype/workflow/workflow.py index 4a8ae05e6b..d524d43d41 100644 --- a/frappe/workflow/doctype/workflow/workflow.py +++ b/frappe/workflow/doctype/workflow/workflow.py @@ -3,9 +3,9 @@ import frappe from frappe import _ - -from frappe.model.document import Document from frappe.model import no_value_fields +from frappe.model.document import Document + class Workflow(Document): def validate(self): @@ -17,50 +17,57 @@ class Workflow(Document): def on_update(self): self.update_doc_status() frappe.clear_cache(doctype=self.document_type) - frappe.cache().delete_key('workflow_' + self.name) # clear cache created in model/workflow.py + frappe.cache().delete_key("workflow_" + self.name) # clear cache created in model/workflow.py def create_custom_field_for_workflow_state(self): frappe.clear_cache(doctype=self.document_type) meta = frappe.get_meta(self.document_type) if not meta.get_field(self.workflow_state_field): # create custom field - frappe.get_doc({ - "doctype":"Custom Field", - "dt": self.document_type, - "__islocal": 1, - "fieldname": self.workflow_state_field, - "label": self.workflow_state_field.replace("_", " ").title(), - "hidden": 1, - "allow_on_submit": 1, - "no_copy": 1, - "fieldtype": "Link", - "options": "Workflow State", - "owner": "Administrator" - }).save() - - frappe.msgprint(_("Created Custom Field {0} in {1}").format(self.workflow_state_field, - self.document_type)) + frappe.get_doc( + { + "doctype": "Custom Field", + "dt": self.document_type, + "__islocal": 1, + "fieldname": self.workflow_state_field, + "label": self.workflow_state_field.replace("_", " ").title(), + "hidden": 1, + "allow_on_submit": 1, + "no_copy": 1, + "fieldtype": "Link", + "options": "Workflow State", + "owner": "Administrator", + } + ).save() + + frappe.msgprint( + _("Created Custom Field {0} in {1}").format(self.workflow_state_field, self.document_type) + ) def update_default_workflow_status(self): docstatus_map = {} states = self.get("states") for d in states: if not d.doc_status in docstatus_map: - frappe.db.sql(""" + frappe.db.sql( + """ UPDATE `tab{doctype}` SET `{field}` = %s WHERE ifnull(`{field}`, '') = '' AND `docstatus` = %s - """.format(doctype=self.document_type, field=self.workflow_state_field), - (d.state, d.doc_status)) + """.format( + doctype=self.document_type, field=self.workflow_state_field + ), + (d.state, d.doc_status), + ) docstatus_map[d.doc_status] = d.state def update_doc_status(self): - ''' - Checks if the docstatus of a state was updated. - If yes then the docstatus of the document with same state will be updated - ''' + """ + Checks if the docstatus of a state was updated. + If yes then the docstatus of the document with same state will be updated + """ doc_before_save = self.get_doc_before_save() before_save_states, new_states = {}, {} if doc_before_save: @@ -72,17 +79,18 @@ class Workflow(Document): for key in new_states: if key in before_save_states: if not new_states[key].doc_status == before_save_states[key].doc_status: - frappe.db.set_value(self.document_type, { - self.workflow_state_field: before_save_states[key].state - }, - 'docstatus', + frappe.db.set_value( + self.document_type, + {self.workflow_state_field: before_save_states[key].state}, + "docstatus", new_states[key].doc_status, - update_modified = False) + update_modified=False, + ) def validate_docstatus(self): def get_state(state): for s in self.states: - if s.state==state: + if s.state == state: return s frappe.throw(frappe._("{0} not a valid State").format(state)) @@ -91,37 +99,45 @@ class Workflow(Document): state = get_state(t.state) next_state = get_state(t.next_state) - if state.doc_status=="2": - frappe.throw(frappe._("Cannot change state of Cancelled Document. Transition row {0}").format(t.idx)) + if state.doc_status == "2": + frappe.throw( + frappe._("Cannot change state of Cancelled Document. Transition row {0}").format(t.idx) + ) - if state.doc_status=="1" and next_state.doc_status=="0": - frappe.throw(frappe._("Submitted Document cannot be converted back to draft. Transition row {0}").format(t.idx)) + if state.doc_status == "1" and next_state.doc_status == "0": + frappe.throw( + frappe._("Submitted Document cannot be converted back to draft. Transition row {0}").format( + t.idx + ) + ) - if state.doc_status=="0" and next_state.doc_status=="2": + if state.doc_status == "0" and next_state.doc_status == "2": frappe.throw(frappe._("Cannot cancel before submitting. See Transition {0}").format(t.idx)) def set_active(self): if int(self.is_active or 0): # clear all other - frappe.db.sql("""UPDATE `tabWorkflow` SET `is_active`=0 + frappe.db.sql( + """UPDATE `tabWorkflow` SET `is_active`=0 WHERE `document_type`=%s""", - self.document_type) + self.document_type, + ) + @frappe.whitelist() def get_fieldnames_for(doctype): - return [f.fieldname for f in frappe.get_meta(doctype).fields \ - if f.fieldname not in no_value_fields] + return [ + f.fieldname for f in frappe.get_meta(doctype).fields if f.fieldname not in no_value_fields + ] + @frappe.whitelist() def get_workflow_state_count(doctype, workflow_state_field, states): states = frappe.parse_json(states) result = frappe.get_all( doctype, - fields=[workflow_state_field, 'count(*) as count', 'docstatus'], - filters = { - workflow_state_field: ['not in', states] - }, - group_by = workflow_state_field + fields=[workflow_state_field, "count(*) as count", "docstatus"], + filters={workflow_state_field: ["not in", states]}, + group_by=workflow_state_field, ) return [r for r in result if r[workflow_state_field]] - diff --git a/frappe/workflow/doctype/workflow_action/__init__.py b/frappe/workflow/doctype/workflow_action/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/workflow/doctype/workflow_action/__init__.py +++ b/frappe/workflow/doctype/workflow_action/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/workflow/doctype/workflow_action/test_workflow_action.py b/frappe/workflow/doctype/workflow_action/test_workflow_action.py index bbd027ba12..04c49134fc 100644 --- a/frappe/workflow/doctype/workflow_action/test_workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/test_workflow_action.py @@ -4,5 +4,6 @@ # import frappe import unittest + class TestWorkflowAction(unittest.TestCase): pass diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 0ab8924a6b..abbcda5c4c 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -2,18 +2,25 @@ # License: MIT. See LICENSE import frappe -from frappe.model.document import Document -from frappe.utils.background_jobs import enqueue -from frappe.utils import get_url, get_datetime -from frappe.desk.form.utils import get_pdf_link -from frappe.utils.verified_command import get_signed_params, verify_request from frappe import _ -from frappe.model.workflow import apply_workflow, get_workflow_name, has_approval_access, \ - get_workflow_state_field, send_email_alert, is_transition_condition_satisfied +from frappe.desk.form.utils import get_pdf_link from frappe.desk.notifications import clear_doctype_notifications -from frappe.utils.user import get_users_with_role -from frappe.utils.data import get_link_to_form +from frappe.model.document import Document +from frappe.model.workflow import ( + apply_workflow, + get_workflow_name, + get_workflow_state_field, + has_approval_access, + is_transition_condition_satisfied, + send_email_alert, +) from frappe.query_builder import DocType +from frappe.utils import get_datetime, get_url +from frappe.utils.background_jobs import enqueue +from frappe.utils.data import get_link_to_form +from frappe.utils.user import get_users_with_role +from frappe.utils.verified_command import get_signed_params, verify_request + class WorkflowAction(Document): pass @@ -25,17 +32,21 @@ def on_doctype_update(): # so even if status is not in the where clause the index will be used frappe.db.add_index("Workflow Action", ["reference_name", "reference_doctype", "status"]) + def get_permission_query_conditions(user): - if not user: user = frappe.session.user + if not user: + user = frappe.session.user - if user == "Administrator": return "" + if user == "Administrator": + return "" roles = frappe.get_roles(user) WorkflowAction = DocType("Workflow Action") WorkflowActionPermittedRole = DocType("Workflow Action Permitted Role") - permitted_workflow_actions = (frappe.qb.from_(WorkflowAction) + permitted_workflow_actions = ( + frappe.qb.from_(WorkflowAction) .join(WorkflowActionPermittedRole) .on(WorkflowAction.name == WorkflowActionPermittedRole.parent) .select(WorkflowAction.name) @@ -46,6 +57,7 @@ def get_permission_query_conditions(user): or `tabWorkflow Action`.`user`='{user}') and `tabWorkflow Action`.`status`='Open'""" + def has_permission(doc, user): user_roles = set(frappe.get_roles(user)) @@ -54,31 +66,43 @@ def has_permission(doc, user): return user == "Administrator" or user_roles.intersection(permitted_roles) + def process_workflow_actions(doc, state): - workflow = get_workflow_name(doc.get('doctype')) - if not workflow: return + workflow = get_workflow_name(doc.get("doctype")) + if not workflow: + return if state == "on_trash": - clear_workflow_actions(doc.get('doctype'), doc.get('name')) + clear_workflow_actions(doc.get("doctype"), doc.get("name")) return - if is_workflow_action_already_created(doc): return + if is_workflow_action_already_created(doc): + return - update_completed_workflow_actions(doc, workflow=workflow, workflow_state=get_doc_workflow_state(doc)) - clear_doctype_notifications('Workflow Action') + update_completed_workflow_actions( + doc, workflow=workflow, workflow_state=get_doc_workflow_state(doc) + ) + clear_doctype_notifications("Workflow Action") - next_possible_transitions = get_next_possible_transitions(workflow, get_doc_workflow_state(doc), doc) + next_possible_transitions = get_next_possible_transitions( + workflow, get_doc_workflow_state(doc), doc + ) - if not next_possible_transitions: return + if not next_possible_transitions: + return user_data_map, roles = get_users_next_action_data(next_possible_transitions, doc) - if not user_data_map: return + if not user_data_map: + return create_workflow_actions_for_roles(roles, doc) if send_email_alert(workflow): - enqueue(send_workflow_action_email, queue='short', users_data=list(user_data_map.values()), doc=doc) + enqueue( + send_workflow_action_email, queue="short", users_data=list(user_data_map.values()), doc=doc + ) + @frappe.whitelist(allow_guest=True) def apply_action(action, doctype, docname, current_state, user=None, last_modified=None): @@ -99,13 +123,14 @@ def apply_action(action, doctype, docname, current_state, user=None, last_modifi else: return_link_expired_page(doc, doc_workflow_state) + @frappe.whitelist(allow_guest=True) def confirm_action(doctype, docname, user, action): if not verify_request(): return logged_in_user = frappe.session.user - if logged_in_user == 'Guest' and user: + if logged_in_user == "Guest" and user: # to allow user to apply action without login frappe.set_user(user) @@ -115,43 +140,51 @@ def confirm_action(doctype, docname, user, action): return_success_page(newdoc) # reset session user - if logged_in_user == 'Guest': + if logged_in_user == "Guest": frappe.set_user(logged_in_user) + def return_success_page(doc): - frappe.respond_as_web_page(_("Success"), + frappe.respond_as_web_page( + _("Success"), _("{0}: {1} is set to state {2}").format( - doc.get('doctype'), - frappe.bold(doc.get('name')), - frappe.bold(get_doc_workflow_state(doc)) - ), indicator_color='green') + doc.get("doctype"), frappe.bold(doc.get("name")), frappe.bold(get_doc_workflow_state(doc)) + ), + indicator_color="green", + ) + def return_action_confirmation_page(doc, action, action_link, alert_doc_change=False): template_params = { - 'title': doc.get('name'), - 'doctype': doc.get('doctype'), - 'docname': doc.get('name'), - 'action': action, - 'action_link': action_link, - 'alert_doc_change': alert_doc_change + "title": doc.get("name"), + "doctype": doc.get("doctype"), + "docname": doc.get("name"), + "action": action, + "action_link": action_link, + "alert_doc_change": alert_doc_change, } - template_params['pdf_link'] = get_pdf_link(doc.get('doctype'), doc.get('name')) + template_params["pdf_link"] = get_pdf_link(doc.get("doctype"), doc.get("name")) - frappe.respond_as_web_page(title=None, + frappe.respond_as_web_page( + title=None, html=None, - indicator_color='blue', - template='confirm_workflow_action', - context=template_params) + indicator_color="blue", + template="confirm_workflow_action", + context=template_params, + ) + def return_link_expired_page(doc, doc_workflow_state): - frappe.respond_as_web_page(_("Link Expired"), - _("Document {0} has been set to state {1} by {2}") - .format( - frappe.bold(doc.get('name')), - frappe.bold(doc_workflow_state), - frappe.bold(frappe.get_value('User', doc.get("modified_by"), 'full_name')) - ), indicator_color='blue') + frappe.respond_as_web_page( + _("Link Expired"), + _("Document {0} has been set to state {1} by {2}").format( + frappe.bold(doc.get("name")), + frappe.bold(doc_workflow_state), + frappe.bold(frappe.get_value("User", doc.get("modified_by"), "full_name")), + ), + indicator_color="blue", + ) def update_completed_workflow_actions(doc, user=None, workflow=None, workflow_state=None): @@ -168,76 +201,97 @@ def update_completed_workflow_actions(doc, user=None, workflow=None, workflow_st clear_old_workflow_actions_using_user(doc, user) update_completed_workflow_actions_using_user(doc, user) + def get_allowed_roles(user, workflow, workflow_state): user = user if user else frappe.session.user - allowed_roles = frappe.get_all('Workflow Transition', - fields='allowed', - filters=[ - ['parent', '=', workflow], - ['next_state', '=', workflow_state] - ], - pluck = 'allowed') + allowed_roles = frappe.get_all( + "Workflow Transition", + fields="allowed", + filters=[["parent", "=", workflow], ["next_state", "=", workflow_state]], + pluck="allowed", + ) user_roles = set(frappe.get_roles(user)) return set(allowed_roles).intersection(user_roles) + def get_workflow_action_by_role(doc, allowed_roles): WorkflowAction = DocType("Workflow Action") WorkflowActionPermittedRole = DocType("Workflow Action Permitted Role") - return (frappe.qb.from_(WorkflowAction).join(WorkflowActionPermittedRole) + return ( + frappe.qb.from_(WorkflowAction) + .join(WorkflowActionPermittedRole) .on(WorkflowAction.name == WorkflowActionPermittedRole.parent) .select(WorkflowAction.name, WorkflowActionPermittedRole.role) - .where((WorkflowAction.reference_name == doc.get('name')) - & (WorkflowAction.reference_doctype == doc.get('doctype')) - & (WorkflowAction.status == 'Open') - & (WorkflowActionPermittedRole.role.isin(list(allowed_roles)))) - .orderby(WorkflowActionPermittedRole.role).limit(1)).run(as_dict=True) - -def update_completed_workflow_actions_using_role(doc, user=None, allowed_roles = set(), workflow_action=None): + .where( + (WorkflowAction.reference_name == doc.get("name")) + & (WorkflowAction.reference_doctype == doc.get("doctype")) + & (WorkflowAction.status == "Open") + & (WorkflowActionPermittedRole.role.isin(list(allowed_roles))) + ) + .orderby(WorkflowActionPermittedRole.role) + .limit(1) + ).run(as_dict=True) + + +def update_completed_workflow_actions_using_role( + doc, user=None, allowed_roles=set(), workflow_action=None +): user = user if user else frappe.session.user WorkflowAction = DocType("Workflow Action") if not workflow_action: return - (frappe.qb.update(WorkflowAction) - .set(WorkflowAction.status, 'Completed') + ( + frappe.qb.update(WorkflowAction) + .set(WorkflowAction.status, "Completed") .set(WorkflowAction.completed_by, user) .set(WorkflowAction.completed_by_role, workflow_action[0].role) .where(WorkflowAction.name == workflow_action[0].name) ).run() + def clear_old_workflow_actions_using_user(doc, user=None): user = user if user else frappe.session.user - if frappe.db.has_column('Workflow Action', 'user'): - frappe.db.delete("Workflow Action", { - "reference_name": doc.get("name"), - "reference_doctype": doc.get("doctype"), - "status": "Open", - "user": ("!=", user) - }) + if frappe.db.has_column("Workflow Action", "user"): + frappe.db.delete( + "Workflow Action", + { + "reference_name": doc.get("name"), + "reference_doctype": doc.get("doctype"), + "status": "Open", + "user": ("!=", user), + }, + ) + def update_completed_workflow_actions_using_user(doc, user=None): user = user or frappe.session.user - if frappe.db.has_column('Workflow Action', 'user'): + if frappe.db.has_column("Workflow Action", "user"): WorkflowAction = DocType("Workflow Action") - (frappe.qb.update(WorkflowAction) - .set(WorkflowAction.status, 'Completed') + ( + frappe.qb.update(WorkflowAction) + .set(WorkflowAction.status, "Completed") .set(WorkflowAction.completed_by, user) - .where((WorkflowAction.reference_name == doc.get('name')) - & (WorkflowAction.reference_doctype == doc.get('doctype')) - & (WorkflowAction.status == 'Open') - & (WorkflowAction.user == user)) + .where( + (WorkflowAction.reference_name == doc.get("name")) + & (WorkflowAction.reference_doctype == doc.get("doctype")) + & (WorkflowAction.status == "Open") + & (WorkflowAction.user == user) + ) ).run() + def get_next_possible_transitions(workflow_name, state, doc=None): - transitions = frappe.get_all('Workflow Transition', - fields=['allowed', 'action', 'state', 'allow_self_approval', 'next_state', '`condition`'], - filters=[['parent', '=', workflow_name], - ['state', '=', state]]) + transitions = frappe.get_all( + "Workflow Transition", + fields=["allowed", "action", "state", "allow_self_approval", "next_state", "`condition`"], + filters=[["parent", "=", workflow_name], ["state", "=", state]], + ) transitions_to_return = [] @@ -252,6 +306,7 @@ def get_next_possible_transitions(workflow_name, state, doc=None): return transitions_to_return + def get_users_next_action_data(transitions, doc): roles = set() user_data_map = {} @@ -261,49 +316,54 @@ def get_users_next_action_data(transitions, doc): filtered_users = filter_allowed_users(users, doc, transition) for user in filtered_users: if not user_data_map.get(user): - user_data_map[user] = frappe._dict({ - 'possible_actions': [], - 'email': frappe.db.get_value('User', user, 'email'), - }) - - user_data_map[user].get('possible_actions').append(frappe._dict({ - 'action_name': transition.action, - 'action_link': get_workflow_action_url(transition.action, doc, user) - })) + user_data_map[user] = frappe._dict( + { + "possible_actions": [], + "email": frappe.db.get_value("User", user, "email"), + } + ) + + user_data_map[user].get("possible_actions").append( + frappe._dict( + { + "action_name": transition.action, + "action_link": get_workflow_action_url(transition.action, doc, user), + } + ) + ) return user_data_map, roles def create_workflow_actions_for_roles(roles, doc): - workflow_action = frappe.get_doc({ - 'doctype': 'Workflow Action', - 'reference_doctype': doc.get('doctype'), - 'reference_name': doc.get('name'), - 'workflow_state': get_doc_workflow_state(doc), - 'status': 'Open', - }) + workflow_action = frappe.get_doc( + { + "doctype": "Workflow Action", + "reference_doctype": doc.get("doctype"), + "reference_name": doc.get("name"), + "workflow_state": get_doc_workflow_state(doc), + "status": "Open", + } + ) for role in roles: - workflow_action.append('permitted_roles', { - 'role': role - }) + workflow_action.append("permitted_roles", {"role": role}) workflow_action.insert(ignore_permissions=True) + def send_workflow_action_email(users_data, doc): common_args = get_common_email_args(doc) - message = common_args.pop('message', None) + message = common_args.pop("message", None) for d in users_data: email_args = { - 'recipients': [d.get('email')], - 'args': { - 'actions': list(deduplicate_actions(d.get('possible_actions'))), - 'message': message - }, - 'reference_name': doc.name, - 'reference_doctype': doc.doctype + "recipients": [d.get("email")], + "args": {"actions": list(deduplicate_actions(d.get("possible_actions"))), "message": message}, + "reference_name": doc.name, + "reference_doctype": doc.doctype, } email_args.update(common_args) - enqueue(method=frappe.sendmail, queue='short', **email_args) + enqueue(method=frappe.sendmail, queue="short", **email_args) + def deduplicate_actions(action_list): action_map = {} @@ -313,103 +373,121 @@ def deduplicate_actions(action_list): return action_map.values() + def get_workflow_action_url(action, doc, user): - apply_action_method = "/api/method/frappe.workflow.doctype.workflow_action.workflow_action.apply_action" + apply_action_method = ( + "/api/method/frappe.workflow.doctype.workflow_action.workflow_action.apply_action" + ) params = { - "doctype": doc.get('doctype'), - "docname": doc.get('name'), + "doctype": doc.get("doctype"), + "docname": doc.get("name"), "action": action, "current_state": get_doc_workflow_state(doc), "user": user, - "last_modified": doc.get('modified') + "last_modified": doc.get("modified"), } return get_url(apply_action_method + "?" + get_signed_params(params)) + def get_confirm_workflow_action_url(doc, action, user): - confirm_action_method = "/api/method/frappe.workflow.doctype.workflow_action.workflow_action.confirm_action" + confirm_action_method = ( + "/api/method/frappe.workflow.doctype.workflow_action.workflow_action.confirm_action" + ) params = { "action": action, - "doctype": doc.get('doctype'), - "docname": doc.get('name'), - "user": user + "doctype": doc.get("doctype"), + "docname": doc.get("name"), + "user": user, } return get_url(confirm_action_method + "?" + get_signed_params(params)) + def is_workflow_action_already_created(doc): - return frappe.db.exists({ - 'doctype': 'Workflow Action', - 'reference_name': doc.get('name'), - 'reference_doctype': doc.get('doctype'), - 'workflow_state': get_doc_workflow_state(doc), - }) + return frappe.db.exists( + { + "doctype": "Workflow Action", + "reference_name": doc.get("name"), + "reference_doctype": doc.get("doctype"), + "workflow_state": get_doc_workflow_state(doc), + } + ) + def clear_workflow_actions(doctype, name): if not (doctype and name): return - frappe.db.delete("Workflow Action", filters = { + frappe.db.delete( + "Workflow Action", + filters={ "reference_name": name, "reference_doctype": doctype, - } + }, ) + def get_doc_workflow_state(doc): - workflow_name = get_workflow_name(doc.get('doctype')) + workflow_name = get_workflow_name(doc.get("doctype")) workflow_state_field = get_workflow_state_field(workflow_name) return doc.get(workflow_state_field) + def filter_allowed_users(users, doc, transition): """Filters list of users by checking if user has access to doc and if the user satisfies 'workflow transision self approval' condition """ from frappe.permissions import has_permission + filtered_users = [] for user in users: - if (has_approval_access(user, doc, transition) - and has_permission(doctype=doc, user=user)): + if has_approval_access(user, doc, transition) and has_permission(doctype=doc, user=user): filtered_users.append(user) return filtered_users + def get_common_email_args(doc): - doctype = doc.get('doctype') - docname = doc.get('name') + doctype = doc.get("doctype") + docname = doc.get("name") email_template = get_email_template(doc) if email_template: subject = frappe.render_template(email_template.subject, vars(doc)) response = frappe.render_template(email_template.response, vars(doc)) else: - subject = _('Workflow Action') + f" on {doctype}: {docname}" + subject = _("Workflow Action") + f" on {doctype}: {docname}" response = get_link_to_form(doctype, docname, f"{doctype}: {docname}") common_args = { - 'template': 'workflow_action', - 'header': 'Workflow Action', - 'attachments': [frappe.attach_print(doctype, docname, file_name=docname, doc=doc)], - 'subject': subject, - 'message': response + "template": "workflow_action", + "header": "Workflow Action", + "attachments": [frappe.attach_print(doctype, docname, file_name=docname, doc=doc)], + "subject": subject, + "message": response, } return common_args + def get_email_template(doc): """Returns next_action_email_template for workflow state (if available) based on doc current workflow state """ - workflow_name = get_workflow_name(doc.get('doctype')) + workflow_name = get_workflow_name(doc.get("doctype")) doc_state = get_doc_workflow_state(doc) - template_name = frappe.db.get_value('Workflow Document State', { - 'parent': workflow_name, - 'state': doc_state - }, 'next_action_email_template') + template_name = frappe.db.get_value( + "Workflow Document State", + {"parent": workflow_name, "state": doc_state}, + "next_action_email_template", + ) + + if not template_name: + return + return frappe.get_doc("Email Template", template_name) - if not template_name: return - return frappe.get_doc('Email Template', template_name) def get_state_optional_field_value(workflow_name, state): - return frappe.get_cached_value('Workflow Document State', { - 'parent': workflow_name, - 'state': state - }, 'is_optional_state') + return frappe.get_cached_value( + "Workflow Document State", {"parent": workflow_name, "state": state}, "is_optional_state" + ) diff --git a/frappe/workflow/doctype/workflow_action_master/workflow_action_master.py b/frappe/workflow/doctype/workflow_action_master/workflow_action_master.py index 3f71fb7fe8..eb227b6125 100644 --- a/frappe/workflow/doctype/workflow_action_master/workflow_action_master.py +++ b/frappe/workflow/doctype/workflow_action_master/workflow_action_master.py @@ -4,5 +4,6 @@ from frappe.model.document import Document + class WorkflowActionMaster(Document): pass diff --git a/frappe/workflow/doctype/workflow_action_permitted_role/workflow_action_permitted_role.py b/frappe/workflow/doctype/workflow_action_permitted_role/workflow_action_permitted_role.py index 0370f6a4c8..4266d96b7e 100644 --- a/frappe/workflow/doctype/workflow_action_permitted_role/workflow_action_permitted_role.py +++ b/frappe/workflow/doctype/workflow_action_permitted_role/workflow_action_permitted_role.py @@ -4,5 +4,6 @@ # import frappe from frappe.model.document import Document + class WorkflowActionPermittedRole(Document): pass diff --git a/frappe/workflow/doctype/workflow_document_state/__init__.py b/frappe/workflow/doctype/workflow_document_state/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/workflow/doctype/workflow_document_state/__init__.py +++ b/frappe/workflow/doctype/workflow_document_state/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py b/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py index 2764036614..9da45ef278 100644 --- a/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py +++ b/frappe/workflow/doctype/workflow_document_state/workflow_document_state.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class WorkflowDocumentState(Document): - pass \ No newline at end of file + pass diff --git a/frappe/workflow/doctype/workflow_state/__init__.py b/frappe/workflow/doctype/workflow_state/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/workflow/doctype/workflow_state/__init__.py +++ b/frappe/workflow/doctype/workflow_state/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/workflow/doctype/workflow_state/test_workflow_state.py b/frappe/workflow/doctype/workflow_state/test_workflow_state.py index f38461c1a7..916e695916 100644 --- a/frappe/workflow/doctype/workflow_state/test_workflow_state.py +++ b/frappe/workflow/doctype/workflow_state/test_workflow_state.py @@ -2,4 +2,4 @@ # License: MIT. See LICENSE import frappe -test_records = frappe.get_test_records('Workflow State') \ No newline at end of file +test_records = frappe.get_test_records("Workflow State") diff --git a/frappe/workflow/doctype/workflow_state/workflow_state.py b/frappe/workflow/doctype/workflow_state/workflow_state.py index 495876bddb..8ee56e9803 100644 --- a/frappe/workflow/doctype/workflow_state/workflow_state.py +++ b/frappe/workflow/doctype/workflow_state/workflow_state.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class WorkflowState(Document): - pass \ No newline at end of file + pass diff --git a/frappe/workflow/doctype/workflow_transition/__init__.py b/frappe/workflow/doctype/workflow_transition/__init__.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/workflow/doctype/workflow_transition/__init__.py +++ b/frappe/workflow/doctype/workflow_transition/__init__.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/workflow/doctype/workflow_transition/workflow_transition.py b/frappe/workflow/doctype/workflow_transition/workflow_transition.py index 5043b765a9..2649fd7bde 100644 --- a/frappe/workflow/doctype/workflow_transition/workflow_transition.py +++ b/frappe/workflow/doctype/workflow_transition/workflow_transition.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE import frappe - from frappe.model.document import Document + class WorkflowTransition(Document): - pass \ No newline at end of file + pass diff --git a/frappe/www/404.py b/frappe/www/404.py index d2a5eeb700..ecf4ec8901 100644 --- a/frappe/www/404.py +++ b/frappe/www/404.py @@ -1,5 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE + def get_context(context): context.http_status_code = 404 diff --git a/frappe/www/_test/_test_folder/_test_page.py b/frappe/www/_test/_test_folder/_test_page.py index 3d4a645f9b..edaea99091 100644 --- a/frappe/www/_test/_test_folder/_test_page.py +++ b/frappe/www/_test/_test_folder/_test_page.py @@ -1,3 +1,3 @@ def get_context(context): - context.base_template_path = 'frappe/templates/test/_test_base_breadcrumbs.html' + context.base_template_path = "frappe/templates/test/_test_base_breadcrumbs.html" context.add_breadcrumbs = 1 diff --git a/frappe/www/_test/_test_home_page.py b/frappe/www/_test/_test_home_page.py index 936399c700..6ac86d76c2 100644 --- a/frappe/www/_test/_test_home_page.py +++ b/frappe/www/_test/_test_home_page.py @@ -1,2 +1,2 @@ def get_website_user_home_page(user): - return '/_test/_test_folder' \ No newline at end of file + return "/_test/_test_folder" diff --git a/frappe/www/_test/_test_no_context.py b/frappe/www/_test/_test_no_context.py index 2ecb1eb828..01ccd2db64 100644 --- a/frappe/www/_test/_test_no_context.py +++ b/frappe/www/_test/_test_no_context.py @@ -1,7 +1,8 @@ import frappe + # no context object is accepted def get_context(): context = frappe._dict() context.body = "Custom Content" - return context \ No newline at end of file + return context diff --git a/frappe/www/about.py b/frappe/www/about.py index 7c33ed0140..fd2331d9f6 100644 --- a/frappe/www/about.py +++ b/frappe/www/about.py @@ -5,6 +5,7 @@ import frappe sitemap = 1 + def get_context(context): context.doc = frappe.get_doc("About Us Settings", "About Us Settings") diff --git a/frappe/www/app.py b/frappe/www/app.py index 92107816c7..ae0dad3326 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -2,12 +2,15 @@ # License: MIT. See LICENSE no_cache = 1 -import os, re +import os +import re + import frappe -from frappe import _ import frappe.sessions +from frappe import _ from frappe.utils.jinja_globals import is_rtl + def get_context(context): if frappe.session.user == "Guest": frappe.throw(_("Log in to access this page."), frappe.PermissionError) @@ -18,7 +21,7 @@ def get_context(context): try: boot = frappe.sessions.get() except Exception as e: - boot = frappe._dict(status='failed', error = str(e)) + boot = frappe._dict(status="failed", error=str(e)) print(frappe.get_traceback()) # this needs commit @@ -36,24 +39,27 @@ def get_context(context): # TODO: Find better fix boot_json = re.sub(r"", "", boot_json) - context.update({ - "no_cache": 1, - "build_version": frappe.utils.get_build_version(), - "include_js": hooks["app_include_js"], - "include_css": hooks["app_include_css"], - "layout_direction": "rtl" if is_rtl() else "ltr", - "lang": frappe.local.lang, - "sounds": hooks["sounds"], - "boot": boot if context.get("for_mobile") else boot_json, - "desk_theme": desk_theme or "Light", - "csrf_token": csrf_token, - "google_analytics_id": frappe.conf.get("google_analytics_id"), - "google_analytics_anonymize_ip": frappe.conf.get("google_analytics_anonymize_ip"), - "mixpanel_id": frappe.conf.get("mixpanel_id") - }) + context.update( + { + "no_cache": 1, + "build_version": frappe.utils.get_build_version(), + "include_js": hooks["app_include_js"], + "include_css": hooks["app_include_css"], + "layout_direction": "rtl" if is_rtl() else "ltr", + "lang": frappe.local.lang, + "sounds": hooks["sounds"], + "boot": boot if context.get("for_mobile") else boot_json, + "desk_theme": desk_theme or "Light", + "csrf_token": csrf_token, + "google_analytics_id": frappe.conf.get("google_analytics_id"), + "google_analytics_anonymize_ip": frappe.conf.get("google_analytics_anonymize_ip"), + "mixpanel_id": frappe.conf.get("mixpanel_id"), + } + ) return context + @frappe.whitelist() def get_desk_assets(build_version): """Get desk assets to be loaded for mobile app""" @@ -65,25 +71,21 @@ def get_desk_assets(build_version): for path in data["include_js"]: # assets path shouldn't start with / # as it points to different location altogether - if path.startswith('/assets/'): - path = path.replace('/assets/', 'assets/') + if path.startswith("/assets/"): + path = path.replace("/assets/", "assets/") try: - with open(os.path.join(frappe.local.sites_path, path) ,"r") as f: + with open(os.path.join(frappe.local.sites_path, path), "r") as f: assets[0]["data"] = assets[0]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8") except IOError: pass for path in data["include_css"]: - if path.startswith('/assets/'): - path = path.replace('/assets/', 'assets/') + if path.startswith("/assets/"): + path = path.replace("/assets/", "assets/") try: - with open(os.path.join(frappe.local.sites_path, path) ,"r") as f: + with open(os.path.join(frappe.local.sites_path, path), "r") as f: assets[1]["data"] = assets[1]["data"] + "\n" + frappe.safe_decode(f.read(), "utf-8") except IOError: pass - return { - "build_version": data["build_version"], - "boot": data["boot"], - "assets": assets - } + return {"build_version": data["build_version"], "boot": data["boot"], "assets": assets} diff --git a/frappe/www/complete_signup.py b/frappe/www/complete_signup.py index eb5ba62e5c..98029dd956 100644 --- a/frappe/www/complete_signup.py +++ b/frappe/www/complete_signup.py @@ -1,3 +1,2 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE - diff --git a/frappe/www/contact.py b/frappe/www/contact.py index 2caf69a53a..11be5e86da 100644 --- a/frappe/www/contact.py +++ b/frappe/www/contact.py @@ -2,11 +2,12 @@ # License: MIT. See LICENSE import frappe -from frappe.utils import now from frappe import _ +from frappe.utils import now sitemap = 1 + def get_context(context): doc = frappe.get_doc("Contact Us Settings", "Contact Us Settings") @@ -15,33 +16,38 @@ def get_context(context): else: query_options = ["Sales", "Support", "General"] - out = { - "query_options": query_options, - "parents": [ - { "name": _("Home"), "route": "/" } - ] - } + out = {"query_options": query_options, "parents": [{"name": _("Home"), "route": "/"}]} out.update(doc.as_dict()) return out + max_communications_per_hour = 1000 + @frappe.whitelist(allow_guest=True) def send_message(subject="Website Query", message="", sender=""): if not message: - frappe.response["message"] = 'Please write something' + frappe.response["message"] = "Please write something" return if not sender: - frappe.response["message"] = 'Email Address Required' + frappe.response["message"] = "Email Address Required" return # guest method, cap max writes per hour - if frappe.db.sql("""select count(*) from `tabCommunication` + if ( + frappe.db.sql( + """select count(*) from `tabCommunication` where `sent_or_received`="Received" - and TIMEDIFF(%s, modified) < '01:00:00'""", now())[0][0] > max_communications_per_hour: - frappe.response["message"] = "Sorry: we believe we have received an unreasonably high number of requests of this kind. Please try later" + and TIMEDIFF(%s, modified) < '01:00:00'""", + now(), + )[0][0] + > max_communications_per_hour + ): + frappe.response[ + "message" + ] = "Sorry: we believe we have received an unreasonably high number of requests of this kind. Please try later" return # send email @@ -49,15 +55,16 @@ def send_message(subject="Website Query", message="", sender=""): if forward_to_email: frappe.sendmail(recipients=forward_to_email, sender=sender, content=message, subject=subject) - # add to to-do ? - frappe.get_doc(dict( - doctype = 'Communication', - sender=sender, - subject= _('New Message from Website Contact Page'), - sent_or_received='Received', - content=message, - status='Open', - )).insert(ignore_permissions=True) + frappe.get_doc( + dict( + doctype="Communication", + sender=sender, + subject=_("New Message from Website Contact Page"), + sent_or_received="Received", + content=message, + status="Open", + ) + ).insert(ignore_permissions=True) return "okay" diff --git a/frappe/www/error.py b/frappe/www/error.py index 513af881fa..9b9d101cc2 100644 --- a/frappe/www/error.py +++ b/frappe/www/error.py @@ -4,9 +4,11 @@ import frappe no_cache = 1 + def get_context(context): - if frappe.flags.in_migrate: return + if frappe.flags.in_migrate: + return context.http_status_code = 500 print(frappe.get_traceback()) - return {"error": frappe.get_traceback().replace("<", "<").replace(">", ">") } + return {"error": frappe.get_traceback().replace("<", "<").replace(">", ">")} diff --git a/frappe/www/list.py b/frappe/www/list.py index 715f099f33..2048f1223e 100644 --- a/frappe/www/list.py +++ b/frappe/www/list.py @@ -1,26 +1,30 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, json +import json + +import frappe +from frappe import _ +from frappe.model.document import Document, get_controller from frappe.utils import cint, quoted from frappe.website.path_resolver import resolve_path -from frappe.model.document import get_controller, Document -from frappe import _ no_cache = 1 + def get_context(context, **dict_params): """Returns context for a list standard list page. Will also update `get_list_context` from the doctype module file""" frappe.local.form_dict.update(dict_params) doctype = frappe.local.form_dict.doctype - context.parents = [{"route":"me", "title":_("My Account")}] + context.parents = [{"route": "me", "title": _("My Account")}] context.meta = frappe.get_meta(doctype) context.update(get_list_context(context, doctype) or {}) context.doctype = doctype context.txt = frappe.local.form_dict.txt context.update(get(**frappe.local.form_dict)) + @frappe.whitelist(allow_guest=True) def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): """Returns processed HTML page for a standard listing.""" @@ -33,7 +37,8 @@ def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): meta = frappe.get_meta(doctype) list_context = frappe.flags.list_context - if not raw_result: return {"result": []} + if not raw_result: + return {"result": []} if txt: list_context.default_subtitle = _('Filtered by "{0}"').format(txt) @@ -44,8 +49,7 @@ def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): for doc in raw_result: doc.doctype = doctype - new_context = frappe._dict(doc=doc, meta=meta, - list_view_fields=list_view_fields) + new_context = frappe._dict(doc=doc, meta=meta, list_view_fields=list_view_fields) if not list_context.get_list and not isinstance(new_context.doc, Document): new_context.doc = frappe.get_doc(doc.doctype, doc.name) @@ -60,6 +64,7 @@ def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): result.append(rendered_row) from frappe.utils.response import json_handler + return { "raw_result": json.dumps(raw_result, default=json_handler), "result": result, @@ -67,8 +72,11 @@ def get(doctype, txt=None, limit_start=0, limit=20, pathname=None, **kwargs): "next_start": limit_start + limit, } + @frappe.whitelist(allow_guest=True) -def get_list_data(doctype, txt=None, limit_start=0, fields=None, cmd=None, limit=20, web_form_name=None, **kwargs): +def get_list_data( + doctype, txt=None, limit_start=0, fields=None, cmd=None, limit=20, web_form_name=None, **kwargs +): """Returns processed HTML page for a standard listing.""" limit_start = cint(limit_start) @@ -77,28 +85,34 @@ def get_list_data(doctype, txt=None, limit_start=0, fields=None, cmd=None, limit if not txt and frappe.form_dict.search: txt = frappe.form_dict.search - del frappe.form_dict['search'] + del frappe.form_dict["search"] controller = get_controller(doctype) meta = frappe.get_meta(doctype) filters = prepare_filters(doctype, controller, kwargs) list_context = get_list_context(frappe._dict(), doctype, web_form_name) - list_context.title_field = getattr(controller, 'website', - {}).get('page_title_field', meta.title_field or 'name') + list_context.title_field = getattr(controller, "website", {}).get( + "page_title_field", meta.title_field or "name" + ) if list_context.filters: filters.update(list_context.filters) _get_list = list_context.get_list or get_list - kwargs = dict(doctype=doctype, txt=txt, filters=filters, - limit_start=limit_start, limit_page_length=limit, - order_by = list_context.order_by or 'modified desc') + kwargs = dict( + doctype=doctype, + txt=txt, + filters=filters, + limit_start=limit_start, + limit_page_length=limit, + order_by=list_context.order_by or "modified desc", + ) # allow guest if flag is set if not list_context.get_list and (list_context.allow_guest or meta.allow_guest_to_view): - kwargs['ignore_permissions'] = True + kwargs["ignore_permissions"] = True raw_result = _get_list(**kwargs) @@ -107,15 +121,18 @@ def get_list_data(doctype, txt=None, limit_start=0, fields=None, cmd=None, limit return raw_result + def set_route(context): - '''Set link for the list item''' + """Set link for the list item""" if context.web_form_name: context.route = "{0}?name={1}".format(context.pathname, quoted(context.doc.name)) - elif context.doc and getattr(context.doc, 'route', None): + elif context.doc and getattr(context.doc, "route", None): context.route = context.doc.route else: - context.route = "{0}/{1}".format(context.pathname or quoted(context.doc.doctype), - quoted(context.doc.name)) + context.route = "{0}/{1}".format( + context.pathname or quoted(context.doc.doctype), quoted(context.doc.name) + ) + def prepare_filters(doctype, controller, kwargs): for key in kwargs.keys(): @@ -126,14 +143,14 @@ def prepare_filters(doctype, controller, kwargs): filters = frappe._dict(kwargs) meta = frappe.get_meta(doctype) - if hasattr(controller, 'website') and controller.website.get('condition_field'): - filters[controller.website['condition_field']] = 1 + if hasattr(controller, "website") and controller.website.get("condition_field"): + filters[controller.website["condition_field"]] = 1 if filters.pathname: # resolve additional filters from path resolve_path(filters.pathname) for key, val in frappe.local.form_dict.items(): - if key not in filters and key != 'flags': + if key not in filters and key != "flags": filters[key] = val # filter the filters to include valid fields only @@ -143,6 +160,7 @@ def prepare_filters(doctype, controller, kwargs): return filters + def get_list_context(context, doctype, web_form_name=None): from frappe.modules import load_doctype_module @@ -166,7 +184,7 @@ def get_list_context(context, doctype, web_form_name=None): # get context for custom webform if meta.custom and web_form_name: - webform_list_contexts = frappe.get_hooks('webform_list_context') + webform_list_contexts = frappe.get_hooks("webform_list_context") if webform_list_contexts: out = frappe._dict(frappe.get_attr(webform_list_contexts[0])(meta.module) or {}) if out: @@ -174,7 +192,7 @@ def get_list_context(context, doctype, web_form_name=None): # get context from web form module if web_form_name: - web_form = frappe.get_doc('Web Form', web_form_name) + web_form = frappe.get_doc("Web Form", web_form_name) list_context = update_context_from_module(web_form.get_web_form_module(), list_context) # get path from '/templates/' folder of the doctype @@ -186,7 +204,17 @@ def get_list_context(context, doctype, web_form_name=None): return list_context -def get_list(doctype, txt, filters, limit_start, limit_page_length=20, ignore_permissions=False, fields=None, order_by=None): + +def get_list( + doctype, + txt, + filters, + limit_start, + limit_page_length=20, + ignore_permissions=False, + fields=None, + order_by=None, +): meta = frappe.get_meta(doctype) if not filters: filters = [] @@ -199,7 +227,7 @@ def get_list(doctype, txt, filters, limit_start, limit_page_length=20, ignore_pe if txt: if meta.search_fields: for f in meta.get_search_fields(): - if f == 'name' or meta.get_field(f).fieldtype in ('Data', 'Text', 'Small Text', 'Text Editor'): + if f == "name" or meta.get_field(f).fieldtype in ("Data", "Text", "Small Text", "Text Editor"): or_filters.append([doctype, f, "like", "%" + txt + "%"]) else: if isinstance(filters, dict): @@ -207,8 +235,13 @@ def get_list(doctype, txt, filters, limit_start, limit_page_length=20, ignore_pe else: filters.append([doctype, "name", "like", "%" + txt + "%"]) - return frappe.get_list(doctype, fields = fields, - filters=filters, or_filters=or_filters, limit_start=limit_start, - limit_page_length = limit_page_length, ignore_permissions=ignore_permissions, - order_by=order_by) - + return frappe.get_list( + doctype, + fields=fields, + filters=filters, + or_filters=or_filters, + limit_start=limit_start, + limit_page_length=limit_page_length, + ignore_permissions=ignore_permissions, + order_by=order_by, + ) diff --git a/frappe/www/login.py b/frappe/www/login.py index 5b241132f4..c2eb5d2a6b 100644 --- a/frappe/www/login.py +++ b/frappe/www/login.py @@ -1,45 +1,56 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import json + import frappe import frappe.utils -from frappe.utils.oauth import get_oauth2_authorize_url, get_oauth_keys, login_via_oauth2, login_via_oauth2_id_token, login_oauth_user as _login_oauth_user, redirect_post_login -import json from frappe import _ from frappe.auth import LoginManager from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings -from frappe.utils.password import get_decrypted_password -from frappe.utils.html_utils import get_icon_html from frappe.integrations.oauth2_logins import decoder_compat -from frappe.website.utils import get_home_page +from frappe.utils.html_utils import get_icon_html from frappe.utils.jinja import guess_is_path +from frappe.utils.oauth import get_oauth2_authorize_url, get_oauth_keys +from frappe.utils.oauth import login_oauth_user as _login_oauth_user +from frappe.utils.oauth import login_via_oauth2, login_via_oauth2_id_token, redirect_post_login +from frappe.utils.password import get_decrypted_password +from frappe.website.utils import get_home_page no_cache = True + def get_context(context): redirect_to = frappe.local.request.args.get("redirect-to") if frappe.session.user != "Guest": if not redirect_to: - if frappe.session.data.user_type=="Website User": + if frappe.session.data.user_type == "Website User": redirect_to = get_home_page() else: redirect_to = "/app" - if redirect_to != 'login': + if redirect_to != "login": frappe.local.flags.redirect_location = redirect_to raise frappe.Redirect # get settings from site config context.no_header = True - context.for_test = 'login.html' + context.for_test = "login.html" context["title"] = "Login" context["provider_logins"] = [] - context["disable_signup"] = frappe.utils.cint(frappe.db.get_single_value("Website Settings", "disable_signup")) - context["logo"] = (frappe.db.get_single_value('Website Settings', 'app_logo') or - frappe.get_hooks("app_logo_url")[-1]) - context["app_name"] = (frappe.db.get_single_value('Website Settings', 'app_name') or - frappe.get_system_settings("app_name") or _("Frappe")) + context["disable_signup"] = frappe.utils.cint( + frappe.db.get_single_value("Website Settings", "disable_signup") + ) + context["logo"] = ( + frappe.db.get_single_value("Website Settings", "app_logo") + or frappe.get_hooks("app_logo_url")[-1] + ) + context["app_name"] = ( + frappe.db.get_single_value("Website Settings", "app_name") + or frappe.get_system_settings("app_name") + or _("Frappe") + ) signup_form_template = frappe.get_hooks("signup_form_template") if signup_form_template and len(signup_form_template) and signup_form_template[0]: @@ -51,7 +62,10 @@ def get_context(context): if path: context["signup_form_template"] = frappe.get_template(path).render() - providers = [i.name for i in frappe.get_all("Social Login Key", filters={"enable_social_login":1}, order_by="name")] + providers = [ + i.name + for i in frappe.get_all("Social Login Key", filters={"enable_social_login": 1}, order_by="name") + ] for provider in providers: client_id, base_url = frappe.get_value("Social Login Key", provider, ["client_id", "base_url"]) client_secret = get_decrypted_password("Social Login Key", provider, "client_secret") @@ -65,13 +79,15 @@ def get_context(context): else: icon = get_icon_html(icon_url, small=True) - if (get_oauth_keys(provider) and client_secret and client_id and base_url): - context.provider_logins.append({ - "name": provider, - "provider_name": provider_name, - "auth_url": get_oauth2_authorize_url(provider, redirect_to), - "icon": icon - }) + if get_oauth_keys(provider) and client_secret and client_id and base_url: + context.provider_logins.append( + { + "name": provider, + "provider_name": provider_name, + "auth_url": get_oauth2_authorize_url(provider, redirect_to), + "icon": icon, + } + ) context["social_login"] = True ldap_settings = LDAPSettings.get_ldap_client_settings() context["ldap_settings"] = ldap_settings @@ -84,30 +100,36 @@ def get_context(context): if frappe.utils.cint(frappe.get_system_settings("allow_login_using_user_name")): login_label.append(_("Username")) - context['login_label'] = ' {0} '.format(_('or')).join(login_label) + context["login_label"] = " {0} ".format(_("or")).join(login_label) return context + @frappe.whitelist(allow_guest=True) def login_via_google(code, state): login_via_oauth2("google", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_github(code, state): login_via_oauth2("github", code, state) + @frappe.whitelist(allow_guest=True) def login_via_facebook(code, state): login_via_oauth2("facebook", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_frappe(code, state): login_via_oauth2("frappe", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_office365(code, state): login_via_oauth2_id_token("office_365", code, state, decoder=decoder_compat) + @frappe.whitelist(allow_guest=True) def login_via_token(login_token): sid = frappe.cache().get_value("login_token:{0}".format(login_token), expires=True) @@ -118,4 +140,6 @@ def login_via_token(login_token): frappe.local.form_dict.sid = sid frappe.local.login_manager = LoginManager() - redirect_post_login(desk_user = frappe.db.get_value("User", frappe.session.user, "user_type")=="System User") + redirect_post_login( + desk_user=frappe.db.get_value("User", frappe.session.user, "user_type") == "System User" + ) diff --git a/frappe/www/me.py b/frappe/www/me.py index 1336008ade..fc8e920ce2 100644 --- a/frappe/www/me.py +++ b/frappe/www/me.py @@ -2,13 +2,14 @@ # License: MIT. See LICENSE import frappe -from frappe import _ import frappe.www.list +from frappe import _ no_cache = 1 + def get_context(context): - if frappe.session.user=='Guest': + if frappe.session.user == "Guest": frappe.throw(_("You need to be logged in to access this page"), frappe.PermissionError) context.current_user = frappe.get_doc("User", frappe.session.user) diff --git a/frappe/www/message.py b/frappe/www/message.py index b73f162dcf..fa13f3ab75 100644 --- a/frappe/www/message.py +++ b/frappe/www/message.py @@ -2,11 +2,11 @@ # License: MIT. See LICENSE import frappe - from frappe.utils import strip_html_tags no_cache = 1 + def get_context(context): message_context = frappe._dict() if hasattr(frappe.local, "message"): @@ -21,9 +21,9 @@ def get_context(context): key = "message_id:{0}".format(message_id) message = frappe.cache().get_value(key, expires=True) if message: - message_context.update(message.get('context', {})) - if message.get('http_status_code'): - frappe.local.response['http_status_code'] = message['http_status_code'] + message_context.update(message.get("context", {})) + if message.get("http_status_code"): + frappe.local.response["http_status_code"] = message["http_status_code"] if not message_context.title: message_context.title = frappe.form_dict.title diff --git a/frappe/www/printview.py b/frappe/www/printview.py index a32a208107..bed0ad15bf 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -1,9 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe, os, copy, json, re -from frappe import _, get_module_path +import copy +import json +import os +import re +import frappe +from frappe import _, get_module_path from frappe.core.doctype.access_log.access_log import make_access_log from frappe.utils import cint, sanitize_html, strip_html from frappe.utils.jinja_globals import is_rtl @@ -12,13 +16,17 @@ no_cache = 1 standard_format = "templates/print_formats/standard.html" + def get_context(context): """Build context for print""" if not ((frappe.form_dict.doctype and frappe.form_dict.name) or frappe.form_dict.doc): return { - "body": sanitize_html("""

Error

+ "body": sanitize_html( + """

Error

Parameters doctype and name required

- 
%s
""" % repr(frappe.form_dict)) + 
%s
""" + % repr(frappe.form_dict) + ) } if frappe.form_dict.doc: @@ -32,27 +40,34 @@ def get_context(context): meta = frappe.get_meta(doc.doctype) - print_format = get_print_format_doc(None, meta = meta) + print_format = get_print_format_doc(None, meta=meta) - make_access_log(doctype=frappe.form_dict.doctype, document=frappe.form_dict.name, file_type='PDF', method='Print') + make_access_log( + doctype=frappe.form_dict.doctype, document=frappe.form_dict.name, file_type="PDF", method="Print" + ) return { - "body": get_rendered_template(doc, print_format = print_format, - meta=meta, trigger_print = frappe.form_dict.trigger_print, - no_letterhead=frappe.form_dict.no_letterhead, letterhead=letterhead, - settings=settings), + "body": get_rendered_template( + doc, + print_format=print_format, + meta=meta, + trigger_print=frappe.form_dict.trigger_print, + no_letterhead=frappe.form_dict.no_letterhead, + letterhead=letterhead, + settings=settings, + ), "css": get_print_style(frappe.form_dict.style, print_format), "comment": frappe.session.user, "title": doc.get(meta.title_field) if meta.title_field else doc.name, "lang": frappe.local.lang, - "layout_direction": "rtl" if is_rtl() else "ltr" + "layout_direction": "rtl" if is_rtl() else "ltr", } + def get_print_format_doc(print_format_name, meta): """Returns print format document""" if not print_format_name: - print_format_name = frappe.form_dict.format \ - or meta.default_print_format or "Standard" + print_format_name = frappe.form_dict.format or meta.default_print_format or "Standard" if print_format_name == "Standard": return None @@ -63,9 +78,17 @@ def get_print_format_doc(print_format_name, meta): # if old name, return standard! return None -def get_rendered_template(doc, name=None, print_format=None, meta=None, - no_letterhead=None, letterhead=None, trigger_print=False, - settings=None): + +def get_rendered_template( + doc, + name=None, + print_format=None, + meta=None, + no_letterhead=None, + letterhead=None, + trigger_print=False, + settings=None, +): print_settings = frappe.get_single("Print Settings").as_dict() print_settings.update(settings or {}) @@ -83,16 +106,18 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None, validate_print_permission(doc) if doc.meta.is_submittable: - if doc.docstatus==0 and not cint(print_settings.allow_print_for_draft): + if doc.docstatus == 0 and not cint(print_settings.allow_print_for_draft): frappe.throw(_("Not allowed to print draft documents"), frappe.PermissionError) - if doc.docstatus==2 and not cint(print_settings.allow_print_for_cancelled): + if doc.docstatus == 2 and not cint(print_settings.allow_print_for_cancelled): frappe.throw(_("Not allowed to print cancelled documents"), frappe.PermissionError) doc.run_method("before_print", print_settings) - if not hasattr(doc, "print_heading"): doc.print_heading = None - if not hasattr(doc, "sub_heading"): doc.sub_heading = None + if not hasattr(doc, "print_heading"): + doc.print_heading = None + if not hasattr(doc, "sub_heading"): + doc.sub_heading = None if not meta: meta = frappe.get_meta(doc.doctype) @@ -108,8 +133,7 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None, doc.absolute_value = print_format.absolute_value def get_template_from_string(): - return jenv.from_string(get_print_format(doc.doctype, - print_format)) + return jenv.from_string(get_print_format(doc.doctype, print_format)) if print_format.custom_format: template = get_template_from_string() @@ -127,7 +151,7 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None, template = "standard" - elif print_format.standard=="Yes": + elif print_format.standard == "Yes": template = get_template_from_string() else: @@ -137,17 +161,20 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None, else: template = "standard" - if template == "standard": template = jenv.get_template(standard_format) letter_head = frappe._dict(get_letter_head(doc, no_letterhead, letterhead) or {}) if letter_head.content: - letter_head.content = frappe.utils.jinja.render_template(letter_head.content, {"doc": doc.as_dict()}) + letter_head.content = frappe.utils.jinja.render_template( + letter_head.content, {"doc": doc.as_dict()} + ) if letter_head.footer: - letter_head.footer = frappe.utils.jinja.render_template(letter_head.footer, {"doc": doc.as_dict()}) + letter_head.footer = frappe.utils.jinja.render_template( + letter_head.footer, {"doc": doc.as_dict()} + ) convert_markdown(doc, meta) @@ -156,16 +183,18 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None, if format_data and format_data[0].get("fieldname") == "print_heading_template": args["print_heading_template"] = format_data.pop(0).get("options") - args.update({ - "doc": doc, - "meta": frappe.get_meta(doc.doctype), - "layout": make_layout(doc, meta, format_data), - "no_letterhead": no_letterhead, - "trigger_print": cint(trigger_print), - "letter_head": letter_head.content, - "footer": letter_head.footer, - "print_settings": print_settings - }) + args.update( + { + "doc": doc, + "meta": frappe.get_meta(doc.doctype), + "layout": make_layout(doc, meta, format_data), + "no_letterhead": no_letterhead, + "trigger_print": cint(trigger_print), + "letter_head": letter_head.content, + "footer": letter_head.footer, + "print_settings": print_settings, + } + ) html = template.render(args, filters={"len": len}) @@ -174,6 +203,7 @@ def get_rendered_template(doc, name=None, print_format=None, meta=None, return html + def set_link_titles(doc): # Adds name with title of link field doctype to __link_titles if not doc.get("__link_titles"): @@ -183,6 +213,7 @@ def set_link_titles(doc): set_title_values_for_link_and_dynamic_link_fields(meta, doc) set_title_values_for_table_and_multiselect_fields(meta, doc) + def set_title_values_for_link_and_dynamic_link_fields(meta, doc, parent_doc=None): if parent_doc and not parent_doc.get("__link_titles"): setattr(parent_doc, "__link_titles", {}) @@ -207,6 +238,7 @@ def set_title_values_for_link_and_dynamic_link_fields(meta, doc, parent_doc=None elif doc: doc.__link_titles["{0}::{1}".format(doctype, doc.get(field.fieldname))] = link_title + def set_title_values_for_table_and_multiselect_fields(meta, doc): for field in meta.get_table_fields(): if not doc.get(field.fieldname): @@ -216,18 +248,29 @@ def set_title_values_for_table_and_multiselect_fields(meta, doc): for value in doc.get(field.fieldname): set_title_values_for_link_and_dynamic_link_fields(_meta, value, doc) + def convert_markdown(doc, meta): - '''Convert text field values to markdown if necessary''' + """Convert text field values to markdown if necessary""" for field in meta.fields: - if field.fieldtype=='Text Editor': + if field.fieldtype == "Text Editor": value = doc.get(field.fieldname) - if value and '' in value: + if value and "" in value: doc.set(field.fieldname, frappe.utils.md_to_html(value)) + @frappe.whitelist() -def get_html_and_style(doc, name=None, print_format=None, meta=None, - no_letterhead=None, letterhead=None, trigger_print=False, style=None, - settings=None, templates=None): +def get_html_and_style( + doc, + name=None, + print_format=None, + meta=None, + no_letterhead=None, + letterhead=None, + trigger_print=False, + style=None, + settings=None, + templates=None, +): """Returns `html` and `style` of print format, used in PDF etc""" if isinstance(doc, str) and isinstance(name, str): @@ -240,17 +283,22 @@ def get_html_and_style(doc, name=None, print_format=None, meta=None, set_link_titles(doc) try: - html = get_rendered_template(doc, name=name, print_format=print_format, meta=meta, - no_letterhead=no_letterhead, letterhead=letterhead, trigger_print=trigger_print, - settings=frappe.parse_json(settings)) + html = get_rendered_template( + doc, + name=name, + print_format=print_format, + meta=meta, + no_letterhead=no_letterhead, + letterhead=letterhead, + trigger_print=trigger_print, + settings=frappe.parse_json(settings), + ) except frappe.TemplateNotFoundError: frappe.clear_last_message() html = None - return { - "html": html, - "style": get_print_style(style=style, print_format=print_format) - } + return {"html": html, "style": get_print_style(style=style, print_format=print_format)} + @frappe.whitelist() def get_rendered_raw_commands(doc, name=None, print_format=None, meta=None, lang=None): @@ -265,23 +313,25 @@ def get_rendered_raw_commands(doc, name=None, print_format=None, meta=None, lang print_format = get_print_format_doc(print_format, meta=meta or frappe.get_meta(doc.doctype)) if not print_format or (print_format and not print_format.raw_printing): - frappe.throw(_("{0} is not a raw printing format.").format(print_format), - frappe.TemplateNotFoundError) + frappe.throw( + _("{0} is not a raw printing format.").format(print_format), frappe.TemplateNotFoundError + ) return { "raw_commands": get_rendered_template(doc, name=name, print_format=print_format, meta=meta) } + def validate_print_permission(doc): if frappe.form_dict.get("key"): if frappe.form_dict.key == doc.get_signature(): return for ptype in ("read", "print"): - if (not frappe.has_permission(doc.doctype, ptype, doc) - and not frappe.has_website_permission(doc)): + if not frappe.has_permission(doc.doctype, ptype, doc) and not frappe.has_website_permission(doc): raise frappe.PermissionError(_("No {0} permission").format(ptype)) + def get_letter_head(doc, no_letterhead, letterhead=None): if no_letterhead: return {} @@ -290,17 +340,23 @@ def get_letter_head(doc, no_letterhead, letterhead=None): if doc.get("letter_head"): return frappe.db.get_value("Letter Head", doc.letter_head, ["content", "footer"], as_dict=True) else: - return frappe.db.get_value("Letter Head", {"is_default": 1}, ["content", "footer"], as_dict=True) or {} + return ( + frappe.db.get_value("Letter Head", {"is_default": 1}, ["content", "footer"], as_dict=True) or {} + ) + def get_print_format(doctype, print_format): if print_format.disabled: - frappe.throw(_("Print Format {0} is disabled").format(print_format.name), - frappe.DoesNotExistError) + frappe.throw( + _("Print Format {0} is disabled").format(print_format.name), frappe.DoesNotExistError + ) # server, find template module = print_format.module or frappe.db.get_value("DocType", doctype, "module") - path = os.path.join(get_module_path(module, "Print Format", print_format.name), - frappe.scrub(print_format.name) + ".html") + path = os.path.join( + get_module_path(module, "Print Format", print_format.name), + frappe.scrub(print_format.name) + ".html", + ) if os.path.exists(path): with open(path, "r") as pffile: @@ -311,8 +367,8 @@ def get_print_format(doctype, print_format): if print_format.html: return print_format.html - frappe.throw(_("No template found at path: {0}").format(path), - frappe.TemplateNotFoundError) + frappe.throw(_("No template found at path: {0}").format(path), frappe.TemplateNotFoundError) + def make_layout(doc, meta, format_data=None): """Builds a hierarchical layout object from the fields list to be rendered @@ -324,12 +380,13 @@ def make_layout(doc, meta, format_data=None): layout, page = [], [] layout.append(page) - def get_new_section(): return {'columns': [], 'has_data': False} + def get_new_section(): + return {"columns": [], "has_data": False} def append_empty_field_dict_to_page_column(page): - """ append empty columns dict to page layout """ - if not page[-1]['columns']: - page[-1]['columns'].append({'fields': []}) + """append empty columns dict to page layout""" + if not page[-1]["columns"]: + page[-1]["columns"].append({"fields": []}) for df in format_data or meta.fields: if format_data: @@ -339,50 +396,50 @@ def make_layout(doc, meta, format_data=None): original = meta.get_field(df.fieldname) if original: newdf = original.as_dict() - newdf.hide_in_print_layout = original.get('hide_in_print_layout') + newdf.hide_in_print_layout = original.get("hide_in_print_layout") newdf.update(df) df = newdf df.print_hide = 0 - if df.fieldtype=="Section Break" or page==[]: + if df.fieldtype == "Section Break" or page == []: if len(page) > 1: - if page[-1]['has_data']==False: + if page[-1]["has_data"] == False: # truncate last section if empty del page[-1] section = get_new_section() - if df.fieldtype=='Section Break' and df.label: - section['label'] = df.label + if df.fieldtype == "Section Break" and df.label: + section["label"] = df.label page.append(section) - elif df.fieldtype=="Column Break": + elif df.fieldtype == "Column Break": # if last column break and last column is not empty - page[-1]['columns'].append({'fields': []}) + page[-1]["columns"].append({"fields": []}) else: # add a column if not yet added append_empty_field_dict_to_page_column(page) - if df.fieldtype=="HTML" and df.options: - doc.set(df.fieldname, True) # show this field + if df.fieldtype == "HTML" and df.options: + doc.set(df.fieldname, True) # show this field - if df.fieldtype=='Signature' and not doc.get(df.fieldname): - placeholder_image = '/assets/frappe/images/signature-placeholder.png' + if df.fieldtype == "Signature" and not doc.get(df.fieldname): + placeholder_image = "/assets/frappe/images/signature-placeholder.png" doc.set(df.fieldname, placeholder_image) if is_visible(df, doc) and has_value(df, doc): append_empty_field_dict_to_page_column(page) - page[-1]['columns'][-1]['fields'].append(df) + page[-1]["columns"][-1]["fields"].append(df) # section has fields - page[-1]['has_data'] = True + page[-1]["has_data"] = True # if table, add the row info in the field # if a page break is found, create a new docfield - if df.fieldtype=="Table": + if df.fieldtype == "Table": df.rows = [] df.start = 0 df.end = None @@ -400,10 +457,11 @@ def make_layout(doc, meta, format_data=None): df = copy.copy(df) df.start = i df.end = None - page[-1]['columns'][-1]['fields'].append(df) + page[-1]["columns"][-1]["fields"].append(df) return layout + def is_visible(df, doc): """Returns True if docfield is visible in print layout and does not have print_hide set.""" if df.fieldtype in ("Section Break", "Column Break", "Button"): @@ -414,6 +472,7 @@ def is_visible(df, doc): return not doc.is_print_hide(df.fieldname, df) + def has_value(df, doc): value = doc.get(df.fieldname) if value in (None, ""): @@ -430,22 +489,23 @@ def has_value(df, doc): return True + def get_print_style(style=None, print_format=None, for_legacy=False): print_settings = frappe.get_doc("Print Settings") if not style: - style = print_settings.print_style or '' + style = print_settings.print_style or "" context = { "print_settings": print_settings, "print_style": style, - "font": get_font(print_settings, print_format, for_legacy) + "font": get_font(print_settings, print_format, for_legacy), } css = frappe.get_template("templates/styles/standard.css").render(context) - if style and frappe.db.exists('Print Style', style): - css = css + '\n' + frappe.db.get_value('Print Style', style, 'css') + if style and frappe.db.exists("Print Style", style): + css = css + "\n" + frappe.db.get_value("Print Style", style, "css") # move @import to top for at_import in list(set(re.findall(r"(@import url\([^\)]+\)[;]?)", css))): @@ -459,6 +519,7 @@ def get_print_style(style=None, print_format=None, for_legacy=False): return css + def get_font(print_settings, print_format=None, for_legacy=False): default = 'Inter, "Helvetica Neue", Helvetica, Arial, "Open Sans", sans-serif' if for_legacy: @@ -466,30 +527,30 @@ def get_font(print_settings, print_format=None, for_legacy=False): font = None if print_format: - if print_format.font and print_format.font!="Default": - font = '{0}, sans-serif'.format(print_format.font) + if print_format.font and print_format.font != "Default": + font = "{0}, sans-serif".format(print_format.font) if not font: - if print_settings.font and print_settings.font!="Default": - font = '{0}, sans-serif'.format(print_settings.font) + if print_settings.font and print_settings.font != "Default": + font = "{0}, sans-serif".format(print_settings.font) else: font = default return font + def get_visible_columns(data, table_meta, df): """Returns list of visible columns based on print_hide and if all columns have value.""" columns = [] doc = data[0] or frappe.new_doc(df.options) - hide_in_print_layout = df.get('hide_in_print_layout') or [] + hide_in_print_layout = df.get("hide_in_print_layout") or [] def add_column(col_df): if col_df.fieldname in hide_in_print_layout: return False - return is_visible(col_df, doc) \ - and column_has_value(data, col_df.get("fieldname"), col_df) + return is_visible(col_df, doc) and column_has_value(data, col_df.get("fieldname"), col_df) if df.get("visible_columns"): # columns specified by column builder @@ -509,11 +570,12 @@ def get_visible_columns(data, table_meta, df): return columns + def column_has_value(data, fieldname, col_df): """Check if at least one cell in column has non-zero and non-blank value""" has_value = False - if col_df.fieldtype in ['Float', 'Currency'] and not col_df.print_hide_if_no_value: + if col_df.fieldtype in ["Float", "Currency"] and not col_df.print_hide_if_no_value: return True for row in data: @@ -529,6 +591,7 @@ def column_has_value(data, fieldname, col_df): return has_value + trigger_print_script = """