diff --git a/.eslintrc b/.eslintrc index c55acc5bac..a80d2910fa 100644 --- a/.eslintrc +++ b/.eslintrc @@ -80,6 +80,7 @@ "validate_email": true, "validate_name": true, "validate_phone": true, + "validate_url": true, "get_number_format": true, "format_number": true, "format_currency": true, diff --git a/.flake8 b/.flake8 index 399b176e1d..56c9b9a369 100644 --- a/.flake8 +++ b/.flake8 @@ -29,4 +29,5 @@ ignore = B950, W191, -max-line-length = 200 \ No newline at end of file +max-line-length = 200 +exclude=.github/helper/semgrep_rules diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..4faece896a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,12 @@ +# Since version 2.23 (released in August 2019), git-blame has a feature +# to ignore or bypass certain commits. +# +# This file contains a list of commits that are not likely what you +# are looking for in a blame, such as mass reformatting or renaming. +# You can set this file as a default ignore file for blame by running +# the following command. +# +# $ git config blame.ignoreRevsFile .git-blame-ignore-revs + +# Replace use of Class.extend with native JS class +fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85 diff --git a/.github/helper/semgrep_rules/frappe_correctness.py b/.github/helper/semgrep_rules/frappe_correctness.py index 37889fbbb1..745e6463b8 100644 --- a/.github/helper/semgrep_rules/frappe_correctness.py +++ b/.github/helper/semgrep_rules/frappe_correctness.py @@ -4,25 +4,61 @@ from frappe import _, flt from frappe.model.document import Document +# ruleid: frappe-modifying-but-not-comitting def on_submit(self): if self.value_of_goods == 0: frappe.throw(_('Value of goods cannot be 0')) - # ruleid: frappe-modifying-after-submit self.status = 'Submitted' -def on_submit(self): # noqa - if flt(self.per_billed) < 100: - self.update_billing_status() - else: - # todook: frappe-modifying-after-submit - self.status = "Completed" - self.db_set("status", "Completed") - -class TestDoc(Document): - pass - - def validate(self): - #ruleid: frappe-modifying-child-tables-while-iterating - for item in self.child_table: - if item.value < 0: - self.remove(item) + +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + self.status = 'Submitted' + self.db_set('status', 'Submitted') + +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + x = "y" + self.status = x + self.db_set('status', x) + + +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + x = "y" + self.status = x + self.save() + +# ruleid: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + + def tainted_method(self): + self.status = "uptate" + + +# ok: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + + def tainted_method(self): + self.status = "update" + self.db_set("status", "update") + +# ok: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + self.save() + + def tainted_method(self): + self.status = "uptate" diff --git a/.github/helper/semgrep_rules/translate.js b/.github/helper/semgrep_rules/translate.js index 7b92fe2dff..9cdfb75d0b 100644 --- a/.github/helper/semgrep_rules/translate.js +++ b/.github/helper/semgrep_rules/translate.js @@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.') // ruleid: frappe-translation-js-splitting __('You have {0} subscribers' + 'in your mailing list', [subscribers.length]) + +// ok: frappe-translation-js-splitting +__("Ctrl+Enter to add comment") + +// ruleid: frappe-translation-js-splitting +__('You have {0} subscribers \ + in your mailing list', [subscribers.length]) diff --git a/.github/helper/semgrep_rules/translate.py b/.github/helper/semgrep_rules/translate.py index bd6cd9126c..9de6aa94f0 100644 --- a/.github/helper/semgrep_rules/translate.py +++ b/.github/helper/semgrep_rules/translate.py @@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool") _("") # ruleid: frappe-translation-empty-string _('') + + +class Test: + # ok: frappe-translation-python-splitting + def __init__( + args + ): + pass diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml index df55089b9f..5f03fb9fd0 100644 --- a/.github/helper/semgrep_rules/translate.yml +++ b/.github/helper/semgrep_rules/translate.yml @@ -42,10 +42,10 @@ rules: - id: frappe-translation-python-splitting pattern-either: - - pattern: _(...) + ... + _(...) + - pattern: _(...) + _(...) - pattern: _("..." + "...") - - pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\` - - pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( ) + - pattern-regex: '[\s\.]_\([^\)]*\\\s*' # lines broken by `\` + - pattern-regex: '[\s\.]_\(\s*\n' # line breaks allowed by python for using ( ) message: | Do not split strings inside translate function. Do not concatenate using translate functions. Please refer: https://frappeframework.com/docs/user/en/translations @@ -54,8 +54,8 @@ rules: - id: frappe-translation-js-splitting pattern-either: - - pattern-regex: '__\([^\)]*[\+\\]\s*' - - pattern: __('...' + '...') + - pattern-regex: '__\([^\)]*[\\]\s+' + - pattern: __('...' + '...', ...) - pattern: __('...') + __('...') message: | Do not split strings inside translate function. Do not concatenate using translate functions. diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 2a934a6795..a23885b508 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -15,11 +15,11 @@ jobs: path: 'frappe' - uses: actions/setup-node@v1 with: - python-version: '12.x' + node-version: 14 - uses: actions/setup-python@v2 with: python-version: '3.6' - - name: Set up bench for current push + - name: Set up bench and build assets run: | npm install -g yarn pip3 install -U frappe-bench @@ -29,7 +29,7 @@ jobs: - name: Package assets run: | mkdir -p $GITHUB_WORKSPACE/build - tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css + tar -cvpzf $GITHUB_WORKSPACE/build/$GITHUB_SHA.tar.gz ./frappe-bench/sites/assets/frappe/dist - name: Publish assets to S3 uses: jakejarvis/s3-sync-action@master diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/publish-assets-releases.yml index e86f884f35..a697517c23 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/publish-assets-releases.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.6' - - name: Set up bench for current push + - name: Set up bench and build assets run: | npm install -g yarn pip3 install -U frappe-bench @@ -32,7 +32,7 @@ jobs: - name: Package assets run: | mkdir -p $GITHUB_WORKSPACE/build - tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/js ./frappe-bench/sites/assets/css + tar -cvpzf $GITHUB_WORKSPACE/build/assets.tar.gz ./frappe-bench/sites/assets/frappe/dist - name: Get release id: get_release diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 5092bf4705..389524e968 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -4,6 +4,8 @@ on: pull_request: branches: - develop + - version-13-hotfix + - version-13-pre-release jobs: semgrep: name: Frappe Linter diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/server-mariadb-tests.yml similarity index 51% rename from .github/workflows/ci-tests.yml rename to .github/workflows/server-mariadb-tests.yml index d2fbef528b..1742e813c6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -1,10 +1,8 @@ -name: CI +name: Server on: pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] workflow_dispatch: - push: jobs: test: @@ -13,23 +11,9 @@ jobs: strategy: fail-fast: false matrix: - include: - - DB: "mariadb" - TYPE: "server" - JOB_NAME: "Python MariaDB" - RUN_COMMAND: bench --site test_site run-tests --coverage + container: [1, 2] - - DB: "postgres" - TYPE: "server" - JOB_NAME: "Python PostgreSQL" - RUN_COMMAND: bench --site test_site run-tests --coverage - - - DB: "mariadb" - TYPE: "ui" - JOB_NAME: "UI MariaDB" - RUN_COMMAND: bench --site test_site run-ui-tests frappe --headless - - name: ${{ matrix.JOB_NAME }} + name: Python Unit Tests (MariaDB) services: mysql: @@ -40,18 +24,6 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 - postgres: - image: postgres:12.4 - env: - POSTGRES_PASSWORD: travis - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - name: Clone uses: actions/checkout@v2 @@ -63,7 +35,7 @@ jobs: - uses: actions/setup-node@v2 with: - node-version: '12' + node-version: 14 check-latest: true - name: Add to Hosts @@ -104,68 +76,54 @@ jobs: restore-keys: | ${{ runner.os }}-yarn- - - name: Cache cypress binary - if: matrix.TYPE == 'ui' - uses: actions/cache@v2 - with: - path: ~/.cache - key: ${{ runner.os }}-cypress- - restore-keys: | - ${{ runner.os }}-cypress- - ${{ runner.os }}- - - name: Install Dependencies run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: ${{ matrix.TYPE }} + TYPE: server - name: Install run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} + DB: mariadb + TYPE: server - - name: Run Set-Up - if: matrix.TYPE == 'ui' - run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard - env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} - - - name: Setup tmate session - if: contains(github.event.pull_request.labels.*.name, 'debug-gha') - uses: mxschmitt/action-tmate@v3 - name: Run Tests - run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }} + run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} + CI_BUILD_ID: ${{ github.run_id }} + ORCHESTRATOR_URL: http://test-orchestrator.frappe.io - - name: Coverage - Pull Request - if: matrix.TYPE == 'server' && github.event_name == 'pull_request' + - name: Upload Coverage Data run: | cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} cd ${GITHUB_WORKSPACE} - pip install coveralls==2.2.0 - pip install coverage==4.5.4 - coveralls --service=github + pip3 install coverage==5.5 + pip3 install coveralls==3.0.1 + coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - COVERALLS_SERVICE_NAME: github - - - name: Coverage - Push - if: matrix.TYPE == 'server' && github.event_name == 'push' + COVERALLS_FLAG_NAME: run-${{ matrix.container }} + COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }} + COVERALLS_PARALLEL: true + + coveralls: + name: Coverage Wrap Up + needs: test + container: python:3-slim + runs-on: ubuntu-18.04 + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Coveralls Finished run: | - cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} cd ${GITHUB_WORKSPACE} - pip install coveralls==2.2.0 - pip install coverage==4.5.4 - coveralls --service=github-actions + pip3 install coverage==5.5 + pip3 install coveralls==3.0.1 + coveralls --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - COVERALLS_SERVICE_NAME: github-actions diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml new file mode 100644 index 0000000000..4325eebaad --- /dev/null +++ b/.github/workflows/server-postgres-tests.yml @@ -0,0 +1,100 @@ +name: Server + +on: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-18.04 + + strategy: + fail-fast: false + matrix: + container: [1, 2] + + name: Python Unit Tests (Postgres) + + services: + postgres: + image: postgres:12.4 + env: + POSTGRES_PASSWORD: travis + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - uses: actions/setup-node@v2 + with: + node-version: '14' + check-latest: true + + - name: Add to Hosts + run: | + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: server + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: postgres + TYPE: server + + - name: Run Tests + run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator + env: + CI_BUILD_ID: ${{ github.run_id }} + ORCHESTRATOR_URL: http://test-orchestrator.frappe.io diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml new file mode 100644 index 0000000000..d9ccb07da0 --- /dev/null +++ b/.github/workflows/ui-tests.yml @@ -0,0 +1,105 @@ +name: UI + +on: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-18.04 + + strategy: + fail-fast: false + matrix: + containers: [1, 2] + + name: UI Tests (Cypress) + + services: + mysql: + image: mariadb:10.3 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: YES + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - uses: actions/setup-node@v2 + with: + node-version: 14 + check-latest: true + + - name: Add to Hosts + run: | + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Cache cypress binary + uses: actions/cache@v2 + with: + path: ~/.cache + key: ${{ runner.os }}-cypress- + restore-keys: | + ${{ runner.os }}-cypress- + ${{ runner.os }}- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: ui + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: mariadb + TYPE: ui + + - name: Site Setup + run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard + + - name: UI Tests + run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID diff --git a/.gitignore b/.gitignore index 766288fe2e..1ff3122d70 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ locale dist/ # build/ frappe/docs/current +frappe/public/dist .vscode node_modules .kdev4/ diff --git a/.mergify.yml b/.mergify.yml index 82f710a5a8..c759c1e3ec 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,12 @@ pull_request_rules: conditions: - status-success=Sider - status-success=Semantic Pull Request - - status-success=Python MariaDB - - status-success=Python PostgreSQL - - status-success=UI MariaDB + - status-success=Python Unit Tests (MariaDB) (1) + - status-success=Python Unit Tests (MariaDB) (2) + - status-success=Python Unit Tests (Postgres) (1) + - status-success=Python Unit Tests (Postgres) (2) + - status-success=UI Tests (Cypress) (1) + - status-success=UI Tests (Cypress) (2) - status-success=security/snyk (frappe) - label!=dont-merge - label!=squash @@ -16,9 +19,12 @@ pull_request_rules: - name: Automatic squash on CI success and review conditions: - status-success=Sider - - status-success=Python MariaDB - - status-success=Python PostgreSQL - - status-success=UI MariaDB + - status-success=Python Unit Tests (MariaDB) (1) + - status-success=Python Unit Tests (MariaDB) (2) + - status-success=Python Unit Tests (Postgres) (1) + - status-success=Python Unit Tests (Postgres) (2) + - status-success=UI Tests (Cypress) (1) + - status-success=UI Tests (Cypress) (2) - status-success=security/snyk (frappe) - label!=dont-merge - label=squash diff --git a/cypress/fixtures/data_field_validation_doctype.js b/cypress/fixtures/data_field_validation_doctype.js new file mode 100644 index 0000000000..da091af7e5 --- /dev/null +++ b/cypress/fixtures/data_field_validation_doctype.js @@ -0,0 +1,65 @@ +export default { + name: 'Validation Test', + custom: 1, + actions: [], + creation: '2019-03-15 06:29:07.215072', + doctype: 'DocType', + editable_grid: 1, + engine: 'InnoDB', + fields: [ + { + fieldname: 'email', + fieldtype: 'Data', + label: 'Email', + options: 'Email' + }, + { + fieldname: 'URL', + fieldtype: 'Data', + label: 'URL', + options: 'URL' + }, + { + fieldname: 'Phone', + fieldtype: 'Data', + label: 'Phone', + options: 'Phone' + }, + { + fieldname: 'person_name', + fieldtype: 'Data', + label: 'Person Name', + options: 'Name' + }, + { + fieldname: 'read_only_url', + fieldtype: 'Data', + label: 'Read Only URL', + options: 'URL', + read_only: '1', + default: 'https://frappe.io' + } + ], + issingle: 1, + links: [], + modified: '2021-04-19 14:40:53.127615', + modified_by: 'Administrator', + module: 'Custom', + owner: 'Administrator', + permissions: [ + { + create: 1, + delete: 1, + email: 1, + print: 1, + read: 1, + role: 'System Manager', + share: 1, + write: 1 + } + ], + quick_entry: 1, + sort_field: 'modified', + sort_order: 'ASC', + track_changes: 1 +}; diff --git a/cypress/integration/data_field_form_validation.js b/cypress/integration/data_field_form_validation.js new file mode 100644 index 0000000000..c6feea5550 --- /dev/null +++ b/cypress/integration/data_field_form_validation.js @@ -0,0 +1,43 @@ +import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; +const doctype_name = data_field_validation_doctype.name; + + +context('Data Field Input Validation in New Form', () => { + before(() => { + cy.login(); + cy.visit('/app/website'); + return cy.insert_doc('DocType', data_field_validation_doctype, true); + }); + + function validateField(fieldname, invalid_value, valid_value) { + // Invalid, should have has-error class + cy.get_field(fieldname).clear().type(invalid_value).blur(); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('have.class', 'has-error'); + // Valid value, should not have has-error class + cy.get_field(fieldname).clear().type(valid_value); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('not.have.class', 'has-error'); + } + + describe('Data Field Options', () => { + it('should validate email address', () => { + cy.new_form(doctype_name); + validateField('email', 'captian', 'hello@test.com'); + }); + + it('should validate URL', () => { + validateField('url', 'jkl', 'https://frappe.io'); + validateField('url', 'abcd.com', 'http://google.com/home'); + validateField('url', '&&http://google.uae', 'gopher://frappe.io'); + validateField('url', 'ftt2:://google.in?q=news', 'ftps2://frappe.io/__/#home'); + validateField('url', 'ftt2://', 'ntps://localhost'); // For intranet URLs + }); + + it('should validate phone number', () => { + validateField('phone', 'america', '89787878'); + }); + + it('should validate name', () => { + validateField('person_name', ' 777Hello', 'James Bond'); + }); + }); +}); \ No newline at end of file diff --git a/cypress/integration/recorder.js b/cypress/integration/recorder.js index d30cc3568c..5b7692d8ff 100644 --- a/cypress/integration/recorder.js +++ b/cypress/integration/recorder.js @@ -50,7 +50,7 @@ context('Recorder', () => { cy.get('.result-list').should('contain', '/api/method/frappe.desk.reportview.get'); }); - it.only('Recorder View Request', () => { + it('Recorder View Request', () => { cy.get('.primary-action').should('contain', 'Start').click(); cy.visit('/app/List/DocType/List'); diff --git a/cypress/integration/url_data_field.js b/cypress/integration/url_data_field.js new file mode 100644 index 0000000000..cf22c62363 --- /dev/null +++ b/cypress/integration/url_data_field.js @@ -0,0 +1,43 @@ +import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; + +const doctype_name = data_field_validation_doctype.name; + +context('URL Data Field Input', () => { + before(() => { + cy.login(); + cy.visit('/app/website'); + return cy.insert_doc('DocType', data_field_validation_doctype, true); + }); + + + describe('URL Data Field Input ', () => { + it('should not show URL link button without focus', () => { + cy.new_form(doctype_name); + cy.get_field('url').clear().type('https://frappe.io'); + cy.get_field('url').blur().wait(500); + cy.get('.link-btn').should('not.be.visible'); + }); + + it('should show URL link button on focus', () => { + cy.get_field('url').focus().wait(500); + cy.get('.link-btn').should('be.visible'); + }); + + it('should not show URL link button for invalid URL', () => { + cy.get_field('url').clear().type('fuzzbuzz'); + cy.get('.link-btn').should('not.be.visible'); + }); + + it('should have valid URL link with target _blank', () => { + cy.get_field('url').clear().type('https://frappe.io'); + cy.get('.link-btn .btn-open').should('have.attr', 'href', 'https://frappe.io'); + cy.get('.link-btn .btn-open').should('have.attr', 'target', '_blank'); + }); + + it('should inject anchor tag in read-only URL data field', () => { + cy.get('[data-fieldname="read_only_url"]') + .find('a') + .should('have.attr', 'target', '_blank'); + }); + }); +}); \ No newline at end of file diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js new file mode 100644 index 0000000000..ecf0d49511 --- /dev/null +++ b/esbuild/esbuild.js @@ -0,0 +1,486 @@ +/* eslint-disable no-console */ +let path = require("path"); +let fs = require("fs"); +let glob = require("fast-glob"); +let esbuild = require("esbuild"); +let vue = require("esbuild-vue"); +let yargs = require("yargs"); +let cliui = require("cliui")(); +let chalk = require("chalk"); +let html_plugin = require("./frappe-html"); +let postCssPlugin = require("esbuild-plugin-postcss2").default; +let ignore_assets = require("./ignore-assets"); +let sass_options = require("./sass_options"); +let { + app_list, + assets_path, + apps_path, + sites_path, + get_app_path, + get_public_path, + log, + log_warn, + log_error, + bench_path, + get_redis_subscriber +} = require("./utils"); + +let argv = yargs + .usage("Usage: node esbuild [options]") + .option("apps", { + type: "string", + description: "Run build for specific apps" + }) + .option("skip_frappe", { + type: "boolean", + description: "Skip building frappe assets" + }) + .option("files", { + type: "string", + description: "Run build for specified bundles" + }) + .option("watch", { + type: "boolean", + description: "Run in watch mode and rebuild on file changes" + }) + .option("production", { + type: "boolean", + description: "Run build in production mode" + }) + .option("run-build-command", { + type: "boolean", + description: "Run build command for apps" + }) + .example( + "node esbuild --apps frappe,erpnext", + "Run build only for frappe and erpnext" + ) + .example( + "node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js", + "Run build only for specified bundles" + ) + .version(false).argv; + +const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter( + app => !(argv.skip_frappe && app == "frappe") +); +const FILES_TO_BUILD = argv.files ? argv.files.split(",") : []; +const WATCH_MODE = Boolean(argv.watch); +const PRODUCTION = Boolean(argv.production); +const RUN_BUILD_COMMAND = !WATCH_MODE && Boolean(argv["run-build-command"]); + +const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`; +const NODE_PATHS = [].concat( + // node_modules of apps directly importable + app_list + .map(app => path.resolve(get_app_path(app), "../node_modules")) + .filter(fs.existsSync), + // import js file of any app if you provide the full path + app_list + .map(app => path.resolve(get_app_path(app), "..")) + .filter(fs.existsSync) +); + +execute() + .then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS)) + .catch(e => console.error(e)); + +if (WATCH_MODE) { + // listen for open files in editor event + open_in_editor(); +} + +async function execute() { + console.time(TOTAL_BUILD_TIME); + if (!FILES_TO_BUILD.length) { + await clean_dist_folders(APPS); + } + + let result; + try { + result = await build_assets_for_apps(APPS, FILES_TO_BUILD); + } catch (e) { + log_error("There were some problems during build"); + log(); + log(chalk.dim(e.stack)); + return; + } + + if (!WATCH_MODE) { + log_built_assets(result.metafile); + console.timeEnd(TOTAL_BUILD_TIME); + log(); + } else { + log("Watching for changes..."); + } + return await write_assets_json(result.metafile); +} + +function build_assets_for_apps(apps, files) { + let { include_patterns, ignore_patterns } = files.length + ? get_files_to_build(files) + : get_all_files_to_build(apps); + + return glob(include_patterns, { ignore: ignore_patterns }).then(files => { + let output_path = assets_path; + + let file_map = {}; + for (let file of files) { + let relative_app_path = path.relative(apps_path, file); + let app = relative_app_path.split(path.sep)[0]; + + let extension = path.extname(file); + let output_name = path.basename(file, extension); + if ( + [".css", ".scss", ".less", ".sass", ".styl"].includes(extension) + ) { + output_name = path.join("css", output_name); + } else if ([".js", ".ts"].includes(extension)) { + output_name = path.join("js", output_name); + } + output_name = path.join(app, "dist", output_name); + + if (Object.keys(file_map).includes(output_name)) { + log_warn( + `Duplicate output file ${output_name} generated from ${file}` + ); + } + + file_map[output_name] = file; + } + + return build_files({ + files: file_map, + outdir: output_path + }); + }); +} + +function get_all_files_to_build(apps) { + let include_patterns = []; + let ignore_patterns = []; + + for (let app of apps) { + let public_path = get_public_path(app); + include_patterns.push( + path.resolve( + public_path, + "**", + "*.bundle.{js,ts,css,sass,scss,less,styl}" + ) + ); + ignore_patterns.push( + path.resolve(public_path, "node_modules"), + path.resolve(public_path, "dist") + ); + } + + return { + include_patterns, + ignore_patterns + }; +} + +function get_files_to_build(files) { + // files: ['frappe/website.bundle.js', 'erpnext/main.bundle.js'] + let include_patterns = []; + let ignore_patterns = []; + + for (let file of files) { + let [app, bundle] = file.split("/"); + let public_path = get_public_path(app); + include_patterns.push(path.resolve(public_path, "**", bundle)); + ignore_patterns.push( + path.resolve(public_path, "node_modules"), + path.resolve(public_path, "dist") + ); + } + + return { + include_patterns, + ignore_patterns + }; +} + +function build_files({ files, outdir }) { + return esbuild.build({ + entryPoints: files, + entryNames: "[dir]/[name].[hash]", + outdir, + sourcemap: true, + bundle: true, + metafile: true, + minify: PRODUCTION, + nodePaths: NODE_PATHS, + define: { + "process.env.NODE_ENV": JSON.stringify( + PRODUCTION ? "production" : "development" + ) + }, + plugins: [ + html_plugin, + ignore_assets, + vue(), + postCssPlugin({ + plugins: [require("autoprefixer")], + sassOptions: sass_options + }) + ], + watch: get_watch_config() + }); +} + +function get_watch_config() { + if (WATCH_MODE) { + return { + async onRebuild(error, result) { + if (error) { + log_error("There was an error during rebuilding changes."); + log(); + log(chalk.dim(error.stack)); + notify_redis({ error }); + } else { + let { + assets_json, + prev_assets_json + } = await write_assets_json(result.metafile); + if (prev_assets_json) { + log_rebuilt_assets(prev_assets_json, assets_json); + } + notify_redis({ success: true }); + } + } + }; + } + return null; +} + +async function clean_dist_folders(apps) { + for (let app of apps) { + let public_path = get_public_path(app); + await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), { + recursive: true + }); + await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), { + recursive: true + }); + } +} + +function log_built_assets(metafile) { + let column_widths = [60, 20]; + cliui.div( + { + text: chalk.cyan.bold("File"), + width: column_widths[0] + }, + { + text: chalk.cyan.bold("Size"), + width: column_widths[1] + } + ); + cliui.div(""); + + let output_by_dist_path = {}; + for (let outfile in metafile.outputs) { + if (outfile.endsWith(".map")) continue; + let data = metafile.outputs[outfile]; + outfile = path.resolve(outfile); + outfile = path.relative(assets_path, outfile); + let filename = path.basename(outfile); + let dist_path = outfile.replace(filename, ""); + output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || []; + output_by_dist_path[dist_path].push({ + name: filename, + size: (data.bytes / 1000).toFixed(2) + " Kb" + }); + } + + for (let dist_path in output_by_dist_path) { + let files = output_by_dist_path[dist_path]; + cliui.div({ + text: dist_path, + width: column_widths[0] + }); + + for (let i in files) { + let file = files[i]; + let branch = ""; + if (i < files.length - 1) { + branch = "├─ "; + } else { + branch = "└─ "; + } + let color = file.name.endsWith(".js") ? "green" : "blue"; + cliui.div( + { + text: branch + chalk[color]("" + file.name), + width: column_widths[0] + }, + { + text: file.size, + width: column_widths[1] + } + ); + } + cliui.div(""); + } + log(cliui.toString()); +} + +// to store previous build's assets.json for comparison +let prev_assets_json; +let curr_assets_json; + +async function write_assets_json(metafile) { + prev_assets_json = curr_assets_json; + let out = {}; + for (let output in metafile.outputs) { + let info = metafile.outputs[output]; + let asset_path = "/" + path.relative(sites_path, output); + if (info.entryPoint) { + out[path.basename(info.entryPoint)] = asset_path; + } + } + + let assets_json_path = path.resolve( + assets_path, + "frappe", + "dist", + "assets.json" + ); + let assets_json; + try { + assets_json = await fs.promises.readFile(assets_json_path, "utf-8"); + } catch (error) { + assets_json = "{}"; + } + assets_json = JSON.parse(assets_json); + // update with new values + assets_json = Object.assign({}, assets_json, out); + curr_assets_json = assets_json; + + await fs.promises.writeFile( + assets_json_path, + JSON.stringify(assets_json, null, 4) + ); + await update_assets_json_in_cache(assets_json); + return { + assets_json, + prev_assets_json + }; +} + +function update_assets_json_in_cache(assets_json) { + // update assets_json cache in redis, so that it can be read directly by python + return new Promise(resolve => { + let client = get_redis_subscriber("redis_cache"); + // handle error event to avoid printing stack traces + client.on("error", _ => { + log_warn("Cannot connect to redis_cache to update assets_json"); + }); + client.set("assets_json", JSON.stringify(assets_json), err => { + client.unref(); + resolve(); + }); + }); +} + +function run_build_command_for_apps(apps) { + let cwd = process.cwd(); + let { execSync } = require("child_process"); + + for (let app of apps) { + if (app === "frappe") continue; + + let root_app_path = path.resolve(get_app_path(app), ".."); + let package_json = path.resolve(root_app_path, "package.json"); + if (fs.existsSync(package_json)) { + let { scripts } = require(package_json); + if (scripts && scripts.build) { + log("\nRunning build command for", chalk.bold(app)); + process.chdir(root_app_path); + execSync("yarn build", { encoding: "utf8", stdio: "inherit" }); + } + } + } + + process.chdir(cwd); +} + +async function notify_redis({ error, success }) { + // notify redis which in turns tells socketio to publish this to browser + let subscriber = get_redis_subscriber("redis_socketio"); + subscriber.on("error", _ => { + log_warn("Cannot connect to redis_socketio for browser events"); + }); + + let payload = null; + if (error) { + let formatted = await esbuild.formatMessages(error.errors, { + kind: "error", + terminalWidth: 100 + }); + let stack = error.stack.replace(new RegExp(bench_path, "g"), ""); + payload = { + error, + formatted, + stack + }; + } + if (success) { + payload = { + success: true + }; + } + + subscriber.publish( + "events", + JSON.stringify({ + event: "build_event", + message: payload + }) + ); +} + +function open_in_editor() { + let subscriber = get_redis_subscriber("redis_socketio"); + subscriber.on("error", _ => { + log_warn("Cannot connect to redis_socketio for open_in_editor events"); + }); + subscriber.on("message", (event, file) => { + if (event === "open_in_editor") { + file = JSON.parse(file); + let file_path = path.resolve(file.file); + log("Opening file in editor:", file_path); + let launch = require("launch-editor"); + launch(`${file_path}:${file.line}:${file.column}`); + } + }); + subscriber.subscribe("open_in_editor"); +} + +function log_rebuilt_assets(prev_assets, new_assets) { + let added_files = []; + let old_files = Object.values(prev_assets); + let new_files = Object.values(new_assets); + + for (let filepath of new_files) { + if (!old_files.includes(filepath)) { + added_files.push(filepath); + } + } + + log( + chalk.yellow( + `${new Date().toLocaleTimeString()}: Compiled ${ + added_files.length + } files...` + ) + ); + for (let filepath of added_files) { + let filename = path.basename(filepath); + log(" " + filename); + } + log(); +} diff --git a/esbuild/frappe-html.js b/esbuild/frappe-html.js new file mode 100644 index 0000000000..8c4b7ca3d7 --- /dev/null +++ b/esbuild/frappe-html.js @@ -0,0 +1,43 @@ +module.exports = { + name: "frappe-html", + setup(build) { + let path = require("path"); + let fs = require("fs/promises"); + + build.onResolve({ filter: /\.html$/ }, args => { + return { + path: path.join(args.resolveDir, args.path), + namespace: "frappe-html" + }; + }); + + build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => { + let filepath = args.path; + let filename = path.basename(filepath).split(".")[0]; + + return fs + .readFile(filepath, "utf-8") + .then(content => { + content = scrub_html_template(content); + return { + contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n` + }; + }) + .catch(() => { + return { + contents: "", + warnings: [ + { + text: `There was an error importing ${filepath}` + } + ] + }; + }); + }); + } +}; + +function scrub_html_template(content) { + content = content.replace(/`/g, "\\`"); + return content; +} diff --git a/esbuild/ignore-assets.js b/esbuild/ignore-assets.js new file mode 100644 index 0000000000..5edfef2110 --- /dev/null +++ b/esbuild/ignore-assets.js @@ -0,0 +1,11 @@ +module.exports = { + name: "frappe-ignore-asset", + setup(build) { + build.onResolve({ filter: /^\/assets\// }, args => { + return { + path: args.path, + external: true + }; + }); + } +}; diff --git a/esbuild/index.js b/esbuild/index.js new file mode 100644 index 0000000000..2721673702 --- /dev/null +++ b/esbuild/index.js @@ -0,0 +1 @@ +require("./esbuild"); diff --git a/esbuild/sass_options.js b/esbuild/sass_options.js new file mode 100644 index 0000000000..fcc7e04ccd --- /dev/null +++ b/esbuild/sass_options.js @@ -0,0 +1,29 @@ +let path = require("path"); +let { get_app_path, app_list } = require("./utils"); + +let node_modules_path = path.resolve( + get_app_path("frappe"), + "..", + "node_modules" +); +let app_paths = app_list + .map(get_app_path) + .map(app_path => path.resolve(app_path, "..")); + +module.exports = { + includePaths: [node_modules_path, ...app_paths], + importer: function(url) { + if (url.startsWith("~")) { + // strip ~ so that it can resolve from node_modules + url = url.slice(1); + } + if (url.endsWith(".css")) { + // strip .css from end of path + url = url.slice(0, -4); + } + // normal file, let it go + return { + file: url + }; + } +}; diff --git a/esbuild/utils.js b/esbuild/utils.js new file mode 100644 index 0000000000..82490adb36 --- /dev/null +++ b/esbuild/utils.js @@ -0,0 +1,145 @@ +const path = require("path"); +const fs = require("fs"); +const chalk = require("chalk"); + +const frappe_path = path.resolve(__dirname, ".."); +const bench_path = path.resolve(frappe_path, "..", ".."); +const sites_path = path.resolve(bench_path, "sites"); +const apps_path = path.resolve(bench_path, "apps"); +const assets_path = path.resolve(sites_path, "assets"); +const app_list = get_apps_list(); + +const app_paths = app_list.reduce((out, app) => { + out[app] = path.resolve(apps_path, app, app); + return out; +}, {}); +const public_paths = app_list.reduce((out, app) => { + out[app] = path.resolve(app_paths[app], "public"); + return out; +}, {}); +const public_js_paths = app_list.reduce((out, app) => { + out[app] = path.resolve(app_paths[app], "public/js"); + return out; +}, {}); + +const bundle_map = app_list.reduce((out, app) => { + const public_js_path = public_js_paths[app]; + if (fs.existsSync(public_js_path)) { + const all_files = fs.readdirSync(public_js_path); + const js_files = all_files.filter(file => file.endsWith(".js")); + + for (let js_file of js_files) { + const filename = path.basename(js_file).split(".")[0]; + out[path.join(app, "js", filename)] = path.resolve( + public_js_path, + js_file + ); + } + } + + return out; +}, {}); + +const get_public_path = app => public_paths[app]; + +const get_build_json_path = app => + path.resolve(get_public_path(app), "build.json"); + +function get_build_json(app) { + try { + return require(get_build_json_path(app)); + } catch (e) { + // build.json does not exist + return null; + } +} + +function delete_file(path) { + if (fs.existsSync(path)) { + fs.unlinkSync(path); + } +} + +function run_serially(tasks) { + let result = Promise.resolve(); + tasks.forEach(task => { + if (task) { + result = result.then ? result.then(task) : Promise.resolve(); + } + }); + return result; +} + +const get_app_path = app => app_paths[app]; + +function get_apps_list() { + return fs + .readFileSync(path.resolve(sites_path, "apps.txt"), { + encoding: "utf-8" + }) + .split("\n") + .filter(Boolean); +} + +function get_cli_arg(name) { + let args = process.argv.slice(2); + let arg = `--${name}`; + let index = args.indexOf(arg); + + let value = null; + if (index != -1) { + value = true; + } + if (value && args[index + 1]) { + value = args[index + 1]; + } + return value; +} + +function log_error(message, badge = "ERROR") { + badge = chalk.white.bgRed(` ${badge} `); + console.error(`${badge} ${message}`); // eslint-disable-line no-console +} + +function log_warn(message, badge = "WARN") { + badge = chalk.black.bgYellowBright(` ${badge} `); + console.warn(`${badge} ${message}`); // eslint-disable-line no-console +} + +function log(...args) { + console.log(...args); // eslint-disable-line no-console +} + +function get_redis_subscriber(kind) { + // get redis subscriber that aborts after 10 connection attempts + let { get_redis_subscriber: get_redis } = require("../node_utils"); + return get_redis(kind, { + retry_strategy: function(options) { + // abort after 10 connection attempts + if (options.attempt > 10) { + return undefined; + } + return Math.min(options.attempt * 100, 2000); + } + }); +} + +module.exports = { + app_list, + bench_path, + assets_path, + sites_path, + apps_path, + bundle_map, + get_public_path, + get_build_json_path, + get_build_json, + get_app_path, + delete_file, + run_serially, + get_cli_arg, + log, + log_warn, + log_error, + get_redis_subscriber +}; diff --git a/frappe/__init__.py b/frappe/__init__.py index 5680ba86b5..9b208f7c2d 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -10,11 +10,16 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ -from __future__ import unicode_literals, print_function +import os, warnings + +_dev_server = os.environ.get('DEV_SERVER', False) + +if _dev_server: + warnings.simplefilter('always', DeprecationWarning) + warnings.simplefilter('always', PendingDeprecationWarning) -from six import iteritems, binary_type, text_type, string_types, PY2 from werkzeug.local import Local, release_local -import os, sys, importlib, inspect, json +import sys, importlib, inspect, json import typing from past.builtins import cmp import click @@ -27,13 +32,6 @@ from .utils.lazy_loader import lazy_import # Lazy imports faker = lazy_import('faker') - -# Harmless for Python 3 -# For Python 2 set default encoding to utf-8 -if PY2: - reload(sys) - sys.setdefaultencoding("utf-8") - __version__ = '14.0.0-dev' __title__ = "Frappe Framework" @@ -97,14 +95,14 @@ def _(msg, lang=None, context=None): def as_unicode(text, encoding='utf-8'): '''Convert to unicode if required''' - if isinstance(text, text_type): + if isinstance(text, str): return text elif text==None: return '' - elif isinstance(text, binary_type): - return text_type(text, encoding) + elif isinstance(text, bytes): + return str(text, encoding) else: - return text_type(text) + return str(text) def get_lang_dict(fortype, name=None): """Returns the translated language dict for the given type and name. @@ -204,7 +202,7 @@ def init(site, sites_path=None, new_site=False): local.meta_cache = {} local.form_dict = _dict() local.session = _dict() - local.dev_server = os.environ.get('DEV_SERVER', False) + local.dev_server = _dev_server setup_module_map() @@ -597,7 +595,7 @@ def is_whitelisted(method): # strictly sanitize form_dict # escapes html characters like <> except for predefined tags like a, b, ul etc. for key, value in form_dict.items(): - if isinstance(value, string_types): + if isinstance(value, str): form_dict[key] = sanitize_html(value) def read_only(): @@ -721,7 +719,7 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc user = session.user if doc: - if isinstance(doc, string_types): + if isinstance(doc, str): doc = get_doc(doctype, doc) doctype = doc.doctype @@ -790,7 +788,7 @@ def set_value(doctype, docname, fieldname, value=None): return frappe.client.set_value(doctype, docname, fieldname, value) def get_cached_doc(*args, **kwargs): - if args and len(args) > 1 and isinstance(args[1], text_type): + if args and len(args) > 1 and isinstance(args[1], str): key = get_document_cache_key(args[0], args[1]) # local cache doc = local.document_cache.get(key) @@ -821,7 +819,7 @@ def clear_document_cache(doctype, name): def get_cached_value(doctype, name, fieldname, as_dict=False): doc = get_cached_doc(doctype, name) - if isinstance(fieldname, string_types): + if isinstance(fieldname, str): if as_dict: throw('Cannot make dict for single fieldname') return doc.get(fieldname) @@ -1027,7 +1025,7 @@ def get_doc_hooks(): if not hasattr(local, 'doc_events_hooks'): hooks = get_hooks('doc_events', {}) out = {} - for key, value in iteritems(hooks): + for key, value in hooks.items(): if isinstance(key, tuple): for doctype in key: append_hook(out, doctype, value) @@ -1144,7 +1142,7 @@ def get_file_json(path): def read_file(path, raise_not_found=False): """Open a file and return its content as Unicode.""" - if isinstance(path, text_type): + if isinstance(path, str): path = path.encode("utf-8") if os.path.exists(path): @@ -1167,7 +1165,7 @@ def get_attr(method_string): def call(fn, *args, **kwargs): """Call a function and match arguments.""" - if isinstance(fn, string_types): + if isinstance(fn, str): fn = get_attr(fn) newargs = get_newargs(fn, kwargs) @@ -1178,13 +1176,9 @@ def get_newargs(fn, kwargs): if hasattr(fn, 'fnargs'): fnargs = fn.fnargs else: - try: - fnargs, varargs, varkw, defaults = inspect.getargspec(fn) - except ValueError: - fnargs = inspect.getfullargspec(fn).args - varargs = inspect.getfullargspec(fn).varargs - varkw = inspect.getfullargspec(fn).varkw - defaults = inspect.getfullargspec(fn).defaults + fnargs = inspect.getfullargspec(fn).args + fnargs.extend(inspect.getfullargspec(fn).kwonlyargs) + varkw = inspect.getfullargspec(fn).varkw newargs = {} for a in kwargs: @@ -1626,6 +1620,12 @@ def enqueue(*args, **kwargs): import frappe.utils.background_jobs return frappe.utils.background_jobs.enqueue(*args, **kwargs) +def task(**task_kwargs): + def decorator_task(f): + f.enqueue = lambda **fun_kwargs: enqueue(f, **task_kwargs, **fun_kwargs) + return f + return decorator_task + def enqueue_doc(*args, **kwargs): ''' Enqueue method to be executed using a background worker diff --git a/frappe/app.py b/frappe/app.py index c9e993a853..64befdf531 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,17 +99,7 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: - frappe.logger("frappe.web", allow_site=frappe.local.site).info({ - "site": get_site_name(request.host), - "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), - "base_url": getattr(request, "base_url", "NOTFOUND"), - "full_path": getattr(request, "full_path", "NOTFOUND"), - "method": getattr(request, "method", "NOTFOUND"), - "scheme": getattr(request, "scheme", "NOTFOUND"), - "http_status_code": getattr(response, "status_code", "NOTFOUND") - }) - + log_request(request, response) process_response(response) frappe.destroy() @@ -137,6 +127,19 @@ def init_request(request): if request.method != "OPTIONS": frappe.local.http_request = frappe.auth.HTTPRequest() +def log_request(request, response): + if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), + "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), + "base_url": getattr(request, "base_url", "NOTFOUND"), + "full_path": getattr(request, "full_path", "NOTFOUND"), + "method": getattr(request, "method", "NOTFOUND"), + "scheme": getattr(request, "scheme", "NOTFOUND"), + "http_status_code": getattr(response, "status_code", "NOTFOUND") + }) + + def process_response(response): if not response: return @@ -185,7 +188,7 @@ def make_form_dict(request): args = request.form or request.args if not isinstance(args, dict): - frappe.throw("Invalid request arguments") + frappe.throw(_("Invalid request arguments")) try: frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \ @@ -201,12 +204,20 @@ def handle_exception(e): response = None http_status_code = getattr(e, "http_status_code", 500) return_as_message = False + accept_header = frappe.get_request_header("Accept") or "" + respond_as_json = ( + frappe.get_request_header('Accept') + and (frappe.local.is_ajax or 'application/json' in accept_header) + or ( + frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text") + ) + ) if frappe.conf.get('developer_mode'): # don't fail silently print(frappe.get_traceback()) - if frappe.get_request_header('Accept') and (frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept')): + if respond_as_json: # handle ajax responses first # if the request is ajax, send back the trace or error message response = frappe.utils.response.report_error(http_status_code) @@ -286,8 +297,9 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No _sites_path = sites_path from werkzeug.serving import run_simple + patch_werkzeug_reloader() - if profile: + if profile or os.environ.get('USE_PROFILER'): application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls')) if not os.environ.get('NO_STATICS'): @@ -316,3 +328,23 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No use_debugger=not in_test_env, use_evalex=not in_test_env, threaded=not no_threading) + +def patch_werkzeug_reloader(): + """ + This function monkey patches Werkzeug reloader to ignore reloading files in + the __pycache__ directory. + + To be deprecated when upgrading to Werkzeug 2. + """ + + from werkzeug._reloader import WatchdogReloaderLoop + + trigger_reload = WatchdogReloaderLoop.trigger_reload + + def custom_trigger_reload(self, filename): + if os.path.basename(os.path.dirname(filename)) == "__pycache__": + return + + return trigger_reload(self, filename) + + WatchdogReloaderLoop.trigger_reload = custom_trigger_reload diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.js b/frappe/automation/doctype/auto_repeat/auto_repeat.js index 7028ac486d..896a10dfe0 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.js +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.js @@ -103,7 +103,7 @@ frappe.ui.form.on('Auto Repeat', { frappe.auto_repeat.render_schedule = function(frm) { if (!frm.is_dirty() && frm.doc.status !== 'Disabled') { frm.call("get_auto_repeat_schedule").then(r => { - frm.dashboard.wrapper.empty(); + frm.dashboard.reset(); frm.dashboard.add_section( frappe.render_template("auto_repeat_schedule", { schedule_details: r.message || [] diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py index f41f31f3bb..6ceb4dba72 100644 --- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py @@ -173,7 +173,7 @@ class TestAutoRepeat(unittest.TestCase): fields=['docstatus'], limit=1 ) - self.assertEquals(docnames[0].docstatus, 1) + self.assertEqual(docnames[0].docstatus, 1) def make_auto_repeat(**args): diff --git a/frappe/boot.py b/frappe/boot.py index 65a07b15e5..0dfcb8d1b4 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -42,8 +42,6 @@ def get_bootinfo(): bootinfo.user_info = get_user_info() bootinfo.sid = frappe.session['sid'] - bootinfo.user_groups = frappe.get_all('User Group', pluck="name") - bootinfo.modules = {} bootinfo.module_list = [] load_desktop_data(bootinfo) diff --git a/frappe/build.py b/frappe/build.py index baedb633b6..c970ae3a28 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -1,14 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import print_function, unicode_literals - import os import re import json import shutil -import warnings -import tempfile +import subprocess +from tempfile import mkdtemp, mktemp from distutils.spawn import find_executable import frappe @@ -16,8 +14,9 @@ from frappe.utils.minify import JavascriptMinify import click import psutil -from six import iteritems, text_type -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse +from simple_chalk import green +from semantic_version import Version timestamps = {} @@ -39,35 +38,36 @@ def download_file(url, prefix): def build_missing_files(): - # check which files dont exist yet from the build.json and tell build.js to build only those! + '''Check which files dont exist yet from the assets.json and run build for those files''' + missing_assets = [] current_asset_files = [] - frappe_build = os.path.join("..", "apps", "frappe", "frappe", "public", "build.json") for type in ["css", "js"]: - current_asset_files.extend( - [ - "{0}/{1}".format(type, name) - for name in os.listdir(os.path.join(sites_path, "assets", type)) - ] - ) + folder = os.path.join(sites_path, "assets", "frappe", "dist", type) + current_asset_files.extend(os.listdir(folder)) - with open(frappe_build) as f: - all_asset_files = json.load(f).keys() + development = frappe.local.conf.developer_mode or frappe.local.dev_server + build_mode = "development" if development else "production" - for asset in all_asset_files: - if asset.replace("concat:", "") not in current_asset_files: - missing_assets.append(asset) + assets_json = frappe.read_file(frappe.get_app_path('frappe', 'public', 'dist', 'assets.json')) + if assets_json: + assets_json = frappe.parse_json(assets_json) - if missing_assets: - from subprocess import check_call - from shlex import split + for bundle_file, output_file in assets_json.items(): + if not output_file.startswith('/assets/frappe'): + continue - click.secho("\nBuilding missing assets...\n", fg="yellow") - command = split( - "node rollup/build.js --files {0} --no-concat".format(",".join(missing_assets)) - ) - check_call(command, cwd=os.path.join("..", "apps", "frappe")) + if os.path.basename(output_file) not in current_asset_files: + missing_assets.append(bundle_file) + + if missing_assets: + click.secho("\nBuilding missing assets...\n", fg="yellow") + files_to_build = ["frappe/" + name for name in missing_assets] + bundle(build_mode, files=files_to_build) + else: + # no assets.json, run full build + bundle(build_mode, apps="frappe") def get_assets_link(frappe_head): @@ -75,8 +75,8 @@ def get_assets_link(frappe_head): from requests import head tag = getoutput( - "cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" - " refs/tags/,,' -e 's/\^{}//'" + r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" + r" refs/tags/,,' -e 's/\^{}//'" % frappe_head ) @@ -97,9 +97,7 @@ def download_frappe_assets(verbose=True): commit HEAD. Returns True if correctly setup else returns False. """ - from simple_chalk import green from subprocess import getoutput - from tempfile import mkdtemp assets_setup = False frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") @@ -166,7 +164,7 @@ def symlink(target, link_name, overwrite=False): # Create link to target with temporary filename while True: - temp_link_name = tempfile.mktemp(dir=link_dir) + temp_link_name = mktemp(dir=link_dir) # os.* functions mimic as closely as possible system functions # The POSIX symlink() returns EEXIST if link_name already exists @@ -193,7 +191,8 @@ def symlink(target, link_name, overwrite=False): def setup(): - global app_paths + global app_paths, assets_path + pymodules = [] for app in frappe.get_all_apps(True): try: @@ -201,51 +200,54 @@ def setup(): except ImportError: pass app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules] + assets_path = os.path.join(frappe.local.sites_path, "assets") -def get_node_pacman(): - exec_ = find_executable("yarn") - if exec_: - return exec_ - raise ValueError("Yarn not found") - - -def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False): +def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None): """concat / minify js files""" setup() - make_asset_dirs(make_copy=make_copy, restore=restore) + make_asset_dirs(hard_link=hard_link) - pacman = get_node_pacman() - mode = "build" if no_compress else "production" - command = "{pacman} run {mode}".format(pacman=pacman, mode=mode) + mode = "production" if mode == "production" else "build" + command = "yarn run {mode}".format(mode=mode) - if app: - command += " --app {app}".format(app=app) + if apps: + command += " --apps {apps}".format(apps=apps) if skip_frappe: command += " --skip_frappe" - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) - check_yarn() + if files: + command += " --files {files}".format(files=','.join(files)) + + command += " --run-build-command" + + check_node_executable() + frappe_app_path = frappe.get_app_path("frappe", "..") frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env()) -def watch(no_compress): +def watch(apps=None): """watch and rebuild if necessary""" setup() - pacman = get_node_pacman() + command = "yarn run watch" + if apps: + command += " --apps {apps}".format(apps=apps) - frappe_app_path = os.path.abspath(os.path.join(app_paths[0], "..")) - check_yarn() + check_node_executable() frappe_app_path = frappe.get_app_path("frappe", "..") - frappe.commands.popen("{pacman} run watch".format(pacman=pacman), - cwd=frappe_app_path, env=get_node_env()) + frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env()) -def check_yarn(): +def check_node_executable(): + node_version = Version(subprocess.getoutput('node -v')[1:]) + warn = '⚠️ ' + if node_version.major < 14: + click.echo(f"{warn} Please update your node version to 14") if not find_executable("yarn"): - print("Please install yarn using below command and try again.\nnpm install -g yarn") + click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn") + click.echo() def get_node_env(): node_env = { @@ -266,75 +268,109 @@ def get_safe_max_old_space_size(): return safe_max_old_space_size -def make_asset_dirs(make_copy=False, restore=False): - # don't even think of making assets_path absolute - rm -rf ahead. - assets_path = os.path.join(frappe.local.sites_path, "assets") +def generate_assets_map(): + symlinks = {} - for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]: - if not os.path.exists(dir_path): - os.makedirs(dir_path) + for app_name in frappe.get_all_apps(): + app_doc_path = None - for app_name in frappe.get_all_apps(True): pymodule = frappe.get_module(app_name) app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__)) - - symlinks = [] app_public_path = os.path.join(app_base_path, "public") - # app/public > assets/app - symlinks.append([app_public_path, os.path.join(assets_path, app_name)]) - # app/node_modules > assets/app/node_modules - if os.path.exists(os.path.abspath(app_public_path)): - symlinks.append( - [ - os.path.join(app_base_path, "..", "node_modules"), - os.path.join(assets_path, app_name, "node_modules"), - ] - ) + app_node_modules_path = os.path.join(app_base_path, "..", "node_modules") + app_docs_path = os.path.join(app_base_path, "docs") + app_www_docs_path = os.path.join(app_base_path, "www", "docs") - app_doc_path = None - if os.path.isdir(os.path.join(app_base_path, "docs")): - app_doc_path = os.path.join(app_base_path, "docs") + app_assets = os.path.abspath(app_public_path) + app_node_modules = os.path.abspath(app_node_modules_path) - elif os.path.isdir(os.path.join(app_base_path, "www", "docs")): - app_doc_path = os.path.join(app_base_path, "www", "docs") + # {app}/public > assets/{app} + if os.path.isdir(app_assets): + symlinks[app_assets] = os.path.join(assets_path, app_name) + # {app}/node_modules > assets/{app}/node_modules + if os.path.isdir(app_node_modules): + symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules") + + # {app}/docs > assets/{app}_docs + if os.path.isdir(app_docs_path): + app_doc_path = os.path.join(app_base_path, "docs") + elif os.path.isdir(app_www_docs_path): + app_doc_path = os.path.join(app_base_path, "www", "docs") if app_doc_path: - symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")]) - - for source, target in symlinks: - source = os.path.abspath(source) - if os.path.exists(source): - if restore: - if os.path.exists(target): - if os.path.islink(target): - os.unlink(target) - else: - shutil.rmtree(target) - shutil.copytree(source, target) - elif make_copy: - if os.path.exists(target): - warnings.warn("Target {target} already exists.".format(target=target)) - else: - shutil.copytree(source, target) - else: - if os.path.exists(target): - if os.path.islink(target): - os.unlink(target) - else: - shutil.rmtree(target) - try: - symlink(source, target, overwrite=True) - except OSError: - print("Cannot link {} to {}".format(source, target)) - else: - # warnings.warn('Source {source} does not exist.'.format(source = source)) - pass + app_docs = os.path.abspath(app_doc_path) + symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs") + return symlinks -def build(no_compress=False, verbose=False): - assets_path = os.path.join(frappe.local.sites_path, "assets") - for target, sources in iteritems(get_build_maps()): +def setup_assets_dirs(): + for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")): + os.makedirs(dir_path, exist_ok=True) + + +def clear_broken_symlinks(): + for path in os.listdir(assets_path): + path = os.path.join(assets_path, path) + if os.path.islink(path) and not os.path.exists(path): + os.remove(path) + + + +def unstrip(message: str) -> str: + """Pads input string on the right side until the last available column in the terminal + """ + _len = len(message) + try: + max_str = os.get_terminal_size().columns + except Exception: + max_str = 80 + + if _len < max_str: + _rem = max_str - _len + else: + _rem = max_str % _len + + return f"{message}{' ' * _rem}" + + +def make_asset_dirs(hard_link=False): + setup_assets_dirs() + clear_broken_symlinks() + symlinks = generate_assets_map() + + for source, target in symlinks.items(): + start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}") + fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}") + + # Used '\r' instead of '\x1b[1K\r' to print entire lines in smaller terminal sizes + try: + print(start_message, end="\r") + link_assets_dir(source, target, hard_link=hard_link) + except Exception: + print(fail_message, end="\r") + + print(unstrip(f"{green('✔')} Application Assets Linked") + "\n") + + +def link_assets_dir(source, target, hard_link=False): + if not os.path.exists(source): + return + + if os.path.exists(target): + if os.path.islink(target): + os.unlink(target) + else: + shutil.rmtree(target) + + if hard_link: + shutil.copytree(source, target, dirs_exist_ok=True) + else: + symlink(source, target, overwrite=True) + + +def build(no_compress=False, verbose=False): + for target, sources in get_build_maps().items(): pack(os.path.join(assets_path, target), sources, no_compress, verbose) @@ -348,7 +384,7 @@ def get_build_maps(): if os.path.exists(path): with open(path) as f: try: - for target, sources in iteritems(json.loads(f.read())): + for target, sources in (json.loads(f.read() or "{}")).items(): # update app path source_paths = [] for source in sources: @@ -381,7 +417,7 @@ def pack(target, sources, no_compress, verbose): timestamps[f] = os.path.getmtime(f) try: with open(f, "r") as sourcefile: - data = text_type(sourcefile.read(), "utf-8", errors="ignore") + data = str(sourcefile.read(), "utf-8", errors="ignore") extn = f.rsplit(".", 1)[1] @@ -396,7 +432,7 @@ def pack(target, sources, no_compress, verbose): jsm.minify(tmpin, tmpout) minified = tmpout.getvalue() if minified: - outtxt += text_type(minified or "", "utf-8").strip("\n") + ";" + outtxt += str(minified or "", "utf-8").strip("\n") + ";" if verbose: print("{0}: {1}k".format(f, int(len(minified) / 1024))) @@ -426,16 +462,16 @@ def html_to_js_template(path, content): def scrub_html_template(content): """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space - content = re.sub("\s+", " ", content) + content = re.sub(r"\s+", " ", content) # strip comments - content = re.sub("()", "", content) + content = re.sub(r"()", "", content) return content.replace("'", "\'") def files_dirty(): - for target, sources in iteritems(get_build_maps()): + for target, sources in get_build_maps().items(): for f in sources: if ":" in f: f, suffix = f.split(":") diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 4e0fe0cf44..7330c83102 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -13,6 +13,8 @@ common_default_keys = ["__default", "__global"] doctype_map_keys = ('energy_point_rule_map', 'assignment_rule_map', 'milestone_tracker_map', 'event_consumer_document_type_map') +bench_cache_keys = ('assets_json',) + global_cache_keys = ("app_hooks", "installed_apps", 'all_apps', "app_modules", "module_app", "system_settings", 'scheduler_events', 'time_zone', 'webhooks', 'active_domains', @@ -58,6 +60,7 @@ def clear_global_cache(): clear_doctype_cache() clear_website_cache() frappe.cache().delete_value(global_cache_keys) + frappe.cache().delete_value(bench_cache_keys) frappe.setup_module_map() def clear_defaults_cache(user=None): diff --git a/frappe/change_log/v13/v13_2_0.md b/frappe/change_log/v13/v13_2_0.md new file mode 100644 index 0000000000..6fc3eec5e3 --- /dev/null +++ b/frappe/change_log/v13/v13_2_0.md @@ -0,0 +1,32 @@ +# Version 13.2.0 Release Notes + +### Features & Enhancements + +- Add option to mention a group of users ([#12844](https://github.com/frappe/frappe/pull/12844)) +- Copy DocType / documents across sites ([#12872](https://github.com/frappe/frappe/pull/12872)) +- Scheduler log in notifications ([#1135](https://github.com/frappe/frappe/pull/1135)) +- Add Enable/Disable Webhook via Check Field ([#12842](https://github.com/frappe/frappe/pull/12842)) +- Allow query/custom reports to save custom data in the json field ([#12534](https://github.com/frappe/frappe/pull/12534)) + +### Fixes + +- Load server translations in boot (backport #12848) ([#12852](https://github.com/frappe/frappe/pull/12852)) +- Allow to override dashboard chart properties type/color ([#12846](https://github.com/frappe/frappe/pull/12846)) +- Multi-column paste in grid ([#12861](https://github.com/frappe/frappe/pull/12861)) +- Add log_error and FrappeClient to restricted python ([#12857](https://github.com/frappe/frappe/pull/12857)) +- Redirect Web Form user directly to success URL, if no amount is due ([#12661](https://github.com/frappe/frappe/pull/12661)) +- Attachment pill lock icon redirects to File ([#12864](https://github.com/frappe/frappe/pull/12864)) +- Redirect Web Form user directly to success URL, if no amount is due (backport #12661) ([#12856](https://github.com/frappe/frappe/pull/12856)) +- Remove events to redraw charts ([#12973](https://github.com/frappe/frappe/pull/12973)) +- Don't allow user to remove/change data source file in data import ([#12827](https://github.com/frappe/frappe/pull/12827)) +- Load server translations in boot ([#12848](https://github.com/frappe/frappe/pull/12848)) +- Newly created Workspace not being accessible unless a shortcut u… ([#12866](https://github.com/frappe/frappe/pull/12866)) +- Currency labels in grids ([#12974](https://github.com/frappe/frappe/pull/12974)) +- Handle error while session start ([#12933](https://github.com/frappe/frappe/pull/12933)) +- Add field type check in custom field validation ([#12858](https://github.com/frappe/frappe/pull/12858)) +- Make language select optional and fix breakpoint issues ([#12860](https://github.com/frappe/frappe/pull/12860)) +- Form Dashboard reference link ([#12945](https://github.com/frappe/frappe/pull/12945)) +- Invalid HTML generated by the base template ([#12953](https://github.com/frappe/frappe/pull/12953)) +- Default values were not triggering change event ([#12975](https://github.com/frappe/frappe/pull/12975)) +- Make strings translatable ([#12877](https://github.com/frappe/frappe/pull/12877)) +- Added build-message-files command ([#12950](https://github.com/frappe/frappe/pull/12950)) \ No newline at end of file diff --git a/frappe/change_log/v13/v13_3_0.md b/frappe/change_log/v13/v13_3_0.md new file mode 100644 index 0000000000..6ab181ef09 --- /dev/null +++ b/frappe/change_log/v13/v13_3_0.md @@ -0,0 +1,49 @@ +# Version 13.3.0 Release Notes + +### Features & Enhancements + +- Deletion Steps in Data Deletion Tool ([#13124](https://github.com/frappe/frappe/pull/13124)) +- Format Option for list-apps in bench CLI ([#13125](https://github.com/frappe/frappe/pull/13125)) +- Add password fieldtype option for Web Form ([#13093](https://github.com/frappe/frappe/pull/13093)) +- Add simple __repr__ for DocTypes ([#13151](https://github.com/frappe/frappe/pull/13151)) +- Switch theme with left/right keys ([#13077](https://github.com/frappe/frappe/pull/13077)) +- sourceURL for injected javascript ([#13022](https://github.com/frappe/frappe/pull/13022)) + +### Fixes + +- Decode uri before importing file via weblink ([#13026](https://github.com/frappe/frappe/pull/13026)) +- Respond to /api requests as JSON by default ([#13053](https://github.com/frappe/frappe/pull/13053)) +- Disabled checkbox should be disabled ([#13021](https://github.com/frappe/frappe/pull/13021)) +- Moving Site folder across different FileSystems failed ([#13038](https://github.com/frappe/frappe/pull/13038)) +- Freeze screen till the background request is complete ([#13078](https://github.com/frappe/frappe/pull/13078)) +- Added conditional rendering for content field in split section w… ([#13075](https://github.com/frappe/frappe/pull/13075)) +- Show delete button on portal if user has permission to delete document ([#13149](https://github.com/frappe/frappe/pull/13149)) +- Dont disable dialog scroll on focusing a Link/Autocomplete field ([#13119](https://github.com/frappe/frappe/pull/13119)) +- Typo in RecorderDetail.vue ([#13086](https://github.com/frappe/frappe/pull/13086)) +- Error for bench drop-site. Added missing import. ([#13064](https://github.com/frappe/frappe/pull/13064)) +- Report column context ([#13090](https://github.com/frappe/frappe/pull/13090)) +- Different service name for push and pull request events ([#13094](https://github.com/frappe/frappe/pull/13094)) +- Moving Site folder across different FileSystems failed ([#13033](https://github.com/frappe/frappe/pull/13033)) +- Consistent checkboxes on all browsers ([#13042](https://github.com/frappe/frappe/pull/13042)) +- Changed shorcut widgets color picker to dropdown ([#13073](https://github.com/frappe/frappe/pull/13073)) +- Error while exporting reports with duration field ([#13118](https://github.com/frappe/frappe/pull/13118)) +- Add margin to download backup card ([#13079](https://github.com/frappe/frappe/pull/13079)) +- Move mention list generation logic to server-side ([#13074](https://github.com/frappe/frappe/pull/13074)) +- Make strings translatable ([#13046](https://github.com/frappe/frappe/pull/13046)) +- Don't evaluate dynamic properties to check if conflicts exist ([#13186](https://github.com/frappe/frappe/pull/13186)) +- Add __ function in vue global for translation in recorder ([#13089](https://github.com/frappe/frappe/pull/13089)) +- Make strings translatable ([#13076](https://github.com/frappe/frappe/pull/13076)) +- Show config in bench CLI ([#13128](https://github.com/frappe/frappe/pull/13128)) +- Add breadcrumbs for list view ([#13091](https://github.com/frappe/frappe/pull/13091)) +- Do not skip data in save while using shortcut ([#13182](https://github.com/frappe/frappe/pull/13182)) +- Use docfields from options if no docfields are returned from meta ([#13188](https://github.com/frappe/frappe/pull/13188)) +- Disable reloading files in `__pycache__` directory ([#13109](https://github.com/frappe/frappe/pull/13109)) +- RTL stylesheet route to load RTL style on demand. ([#13007](https://github.com/frappe/frappe/pull/13007)) +- Do not show messsage when exception is handled ([#13111](https://github.com/frappe/frappe/pull/13111)) +- Replace parseFloat by Number ([#13082](https://github.com/frappe/frappe/pull/13082)) +- Add margin to download backup card ([#13050](https://github.com/frappe/frappe/pull/13050)) +- Translate report column labels ([#13083](https://github.com/frappe/frappe/pull/13083)) +- Grid row color picker field not working ([#13040](https://github.com/frappe/frappe/pull/13040)) +- Improve oauthlib implementation ([#13045](https://github.com/frappe/frappe/pull/13045)) +- Replace filter_by like with full text filter ([#13126](https://github.com/frappe/frappe/pull/13126)) +- Focus jumps to first field ([#13067](https://github.com/frappe/frappe/pull/13067)) \ No newline at end of file diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index 61ee62d352..e521acc9ad 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -28,6 +28,10 @@ def pass_context(f): except frappe.exceptions.SiteNotSpecifiedError as e: click.secho(str(e), fg='yellow') sys.exit(1) + except frappe.exceptions.IncorrectSitePath: + site = ctx.obj.get("sites", "")[0] + click.secho(f'Site {site} does not exist!', fg='yellow') + sys.exit(1) if profile: pr.disable() diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 0102d3ac40..22a063651c 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -1,6 +1,7 @@ # imports - standard imports import os import sys +import shutil # imports - third party imports import click @@ -202,10 +203,13 @@ def install_app(context, apps): @click.command("list-apps") +@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text") @pass_context -def list_apps(context): +def list_apps(context, format): "List apps in site" + summary_dict = {} + def fix_whitespaces(text): if site == context.sites[-1]: text = text.rstrip() @@ -234,18 +238,23 @@ def list_apps(context): ] applications_summary = "\n".join(installed_applications) summary = f"{site_title}\n{applications_summary}\n" + summary_dict[site] = [app.app_name for app in apps] else: - applications_summary = "\n".join(frappe.get_installed_apps()) + installed_applications = frappe.get_installed_apps() + applications_summary = "\n".join(installed_applications) summary = f"{site_title}\n{applications_summary}\n" + summary_dict[site] = installed_applications summary = fix_whitespaces(summary) - if applications_summary and summary: + if format == "text" and applications_summary and summary: print(summary) frappe.destroy() + if format == "json": + click.echo(frappe.as_json(summary_dict)) @click.command('add-system-manager') @click.argument('email') @@ -547,7 +556,7 @@ def move(dest_dir, site): site_dump_exists = os.path.exists(final_new_path) count = int(count or 0) + 1 - os.rename(old_path, final_new_path) + shutil.move(old_path, final_new_path) frappe.destroy() return final_new_path diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index c4b6cf4655..4da0f6bb78 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -16,33 +16,52 @@ from frappe.utils import get_bench_path, update_progress_bar, cint @click.command('build') @click.option('--app', help='Build assets for app') -@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking') -@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force') +@click.option('--apps', help='Build assets for specific apps') +@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking') +@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking') +@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force') +@click.option('--production', is_flag=True, default=False, help='Build assets in production mode') @click.option('--verbose', is_flag=True, default=False, help='Verbose') @click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') -def build(app=None, make_copy=False, restore=False, verbose=False, force=False): - "Minify + concatenate JS and CSS files, build translations" - import frappe.build +def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False): + "Compile JS and CSS source files" + from frappe.build import bundle, download_frappe_assets frappe.init('') - # don't minify in developer_mode for faster builds - no_compress = frappe.local.conf.developer_mode or False + + if not apps and app: + apps = app # dont try downloading assets if force used, app specified or running via CI - if not (force or app or os.environ.get('CI')): + if not (force or apps or os.environ.get('CI')): # skip building frappe if assets exist remotely - skip_frappe = frappe.build.download_frappe_assets(verbose=verbose) + skip_frappe = download_frappe_assets(verbose=verbose) else: skip_frappe = False - frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe) + # don't minify in developer_mode for faster builds + development = frappe.local.conf.developer_mode or frappe.local.dev_server + mode = "development" if development else "production" + if production: + mode = "production" + + if make_copy or restore: + hard_link = make_copy or restore + click.secho( + "bench build: --make-copy and --restore options are deprecated in favour of --hard-link", + fg="yellow", + ) + + bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) + @click.command('watch') -def watch(): - "Watch and concatenate JS and CSS files as and when they change" - import frappe.build +@click.option('--apps', help='Watch assets for specific apps') +def watch(apps=None): + "Watch and compile JS and CSS files as and when they change" + from frappe.build import watch frappe.init('') - frappe.build.watch(True) + watch(apps) @click.command('clear-cache') @@ -96,22 +115,54 @@ def destroy_all_sessions(context, reason=None): raise SiteNotSpecifiedError @click.command('show-config') +@click.option("--format", "-f", type=click.Choice(["text", "json"]), default="text") @pass_context -def show_config(context): - "print configuration file" - print("\t\033[92m{:<50}\033[0m \033[92m{:<15}\033[0m".format('Config','Value')) - sites_path = os.path.join(frappe.utils.get_bench_path(), 'sites') - site_path = context.sites[0] - configuration = frappe.get_site_config(sites_path=sites_path, site_path=site_path) - print_config(configuration) +def show_config(context, format): + "Print configuration file to STDOUT in speified format" + + if not context.sites: + raise SiteNotSpecifiedError + + sites_config = {} + sites_path = os.getcwd() + + from frappe.utils.commands import render_table + + def transform_config(config, prefix=None): + prefix = f"{prefix}." if prefix else "" + site_config = [] + + for conf, value in config.items(): + if isinstance(value, dict): + site_config += transform_config(value, prefix=f"{prefix}{conf}") + else: + log_value = json.dumps(value) if isinstance(value, list) else value + site_config += [[f"{prefix}{conf}", log_value]] + + return site_config + + for site in context.sites: + frappe.init(site) + + if len(context.sites) != 1 and format == "text": + if context.sites.index(site) != 0: + click.echo() + click.secho(f"Site {site}", fg="yellow") + + configuration = frappe.get_site_config(sites_path=sites_path, site_path=site) + + if format == "text": + data = transform_config(configuration) + data.insert(0, ['Config','Value']) + render_table(data) + if format == "json": + sites_config[site] = configuration -def print_config(config): - for conf, value in config.items(): - if isinstance(value, dict): - print_config(value) - else: - print("\t{:<50} {:<15}".format(conf, value)) + frappe.destroy() + + if format == "json": + click.echo(frappe.as_json(sites_config)) @click.command('reset-perms') @@ -470,6 +521,7 @@ def console(context): locals()[app] = __import__(app) except ModuleNotFoundError: failed_to_import.append(app) + all_apps.remove(app) print("Apps in this namespace:\n{}".format(", ".join(all_apps))) if failed_to_import: @@ -552,12 +604,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal if os.environ.get('CI'): sys.exit(ret) +@click.command('run-parallel-tests') +@click.option('--app', help="For App", default='frappe') +@click.option('--build-number', help="Build number", default=1) +@click.option('--total-builds', help="Total number of builds", default=1) +@click.option('--with-coverage', is_flag=True, help="Build coverage file") +@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests") +@pass_context +def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False): + site = get_site(context) + if use_orchestrator: + from frappe.parallel_test_runner import ParallelTestWithOrchestrator + ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage) + else: + from frappe.parallel_test_runner import ParallelTestRunner + ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage) @click.command('run-ui-tests') @click.argument('app') @click.option('--headless', is_flag=True, help="Run UI Test in headless mode") +@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode") +@click.option('--ci-build-id') @pass_context -def run_ui_tests(context, app, headless=False): +def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None): "Run UI tests" site = get_site(context) app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..')) @@ -589,6 +658,12 @@ def run_ui_tests(context, app, headless=False): command = '{site_env} {password_env} {cypress} {run_or_open}' formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open) + if parallel: + formatted_command += ' --parallel' + + if ci_build_id: + formatted_command += ' --ci-build-id {}'.format(ci_build_id) + click.secho("Running Cypress...", fg="yellow") frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True) @@ -657,20 +732,27 @@ def make_app(destination, app_name): @click.command('set-config') @click.argument('key') @click.argument('value') -@click.option('-g', '--global', 'global_', is_flag = True, default = False, help = 'Set Global Site Config') -@click.option('--as-dict', is_flag=True, default=False) +@click.option('-g', '--global', 'global_', is_flag=True, default=False, help='Set value in bench config') +@click.option('-p', '--parse', is_flag=True, default=False, help='Evaluate as Python Object') +@click.option('--as-dict', is_flag=True, default=False, help='Legacy: Evaluate as Python Object') @pass_context -def set_config(context, key, value, global_ = False, as_dict=False): +def set_config(context, key, value, global_=False, parse=False, as_dict=False): "Insert/Update a value in site_config.json" from frappe.installer import update_site_config - import ast + if as_dict: + from frappe.utils.commands import warn + warn("--as-dict will be deprecated in v14. Use --parse instead", category=PendingDeprecationWarning) + parse = as_dict + + if parse: + import ast value = ast.literal_eval(value) if global_: - sites_path = os.getcwd() # big assumption. + sites_path = os.getcwd() common_site_config_path = os.path.join(sites_path, 'common_site_config.json') - update_site_config(key, value, validate = False, site_config_path = common_site_config_path) + update_site_config(key, value, validate=False, site_config_path=common_site_config_path) else: for site in context.sites: frappe.init(site=site) @@ -727,50 +809,6 @@ def rebuild_global_search(context, static_pages=False): if not context.sites: raise SiteNotSpecifiedError -@click.command('auto-deploy') -@click.argument('app') -@click.option('--migrate', is_flag=True, default=False, help='Migrate after pulling') -@click.option('--restart', is_flag=True, default=False, help='Restart after migration') -@click.option('--remote', default='upstream', help='Remote, default is "upstream"') -@pass_context -def auto_deploy(context, app, migrate=False, restart=False, remote='upstream'): - '''Pull and migrate sites that have new version''' - from frappe.utils.gitutils import get_app_branch - from frappe.utils import get_sites - - branch = get_app_branch(app) - app_path = frappe.get_app_path(app) - - # fetch - subprocess.check_output(['git', 'fetch', remote, branch], cwd = app_path) - - # get diff - if subprocess.check_output(['git', 'diff', '{0}..{1}/{0}'.format(branch, remote)], cwd = app_path): - print('Updates found for {0}'.format(app)) - if app=='frappe': - # run bench update - import shlex - subprocess.check_output(shlex.split('bench update --no-backup'), cwd = '..') - else: - updated = False - subprocess.check_output(['git', 'pull', '--rebase', remote, branch], - cwd = app_path) - # find all sites with that app - for site in get_sites(): - frappe.init(site) - if app in frappe.get_installed_apps(): - print('Updating {0}'.format(site)) - updated = True - subprocess.check_output(['bench', '--site', site, 'clear-cache'], cwd = '..') - if migrate: - subprocess.check_output(['bench', '--site', site, 'migrate'], cwd = '..') - frappe.destroy() - - if updated or restart: - subprocess.check_output(['bench', 'restart'], cwd = '..') - else: - print('No Updates') - commands = [ build, @@ -801,5 +839,6 @@ commands = [ watch, bulk_rename, add_to_email_queue, - rebuild_global_search + rebuild_global_search, + run_parallel_tests ] diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py index 4929873dc4..b131428696 100644 --- a/frappe/contacts/doctype/contact/test_contact.py +++ b/frappe/contacts/doctype/contact/test_contact.py @@ -5,7 +5,8 @@ from __future__ import unicode_literals import frappe import unittest -from frappe.exceptions import ValidationError + +test_dependencies = ['Contact', 'Salutation'] class TestContact(unittest.TestCase): @@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True): if save: doc.insert() - return doc \ No newline at end of file + return doc diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index bd0ea08cc7..f33c7a1c85 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -65,12 +65,12 @@ class TestActivityLog(unittest.TestCase): frappe.local.login_manager = LoginManager() auth_log = self.get_auth_log() - self.assertEquals(auth_log.status, 'Success') + self.assertEqual(auth_log.status, 'Success') # test user logout log frappe.local.login_manager.logout() auth_log = self.get_auth_log(operation='Logout') - self.assertEquals(auth_log.status, 'Success') + self.assertEqual(auth_log.status, 'Success') # test invalid login frappe.form_dict.update({ 'pwd': 'password' }) @@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase): def update_system_settings(args): doc = frappe.get_doc('System Settings') doc.update(args) + doc.flags.ignore_mandatory = 1 doc.save() diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 731cb85d7c..d3017055cf 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -272,22 +272,13 @@ def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None) doc.attachments.append(a) def set_incoming_outgoing_accounts(doc): - doc.incoming_email_account = doc.outgoing_email_account = None + from frappe.email.doctype.email_account.email_account import EmailAccount + incoming_email_account = EmailAccount.find_incoming( + match_by_email=doc.sender, match_by_doctype=doc.reference_doctype) + doc.incoming_email_account = incoming_email_account.email_id if incoming_email_account else None - if not doc.incoming_email_account and doc.sender: - doc.incoming_email_account = frappe.db.get_value("Email Account", - {"email_id": doc.sender, "enable_incoming": 1}, "email_id") - - if not doc.incoming_email_account and doc.reference_doctype: - doc.incoming_email_account = frappe.db.get_value("Email Account", - {"append_to": doc.reference_doctype, }, "email_id") - - if not doc.incoming_email_account: - doc.incoming_email_account = frappe.db.get_value("Email Account", - {"default_incoming": 1, "enable_incoming": 1}, "email_id") - - doc.outgoing_email_account = frappe.email.smtp.get_outgoing_email_account(raise_exception_not_set=False, - append_to=doc.doctype, sender=doc.sender) + doc.outgoing_email_account = EmailAccount.find_outgoing( + match_by_email=doc.sender, match_by_doctype=doc.reference_doctype) if doc.sent_or_received == "Sent": doc.db_set("email_account", doc.outgoing_email_account.name) diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index bec8cde7ea..5d600cc0db 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -282,7 +282,7 @@ class DataExporter: try: sflags = self.docs_to_export.get("flags", "I,U").upper() flags = 0 - for a in re.split('\W+',sflags): + for a in re.split(r'\W+', sflags): flags = flags | reflags.get(a,0) c = re.compile(names, flags) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index e03c22a898..079bdaa09c 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -203,7 +203,7 @@ frappe.ui.form.on('Data Import', { }, download_template(frm) { - frappe.require('/assets/js/data_import_tools.min.js', () => { + frappe.require('data_import_tools.bundle.js', () => { frm.data_exporter = new frappe.data_import.DataExporter( frm.doc.reference_doctype, frm.doc.import_type @@ -287,7 +287,7 @@ frappe.ui.form.on('Data Import', { return; } - frappe.require('/assets/js/data_import_tools.min.js', () => { + frappe.require('data_import_tools.bundle.js', () => { frm.import_preview = new frappe.data_import.ImportPreview({ wrapper: frm.get_field('import_preview').$wrapper, doctype: frm.doc.reference_doctype, diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 1c56f54303..7e8374a0a2 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -211,7 +211,12 @@ def export_json( doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc" ): def post_process(out): - del_keys = ("modified_by", "creation", "owner", "idx") + # Note on Tree DocTypes: + # The tree structure is maintained in the database via the fields "lft" + # and "rgt". They are automatically set and kept up-to-date. Importing + # them would destroy any existing tree structure. For this reason they + # are not exported as well. + del_keys = ("modified_by", "creation", "owner", "idx", "lft", "rgt") for doc in out: for key in del_keys: if key in doc: diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 388d9389f2..d3f981add4 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -233,7 +233,7 @@ class Importer: return updated_doc else: # throw if no changes - frappe.throw("No changes to update") + frappe.throw(_("No changes to update")) def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -319,7 +319,7 @@ class ImportFile: self.warnings = [] self.file_doc = self.file_path = self.google_sheets_url = None - if isinstance(file, frappe.string_types): + if isinstance(file, str): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) elif "docs.google.com/spreadsheets" in file: @@ -626,7 +626,7 @@ class Row: return elif df.fieldtype in ["Date", "Datetime"]: value = self.get_date(value, col) - if isinstance(value, frappe.string_types): + if isinstance(value, str): # value was not parsed as datetime object self.warnings.append( { @@ -641,7 +641,7 @@ class Row: return elif df.fieldtype == "Duration": import re - is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) + is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) if not is_valid_duration: self.warnings.append( { @@ -929,10 +929,7 @@ class Column: self.warnings.append( { "col": self.column_number, - "message": _( - "Date format could not be determined from the values in" - " this column. Defaulting to yyyy-mm-dd." - ), + "message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."), "type": "info", } ) diff --git a/frappe/core/doctype/docshare/test_docshare.py b/frappe/core/doctype/docshare/test_docshare.py index d4ef1f92f8..9c424eb4d7 100644 --- a/frappe/core/doctype/docshare/test_docshare.py +++ b/frappe/core/doctype/docshare/test_docshare.py @@ -7,6 +7,8 @@ import frappe.share import unittest from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype +test_dependencies = ['User'] + class TestDocShare(unittest.TestCase): def setUp(self): self.user = "test@example.com" @@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase): self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user)) self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user)) - frappe.share.remove(doctype, submittable_doc.name, self.user) \ No newline at end of file + frappe.share.remove(doctype, submittable_doc.name, self.user) diff --git a/frappe/core/doctype/doctype/boilerplate/controller._py b/frappe/core/doctype/doctype/boilerplate/controller._py index 583bd30908..6db99def55 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller._py +++ b/frappe/core/doctype/doctype/boilerplate/controller._py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) {year}, {app_publisher} and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe {base_class_import} diff --git a/frappe/core/doctype/doctype/boilerplate/test_controller._py b/frappe/core/doctype/doctype/boilerplate/test_controller._py index 8ed08ae15a..5f4150ce9b 100644 --- a/frappe/core/doctype/doctype/boilerplate/test_controller._py +++ b/frappe/core/doctype/doctype/boilerplate/test_controller._py @@ -1,7 +1,5 @@ -# -*- coding: utf-8 -*- # Copyright (c) {year}, {app_publisher} and Contributors # See license.txt -from __future__ import unicode_literals # import frappe import unittest diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index fe5038b841..7f93d3130a 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -662,4 +662,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} \ No newline at end of file +} diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3588cc553a..84673f990a 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -83,12 +83,61 @@ class DocType(Document): if not self.is_new(): self.before_update = frappe.get_doc('DocType', self.name) self.setup_fields_to_fetch() + self.validate_field_name_conflicts() check_email_append_to(self) if self.default_print_format and not self.custom: frappe.throw(_('Standard DocType cannot have default print format, use Customize Form')) + if frappe.conf.get('developer_mode'): + self.owner = 'Administrator' + self.modified_by = 'Administrator' + + def validate_field_name_conflicts(self): + """Check if field names dont conflict with controller properties and methods""" + core_doctypes = [ + "Custom DocPerm", + "DocPerm", + "Custom Field", + "Customize Form Field", + "DocField", + ] + + if self.name in core_doctypes: + return + + from frappe.model.base_document import get_controller + + try: + controller = get_controller(self.name) + except ImportError: + controller = Document + + available_objects = {x for x in dir(controller) if isinstance(x, str)} + property_set = { + x for x in available_objects if isinstance(getattr(controller, x, None), property) + } + method_set = { + x for x in available_objects if x not in property_set and callable(getattr(controller, x, None)) + } + + for docfield in self.get("fields") or []: + conflict_type = None + field = docfield.fieldname + field_label = docfield.label or docfield.fieldname + + if docfield.fieldname in method_set: + conflict_type = "controller method" + if docfield.fieldname in property_set: + conflict_type = "class property" + + if conflict_type: + frappe.throw( + _("Fieldname '{0}' conflicting with a {1} of the name {2} in {3}") + .format(field_label, conflict_type, field, self.name) + ) + def after_insert(self): # clear user cache so that on the next reload this doctype is included in boot clear_user_cache(frappe.session.user) @@ -622,12 +671,12 @@ class DocType(Document): flags = {"flags": re.ASCII} if six.PY3 else {} # a DocType name should not start or end with an empty space - if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): + if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError) # a DocType's name should not start with a number or underscore # and should only contain letters, numbers and underscore - if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags): + if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags): frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError) validate_route_conflict(self.doctype, self.name) @@ -915,7 +964,7 @@ def validate_fields(meta): for field in depends_on_fields: depends_on = docfield.get(field, None) if depends_on and ("=" in depends_on) and \ - re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on): + re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on): frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError) def check_table_multiselect_option(docfield): @@ -1174,11 +1223,19 @@ def make_module_and_roles(doc, perm_fieldname="permissions"): else: raise -def check_if_fieldname_conflicts_with_methods(doctype, fieldname): - doc = frappe.get_doc({"doctype": doctype}) - method_list = [method for method in dir(doc) if isinstance(method, str) and callable(getattr(doc, method))] +def check_fieldname_conflicts(doctype, fieldname): + """Checks if fieldname conflicts with methods or properties""" - if fieldname in method_list: + doc = frappe.get_doc({"doctype": doctype}) + available_objects = [x for x in dir(doc) if isinstance(x, str)] + property_list = [ + x for x in available_objects if isinstance(getattr(type(doc), x, None), property) + ] + method_list = [ + x for x in available_objects if x not in property_list and callable(getattr(doc, x)) + ] + + if fieldname in method_list + property_list: frappe.throw(_("Fieldname {0} conflicting with meta object").format(fieldname)) def clear_linked_doctype_cache(): diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index bfa9d0ec8a..9c492d2c36 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase): fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\ "read_only_depends_on", "fieldname", "fieldtype"]) - pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+""" + pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+' for field in docfields: for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]: condition = field.get(depends_on) @@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None): for f in fields: doc.append('fields', f) - return doc \ No newline at end of file + return doc diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 017106e6f5..c4c37e6d13 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -498,7 +498,7 @@ class File(Document): self.file_size = self.check_max_file_size() if ( - self.content_type and "image" in self.content_type + self.content_type and self.content_type == "image/jpeg" and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images") ): self.content = strip_exif_data(self.content, self.content_type) @@ -912,7 +912,7 @@ def extract_images_from_html(doc, content): return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + content = re.sub(r']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) return content diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2f8f437fc9..2596fe94d0 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase): class TestFile(unittest.TestCase): def setUp(self): + frappe.set_user('Administrator') self.delete_test_data() self.upload_file() diff --git a/frappe/core/doctype/report/boilerplate/controller.py b/frappe/core/doctype/report/boilerplate/controller.py index 55c01e4f75..b8e9cb7467 100644 --- a/frappe/core/doctype/report/boilerplate/controller.py +++ b/frappe/core/doctype/report/boilerplate/controller.py @@ -1,7 +1,6 @@ # Copyright (c) 2013, {app_publisher} and contributors # For license information, please see license.txt -from __future__ import unicode_literals # import frappe def execute(filters=None): diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 9c76c839f3..d09799ca69 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -106,7 +106,7 @@ class TestReport(unittest.TestCase): else: report = frappe.get_doc('Report', 'Test Report') - self.assertNotEquals(report.is_permitted(), True) + self.assertNotEqual(report.is_permitted(), True) frappe.set_user('Administrator') # test for the `_format` method if report data doesn't have sort_by parameter diff --git a/frappe/core/doctype/role_profile/test_role_profile.py b/frappe/core/doctype/role_profile/test_role_profile.py index 624b85c315..975453e8d1 100644 --- a/frappe/core/doctype/role_profile/test_role_profile.py +++ b/frappe/core/doctype/role_profile/test_role_profile.py @@ -5,6 +5,8 @@ from __future__ import unicode_literals import frappe import unittest +test_dependencies = ['Role'] + class TestRoleProfile(unittest.TestCase): def test_make_new_role_profile(self): new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert() @@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase): # clear roles new_role_profile.roles = [] new_role_profile.save() - self.assertEqual(new_role_profile.roles, []) \ No newline at end of file + self.assertEqual(new_role_profile.roles, []) diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index d102526a9e..05aaca81de 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -42,7 +42,7 @@ class SystemSettings(Document): def on_update(self): for df in self.meta.get("fields"): - if df.fieldtype not in no_value_fields: + if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): frappe.db.set_default(df.fieldname, self.get(df.fieldname)) if self.language: diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 0462de8643..a4d13a57e0 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -56,6 +56,7 @@ class User(Document): def after_insert(self): create_notification_settings(self.name) + frappe.cache().delete_key('users_for_mentions') def validate(self): self.check_demo() @@ -129,6 +130,9 @@ class User(Document): if self.time_zone: frappe.defaults.set_default("time_zone", self.time_zone, self.name) + if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'): + frappe.cache().delete_key('users_for_mentions') + def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" return self.name == frappe.session.user @@ -389,6 +393,9 @@ class User(Document): # delete notification settings frappe.delete_doc("Notification Settings", self.name, ignore_permissions=True) + if self.get('allow_in_mentions'): + frappe.cache().delete_key('users_for_mentions') + def before_rename(self, old_name, new_name, merge=False): self.check_demo() diff --git a/frappe/core/doctype/user_group/user_group.py b/frappe/core/doctype/user_group/user_group.py index 64bffa06d0..b1d0fede4c 100644 --- a/frappe/core/doctype/user_group/user_group.py +++ b/frappe/core/doctype/user_group/user_group.py @@ -9,7 +9,7 @@ import frappe class UserGroup(Document): def after_insert(self): - frappe.publish_realtime('user_group_added', self.name) + frappe.cache().delete_key('user_groups') def on_trash(self): - frappe.publish_realtime('user_group_deleted', self.name) + frappe.cache().delete_key('user_groups') diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 2e9b832acc..47651fee72 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -46,7 +46,7 @@ class TestUserPermission(unittest.TestCase): frappe.set_user('test_user_perm1@example.com') doc = frappe.new_doc("Blog Post") - self.assertEquals(doc.blog_category, 'general') + self.assertEqual(doc.blog_category, 'general') frappe.set_user('Administrator') def test_apply_to_all(self): @@ -54,7 +54,7 @@ class TestUserPermission(unittest.TestCase): user = create_user('test_bulk_creation_update@example.com') param = get_params(user, 'User', user.name) is_created = add_user_permissions(param) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) def test_for_apply_to_all_on_update_from_apply_all(self): user = create_user('test_bulk_creation_update@example.com') @@ -63,11 +63,11 @@ class TestUserPermission(unittest.TestCase): # Initially create User Permission document with apply_to_all checked is_created = add_user_permissions(param) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) is_created = add_user_permissions(param) # User Permission should not be changed - self.assertEquals(is_created, 0) + self.assertEqual(is_created, 0) def test_for_applicable_on_update_from_apply_to_all(self): ''' Update User Permission from all to some applicable Doctypes''' @@ -77,7 +77,7 @@ class TestUserPermission(unittest.TestCase): # Initially create User Permission document with apply_to_all checked is_created = add_user_permissions(get_params(user, 'User', user.name)) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) is_created = add_user_permissions(param) frappe.db.commit() @@ -92,7 +92,7 @@ class TestUserPermission(unittest.TestCase): # Check that User Permissions for applicable is created self.assertIsNotNone(is_created_applicable_first) self.assertIsNotNone(is_created_applicable_second) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) def test_for_apply_to_all_on_update_from_applicable(self): ''' Update User Permission from some to all applicable Doctypes''' @@ -102,7 +102,7 @@ class TestUserPermission(unittest.TestCase): # create User permissions that with applicable is_created = add_user_permissions(get_params(user, 'User', user.name, applicable = ["Chat Room", "Chat Message"])) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) is_created = add_user_permissions(param) is_created_apply_to_all = frappe.db.exists("User Permission", get_exists_param(user)) @@ -115,7 +115,7 @@ class TestUserPermission(unittest.TestCase): # Check that all User Permission with applicable is removed self.assertIsNone(removed_applicable_first) self.assertIsNone(removed_applicable_second) - self.assertEquals(is_created, 1) + self.assertEqual(is_created, 1) def test_user_perm_for_nested_doctype(self): """Test if descendants' visibility is controlled for a nested DocType.""" @@ -183,7 +183,7 @@ class TestUserPermission(unittest.TestCase): # User perm is created on ToDo but for doctype Assignment Rule only # it should not have impact on Doc A - self.assertEquals(new_doc.doc, "ToDo") + self.assertEqual(new_doc.doc, "ToDo") frappe.set_user('Administrator') remove_applicable(["Assignment Rule"], "new_doc_test@example.com", "DocType", "ToDo") @@ -228,7 +228,7 @@ class TestUserPermission(unittest.TestCase): # User perm is created on ToDo but for doctype Assignment Rule only # it should not have impact on Doc A - self.assertEquals(new_doc.doc, "ToDo") + self.assertEqual(new_doc.doc, "ToDo") frappe.set_user('Administrator') clear_session_defaults() diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index fbc788f6bf..fec5019ca9 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -191,7 +191,7 @@ def clear_user_permissions(user, for_doctype): def add_user_permissions(data): ''' Add and update the user permissions ''' frappe.only_for('System Manager') - if isinstance(data, frappe.string_types): + if isinstance(data, str): data = json.loads(data) data = frappe._dict(data) diff --git a/frappe/core/page/recorder/recorder.js b/frappe/core/page/recorder/recorder.js index b75ea6a41c..f1f74daf71 100644 --- a/frappe/core/page/recorder/recorder.js +++ b/frappe/core/page/recorder/recorder.js @@ -1,7 +1,7 @@ frappe.pages['recorder'].on_page_load = function(wrapper) { frappe.ui.make_app_page({ parent: wrapper, - title: 'Recorder', + title: __('Recorder'), single_column: true, card_layout: true }); @@ -11,7 +11,7 @@ frappe.pages['recorder'].on_page_load = function(wrapper) { frappe.recorder.show(); }); - frappe.require('/assets/js/frappe-recorder.min.js'); + frappe.require('recorder.bundle.js'); }; class Recorder { diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 3126326636..39aff8b4a7 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -64,18 +64,19 @@ class CustomField(Document): self.translatable = 0 if not self.flags.ignore_validate: - from frappe.core.doctype.doctype.doctype import check_if_fieldname_conflicts_with_methods - check_if_fieldname_conflicts_with_methods(self.dt, self.fieldname) + from frappe.core.doctype.doctype.doctype import check_fieldname_conflicts + check_fieldname_conflicts(self.dt, self.fieldname) def on_update(self): - frappe.clear_cache(doctype=self.dt) + if not frappe.flags.in_setup_wizard: + frappe.clear_cache(doctype=self.dt) if not self.flags.ignore_validate: # validate field from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype validate_fields_for_doctype(self.dt) # update the schema - if not frappe.db.get_value('DocType', self.dt, 'issingle'): + if not frappe.db.get_value('DocType', self.dt, 'issingle') and not frappe.flags.in_setup_wizard: frappe.db.updatedb(self.dt) def on_trash(self): @@ -144,6 +145,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True): '''Add / update multiple custom fields :param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`''' + + if not ignore_validate and frappe.flags.in_setup_wizard: + ignore_validate = True + for doctype, fields in custom_fields.items(): if isinstance(fields, dict): # only one field @@ -163,6 +168,10 @@ def create_custom_fields(custom_fields, ignore_validate = False, update=True): custom_field.update(df) custom_field.save() + frappe.clear_cache(doctype=doctype) + frappe.db.updatedb(doctype) + + @frappe.whitelist() def add_custom_field(doctype, df): diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 442b8dbb31..1807678673 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -278,6 +278,7 @@ }, { "collapsible": 1, + "depends_on": "doc_type", "fieldname": "naming_section", "fieldtype": "Section Break", "label": "Naming" @@ -287,6 +288,16 @@ "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name" + }, + { + "fieldname": "default_email_template", + "fieldtype": "Link", + "label": "Default Email Template", + "options": "Email Template" + }, + { + "fieldname": "column_break_26", + "fieldtype": "Column Break" } ], "hide_toolbar": 1, @@ -295,7 +306,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2021-03-22 12:27:15.462727", + "modified": "2021-04-29 21:21:06.476372", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", @@ -316,4 +327,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index f5e0371c1f..75555a8205 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -47,64 +47,64 @@ class TestCustomizeForm(unittest.TestCase): self.assertEqual(len(d.get("fields")), 0) d = self.get_customize_form("Event") - self.assertEquals(d.doc_type, "Event") - self.assertEquals(len(d.get("fields")), 36) + self.assertEqual(d.doc_type, "Event") + self.assertEqual(len(d.get("fields")), 36) d = self.get_customize_form("Event") - self.assertEquals(d.doc_type, "Event") + self.assertEqual(d.doc_type, "Event") self.assertEqual(len(d.get("fields")), len(frappe.get_doc("DocType", d.doc_type).fields) + 1) - self.assertEquals(d.get("fields")[-1].fieldname, "test_custom_field") - self.assertEquals(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1) + self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field") + self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1) return d def test_save_customization_property(self): d = self.get_customize_form("Event") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value"), None) d.allow_copy = 1 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value"), '1') d.allow_copy = 0 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "allow_copy"}, "value"), None) def test_save_customization_field_property(self): d = self.get_customize_form("Event") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None) repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0] repeat_this_event_field.reqd = 1 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), '1') repeat_this_event_field = d.get("fields", {"fieldname": "repeat_this_event"})[0] repeat_this_event_field.reqd = 0 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Property Setter", + self.assertEqual(frappe.db.get_value("Property Setter", {"doc_type": "Event", "property": "reqd", "field_name": "repeat_this_event"}, "value"), None) def test_save_customization_custom_field_property(self): d = self.get_customize_form("Event") - self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0] custom_field.reqd = 1 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1) + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1) custom_field = d.get("fields", {"is_custom_field": True})[0] custom_field.reqd = 0 d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) def test_save_customization_new_field(self): d = self.get_customize_form("Event") @@ -115,14 +115,14 @@ class TestCustomizeForm(unittest.TestCase): "is_custom_field": 1 }) d.run_method("save_customization") - self.assertEquals(frappe.db.get_value("Custom Field", + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form", "fieldtype"), "Data") - self.assertEquals(frappe.db.get_value("Custom Field", + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form", 'insert_after'), last_fieldname) frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form") - self.assertEquals(frappe.db.get_value("Custom Field", + self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form"), None) @@ -142,7 +142,7 @@ class TestCustomizeForm(unittest.TestCase): d.doc_type = "Event" d.run_method('reset_to_defaults') - self.assertEquals(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0) + self.assertEqual(d.get("fields", {"fieldname": "repeat_this_event"})[0].in_list_view, 0) frappe.local.test_objects["Property Setter"] = [] make_test_records_for_doctype("Property Setter") @@ -156,7 +156,7 @@ class TestCustomizeForm(unittest.TestCase): d = self.get_customize_form("Event") # don't allow for standard fields - self.assertEquals(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0) + self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0) # allow for custom field self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1) diff --git a/frappe/database/database.py b/frappe/database/database.py index 58e5c8a46e..c9c1ec3909 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -858,7 +858,7 @@ class Database(object): if not datetime: return '0001-01-01 00:00:00.000000' - if isinstance(datetime, frappe.string_types): + if isinstance(datetime, str): if ':' not in datetime: datetime = datetime + ' 00:00:00.000000' else: diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 7d1d92408c..879c8394d7 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,5 +1,3 @@ -import warnings - import pymysql from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions, escape_string @@ -55,7 +53,6 @@ class MariaDBDatabase(Database): } def get_connection(self): - warnings.filterwarnings('ignore', category=pymysql.Warning) usessl = 0 if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: usessl = 1 diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 4faea78551..6ac2767a71 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import re import frappe import psycopg2 @@ -13,9 +11,9 @@ from frappe.database.postgres.schema import PostgresTable # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( - psycopg2.extensions.DECIMAL.values, - 'DEC2FLOAT', - lambda value, curs: float(value) if value is not None else None) + psycopg2.extensions.DECIMAL.values, + 'DEC2FLOAT', + lambda value, curs: float(value) if value is not None else None) psycopg2.extensions.register_type(DEC2FLOAT) @@ -65,7 +63,6 @@ class PostgresDatabase(Database): } def get_connection(self): - # warnings.filterwarnings('ignore', category=psycopg2.Warning) conn = psycopg2.connect("host='{}' dbname='{}' user='{}' password='{}' port={}".format( self.host, self.user, self.user, self.password, self.port )) @@ -114,7 +111,7 @@ class PostgresDatabase(Database): if not date: return '0001-01-01' - if not isinstance(date, frappe.string_types): + if not isinstance(date, str): date = date.strftime('%Y-%m-%d') return date diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index d1b5e27a2f..1a3b1ca99b 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -359,15 +359,18 @@ def get_desktop_page(page): Returns: dict: dictionary of cards, charts and shortcuts to be displayed on website """ - wspace = Workspace(page) - wspace.build_workspace() - return { - 'charts': wspace.charts, - 'shortcuts': wspace.shortcuts, - 'cards': wspace.cards, - 'onboarding': wspace.onboarding, - 'allow_customization': not wspace.doc.disable_user_customization - } + try: + wspace = Workspace(page) + wspace.build_workspace() + return { + 'charts': wspace.charts, + 'shortcuts': wspace.shortcuts, + 'cards': wspace.cards, + 'onboarding': wspace.onboarding, + 'allow_customization': not wspace.doc.disable_user_customization + } + except DoesNotExistError: + return {} @frappe.whitelist() def get_desk_sidebar_items(): @@ -608,3 +611,4 @@ def merge_cards_based_on_label(cards): cards_dict[label] = card return list(cards_dict.values()) + diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 20551559fd..25af92f532 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -46,7 +46,7 @@ def enqueue_create_notification(users, doc): doc = frappe._dict(doc) - if isinstance(users, frappe.string_types): + if isinstance(users, str): users = [user.strip() for user in users.split(',') if user.strip()] users = list(set(users)) diff --git a/frappe/desk/doctype/todo/test_todo.py b/frappe/desk/doctype/todo/test_todo.py index b767fd4aef..de5b6724a6 100644 --- a/frappe/desk/doctype/todo/test_todo.py +++ b/frappe/desk/doctype/todo/test_todo.py @@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery from frappe.permissions import add_permission, reset_perms from frappe.core.doctype.doctype.doctype import clear_permissions_cache -# test_records = frappe.get_test_records('ToDo') -test_user_records = frappe.get_test_records('User') +test_dependencies = ['User'] class TestToDo(unittest.TestCase): def test_delete(self): @@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase): frappe.set_user('test4@example.com') #owner and assigned_by is test4 todo3 = create_new_todo('Test3', 'test4@example.com') - + # user without any role to read or write todo document self.assertFalse(todo1.has_permission("read")) self.assertFalse(todo1.has_permission("write")) diff --git a/frappe/desk/doctype/workspace_link/workspace_link.json b/frappe/desk/doctype/workspace_link/workspace_link.json index 010fb3f316..53dadad83d 100644 --- a/frappe/desk/doctype/workspace_link/workspace_link.json +++ b/frappe/desk/doctype/workspace_link/workspace_link.json @@ -8,13 +8,13 @@ "type", "label", "icon", + "only_for", "hidden", "link_details_section", "link_type", "link_to", "column_break_7", "dependencies", - "only_for", "onboard", "is_query_report" ], @@ -84,7 +84,7 @@ { "fieldname": "only_for", "fieldtype": "Link", - "label": "Only for ", + "label": "Only for", "options": "Country" }, { @@ -104,7 +104,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2021-01-12 13:13:12.379443", + "modified": "2021-05-13 13:10:18.128512", "modified_by": "Administrator", "module": "Desk", "name": "Workspace Link", diff --git a/frappe/desk/page/activity/activity.js b/frappe/desk/page/activity/activity.js index 39de414122..7b4e8ddc1a 100644 --- a/frappe/desk/page/activity/activity.js +++ b/frappe/desk/page/activity/activity.js @@ -67,8 +67,8 @@ frappe.pages['activity'].on_page_show = function () { } frappe.activity.last_feed_date = false; -frappe.activity.Feed = Class.extend({ - init: function (row, data) { +frappe.activity.Feed = class Feed { + constructor(row, data) { this.scrub_data(data); this.add_date_separator(row, data); if (!data.add_class) @@ -97,8 +97,9 @@ frappe.activity.Feed = Class.extend({ $(row) .append(frappe.render_template("activity_row", data)) .find("a").addClass("grey"); - }, - scrub_data: function (data) { + } + + scrub_data(data) { data.by = frappe.user.full_name(data.owner); data.avatar = frappe.avatar(data.owner); @@ -113,9 +114,9 @@ frappe.activity.Feed = Class.extend({ data.when = comment_when(data.creation); data.feed_type = data.comment_type || data.communication_medium; - }, + } - add_date_separator: function (row, data) { + add_date_separator(row, data) { var date = frappe.datetime.str_to_obj(data.creation); var last = frappe.activity.last_feed_date; @@ -137,7 +138,7 @@ frappe.activity.Feed = Class.extend({ } frappe.activity.last_feed_date = date; } -}); +}; frappe.activity.render_heatmap = function (page) { $('
\ diff --git a/frappe/desk/page/backups/backups.css b/frappe/desk/page/backups/backups.css index 13f093e0b1..32ccb88c37 100644 --- a/frappe/desk/page/backups/backups.css +++ b/frappe/desk/page/backups/backups.css @@ -5,6 +5,7 @@ .download-backup-card { display: block; text-decoration: none; + margin-bottom: var(--margin-lg); } .download-backup-card:hover { diff --git a/frappe/desk/page/backups/backups.js b/frappe/desk/page/backups/backups.js index c82407c6bd..337ad33f43 100644 --- a/frappe/desk/page/backups/backups.js +++ b/frappe/desk/page/backups/backups.js @@ -1,7 +1,7 @@ frappe.pages['backups'].on_page_load = function(wrapper) { var page = frappe.ui.make_app_page({ parent: wrapper, - title: 'Download Backups', + title: __('Download Backups'), single_column: true }); diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index c38cf47626..1ac5279508 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -124,6 +124,7 @@ def handle_setup_exception(args): frappe.db.rollback() if args: traceback = frappe.get_traceback() + print(traceback) for hook in frappe.get_hooks("setup_wizard_exception"): frappe.get_attr(hook)(traceback, args) diff --git a/frappe/desk/page/translation_tool/translation_tool.js b/frappe/desk/page/translation_tool/translation_tool.js index b3f0c032e3..13f68e647a 100644 --- a/frappe/desk/page/translation_tool/translation_tool.js +++ b/frappe/desk/page/translation_tool/translation_tool.js @@ -1,7 +1,7 @@ frappe.pages['translation-tool'].on_page_load = function(wrapper) { var page = frappe.ui.make_app_page({ parent: wrapper, - title: 'Translation Tool', + title: __('Translation Tool'), single_column: true, card_layout: true, }); diff --git a/frappe/desk/page/user_profile/user_profile.html b/frappe/desk/page/user_profile/user_profile.html index 911ccc702d..f134441b74 100644 --- a/frappe/desk/page/user_profile/user_profile.html +++ b/frappe/desk/page/user_profile/user_profile.html @@ -8,7 +8,7 @@
- No Data to Show + {%=__("No Data to Show") %}
@@ -19,7 +19,7 @@
- No Data to Show + {%=__("No Data to Show") %}
@@ -30,7 +30,7 @@
- No Data to Show + {%=__("No Data to Show") %}
@@ -41,4 +41,4 @@ - \ No newline at end of file + diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 9589507ca6..befaf7b01f 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -377,10 +377,17 @@ def handle_duration_fieldtype_values(result, columns): if fieldtype == "Duration": for entry in range(0, len(result)): - val_in_seconds = result[entry][i] - if val_in_seconds: - duration_val = format_duration(val_in_seconds) - result[entry][i] = duration_val + row = result[entry] + if isinstance(row, dict): + val_in_seconds = row[col.fieldname] + if val_in_seconds: + duration_val = format_duration(val_in_seconds) + row[col.fieldname] = duration_val + else: + val_in_seconds = row[i] + if val_in_seconds: + duration_val = format_duration(val_in_seconds) + row[i] = duration_val return result diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 6181261fc2..3c9109eca9 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -221,3 +221,37 @@ def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): return [] return fn(**kwargs) + + +@frappe.whitelist() +def get_names_for_mentions(search_term): + users_for_mentions = frappe.cache().get_value('users_for_mentions', get_users_for_mentions) + user_groups = frappe.cache().get_value('user_groups', get_user_groups) + + filtered_mentions = [] + for mention_data in users_for_mentions + user_groups: + if search_term.lower() not in mention_data.value.lower(): + continue + + mention_data['link'] = frappe.utils.get_url_to_form( + 'User Group' if mention_data.get('is_group') else 'User Profile', + mention_data['id'] + ) + + filtered_mentions.append(mention_data) + + return sorted(filtered_mentions, key=lambda d: d['value']) + +def get_users_for_mentions(): + return frappe.get_all('User', + fields=['name as id', 'full_name as value'], + filters={ + 'name': ['not in', ('Administrator', 'Guest')], + 'allowed_in_mentions': True, + 'user_type': 'System User', + }) + +def get_user_groups(): + return frappe.get_all('User Group', fields=['name as id', 'name as value'], update={ + 'is_group': True + }) diff --git a/frappe/email/doctype/document_follow/test_document_follow.py b/frappe/email/doctype/document_follow/test_document_follow.py index 1ac2d19305..38aa870232 100644 --- a/frappe/email/doctype/document_follow/test_document_follow.py +++ b/frappe/email/doctype/document_follow/test_document_follow.py @@ -17,14 +17,14 @@ class TestDocumentFollow(unittest.TestCase): document_follow.unfollow_document("Event", event_doc.name, user.name) doc = document_follow.follow_document("Event", event_doc.name, user.name) - self.assertEquals(doc.user, user.name) + self.assertEqual(doc.user, user.name) document_follow.send_hourly_updates() email_queue_entry_name = frappe.get_all("Email Queue", limit=1)[0].name email_queue_entry_doc = frappe.get_doc("Email Queue", email_queue_entry_name) - self.assertEquals((email_queue_entry_doc.recipients[0].recipient), user.name) + self.assertEqual((email_queue_entry_doc.recipients[0].recipient), user.name) self.assertIn(event_doc.doctype, email_queue_entry_doc.message) self.assertIn(event_doc.name, email_queue_entry_doc.message) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 4869c5a9bf..36b662bb39 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -8,9 +8,14 @@ import re import json import socket import time -from frappe import _ +import functools + +import email.utils + +from frappe import _, are_emails_muted from frappe.model.document import Document -from frappe.utils import validate_email_address, cint, cstr, get_datetime, DATE_FORMAT, strip, comma_or, sanitize_html, add_days +from frappe.utils import (validate_email_address, cint, cstr, get_datetime, + DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr) from frappe.utils.user import is_system_user from frappe.utils.jinja import render_template from frappe.email.smtp import SMTPServer @@ -21,17 +26,37 @@ from datetime import datetime, timedelta from frappe.desk.form import assign_to from frappe.utils.user import get_system_managers from frappe.utils.background_jobs import enqueue, get_jobs -from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts from frappe.utils.html_utils import clean_email_html +from frappe.utils.error import raise_error_on_no_output from frappe.email.utils import get_port +OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account") + class SentEmailInInbox(Exception): pass -class InvalidEmailCredentials(frappe.ValidationError): - pass +def cache_email_account(cache_name): + def decorator_cache_email_account(func): + @functools.wraps(func) + def wrapper_cache_email_account(*args, **kwargs): + if not hasattr(frappe.local, cache_name): + setattr(frappe.local, cache_name, {}) + + cached_accounts = getattr(frappe.local, cache_name) + match_by = list(kwargs.values()) + ['default'] + matched_accounts = list(filter(None, [cached_accounts.get(key) for key in match_by])) + if matched_accounts: + return matched_accounts[0] + + matched_accounts = func(*args, **kwargs) + cached_accounts.update(matched_accounts or {}) + return matched_accounts and list(matched_accounts.values())[0] + return wrapper_cache_email_account + return decorator_cache_email_account class EmailAccount(Document): + DOCTYPE = 'Email Account' + def autoname(self): """Set name as `email_account_name` or make title from Email Address.""" if not self.email_account_name: @@ -72,9 +97,8 @@ class EmailAccount(Document): self.get_incoming_server() self.no_failed = 0 - if self.enable_outgoing: - self.check_smtp() + self.validate_smtp_conn() else: if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication): frappe.throw(_("Password is required or select Awaiting Password")) @@ -90,6 +114,13 @@ class EmailAccount(Document): if self.append_to not in valid_doctypes: frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) + def validate_smtp_conn(self): + if not self.smtp_server: + frappe.throw(_("SMTP Server is required")) + + server = self.get_smtp_server() + return server.session + def before_save(self): messages = [] as_list = 1 @@ -151,24 +182,6 @@ class EmailAccount(Document): except Exception: pass - def check_smtp(self): - """Checks SMTP settings.""" - if self.enable_outgoing: - if not self.smtp_server: - frappe.throw(_("{0} is required").format("SMTP Server")) - - server = SMTPServer( - login = getattr(self, "login_id", None) or self.email_id, - server=self.smtp_server, - port=cint(self.smtp_port), - use_tls=cint(self.use_tls), - use_ssl=cint(self.use_ssl_for_outgoing) - ) - if self.password and not self.no_smtp_authentication: - server.password = self.get_password() - - server.sess - def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"): """Returns logged in POP3/IMAP connection object.""" if frappe.cache().get_value("workers:no-internet") == True: @@ -231,7 +244,7 @@ class EmailAccount(Document): return None elif not in_receive and any(map(lambda t: t in message, auth_error_codes)): - self.throw_invalid_credentials_exception() + SMTPServer.throw_invalid_credentials_exception() else: frappe.throw(cstr(e)) @@ -249,13 +262,142 @@ class EmailAccount(Document): else: raise + @property + def _password(self): + raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test) + return self.get_password(raise_exception=raise_exception) + + @property + def default_sender(self): + return email.utils.formataddr((self.name, self.get("email_id"))) + + def is_exists_in_db(self): + """Some of the Email Accounts we create from configs and those doesn't exists in DB. + This is is to check the specific email account exists in DB or not. + """ + return self.find_one_by_filters(name=self.name) + + @classmethod + def from_record(cls, record): + email_account = frappe.new_doc(cls.DOCTYPE) + email_account.update(record) + return email_account + @classmethod - def throw_invalid_credentials_exception(cls): - frappe.throw( - _("Incorrect email or password. Please check your login credentials."), - exc=InvalidEmailCredentials, - title=_("Invalid Credentials") - ) + def find(cls, name): + return frappe.get_doc(cls.DOCTYPE, name) + + @classmethod + def find_one_by_filters(cls, **kwargs): + name = frappe.db.get_value(cls.DOCTYPE, kwargs) + return cls.find(name) if name else None + + @classmethod + def find_from_config(cls): + config = cls.get_account_details_from_site_config() + return cls.from_record(config) if config else None + + @classmethod + def create_dummy(cls): + return cls.from_record({"sender": "notifications@example.com"}) + + @classmethod + @raise_error_on_no_output( + keep_quiet = lambda: not cint(frappe.get_system_settings('setup_complete')), + error_message = OUTGOING_EMAIL_ACCOUNT_MISSING, error_type = frappe.OutgoingEmailError) # noqa + @cache_email_account('outgoing_email_account') + def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False): + """Find the outgoing Email account to use. + + :param match_by_email: Find account using emailID + :param match_by_doctype: Find account by matching `Append To` doctype + :param _raise_error: This is used by raise_error_on_no_output decorator to raise error. + """ + if match_by_email: + match_by_email = parse_addr(match_by_email)[1] + doc = cls.find_one_by_filters(enable_outgoing=1, email_id=match_by_email) + if doc: + return {match_by_email: doc} + + if match_by_doctype: + doc = cls.find_one_by_filters(enable_outgoing=1, enable_incoming=1, append_to=match_by_doctype) + if doc: + return {match_by_doctype: doc} + + doc = cls.find_default_outgoing() + if doc: + return {'default': doc} + + @classmethod + def find_default_outgoing(cls): + """ Find default outgoing account. + """ + doc = cls.find_one_by_filters(enable_outgoing=1, default_outgoing=1) + doc = doc or cls.find_from_config() + return doc or (are_emails_muted() and cls.create_dummy()) + + @classmethod + def find_incoming(cls, match_by_email=None, match_by_doctype=None): + """Find the incoming Email account to use. + :param match_by_email: Find account using emailID + :param match_by_doctype: Find account by matching `Append To` doctype + """ + doc = cls.find_one_by_filters(enable_incoming=1, email_id=match_by_email) + if doc: + return doc + + doc = cls.find_one_by_filters(enable_incoming=1, append_to=match_by_doctype) + if doc: + return doc + + doc = cls.find_default_incoming() + return doc + + @classmethod + def find_default_incoming(cls): + doc = cls.find_one_by_filters(enable_incoming=1, default_incoming=1) + return doc + + @classmethod + def get_account_details_from_site_config(cls): + if not frappe.conf.get("mail_server"): + return {} + + field_to_conf_name_map = { + 'smtp_server': {'conf_names': ('mail_server',)}, + 'smtp_port': {'conf_names': ('mail_port',)}, + 'use_tls': {'conf_names': ('use_tls', 'mail_login')}, + 'login_id': {'conf_names': ('mail_login',)}, + 'email_id': {'conf_names': ('auto_email_id', 'mail_login'), 'default': 'notifications@example.com'}, + 'password': {'conf_names': ('mail_password',)}, + 'always_use_account_email_id_as_sender': + {'conf_names': ('always_use_account_email_id_as_sender',), 'default': 0}, + 'always_use_account_name_as_sender_name': + {'conf_names': ('always_use_account_name_as_sender_name',), 'default': 0}, + 'name': {'conf_names': ('email_sender_name',), 'default': 'Frappe'}, + 'from_site_config': {'default': True} + } + + account_details = {} + for doc_field_name, d in field_to_conf_name_map.items(): + conf_names, default = d.get('conf_names') or [], d.get('default') + value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)] + account_details[doc_field_name] = (value and value[0]) or default + return account_details + + def sendmail_config(self): + return { + 'server': self.smtp_server, + 'port': cint(self.smtp_port), + 'login': getattr(self, "login_id", None) or self.email_id, + 'password': self._password, + 'use_ssl': cint(self.use_ssl_for_outgoing), + 'use_tls': cint(self.use_tls) + } + + def get_smtp_server(self): + config = self.sendmail_config() + return SMTPServer(**config) def handle_incoming_connect_error(self, description): if test_internet(): @@ -642,6 +784,8 @@ class EmailAccount(Document): def send_auto_reply(self, communication, email): """Send auto reply if set.""" + from frappe.core.doctype.communication.email import set_incoming_outgoing_accounts + if self.enable_auto_reply: set_incoming_outgoing_accounts(communication) @@ -653,7 +797,7 @@ class EmailAccount(Document): frappe.sendmail(recipients = [email.from_email], sender = self.email_id, reply_to = communication.incoming_email_account, - subject = _("Re: ") + communication.subject, + subject = " ".join([_("Re:"), communication.subject]), content = render_template(self.auto_reply_message or "", communication.as_dict()) or \ frappe.get_template("templates/emails/auto_reply.html").render(communication.as_dict()), reference_doctype = communication.reference_doctype, diff --git a/frappe/email/doctype/email_domain/test_records.json b/frappe/email/doctype/email_domain/test_records.json index 32bc66e150..a6ccc99f06 100644 --- a/frappe/email/doctype/email_domain/test_records.json +++ b/frappe/email/doctype/email_domain/test_records.json @@ -10,7 +10,8 @@ "incoming_port": "993", "attachment_limit": "1", "smtp_server": "smtp.test.com", - "smtp_port": "587" + "smtp_port": "587", + "password": "password" }, { "doctype": "Email Account", @@ -25,6 +26,7 @@ "incoming_port": "143", "attachment_limit": "1", "smtp_server": "smtp.test.com", - "smtp_port": "587" + "smtp_port": "587", + "password": "password" } ] diff --git a/frappe/email/doctype/email_queue/email_queue.json b/frappe/email/doctype/email_queue/email_queue.json index 4529ea8211..f251786c90 100644 --- a/frappe/email/doctype/email_queue/email_queue.json +++ b/frappe/email/doctype/email_queue/email_queue.json @@ -24,7 +24,8 @@ "unsubscribe_method", "expose_recipients", "attachments", - "retry" + "retry", + "email_account" ], "fields": [ { @@ -139,13 +140,19 @@ "fieldtype": "Int", "label": "Retry", "read_only": 1 + }, + { + "fieldname": "email_account", + "fieldtype": "Link", + "label": "Email Account", + "options": "Email Account" } ], "icon": "fa fa-envelope", "idx": 1, "in_create": 1, "links": [], - "modified": "2020-07-17 15:58:15.369419", + "modified": "2021-04-29 06:33:25.191729", "modified_by": "Administrator", "module": "Email", "name": "Email Queue", diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 267fbdfe9c..076dfc5417 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -2,15 +2,26 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals +import traceback +import json + +from rq.timeouts import JobTimeoutException +import smtplib +import quopri +from email.parser import Parser + import frappe -from frappe import _ +from frappe import _, safe_encode, task from frappe.model.document import Document -from frappe.email.queue import send_one -from frappe.utils import now_datetime - +from frappe.email.queue import get_unsubcribed_url +from frappe.email.email_body import add_attachment +from frappe.utils import cint +from email.policy import SMTPUTF8 +MAX_RETRY_COUNT = 3 class EmailQueue(Document): + DOCTYPE = 'Email Queue' + def set_recipients(self, recipients): self.set("recipients", []) for r in recipients: @@ -30,6 +41,241 @@ class EmailQueue(Document): duplicate.set_recipients(recipients) return duplicate + @classmethod + def find(cls, name): + return frappe.get_doc(cls.DOCTYPE, name) + + def update_db(self, commit=False, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + if commit: + frappe.db.commit() + + def update_status(self, status, commit=False, **kwargs): + self.update_db(status = status, commit = commit, **kwargs) + if self.communication: + communication_doc = frappe.get_doc('Communication', self.communication) + communication_doc.set_delivery_status(commit=commit) + + @property + def cc(self): + return (self.show_as_cc and self.show_as_cc.split(",")) or [] + + @property + def to(self): + return [r.recipient for r in self.recipients if r.recipient not in self.cc] + + @property + def attachments_list(self): + return json.loads(self.attachments) if self.attachments else [] + + def get_email_account(self): + from frappe.email.doctype.email_account.email_account import EmailAccount + + if self.email_account: + return frappe.get_doc('Email Account', self.email_account) + + return EmailAccount.find_outgoing( + match_by_email = self.sender, match_by_doctype = self.reference_doctype) + + def is_to_be_sent(self): + return self.status in ['Not Sent','Partially Sent'] + + def can_send_now(self): + hold_queue = (cint(frappe.defaults.get_defaults().get("hold_queue"))==1) + if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue: + return False + + return True + + def send(self, is_background_task=False): + """ Send emails to recipients. + """ + if not self.can_send_now(): + frappe.db.rollback() + return + + with SendMailContext(self, is_background_task) as ctx: + message = None + for recipient in self.recipients: + if not recipient.is_mail_to_be_sent(): + continue + + message = ctx.build_message(recipient.recipient) + if not frappe.flags.in_test: + ctx.smtp_session.sendmail(recipient.recipient, self.sender, message) + ctx.add_to_sent_list(recipient) + + if frappe.flags.in_test: + frappe.flags.sent_mail = message + return + + if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to: + ctx.email_account_doc.append_email_to_sent_folder(message) + + +@task(queue = 'short') +def send_mail(email_queue_name, is_background_task=False): + """This is equalent to EmqilQueue.send. + + This provides a way to make sending mail as a background job. + """ + record = EmailQueue.find(email_queue_name) + record.send(is_background_task=is_background_task) + +class SendMailContext: + def __init__(self, queue_doc: Document, is_background_task: bool = False): + self.queue_doc = queue_doc + self.is_background_task = is_background_task + self.email_account_doc = queue_doc.get_email_account() + self.smtp_server = self.email_account_doc.get_smtp_server() + self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()] + + def __enter__(self): + self.queue_doc.update_status(status='Sending', commit=True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + exceptions = [ + smtplib.SMTPServerDisconnected, + smtplib.SMTPAuthenticationError, + smtplib.SMTPRecipientsRefused, + smtplib.SMTPConnectError, + smtplib.SMTPHeloError, + JobTimeoutException + ] + + self.smtp_server.quit() + self.log_exception(exc_type, exc_val, exc_tb) + + if exc_type in exceptions: + email_status = (self.sent_to and 'Partially Sent') or 'Not Sent' + self.queue_doc.update_status(status = email_status, commit = True) + elif exc_type: + if self.queue_doc.retry < MAX_RETRY_COUNT: + update_fields = {'status': 'Not Sent', 'retry': self.queue_doc.retry + 1} + else: + update_fields = {'status': (self.sent_to and 'Partially Errored') or 'Error'} + self.queue_doc.update_status(**update_fields, commit = True) + else: + email_status = self.is_mail_sent_to_all() and 'Sent' + email_status = email_status or (self.sent_to and 'Partially Sent') or 'Not Sent' + self.queue_doc.update_status(status = email_status, commit = True) + + def log_exception(self, exc_type, exc_val, exc_tb): + if exc_type: + traceback_string = "".join(traceback.format_tb(exc_tb)) + traceback_string += f"\n Queue Name: {self.queue_doc.name}" + + if self.is_background_task: + frappe.log_error(title = 'frappe.email.queue.flush', message = traceback_string) + else: + frappe.log_error(message = traceback_string) + + @property + def smtp_session(self): + if frappe.flags.in_test: + return + return self.smtp_server.session + + def add_to_sent_list(self, recipient): + # Update recipient status + recipient.update_db(status='Sent', commit=True) + self.sent_to.append(recipient.recipient) + + def is_mail_sent_to_all(self): + return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients]) + + def get_message_object(self, message): + return Parser(policy=SMTPUTF8).parsestr(message) + + def message_placeholder(self, placeholder_key): + map = { + 'tracker': '', + 'unsubscribe_url': '', + 'cc': '', + 'recipient': '', + } + return map.get(placeholder_key) + + def build_message(self, recipient_email): + """Build message specific to the recipient. + """ + message = self.queue_doc.message + if not message: + return "" + + message = message.replace(self.message_placeholder('tracker'), self.get_tracker_str()) + message = message.replace(self.message_placeholder('unsubscribe_url'), + self.get_unsubscribe_str(recipient_email)) + message = message.replace(self.message_placeholder('cc'), self.get_receivers_str()) + message = message.replace(self.message_placeholder('recipient'), + self.get_receipient_str(recipient_email)) + message = self.include_attachments(message) + return message + + def get_tracker_str(self): + tracker_url_html = \ + '' + + message = '' + if frappe.conf.use_ssl and self.queue_doc.track_email_status: + message = quopri.encodestring( + tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode() + ).decode() + return message + + def get_unsubscribe_str(self, recipient_email): + unsubscribe_url = '' + if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype: + doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name + unsubscribe_url = get_unsubcribed_url(doctype, doc_name, recipient_email, + self.queue_doc.unsubscribe_method, self.queue_doc.unsubscribe_param) + + return quopri.encodestring(unsubscribe_url.encode()).decode() + + def get_receivers_str(self): + message = '' + if self.queue_doc.expose_recipients == "footer": + to_str = ', '.join(self.queue_doc.to) + cc_str = ', '.join(self.queue_doc.cc) + message = f"This email was sent to {to_str}" + message = message + f" and copied to {cc_str}" if cc_str else message + return message + + def get_receipient_str(self, recipient_email): + message = '' + if self.queue_doc.expose_recipients != "header": + message = recipient_email + return message + + def include_attachments(self, message): + message_obj = self.get_message_object(message) + attachments = self.queue_doc.attachments_list + + for attachment in attachments: + if attachment.get('fcontent'): + continue + + fid = attachment.get("fid") + if fid: + _file = frappe.get_doc("File", fid) + fcontent = _file.get_content() + attachment.update({ + 'fname': _file.file_name, + 'fcontent': fcontent, + 'parent': message_obj + }) + attachment.pop("fid", None) + add_attachment(**attachment) + + elif attachment.get("print_format_attachment") == 1: + attachment.pop("print_format_attachment", None) + print_format_file = frappe.attach_print(**attachment) + print_format_file.update({"parent": message_obj}) + add_attachment(**print_format_file) + + return safe_encode(message_obj.as_string()) + @frappe.whitelist() def retry_sending(name): doc = frappe.get_doc("Email Queue", name) @@ -42,7 +288,9 @@ def retry_sending(name): @frappe.whitelist() def send_now(name): - send_one(name, now=True) + record = EmailQueue.find(name) + if record: + record.send() def on_doctype_update(): """Add index in `tabCommunication` for `(reference_doctype, reference_name)`""" diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py index 42956a1180..3f07ec58f3 100644 --- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py +++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py @@ -7,4 +7,16 @@ import frappe from frappe.model.document import Document class EmailQueueRecipient(Document): - pass + DOCTYPE = 'Email Queue Recipient' + + def is_mail_to_be_sent(self): + return self.status == 'Not Sent' + + def is_main_sent(self): + return self.status == 'Sent' + + def update_db(self, commit=False, **kwargs): + frappe.db.set_value(self.DOCTYPE, self.name, kwargs) + if commit: + frappe.db.commit() + diff --git a/frappe/email/doctype/notification/notification.json b/frappe/email/doctype/notification/notification.json index c1c877efd4..8b6900a3c9 100644 --- a/frappe/email/doctype/notification/notification.json +++ b/frappe/email/doctype/notification/notification.json @@ -102,7 +102,8 @@ "default": "0", "fieldname": "is_standard", "fieldtype": "Check", - "label": "Is Standard" + "label": "Is Standard", + "no_copy": 1 }, { "depends_on": "is_standard", @@ -281,7 +282,7 @@ "icon": "fa fa-envelope", "index_web_pages_for_search": 1, "links": [], - "modified": "2020-11-24 14:25:43.245677", + "modified": "2021-05-04 11:17:11.882314", "modified_by": "Administrator", "module": "Email", "name": "Notification", diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 87c4b2527a..31d5d9d1cc 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler from frappe.desk.form import assign_to import unittest -test_records = frappe.get_test_records('Notification') - -test_dependencies = ["User"] +test_dependencies = ["User", "Notification"] class TestNotification(unittest.TestCase): def setUp(self): diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 3dcdf00a8e..3b03c42b95 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import frappe, re, os from frappe.utils.pdf import get_pdf -from frappe.email.smtp import get_outgoing_email_account +from frappe.email.doctype.email_account.email_account import EmailAccount from frappe.utils import (get_url, scrub_urls, strip, expand_relative_urls, cint, split_emails, to_markdown, markdown, random_string, parse_addr) import email.utils @@ -75,7 +75,8 @@ class EMail: self.bcc = bcc or [] self.html_set = False - self.email_account = email_account or get_outgoing_email_account(sender=sender) + self.email_account = email_account or \ + EmailAccount.find_outgoing(match_by_email=sender, _raise_error=True) def set_html(self, message, text_content = None, footer=None, print_html=None, formatted=None, inline_images=None, header=None): @@ -249,8 +250,8 @@ class EMail: def get_formatted_html(subject, message, footer=None, print_html=None, email_account=None, header=None, unsubscribe_link=None, sender=None, with_container=False): - if not email_account: - email_account = get_outgoing_email_account(False, sender=sender) + + email_account = email_account or EmailAccount.find_outgoing(match_by_email=sender) signature = None if "" not in message: @@ -291,18 +292,12 @@ def inline_style_in_html(html): ''' Convert email.css and html to inline-styled html ''' from premailer import Premailer + from frappe.utils.jinja_globals import bundled_asset - apps = frappe.get_installed_apps() - - # add frappe email css file - css_files = ['assets/css/email.css'] - if 'frappe' in apps: - apps.remove('frappe') - - for app in apps: - path = 'assets/{0}/css/email.css'.format(app) - css_files.append(path) - + # get email css files from hooks + css_files = frappe.get_hooks('email_css') + css_files = [bundled_asset(path) for path in css_files] + css_files = [path.lstrip('/') for path in css_files] css_files = [css_file for css_file in css_files if os.path.exists(os.path.abspath(css_file))] p = Premailer(html=html, external_styles=css_files, strip_important=False) @@ -480,4 +475,4 @@ def sanitize_email_header(str): return str.replace('\r', '').replace('\n', '') def get_brand_logo(email_account): - return email_account.get('brand_logo') \ No newline at end of file + return email_account.get('brand_logo') diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 2aff04edc9..52c91baf1c 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -7,7 +7,8 @@ import sys from six.moves import html_parser as HTMLParser import smtplib, quopri, json from frappe import msgprint, _, safe_decode, safe_encode, enqueue -from frappe.email.smtp import SMTPServer, get_outgoing_email_account +from frappe.email.smtp import SMTPServer +from frappe.email.doctype.email_account.email_account import EmailAccount from frappe.email.email_body import get_email, get_formatted_html, add_attachment from frappe.utils.verified_command import get_signed_params, verify_request from html2text import html2text @@ -73,7 +74,9 @@ def send(recipients=None, sender=None, subject=None, message=None, text_content= if isinstance(send_after, int): send_after = add_days(nowdate(), send_after) - email_account = get_outgoing_email_account(True, append_to=reference_doctype, sender=sender) + email_account = EmailAccount.find_outgoing( + match_by_doctype=reference_doctype, match_by_email=sender, _raise_error=True) + if not sender or sender == "Administrator": sender = email_account.default_sender @@ -170,19 +173,19 @@ def add(recipients, sender, subject, **kwargs): if not email_queue: email_queue = get_email_queue([r], sender, subject, **kwargs) if kwargs.get('now'): - send_one(email_queue.name, now=True) + email_queue.send() else: duplicate = email_queue.get_duplicate([r]) duplicate.insert(ignore_permissions=True) if kwargs.get('now'): - send_one(duplicate.name, now=True) + duplicate.send() frappe.db.commit() else: email_queue = get_email_queue(recipients, sender, subject, **kwargs) if kwargs.get('now'): - send_one(email_queue.name, now=True) + email_queue.send() def get_email_queue(recipients, sender, subject, **kwargs): '''Make Email Queue object''' @@ -234,6 +237,9 @@ def get_email_queue(recipients, sender, subject, **kwargs): ', '.join(mail.recipients), traceback.format_exc()), 'Email Not Sent') recipients = list(set(recipients + kwargs.get('cc', []) + kwargs.get('bcc', []))) + email_account = kwargs.get('email_account') + email_account_name = email_account and email_account.is_exists_in_db() and email_account.name + e.set_recipients(recipients) e.reference_doctype = kwargs.get('reference_doctype') e.reference_name = kwargs.get('reference_name') @@ -245,8 +251,8 @@ def get_email_queue(recipients, sender, subject, **kwargs): e.send_after = kwargs.get('send_after') e.show_as_cc = ",".join(kwargs.get('cc', [])) e.show_as_bcc = ",".join(kwargs.get('bcc', [])) + e.email_account = email_account_name or None e.insert(ignore_permissions=True) - return e def get_emails_sent_this_month(): @@ -328,44 +334,25 @@ def return_unsubscribed_page(email, doctype, name): indicator_color='green') def flush(from_test=False): - """flush email queue, every time: called from scheduler""" - # additional check - - auto_commit = not from_test + """flush email queue, every time: called from scheduler + """ + from frappe.email.doctype.email_queue.email_queue import send_mail + # To avoid running jobs inside unit tests if frappe.are_emails_muted(): msgprint(_("Emails are muted")) from_test = True - smtpserver_dict = frappe._dict() - - for email in get_queue(): - - if cint(frappe.defaults.get_defaults().get("hold_queue"))==1: - break - - if email.name: - smtpserver = smtpserver_dict.get(email.sender) - if not smtpserver: - smtpserver = SMTPServer() - smtpserver_dict[email.sender] = smtpserver + if cint(frappe.defaults.get_defaults().get("hold_queue"))==1: + return - if from_test: - send_one(email.name, smtpserver, auto_commit) - else: - send_one_args = { - 'email': email.name, - 'smtpserver': smtpserver, - 'auto_commit': auto_commit, - } - enqueue( - method = 'frappe.email.queue.send_one', - queue = 'short', - **send_one_args - ) + for row in get_queue(): + try: + func = send_mail if from_test else send_mail.enqueue + is_background_task = not from_test + func(email_queue_name = row.name, is_background_task = is_background_task) + except Exception: + frappe.log_error() - # NOTE: removing commit here because we pass auto_commit - # finally: - # frappe.db.commit() def get_queue(): return frappe.db.sql('''select name, sender @@ -378,213 +365,6 @@ def get_queue(): by priority desc, creation asc limit 500''', { 'now': now_datetime() }, as_dict=True) - -def send_one(email, smtpserver=None, auto_commit=True, now=False): - '''Send Email Queue with given smtpserver''' - - email = frappe.db.sql('''select - name, status, communication, message, sender, reference_doctype, - reference_name, unsubscribe_param, unsubscribe_method, expose_recipients, - show_as_cc, add_unsubscribe_link, attachments, retry - from - `tabEmail Queue` - where - name=%s - for update''', email, as_dict=True) - - if len(email): - email = email[0] - else: - return - - recipients_list = frappe.db.sql('''select name, recipient, status from - `tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1) - - if frappe.are_emails_muted(): - frappe.msgprint(_("Emails are muted")) - return - - if cint(frappe.defaults.get_defaults().get("hold_queue"))==1 : - return - - if email.status not in ('Not Sent','Partially Sent') : - # rollback to release lock and return - frappe.db.rollback() - return - - frappe.db.sql("""update `tabEmail Queue` set status='Sending', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - email_sent_to_any_recipient = None - - try: - message = None - - if not frappe.flags.in_test: - if not smtpserver: - smtpserver = SMTPServer() - - # to avoid always using default email account for outgoing - if getattr(frappe.local, "outgoing_email_account", None): - frappe.local.outgoing_email_account = {} - - smtpserver.setup_email_account(email.reference_doctype, sender=email.sender) - - for recipient in recipients_list: - if recipient.status != "Not Sent": - continue - - message = prepare_message(email, recipient.recipient, recipients_list) - if not frappe.flags.in_test: - smtpserver.sess.sendmail(email.sender, recipient.recipient, message) - - recipient.status = "Sent" - frappe.db.sql("""update `tabEmail Queue Recipient` set status='Sent', modified=%s where name=%s""", - (now_datetime(), recipient.name), auto_commit=auto_commit) - - email_sent_to_any_recipient = any("Sent" == s.status for s in recipients_list) - - #if all are sent set status - if email_sent_to_any_recipient: - frappe.db.sql("""update `tabEmail Queue` set status='Sent', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - else: - frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s - where name=%s""", ("No recipients to send to", email.name), auto_commit=auto_commit) - if frappe.flags.in_test: - frappe.flags.sent_mail = message - return - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - if smtpserver.append_emails_to_sent_folder and email_sent_to_any_recipient: - smtpserver.email_account.append_email_to_sent_folder(message) - - except (smtplib.SMTPServerDisconnected, - smtplib.SMTPConnectError, - smtplib.SMTPHeloError, - smtplib.SMTPAuthenticationError, - smtplib.SMTPRecipientsRefused, - JobTimeoutException): - - # bad connection/timeout, retry later - - if email_sent_to_any_recipient: - frappe.db.sql("""update `tabEmail Queue` set status='Partially Sent', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - else: - frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - # no need to attempt further - return - - except Exception as e: - frappe.db.rollback() - - if email.retry < 3: - frappe.db.sql("""update `tabEmail Queue` set status='Not Sent', modified=%s, retry=retry+1 where name=%s""", - (now_datetime(), email.name), auto_commit=auto_commit) - else: - if email_sent_to_any_recipient: - frappe.db.sql("""update `tabEmail Queue` set status='Partially Errored', error=%s where name=%s""", - (text_type(e), email.name), auto_commit=auto_commit) - else: - frappe.db.sql("""update `tabEmail Queue` set status='Error', error=%s - where name=%s""", (text_type(e), email.name), auto_commit=auto_commit) - - if email.communication: - frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) - - if now: - print(frappe.get_traceback()) - raise e - - else: - # log to Error Log - frappe.log_error('frappe.email.queue.flush') - -def prepare_message(email, recipient, recipients_list): - message = email.message - if not message: - return "" - - # Parse "Email Account" from "Email Sender" - email_account = get_outgoing_email_account(raise_exception_not_set=False, sender=email.sender) - if frappe.conf.use_ssl and email_account.track_email_status: - # Using SSL => Publically available domain => Email Read Reciept Possible - message = message.replace("", quopri.encodestring(''.format(frappe.local.site, email.communication).encode()).decode()) - else: - # No SSL => No Email Read Reciept - message = message.replace("", quopri.encodestring("".encode()).decode()) - - if email.add_unsubscribe_link and email.reference_doctype: # is missing the check for unsubscribe message but will not add as there will be no unsubscribe url - unsubscribe_url = get_unsubcribed_url(email.reference_doctype, email.reference_name, recipient, - email.unsubscribe_method, email.unsubscribe_params) - message = message.replace("", quopri.encodestring(unsubscribe_url.encode()).decode()) - - if email.expose_recipients == "header": - pass - else: - if email.expose_recipients == "footer": - if isinstance(email.show_as_cc, string_types): - email.show_as_cc = email.show_as_cc.split(",") - email_sent_to = [r.recipient for r in recipients_list] - email_sent_cc = ", ".join([e for e in email_sent_to if e in email.show_as_cc]) - email_sent_to = ", ".join([e for e in email_sent_to if e not in email.show_as_cc]) - - if email_sent_cc: - email_sent_message = _("This email was sent to {0} and copied to {1}").format(email_sent_to,email_sent_cc) - else: - email_sent_message = _("This email was sent to {0}").format(email_sent_to) - message = message.replace("", quopri.encodestring(email_sent_message.encode()).decode()) - - message = message.replace("", recipient) - - message = (message and message.encode('utf8')) or '' - message = safe_decode(message) - - if PY3: - from email.policy import SMTPUTF8 - message = Parser(policy=SMTPUTF8).parsestr(message) - else: - message = Parser().parsestr(message) - - if email.attachments: - # On-demand attachments - - attachments = json.loads(email.attachments) - - for attachment in attachments: - if attachment.get('fcontent'): - continue - - fid = attachment.get("fid") - if fid: - _file = frappe.get_doc("File", fid) - fcontent = _file.get_content() - attachment.update({ - 'fname': _file.file_name, - 'fcontent': fcontent, - 'parent': message - }) - attachment.pop("fid", None) - add_attachment(**attachment) - - elif attachment.get("print_format_attachment") == 1: - attachment.pop("print_format_attachment", None) - print_format_file = frappe.attach_print(**attachment) - print_format_file.update({"parent": message}) - add_attachment(**print_format_file) - - return safe_encode(message.as_string()) - def clear_outbox(days=None): """Remove low priority older than 31 days in Outbox or configured in Log Settings. Note: Used separate query to avoid deadlock diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 949da4a343..6d60007cdb 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -284,7 +284,7 @@ class EmailServer: flags = [] for flag in imaplib.ParseFlags(flag_string) or []: - pattern = re.compile("\w+") + pattern = re.compile(r"\w+") match = re.search(pattern, frappe.as_unicode(flag)) flags.append(match.group(0)) @@ -555,7 +555,7 @@ class Email: def get_thread_id(self): """Extract thread ID from `[]`""" - l = re.findall('(?<=\[)[\w/-]+', self.subject) + l = re.findall(r'(?<=\[)[\w/-]+', self.subject) return l and l[0] or None diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 9ba81fa146..3acb76af23 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -9,11 +9,24 @@ import _socket, sys from frappe import _ from frappe.utils import cint, cstr, parse_addr +CONNECTION_FAILED = _('Could not connect to outgoing email server') +AUTH_ERROR_TITLE = _("Invalid Credentials") +AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.") +SOCKET_ERROR_TITLE = _("Incorrect Configuration") +SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port") +SEND_MAIL_FAILED = _("Unable to send emails at this time") +EMAIL_ACCOUNT_MISSING = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account') + +class InvalidEmailCredentials(frappe.ValidationError): + pass + def send(email, append_to=None, retry=1): """Deprecated: Send the message or add it to Outbox Email""" def _send(retry): + from frappe.email.doctype.email_account.email_account import EmailAccount try: - smtpserver = SMTPServer(append_to=append_to) + email_account = EmailAccount.find_outgoing(match_by_doctype=append_to) + smtpserver = email_account.get_smtp_server() # validate is called in as_string email_body = email.as_string() @@ -34,224 +47,80 @@ def send(email, append_to=None, retry=1): _send(retry) -def get_outgoing_email_account(raise_exception_not_set=True, append_to=None, sender=None): - """Returns outgoing email account based on `append_to` or the default - outgoing account. If default outgoing account is not found, it will - try getting settings from `site_config.json`.""" - - sender_email_id = None - _email_account = None - - if sender: - sender_email_id = parse_addr(sender)[1] - - if not getattr(frappe.local, "outgoing_email_account", None): - frappe.local.outgoing_email_account = {} - - if not (frappe.local.outgoing_email_account.get(append_to) - or frappe.local.outgoing_email_account.get(sender_email_id) - or frappe.local.outgoing_email_account.get("default")): - email_account = None - - if sender_email_id: - # check if the sender has an email account with enable_outgoing - email_account = _get_email_account({"enable_outgoing": 1, - "email_id": sender_email_id}) - - if not email_account and append_to: - # append_to is only valid when enable_incoming is checked - email_accounts = frappe.db.get_values("Email Account", { - "enable_outgoing": 1, - "enable_incoming": 1, - "append_to": append_to, - }, cache=True) - - if email_accounts: - _email_account = email_accounts[0] - - else: - email_account = _get_email_account({ - "enable_outgoing": 1, - "enable_incoming": 1, - "append_to": append_to - }) - - if not email_account: - # sender don't have the outging email account - sender_email_id = None - email_account = get_default_outgoing_email_account(raise_exception_not_set=raise_exception_not_set) - - if not email_account and _email_account: - # if default email account is not configured then setup first email account based on append to - email_account = _email_account - - if not email_account and raise_exception_not_set and cint(frappe.db.get_single_value('System Settings', 'setup_complete')): - frappe.throw(_("Please setup default Email Account from Setup > Email > Email Account"), - frappe.OutgoingEmailError) - - if email_account: - if email_account.enable_outgoing and not getattr(email_account, 'from_site_config', False): - raise_exception = True - if email_account.smtp_server in ['localhost','127.0.0.1'] or email_account.no_smtp_authentication: - raise_exception = False - email_account.password = email_account.get_password(raise_exception=raise_exception) - email_account.default_sender = email.utils.formataddr((email_account.name, email_account.get("email_id"))) - - frappe.local.outgoing_email_account[append_to or sender_email_id or "default"] = email_account - - return frappe.local.outgoing_email_account.get(append_to) \ - or frappe.local.outgoing_email_account.get(sender_email_id) \ - or frappe.local.outgoing_email_account.get("default") - -def get_default_outgoing_email_account(raise_exception_not_set=True): - '''conf should be like: - { - "mail_server": "smtp.example.com", - "mail_port": 587, - "use_tls": 1, - "mail_login": "emails@example.com", - "mail_password": "Super.Secret.Password", - "auto_email_id": "emails@example.com", - "email_sender_name": "Example Notifications", - "always_use_account_email_id_as_sender": 0, - "always_use_account_name_as_sender_name": 0 - } - ''' - email_account = _get_email_account({"enable_outgoing": 1, "default_outgoing": 1}) - if email_account: - email_account.password = email_account.get_password(raise_exception=False) - - if not email_account and frappe.conf.get("mail_server"): - # from site_config.json - email_account = frappe.new_doc("Email Account") - email_account.update({ - "smtp_server": frappe.conf.get("mail_server"), - "smtp_port": frappe.conf.get("mail_port"), - - # legacy: use_ssl was used in site_config instead of use_tls, but meant the same thing - "use_tls": cint(frappe.conf.get("use_tls") or 0) or cint(frappe.conf.get("use_ssl") or 0), - "login_id": frappe.conf.get("mail_login"), - "email_id": frappe.conf.get("auto_email_id") or frappe.conf.get("mail_login") or 'notifications@example.com', - "password": frappe.conf.get("mail_password"), - "always_use_account_email_id_as_sender": frappe.conf.get("always_use_account_email_id_as_sender", 0), - "always_use_account_name_as_sender_name": frappe.conf.get("always_use_account_name_as_sender_name", 0) - }) - email_account.from_site_config = True - email_account.name = frappe.conf.get("email_sender_name") or "Frappe" - - if not email_account and not raise_exception_not_set: - return None - - if frappe.are_emails_muted(): - # create a stub - email_account = frappe.new_doc("Email Account") - email_account.update({ - "email_id": "notifications@example.com" - }) - - return email_account - -def _get_email_account(filters): - name = frappe.db.get_value("Email Account", filters) - return frappe.get_doc("Email Account", name) if name else None - class SMTPServer: - def __init__(self, login=None, password=None, server=None, port=None, use_tls=None, use_ssl=None, append_to=None): - # get defaults from mail settings - - self._sess = None - self.email_account = None - self.server = None - self.append_emails_to_sent_folder = None + def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None): + self.login = login + self.password = password + self._server = server + self._port = port + self.use_tls = use_tls + self.use_ssl = use_ssl + self._session = None + + if not self.server: + frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError) - if server: - self.server = server - self.port = port - self.use_tls = cint(use_tls) - self.use_ssl = cint(use_ssl) - self.login = login - self.password = password + @property + def port(self): + port = self._port or (self.use_ssl and 465) or (self.use_tls and 587) + return cint(port) - else: - self.setup_email_account(append_to) + @property + def server(self): + return cstr(self._server or "") - def setup_email_account(self, append_to=None, sender=None): - self.email_account = get_outgoing_email_account(raise_exception_not_set=False, append_to=append_to, sender=sender) - if self.email_account: - self.server = self.email_account.smtp_server - self.login = (getattr(self.email_account, "login_id", None) or self.email_account.email_id) - if not self.email_account.no_smtp_authentication: - if self.email_account.ascii_encode_password: - self.password = frappe.safe_encode(self.email_account.password, 'ascii') - else: - self.password = self.email_account.password - else: - self.password = None - self.port = self.email_account.smtp_port - self.use_tls = self.email_account.use_tls - self.sender = self.email_account.email_id - self.use_ssl = self.email_account.use_ssl_for_outgoing - self.append_emails_to_sent_folder = self.email_account.append_emails_to_sent_folder - self.always_use_account_email_id_as_sender = cint(self.email_account.get("always_use_account_email_id_as_sender")) - self.always_use_account_name_as_sender_name = cint(self.email_account.get("always_use_account_name_as_sender_name")) + def secure_session(self, conn): + """Secure the connection incase of TLS. + """ + if self.use_tls: + conn.ehlo() + conn.starttls() + conn.ehlo() @property - def sess(self): - """get session""" - if self._sess: - return self._sess + def session(self): + if self.is_session_active(): + return self._session - # check if email server specified - if not getattr(self, 'server'): - err_msg = _('Email Account not setup. Please create a new Email Account from Setup > Email > Email Account') - frappe.msgprint(err_msg) - raise frappe.OutgoingEmailError(err_msg) + SMTP = smtplib.SMTP_SSL if self.use_ssl else smtplib.SMTP try: - if self.use_ssl: - if not self.port: - self.port = 465 - - self._sess = smtplib.SMTP_SSL((self.server or ""), cint(self.port)) - else: - if self.use_tls and not self.port: - self.port = 587 - - self._sess = smtplib.SMTP(cstr(self.server or ""), - cint(self.port) or None) - - if not self._sess: - err_msg = _('Could not connect to outgoing email server') - frappe.msgprint(err_msg) - raise frappe.OutgoingEmailError(err_msg) - - if self.use_tls: - self._sess.ehlo() - self._sess.starttls() - self._sess.ehlo() + self._session = SMTP(self.server, self.port) + if not self._session: + frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError) + self.secure_session(self._session) if self.login and self.password: - ret = self._sess.login(str(self.login or ""), str(self.password or "")) + res = self._session.login(str(self.login or ""), str(self.password or "")) # check if logged correctly - if ret[0]!=235: - frappe.msgprint(ret[1]) - raise frappe.OutgoingEmailError(ret[1]) + if res[0]!=235: + frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError) - return self._sess + return self._session except smtplib.SMTPAuthenticationError as e: - from frappe.email.doctype.email_account.email_account import EmailAccount - EmailAccount.throw_invalid_credentials_exception() + self.throw_invalid_credentials_exception() except _socket.error as e: # Invalid mail server -- due to refusing connection - frappe.throw( - _("Invalid Outgoing Mail Server or Port"), - exc=frappe.ValidationError, - title=_("Incorrect Configuration") - ) + frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE) except smtplib.SMTPException: - frappe.msgprint(_('Unable to send emails at this time')) + frappe.msgprint(SEND_MAIL_FAILED) raise + + def is_session_active(self): + if self._session: + try: + return self._session.noop()[0] == 250 + except Exception: + return False + + def quit(self): + if self.is_session_active(): + self._session.quit() + + @classmethod + def throw_invalid_credentials_exception(cls): + frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials) diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index 3fcabb9495..33668cddba 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -7,10 +7,10 @@ from frappe import safe_decode from frappe.email.receive import Email from frappe.email.email_body import (replace_filename_with_cid, get_email, inline_style_in_html, get_header) -from frappe.email.queue import prepare_message, get_email_queue +from frappe.email.queue import get_email_queue +from frappe.email.doctype.email_queue.email_queue import SendMailContext from six import PY3 - class TestEmailBody(unittest.TestCase): def setUp(self): email_html = ''' @@ -57,7 +57,8 @@ This is the text version of this email content='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', formatted='

' + uni_chr1 + 'abcd' + uni_chr2 + '

', text_content='whatever') - result = prepare_message(email=email, recipient='test@test.com', recipients_list=[]) + mail_ctx = SendMailContext(queue_doc = email) + result = mail_ctx.build_message(recipient_email = 'test@test.com') self.assertTrue(b"

=EA=80=80abcd=DE=B4

" in result) def test_prepare_message_returns_cr_lf(self): @@ -68,8 +69,10 @@ This is the text version of this email content='

\n this is a test of newlines\n' + '

', formatted='

\n this is a test of newlines\n' + '

', text_content='whatever') - result = safe_decode(prepare_message(email=email, - recipient='test@test.com', recipients_list=[])) + + mail_ctx = SendMailContext(queue_doc = email) + result = safe_decode(mail_ctx.build_message(recipient_email='test@test.com')) + if PY3: self.assertTrue(result.count('\n') == result.count("\r")) else: diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py index 0b11c559a2..58e4fdd8a6 100644 --- a/frappe/email/test_smtp.py +++ b/frappe/email/test_smtp.py @@ -4,7 +4,7 @@ import unittest import frappe from frappe.email.smtp import SMTPServer -from frappe.email.smtp import get_outgoing_email_account +from frappe.email.doctype.email_account.email_account import EmailAccount class TestSMTP(unittest.TestCase): def test_smtp_ssl_session(self): @@ -33,13 +33,13 @@ class TestSMTP(unittest.TestCase): frappe.local.outgoing_email_account = {} # lowest preference given to email account with default incoming enabled - create_email_account(email_id="default_outgoing_enabled@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1) - self.assertEqual(get_outgoing_email_account().email_id, "default_outgoing_enabled@gmail.com") + create_email_account(email_id="default_outgoing_enabled@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1) + self.assertEqual(EmailAccount.find_outgoing().email_id, "default_outgoing_enabled@gmail.com") frappe.local.outgoing_email_account = {} # highest preference given to email account with append_to matching - create_email_account(email_id="append_to@gmail.com", password="***", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post") - self.assertEqual(get_outgoing_email_account(append_to="Blog Post").email_id, "append_to@gmail.com") + create_email_account(email_id="append_to@gmail.com", password="password", enable_outgoing = 1, default_outgoing=1, append_to="Blog Post") + self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Blog Post").email_id, "append_to@gmail.com") # add back the mail_server frappe.conf['mail_server'] = mail_server @@ -75,4 +75,4 @@ def make_server(port, ssl, tls): use_tls = tls ) - server.sess + server.session diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index 3d97583549..4836276734 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -55,8 +55,8 @@ class EventProducer(Document): self.reload() def check_url(self): - if not frappe.utils.validate_url(self.producer_url): - frappe.throw(_('Invalid URL')) + valid_url_schemes = ("http", "https") + frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes) # remove '/' from the end of the url like http://test_site.com/ # to prevent mismatch in get_url() results diff --git a/frappe/handler.py b/frappe/handler.py index a38feb90fa..b622667e18 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -228,10 +228,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): is_whitelisted(fn) is_valid_http_method(fn) - try: - fnargs = inspect.getargspec(method_obj)[0] - except ValueError: - fnargs = inspect.getfullargspec(method_obj).args + fnargs = inspect.getfullargspec(method_obj).args if not fnargs or (len(fnargs)==1 and fnargs[0]=="self"): response = doc.run_method(method) diff --git a/frappe/hooks.py b/frappe/hooks.py index 1c78d47755..d0968ce051 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -29,16 +29,16 @@ page_js = { # website app_include_js = [ - "/assets/js/libs.min.js", - "/assets/js/desk.min.js", - "/assets/js/list.min.js", - "/assets/js/form.min.js", - "/assets/js/control.min.js", - "/assets/js/report.min.js", + "libs.bundle.js", + "desk.bundle.js", + "list.bundle.js", + "form.bundle.js", + "controls.bundle.js", + "report.bundle.js", ] app_include_css = [ - "/assets/css/desk.min.css", - "/assets/css/report.min.css", + "desk.bundle.css", + "report.bundle.css", ] doctype_js = { @@ -52,6 +52,8 @@ web_include_js = [ web_include_css = [] +email_css = ['email.bundle.css'] + website_route_rules = [ {"from_route": "/blog/", "to_route": "Blog Post"}, {"from_route": "/kb/", "to_route": "Help Article"}, @@ -226,7 +228,6 @@ scheduler_events = { "frappe.desk.doctype.event.event.send_event_digest", "frappe.sessions.clear_expired_sessions", "frappe.email.doctype.notification.notification.trigger_daily_alerts", - "frappe.realtime.remove_old_task_logs", "frappe.utils.scheduler.restrict_scheduler_events_if_dormant", "frappe.email.doctype.auto_email_report.auto_email_report.send_daily", "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record", diff --git a/frappe/installer.py b/frappe/installer.py index 0cd5b136ae..d7d885d60e 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None): def make_site_dirs(): - site_public_path = os.path.join(frappe.local.site_path, 'public') - site_private_path = os.path.join(frappe.local.site_path, 'private') - for dir_path in ( - os.path.join(site_private_path, 'backups'), - os.path.join(site_public_path, 'files'), - os.path.join(site_private_path, 'files'), - os.path.join(frappe.local.site_path, 'logs'), - os.path.join(frappe.local.site_path, 'task-logs')): - if not os.path.exists(dir_path): - os.makedirs(dir_path) - locks_dir = frappe.get_site_path('locks') - if not os.path.exists(locks_dir): - os.makedirs(locks_dir) + for dir_path in [ + os.path.join("public", "files"), + os.path.join("private", "backups"), + os.path.join("private", "files"), + "error-snapshots", + "locks", + "logs", + ]: + path = frappe.get_site_path(dir_path) + os.makedirs(path, exist_ok=True) def add_module_defs(app): diff --git a/frappe/integrations/doctype/connected_app/connected_app.json b/frappe/integrations/doctype/connected_app/connected_app.json index e5dbb0472a..b5330f4d4f 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.json +++ b/frappe/integrations/doctype/connected_app/connected_app.json @@ -54,7 +54,8 @@ "fieldname": "client_id", "fieldtype": "Data", "in_list_view": 1, - "label": "Client Id" + "label": "Client Id", + "mandatory_depends_on": "eval:doc.redirect_uri" }, { "fieldname": "redirect_uri", @@ -96,12 +97,14 @@ { "fieldname": "authorization_uri", "fieldtype": "Data", - "label": "Authorization URI" + "label": "Authorization URI", + "mandatory_depends_on": "eval:doc.redirect_uri" }, { "fieldname": "token_uri", "fieldtype": "Data", - "label": "Token URI" + "label": "Token URI", + "mandatory_depends_on": "eval:doc.redirect_uri" }, { "fieldname": "revocation_uri", @@ -136,7 +139,7 @@ "link_fieldname": "connected_app" } ], - "modified": "2020-11-16 16:29:50.277405", + "modified": "2021-05-10 05:03:06.296863", "modified_by": "Administrator", "module": "Integrations", "name": "Connected App", diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py index 95077ece77..449e30f6d0 100644 --- a/frappe/integrations/doctype/connected_app/connected_app.py +++ b/frappe/integrations/doctype/connected_app/connected_app.py @@ -26,20 +26,27 @@ class ConnectedApp(Document): self.redirect_uri = urljoin(base_url, callback_path) def get_oauth2_session(self, user=None, init=False): + """Return an auto-refreshing OAuth2 session which is an extension of a requests.Session()""" token = None token_updater = None + auto_refresh_kwargs = None if not init: user = user or frappe.session.user token_cache = self.get_user_token(user) token = token_cache.get_json() token_updater = token_cache.update_data + auto_refresh_kwargs = {'client_id': self.client_id} + client_secret = self.get_password('client_secret') + if client_secret: + auto_refresh_kwargs['client_secret'] = client_secret return OAuth2Session( client_id=self.client_id, token=token, token_updater=token_updater, auto_refresh_url=self.token_uri, + auto_refresh_kwargs=auto_refresh_kwargs, redirect_uri=self.redirect_uri, scope=self.get_scopes() ) diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py index 2ce99d8aa3..2b227f503d 100644 --- a/frappe/integrations/oauth2.py +++ b/frappe/integrations/oauth2.py @@ -1,6 +1,5 @@ import json from urllib.parse import quote, urlencode - from oauthlib.oauth2 import FatalClientError, OAuth2Error from oauthlib.openid.connect.core.endpoints.pre_configured import ( Server as WebApplicationServer, diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index af06696621..205b451336 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -71,7 +71,8 @@ numeric_fieldtypes = ( data_field_options = ( 'Email', 'Name', - 'Phone' + 'Phone', + 'URL' ) default_fields = ( diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 983511f7a4..54d77ba988 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -34,8 +34,9 @@ def get_controller(doctype): from frappe.model.document import Document from frappe.utils.nestedset import NestedSet - module_name, custom = frappe.db.get_value("DocType", doctype, ("module", "custom"), cache=True) \ - or ["Core", False] + module_name, custom = frappe.db.get_value( + "DocType", doctype, ("module", "custom"), cache=True + ) or ["Core", False] if custom: if frappe.db.field_exists("DocType", "is_tree"): @@ -666,6 +667,12 @@ class BaseDocument(object): if data_field_options == "Phone": frappe.utils.validate_phone_number(data, throw=True) + if data_field_options == "URL": + if not data: + continue + + frappe.utils.validate_url(data, throw=True) + def _validate_constants(self): if frappe.flags.in_import or self.is_new() or self.flags.ignore_validate_constants: return @@ -863,7 +870,7 @@ class BaseDocument(object): from frappe.model.meta import get_default_df df = get_default_df(fieldname) - if not currency: + if not currency and df: currency = self.get(df.get("options")) if not frappe.db.exists('Currency', currency, cache=True): currency = None diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 1c863a1577..e0c3406c46 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -465,7 +465,7 @@ class DatabaseQuery(object): elif f.operator.lower() in ('in', 'not in'): values = f.value or '' - if isinstance(values, frappe.string_types): + if isinstance(values, str): values = values.split(",") fallback = "''" diff --git a/frappe/model/document.py b/frappe/model/document.py index 4169919091..623916597e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1347,6 +1347,22 @@ class Document(BaseDocument): from frappe.desk.doctype.tag.tag import DocTags return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def __repr__(self): + name = self.name or "unsaved" + doctype = self.__class__.__name__ + + docstatus = f" docstatus={self.docstatus}" if self.docstatus else "" + parent = f" parent={self.parent}" if self.parent else "" + + return f"<{doctype}: {name}{docstatus}{parent}>" + + def __str__(self): + name = self.name or "unsaved" + doctype = self.__class__.__name__ + + return f"{doctype}({name})" + + def execute_action(doctype, name, action, **kwargs): """Execute an action on a document (called by background worker)""" doc = frappe.get_doc(doctype, name) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 7f58c28397..66e8b08d92 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -118,7 +118,7 @@ class Meta(Document): # non standard list object, skip continue - if (isinstance(value, (frappe.text_type, int, float, datetime, list, tuple)) + if (isinstance(value, (str, int, float, datetime, list, tuple)) or (not no_nulls and value is None)): out[key] = value diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 1a3f90da37..b8d6a6f8d7 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -199,10 +199,39 @@ def getseries(key, digits): def revert_series_if_last(key, name, doc=None): - if ".#" in key: + """ + Reverts the series for particular naming series: + * key is naming series - SINV-.YYYY-.#### + * name is actual name - SINV-2021-0001 + + 1. This function split the key into two parts prefix (SINV-YYYY) & hashes (####). + 2. Use prefix to get the current index of that naming series from Series table + 3. Then revert the current index. + + *For custom naming series:* + 1. hash can exist anywhere, if it exist in hashes then it take normal flow. + 2. If hash doesn't exit in hashes, we get the hash from prefix, then update name and prefix accordingly. + + *Example:* + 1. key = SINV-.YYYY.- + * If key doesn't have hash it will add hash at the end + * prefix will be SINV-YYYY based on this will get current index from Series table. + 2. key = SINV-.####.-2021 + * now prefix = SINV-#### and hashes = 2021 (hash doesn't exist) + * will search hash in key then accordingly get prefix = SINV- + 3. key = ####.-2021 + * prefix = #### and hashes = 2021 (hash doesn't exist) + * will search hash in key then accordingly get prefix = "" + """ + if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: - return + # get the hash part from the key + hash = re.search("#+", key) + if not hash: + return + name = name.replace(hashes, "") + prefix = prefix.replace(hash.group(), "") else: prefix = key @@ -254,7 +283,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" filters.update({fieldname: value}) exists = frappe.db.exists(doctype, filters) - regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator) + regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator) if exists: last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}` diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 5970eae5ca..fdfd00404c 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -107,6 +107,15 @@ def import_doc(docdict, force=False, data_import=False, pre_process=None, doc = frappe.get_doc(docdict) + # Note on Tree DocTypes: + # The tree structure is maintained in the database via the fields "lft" and + # "rgt". They are automatically set and kept up-to-date. Importing them + # would destroy any existing tree structure. + if getattr(doc.meta, 'is_tree', None) and any([doc.lft, doc.rgt]): + print('Ignoring values of `lft` and `rgt` for {} "{}"'.format(doc.doctype, doc.name)) + doc.lft = None + doc.rgt = None + doc.run_method("before_import") doc.flags.ignore_version = ignore_version diff --git a/frappe/oauth.py b/frappe/oauth.py index 3287bf7520..a4c66bf3f2 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -4,11 +4,9 @@ import hashlib import re from http import cookies from urllib.parse import unquote, urlparse - import jwt import pytz from oauthlib.openid import RequestValidator - import frappe from frappe.auth import LoginManager @@ -599,9 +597,10 @@ def get_client_scopes(client_id): def get_userinfo(user): picture = None frappe_server_url = get_server_url() + valid_url_schemes = ("http", "https", "ftp", "ftps") if user.user_image: - if frappe.utils.validate_url(user.user_image): + if frappe.utils.validate_url(user.user_image, valid_schemes=valid_url_schemes): picture = user.user_image else: picture = frappe_server_url + "/" + user.user_image diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py new file mode 100644 index 0000000000..1dbb24f191 --- /dev/null +++ b/frappe/parallel_test_runner.py @@ -0,0 +1,282 @@ +import json +import os +import re +import sys +import time +import unittest +import click +import frappe +import requests + +from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config) + +click_ctx = click.get_current_context(True) +if click_ctx: + click_ctx.color = True + +class ParallelTestRunner(): + def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False): + self.app = app + self.site = site + self.with_coverage = with_coverage + self.build_number = frappe.utils.cint(build_number) or 1 + self.total_builds = frappe.utils.cint(total_builds) + self.setup_test_site() + self.run_tests() + + def setup_test_site(self): + frappe.init(site=self.site) + if not frappe.db: + frappe.connect() + + frappe.flags.in_test = True + frappe.clear_cache() + frappe.utils.scheduler.disable_scheduler() + set_test_email_config() + self.before_test_setup() + + def before_test_setup(self): + start_time = time.time() + for fn in frappe.get_hooks("before_tests", app_name=self.app): + frappe.get_attr(fn)() + + test_module = frappe.get_module(f'{self.app}.tests') + + if hasattr(test_module, "global_test_dependencies"): + for doctype in test_module.global_test_dependencies: + make_test_records(doctype) + + elapsed = time.time() - start_time + elapsed = click.style(f' ({elapsed:.03}s)', fg='red') + click.echo(f'Before Test {elapsed}') + + def run_tests(self): + self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2) + + self.start_coverage() + + for test_file_info in self.get_test_file_list(): + self.run_tests_for_file(test_file_info) + + self.save_coverage() + self.print_result() + + def run_tests_for_file(self, file_info): + if not file_info: return + + frappe.set_user('Administrator') + path, filename = file_info + module = self.get_module(path, filename) + self.create_test_dependency_records(module, path, filename) + test_suite = unittest.TestSuite() + module_test_cases = unittest.TestLoader().loadTestsFromModule(module) + test_suite.addTest(module_test_cases) + test_suite(self.test_result) + + def create_test_dependency_records(self, module, path, filename): + if hasattr(module, "test_dependencies"): + for doctype in module.test_dependencies: + make_test_records(doctype) + + if os.path.basename(os.path.dirname(path)) == "doctype": + # test_data_migration_connector.py > data_migration_connector.json + test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json") + test_record_file_path = os.path.join(path, test_record_filename) + if os.path.exists(test_record_file_path): + with open(test_record_file_path, 'r') as f: + doc = json.loads(f.read()) + doctype = doc["name"] + make_test_records(doctype) + + def get_module(self, path, filename): + app_path = frappe.get_pymodule_path(self.app) + relative_path = os.path.relpath(path, app_path) + if relative_path == '.': + module_name = self.app + else: + relative_path = relative_path.replace('/', '.') + module_name = os.path.splitext(filename)[0] + module_name = f'{self.app}.{relative_path}.{module_name}' + + return frappe.get_module(module_name) + + def print_result(self): + self.test_result.printErrors() + click.echo(self.test_result) + if self.test_result.failures or self.test_result.errors: + if os.environ.get('CI'): + sys.exit(1) + + def start_coverage(self): + if self.with_coverage: + from coverage import Coverage + from frappe.utils import get_bench_path + + # Generate coverage report only for app that is being tested + source_path = os.path.join(get_bench_path(), 'apps', self.app) + omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss', + '*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*'] + + if self.app == 'frappe': + omit.append('*/commands/*') + + self.coverage = Coverage(source=[source_path], omit=omit) + self.coverage.start() + + def save_coverage(self): + if not self.with_coverage: + return + self.coverage.stop() + self.coverage.save() + + def get_test_file_list(self): + test_list = get_all_tests(self.app) + split_size = frappe.utils.ceil(len(test_list) / self.total_builds) + # [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2 + test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)] + return test_chunks[self.build_number - 1] + + +class ParallelTestResult(unittest.TextTestResult): + def startTest(self, test): + self._started_at = time.time() + super(unittest.TextTestResult, self).startTest(test) + test_class = unittest.util.strclass(test.__class__) + if not hasattr(self, 'current_test_class') or self.current_test_class != test_class: + click.echo(f"\n{unittest.util.strclass(test.__class__)}") + self.current_test_class = test_class + + def getTestMethodName(self, test): + return test._testMethodName if hasattr(test, '_testMethodName') else str(test) + + def addSuccess(self, test): + super(unittest.TextTestResult, self).addSuccess(test) + elapsed = time.time() - self._started_at + threshold_passed = elapsed >= SLOW_TEST_THRESHOLD + elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else '' + click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}") + + def addError(self, test, err): + super(unittest.TextTestResult, self).addError(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addFailure(self, test, err): + super(unittest.TextTestResult, self).addFailure(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addSkip(self, test, reason): + super(unittest.TextTestResult, self).addSkip(test, reason) + click.echo(f" {click.style(' = ', fg='white')} {self.getTestMethodName(test)}") + + def addExpectedFailure(self, test, err): + super(unittest.TextTestResult, self).addExpectedFailure(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addUnexpectedSuccess(self, test): + super(unittest.TextTestResult, self).addUnexpectedSuccess(test) + click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}") + + def printErrors(self): + click.echo('\n') + self.printErrorList(' ERROR ', self.errors, 'red') + self.printErrorList(' FAIL ', self.failures, 'red') + + def printErrorList(self, flavour, errors, color): + for test, err in errors: + click.echo(self.separator1) + click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}") + click.echo(self.separator2) + click.echo(err) + + def __str__(self): + return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}" + +def get_all_tests(app): + test_file_list = [] + for path, folders, files in os.walk(frappe.get_pymodule_path(app)): + for dontwalk in ('locals', '.git', 'public', '__pycache__'): + if dontwalk in folders: + folders.remove(dontwalk) + + # for predictability + folders.sort() + files.sort() + + if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: + # in /doctype/doctype/boilerplate/ + continue + + for filename in files: + if filename.startswith("test_") and filename.endswith(".py") \ + and filename != 'test_runner.py': + test_file_list.append([path, filename]) + + return test_file_list + + +class ParallelTestWithOrchestrator(ParallelTestRunner): + ''' + This can be used to balance-out test time across multiple instances + This is dependent on external orchestrator which returns next test to run + + orchestrator endpoints + - register-instance (, , test_spec_list) + - get-next-test-spec (, ) + - test-completed (, ) + ''' + def __init__(self, app, site, with_coverage=False): + self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL') + if not self.orchestrator_url: + click.echo('ORCHESTRATOR_URL environment variable not found!') + click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator') + sys.exit(1) + + self.ci_build_id = os.environ.get('CI_BUILD_ID') + self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10) + if not self.ci_build_id: + click.echo('CI_BUILD_ID environment variable not found!') + sys.exit(1) + + ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage) + + def run_tests(self): + self.test_status = 'ongoing' + self.register_instance() + super().run_tests() + + def get_test_file_list(self): + while self.test_status == 'ongoing': + yield self.get_next_test() + + def register_instance(self): + test_spec_list = get_all_tests(self.app) + response_data = self.call_orchestrator('register-instance', data={ + 'test_spec_list': test_spec_list + }) + self.is_master = response_data.get('is_master') + + def get_next_test(self): + response_data = self.call_orchestrator('get-next-test-spec') + self.test_status = response_data.get('status') + return response_data.get('next_test') + + def print_result(self): + self.call_orchestrator('test-completed') + return super().print_result() + + def call_orchestrator(self, endpoint, data={}): + # add repo token header + # build id in header + headers = { + 'CI-BUILD-ID': self.ci_build_id, + 'CI-INSTANCE-ID': self.ci_instance_id, + 'REPO-TOKEN': '2948288382838DE' + } + url = f'{self.orchestrator_url}/{endpoint}' + res = requests.get(url, json=data, headers=headers) + res.raise_for_status() + response_data = {} + if 'application/json' in res.headers.get('content-type'): + response_data = res.json() + + return response_data diff --git a/frappe/patches.txt b/frappe/patches.txt index 60c3112f4a..e70be0a37b 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -336,3 +336,4 @@ frappe.patches.v13_0.remove_twilio_settings frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook +frappe.patches.v13_0.update_notification_channel_if_empty diff --git a/frappe/patches/v13_0/update_notification_channel_if_empty.py b/frappe/patches/v13_0/update_notification_channel_if_empty.py new file mode 100644 index 0000000000..2c2a40e81b --- /dev/null +++ b/frappe/patches/v13_0/update_notification_channel_if_empty.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + +def execute(): + + frappe.reload_doc("Email", "doctype", "Notification") + + notifications = frappe.get_all('Notification', {'is_standard': 1}, {'name', 'channel'}) + for notification in notifications: + if not notification.channel: + frappe.db.set_value("Notification", notification.name, "channel", "Email", update_modified=False) + frappe.db.commit() diff --git a/frappe/patches/v5_0/fix_text_editor_file_urls.py b/frappe/patches/v5_0/fix_text_editor_file_urls.py index d91aad0234..a6d7d2fb9a 100644 --- a/frappe/patches/v5_0/fix_text_editor_file_urls.py +++ b/frappe/patches/v5_0/fix_text_editor_file_urls.py @@ -33,8 +33,7 @@ def execute(): def scrub_relative_urls(html): """prepend a slash before a relative url""" try: - return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html) - # return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html) + return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html) except: print("Error", html) raise diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py index 7e30bda23e..121916ae5f 100644 --- a/frappe/printing/doctype/print_format/test_print_format.py +++ b/frappe/printing/doctype/print_format/test_print_format.py @@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase): def test_print_user(self, style=None): print_html = frappe.get_print("User", "Administrator", style=style) self.assertTrue("" in print_html) - self.assertTrue(re.findall('
[\s]*administrator[\s]*
', print_html)) + self.assertTrue(re.findall(r'
[\s]*administrator[\s]*
', print_html)) return print_html def test_print_user_standard(self): print_html = self.test_print_user("Standard") - self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html)) - self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html)) + self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html)) + self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html)) self.assertFalse("font-family: serif;" in print_html) def test_print_user_modern(self): diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index dfd93c4efa..233bbe0ce7 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -408,14 +408,17 @@ frappe.ui.form.PrintView = class { setup_print_format_dom(out, $print_format) { this.print_wrapper.find('.print-format-skeleton').remove(); + let base_url = frappe.urllib.get_base_url(); + let print_css = frappe.assets.bundled_asset('print.bundle.css'); this.$print_format_body.find('head').html( ` - ` + ` ); if (frappe.utils.is_rtl(this.lang_code)) { + let rtl_css = frappe.assets.bundled_asset('frappe-rtl.bundle.css'); this.$print_format_body.find('head').append( - `` + `` ); } diff --git a/frappe/printing/page/print_format_builder/print_format_builder.js b/frappe/printing/page/print_format_builder/print_format_builder.js index 7e58e295b5..ca2a8bc378 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder.js +++ b/frappe/printing/page/print_format_builder/print_format_builder.js @@ -23,13 +23,13 @@ frappe.pages['print-format-builder'].on_page_show = function(wrapper) { } } -frappe.PrintFormatBuilder = Class.extend({ - init: function(parent) { +frappe.PrintFormatBuilder = class PrintFormatBuilder { + constructor(parent) { this.parent = parent; this.make(); this.refresh(); - }, - refresh: function() { + } + refresh() { this.custom_html_count = 0; if(!this.print_format) { this.show_start(); @@ -37,8 +37,8 @@ frappe.PrintFormatBuilder = Class.extend({ this.page.set_title(this.print_format.name); this.setup_print_format(); } - }, - make: function() { + } + make() { this.page = frappe.ui.make_app_page({ parent: this.parent, title: __("Print Format Builder"), @@ -56,15 +56,15 @@ frappe.PrintFormatBuilder = Class.extend({ this.setup_edit_custom_html(); // $(this.page.sidebar).css({"position": 'fixed'}); // $(this.page.main).parent().css({"margin-left": '16.67%'}); - }, - show_start: function() { + } + show_start() { this.page.main.html(frappe.render_template("print_format_builder_start", {})); this.page.clear_actions(); this.page.set_title(__("Print Format Builder")); this.start_edit_print_format(); this.start_new_print_format(); - }, - start_edit_print_format: function() { + } + start_edit_print_format() { // print format control var me = this; this.print_format_input = frappe.ui.form.make_control({ @@ -89,8 +89,8 @@ frappe.PrintFormatBuilder = Class.extend({ frappe.set_route('print-format-builder', name); }); }); - }, - start_new_print_format: function() { + } + start_new_print_format() { var me = this; this.doctype_input = frappe.ui.form.make_control({ parent: this.page.main.find(".doctype-selector"), @@ -125,8 +125,8 @@ frappe.PrintFormatBuilder = Class.extend({ me.setup_new_print_format(doctype, name); }); - }, - setup_new_print_format: function(doctype, name, based_on) { + } + setup_new_print_format(doctype, name, based_on) { frappe.call({ method: 'frappe.printing.page.print_format_builder.print_format_builder.create_custom_format', args: { @@ -143,8 +143,8 @@ frappe.PrintFormatBuilder = Class.extend({ } }, }); - }, - setup_print_format: function() { + } + setup_print_format() { var me = this; frappe.model.with_doctype(this.print_format.doc_type, function(doctype) { me.meta = frappe.get_meta(me.print_format.doc_type); @@ -163,23 +163,23 @@ frappe.PrintFormatBuilder = Class.extend({ frappe.set_route("Form", "Print Format", me.print_format.name); }); }); - }, - setup_sidebar: function() { + } + setup_sidebar() { // prepend custom HTML field var fields = [this.get_custom_html_field()].concat(this.meta.fields); this.page.sidebar.html( $(frappe.render_template("print_format_builder_sidebar", {fields: fields})) ); this.setup_field_filter(); - }, - get_custom_html_field: function() { + } + get_custom_html_field() { return { fieldtype: "Custom HTML", fieldname: "_custom_html", label: __("Custom HTML") - } - }, - render_layout: function() { + }; + } + render_layout() { this.page.main.empty(); this.prepare_data(); $(frappe.render_template("print_format_builder_layout", { @@ -190,8 +190,8 @@ frappe.PrintFormatBuilder = Class.extend({ this.setup_edit_heading(); this.setup_field_settings(); this.setup_html_data(); - }, - prepare_data: function() { + } + prepare_data() { this.print_heading_template = null; this.data = JSON.parse(this.print_format.format_data || "[]"); if(!this.data.length) { @@ -280,22 +280,22 @@ frappe.PrintFormatBuilder = Class.extend({ this.layout_data = $.map(this.layout_data, function(s) { return s.has_fields ? s : null }); - }, - get_new_section: function() { + } + get_new_section() { return {columns: [], no_of_columns: 0, label:''}; - }, - get_new_column: function() { + } + get_new_column() { return {fields: []} - }, - add_table_properties: function(f) { + } + add_table_properties(f) { // build table columns and widths in a dict // visible_columns var me = this; if(!f.visible_columns) { me.init_visible_columns(f); } - }, - init_visible_columns: function(f) { + } + init_visible_columns(f) { f.visible_columns = [] $.each(frappe.get_meta(f.options).fields, function(i, _f) { if(!in_list(["Section Break", "Column Break"], _f.fieldtype) && @@ -306,8 +306,8 @@ frappe.PrintFormatBuilder = Class.extend({ print_width: (_f.width || ""), print_hide:0}); } }); - }, - setup_sortable: function() { + } + setup_sortable() { var me = this; // drag from fields library @@ -332,8 +332,8 @@ frappe.PrintFormatBuilder = Class.extend({ Sortable.create(this.page.main.find(".print-format-builder-layout").get(0), { handle: ".print-format-builder-section-head" } ); - }, - setup_sortable_for_column: function(col) { + } + setup_sortable_for_column(col) { var me = this; Sortable.create(col, { group: { @@ -363,8 +363,8 @@ frappe.PrintFormatBuilder = Class.extend({ } }); - }, - setup_field_filter: function() { + } + setup_field_filter() { var me = this; this.page.sidebar.find(".filter-fields").on("keyup", function() { var text = $(this).val(); @@ -373,8 +373,8 @@ frappe.PrintFormatBuilder = Class.extend({ $(this).parent().toggle(show); }) }); - }, - setup_section_settings: function() { + } + setup_section_settings() { var me = this; this.page.main.on("click", ".section-settings", function() { var section = $(this).parent().parent(); @@ -431,8 +431,8 @@ frappe.PrintFormatBuilder = Class.extend({ return false; }); - }, - setup_field_settings: function() { + } + setup_field_settings() { this.page.main.find(".field-settings").on("click", e => { const field = $(e.currentTarget).parent(); // new dialog @@ -482,8 +482,8 @@ frappe.PrintFormatBuilder = Class.extend({ return false; }); - }, - setup_html_data: function() { + } + setup_html_data() { // set JQuery `data` for Custom HTML fields, since editing the HTML // directly causes problem becuase of HTML reformatting // @@ -496,8 +496,8 @@ frappe.PrintFormatBuilder = Class.extend({ var html = me.custom_html_dict[parseInt(content.attr('data-custom-html-id'))].options; content.data('content', html); }) - }, - update_columns_in_section: function(section, no_of_columns, new_no_of_columns) { + } + update_columns_in_section(section, no_of_columns, new_no_of_columns) { var col_size = 12 / new_no_of_columns, me = this, resize = function() { @@ -539,8 +539,8 @@ frappe.PrintFormatBuilder = Class.extend({ resize(); } - }, - setup_add_section: function() { + } + setup_add_section() { var me = this; this.page.main.find(".print-format-builder-add-section").on("click", function() { // boostrap new section info @@ -554,8 +554,8 @@ frappe.PrintFormatBuilder = Class.extend({ me.setup_sortable_for_column($section.find(".print-format-builder-column").get(0)); }); - }, - setup_edit_heading: function() { + } + setup_edit_heading() { var me = this; var $heading = this.page.main.find(".print-format-builder-print-heading"); @@ -565,8 +565,8 @@ frappe.PrintFormatBuilder = Class.extend({ this.page.main.find(".edit-heading").on("click", function() { var d = me.get_edit_html_dialog(__("Edit Heading"), __("Heading"), $heading); }) - }, - setup_column_selector: function() { + } + setup_column_selector() { var me = this; this.page.main.on("click", ".select-columns", function() { var parent = $(this).parents(".print-format-builder-field:first"), @@ -657,24 +657,24 @@ frappe.PrintFormatBuilder = Class.extend({ return false; }); - }, - get_visible_columns_string: function(f) { + } + get_visible_columns_string(f) { if(!f.visible_columns) { this.init_visible_columns(f); } return $.map(f.visible_columns, function(v) { return v.fieldname + "|" + (v.print_width || "") }).join(","); - }, - get_no_content: function() { + } + get_no_content() { return __("Edit to add content") - }, - setup_edit_custom_html: function() { + } + setup_edit_custom_html() { var me = this; this.page.main.on("click", ".edit-html", function() { me.get_edit_html_dialog(__("Edit Custom HTML"), __("Custom HTML"), $(this).parents(".print-format-builder-field:first").find(".html-content")); }); - }, - get_edit_html_dialog: function(title, label, $content) { + } + get_edit_html_dialog(title, label, $content) { var me = this; var d = new frappe.ui.Dialog({ title: title, @@ -710,8 +710,8 @@ frappe.PrintFormatBuilder = Class.extend({ d.show(); return d; - }, - save_print_format: function() { + } + save_print_format() { var data = [], me = this; @@ -789,4 +789,4 @@ frappe.PrintFormatBuilder = Class.extend({ } }); } -}); +}; diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index bdb09541c9..721bec7fa7 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -7,7 +7,7 @@ {{ title }} - + diff --git a/frappe/public/js/barcode_scanner.bundle.js b/frappe/public/js/barcode_scanner.bundle.js new file mode 100644 index 0000000000..294f20c08f --- /dev/null +++ b/frappe/public/js/barcode_scanner.bundle.js @@ -0,0 +1 @@ +import "./frappe/barcode_scanner/quagga"; diff --git a/frappe/public/js/bootstrap-4-web.bundle.js b/frappe/public/js/bootstrap-4-web.bundle.js new file mode 100644 index 0000000000..2e3c4d7145 --- /dev/null +++ b/frappe/public/js/bootstrap-4-web.bundle.js @@ -0,0 +1,64 @@ + +// multilevel dropdown +$('.dropdown-menu a.dropdown-toggle').on('click', function (e) { + e.preventDefault(); + e.stopImmediatePropagation(); + if (!$(this).next().hasClass('show')) { + $(this).parents('.dropdown-menu').first().find('.show').removeClass("show"); + } + var $subMenu = $(this).next(".dropdown-menu"); + $subMenu.toggleClass('show'); + + + $(this).parents('li.nav-item.dropdown.show').on('hidden.bs.dropdown', function () { + $('.dropdown-submenu .show').removeClass("show"); + }); + + return false; +}); + +frappe.get_modal = function (title, content) { + return $( + `` + ); +}; + +frappe.ui.Dialog = class Dialog extends frappe.ui.Dialog { + get_primary_btn() { + return this.$wrapper.find(".modal-footer .btn-primary"); + } + + set_primary_action(label, click) { + this.$wrapper.find('.modal-footer').removeClass('hidden'); + return super.set_primary_action(label, click) + .removeClass('hidden'); + } + + make() { + super.make(); + if (this.fields) { + this.$wrapper.find('.section-body').addClass('w-100'); + } + } +}; diff --git a/frappe/public/js/chat.bundle.js b/frappe/public/js/chat.bundle.js new file mode 100644 index 0000000000..5f9a91ebb7 --- /dev/null +++ b/frappe/public/js/chat.bundle.js @@ -0,0 +1 @@ +import "./frappe/chat"; diff --git a/frappe/public/js/checkout.bundle.js b/frappe/public/js/checkout.bundle.js new file mode 100644 index 0000000000..954e838fa8 --- /dev/null +++ b/frappe/public/js/checkout.bundle.js @@ -0,0 +1 @@ +import "./integrations/razorpay"; diff --git a/frappe/public/js/controls.bundle.js b/frappe/public/js/controls.bundle.js new file mode 100644 index 0000000000..30b5d43905 --- /dev/null +++ b/frappe/public/js/controls.bundle.js @@ -0,0 +1,18 @@ +import "air-datepicker/dist/js/datepicker.min.js"; +import "air-datepicker/dist/js/i18n/datepicker.cs.js"; +import "air-datepicker/dist/js/i18n/datepicker.da.js"; +import "air-datepicker/dist/js/i18n/datepicker.de.js"; +import "air-datepicker/dist/js/i18n/datepicker.en.js"; +import "air-datepicker/dist/js/i18n/datepicker.es.js"; +import "air-datepicker/dist/js/i18n/datepicker.fi.js"; +import "air-datepicker/dist/js/i18n/datepicker.fr.js"; +import "air-datepicker/dist/js/i18n/datepicker.hu.js"; +import "air-datepicker/dist/js/i18n/datepicker.nl.js"; +import "air-datepicker/dist/js/i18n/datepicker.pl.js"; +import "air-datepicker/dist/js/i18n/datepicker.pt-BR.js"; +import "air-datepicker/dist/js/i18n/datepicker.pt.js"; +import "air-datepicker/dist/js/i18n/datepicker.ro.js"; +import "air-datepicker/dist/js/i18n/datepicker.sk.js"; +import "air-datepicker/dist/js/i18n/datepicker.zh.js"; +import "./frappe/ui/capture.js"; +import "./frappe/form/controls/control.js"; diff --git a/frappe/public/js/data_import_tools.bundle.js b/frappe/public/js/data_import_tools.bundle.js new file mode 100644 index 0000000000..b6e4c11968 --- /dev/null +++ b/frappe/public/js/data_import_tools.bundle.js @@ -0,0 +1 @@ +import "./frappe/data_import"; diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js new file mode 100644 index 0000000000..66eb72cda0 --- /dev/null +++ b/frappe/public/js/desk.bundle.js @@ -0,0 +1,105 @@ +import "./frappe/translate.js"; +import "./frappe/class.js"; +import "./frappe/polyfill.js"; +import "./frappe/provide.js"; +import "./frappe/assets.js"; +import "./frappe/format.js"; +import "./frappe/form/formatters.js"; +import "./frappe/dom.js"; +import "./frappe/ui/messages.js"; +import "./frappe/ui/keyboard.js"; +import "./frappe/ui/colors.js"; +import "./frappe/ui/sidebar.js"; +import "./frappe/ui/link_preview.js"; + +import "./frappe/request.js"; +import "./frappe/socketio_client.js"; +import "./frappe/utils/utils.js"; +import "./frappe/event_emitter.js"; +import "./frappe/router.js"; +import "./frappe/router_history.js"; +import "./frappe/defaults.js"; +import "./frappe/roles_editor.js"; +import "./frappe/module_editor.js"; +import "./frappe/microtemplate.js"; + +import "./frappe/ui/page.html"; +import "./frappe/ui/page.js"; +import "./frappe/ui/slides.js"; +// import "./frappe/ui/onboarding_dialog.js"; +import "./frappe/ui/find.js"; +import "./frappe/ui/iconbar.js"; +import "./frappe/form/layout.js"; +import "./frappe/ui/field_group.js"; +import "./frappe/form/link_selector.js"; +import "./frappe/form/multi_select_dialog.js"; +import "./frappe/ui/dialog.js"; +import "./frappe/ui/capture.js"; +import "./frappe/ui/app_icon.js"; +import "./frappe/ui/theme_switcher.js"; + +import "./frappe/model/model.js"; +import "./frappe/db.js"; +import "./frappe/model/meta.js"; +import "./frappe/model/sync.js"; +import "./frappe/model/create_new.js"; +import "./frappe/model/perm.js"; +import "./frappe/model/workflow.js"; +import "./frappe/model/user_settings.js"; + +import "./frappe/utils/user.js"; +import "./frappe/utils/common.js"; +import "./frappe/utils/urllib.js"; +import "./frappe/utils/pretty_date.js"; +import "./frappe/utils/tools.js"; +import "./frappe/utils/datetime.js"; +import "./frappe/utils/number_format.js"; +import "./frappe/utils/help.js"; +import "./frappe/utils/help_links.js"; +import "./frappe/utils/address_and_contact.js"; +import "./frappe/utils/preview_email.js"; +import "./frappe/utils/file_manager.js"; + +import "./frappe/upload.js"; +import "./frappe/ui/tree.js"; + +import "./frappe/views/container.js"; +import "./frappe/views/breadcrumbs.js"; +import "./frappe/views/factory.js"; +import "./frappe/views/pageview.js"; + +import "./frappe/ui/toolbar/awesome_bar.js"; +// import "./frappe/ui/toolbar/energy_points_notifications.js"; +import "./frappe/ui/notifications/notifications.js"; +import "./frappe/ui/toolbar/search.js"; +import "./frappe/ui/toolbar/tag_utils.js"; +import "./frappe/ui/toolbar/search.html"; +import "./frappe/ui/toolbar/search_utils.js"; +import "./frappe/ui/toolbar/about.js"; +import "./frappe/ui/toolbar/navbar.html"; +import "./frappe/ui/toolbar/toolbar.js"; +// import "./frappe/ui/toolbar/notifications.js"; +import "./frappe/views/communication.js"; +import "./frappe/views/translation_manager.js"; +import "./frappe/views/workspace/workspace.js"; + +import "./frappe/widgets/widget_group.js"; + +import "./frappe/ui/sort_selector.html"; +import "./frappe/ui/sort_selector.js"; + +import "./frappe/change_log.html"; +import "./frappe/ui/workspace_loading_skeleton.html"; +import "./frappe/desk.js"; +import "./frappe/query_string.js"; + +// import "./frappe/ui/comment.js"; + +import "./frappe/chat.js"; +import "./frappe/utils/energy_point_utils.js"; +import "./frappe/utils/dashboard_utils.js"; +import "./frappe/ui/chart.js"; +import "./frappe/ui/datatable.js"; +import "./frappe/ui/driver.js"; +import "./frappe/ui/plyr.js"; +import "./frappe/barcode_scanner/index.js"; diff --git a/frappe/public/js/dialog.bundle.js b/frappe/public/js/dialog.bundle.js new file mode 100644 index 0000000000..3100b42ca7 --- /dev/null +++ b/frappe/public/js/dialog.bundle.js @@ -0,0 +1,7 @@ +import "./frappe/dom.js"; +import "./frappe/form/formatters.js"; +import "./frappe/form/layout.js"; +import "./frappe/ui/field_group.js"; +import "./frappe/form/link_selector.js"; +import "./frappe/form/multi_select_dialog.js"; +import "./frappe/ui/dialog.js"; diff --git a/frappe/public/js/form.bundle.js b/frappe/public/js/form.bundle.js new file mode 100644 index 0000000000..5bed5c2cb8 --- /dev/null +++ b/frappe/public/js/form.bundle.js @@ -0,0 +1,17 @@ +import "./frappe/form/templates/address_list.html"; +import "./frappe/form/templates/contact_list.html"; +import "./frappe/form/templates/form_dashboard.html"; +import "./frappe/form/templates/form_footer.html"; +import "./frappe/form/templates/form_links.html"; +import "./frappe/form/templates/form_sidebar.html"; +import "./frappe/form/templates/print_layout.html"; +import "./frappe/form/templates/report_links.html"; +import "./frappe/form/templates/set_sharing.html"; +import "./frappe/form/templates/timeline_message_box.html"; +import "./frappe/form/templates/users_in_sidebar.html"; + +import "./frappe/form/controls/control.js"; +import "./frappe/views/formview.js"; +import "./frappe/form/form.js"; +import "./frappe/meta_tag.js"; + diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js new file mode 100644 index 0000000000..9f7875f96b --- /dev/null +++ b/frappe/public/js/frappe-web.bundle.js @@ -0,0 +1,26 @@ +import "./jquery-bootstrap"; +import "./frappe/class.js"; +import "./frappe/polyfill.js"; +import "./lib/md5.min.js"; +import "./frappe/provide.js"; +import "./frappe/format.js"; +import "./frappe/utils/number_format.js"; +import "./frappe/utils/utils.js"; +import "./frappe/utils/common.js"; +import "./frappe/ui/messages.js"; +import "./frappe/translate.js"; +import "./frappe/utils/pretty_date.js"; +import "./frappe/microtemplate.js"; +import "./frappe/query_string.js"; + +import "./frappe/upload.js"; + +import "./frappe/model/meta.js"; +import "./frappe/model/model.js"; +import "./frappe/model/perm.js"; + +import "./bootstrap-4-web.bundle"; + + +import "../../website/js/website.js"; +import "./frappe/socketio_client.js"; diff --git a/frappe/public/js/frappe/assets.js b/frappe/public/js/frappe/assets.js index 76441af235..3fca8640f3 100644 --- a/frappe/public/js/frappe/assets.js +++ b/frappe/public/js/frappe/assets.js @@ -9,7 +9,14 @@ frappe.require = function(items, callback) { if(typeof items === "string") { items = [items]; } - frappe.assets.execute(items, callback); + items = items.map(item => frappe.assets.bundled_asset(item)); + + return new Promise(resolve => { + frappe.assets.execute(items, () => { + resolve(); + callback && callback(); + }); + }); }; frappe.assets = { @@ -160,4 +167,11 @@ frappe.assets = { frappe.dom.set_style(txt); } }, + + bundled_asset(path) { + if (!path.startsWith('/assets') && path.includes('.bundle.')) { + return frappe.boot.assets_json[path] || path; + } + return path; + } }; diff --git a/frappe/public/js/frappe/barcode_scanner/index.js b/frappe/public/js/frappe/barcode_scanner/index.js index c5e7a7600f..fa3975b578 100644 --- a/frappe/public/js/frappe/barcode_scanner/index.js +++ b/frappe/public/js/frappe/barcode_scanner/index.js @@ -13,7 +13,7 @@ frappe.barcode.scan_barcode = function() { } }, reject); } else { - frappe.require('/assets/js/barcode_scanner.min.js', () => { + frappe.require('barcode_scanner.bundle.js', () => { frappe.barcode.get_barcode().then(barcode => { resolve(barcode); }); diff --git a/frappe/public/js/frappe/build_events/BuildError.vue b/frappe/public/js/frappe/build_events/BuildError.vue new file mode 100644 index 0000000000..6e10852719 --- /dev/null +++ b/frappe/public/js/frappe/build_events/BuildError.vue @@ -0,0 +1,111 @@ + + + diff --git a/frappe/public/js/frappe/build_events/BuildSuccess.vue b/frappe/public/js/frappe/build_events/BuildSuccess.vue new file mode 100644 index 0000000000..75a365fdc2 --- /dev/null +++ b/frappe/public/js/frappe/build_events/BuildSuccess.vue @@ -0,0 +1,52 @@ + + + diff --git a/frappe/public/js/frappe/build_events/build_events.bundle.js b/frappe/public/js/frappe/build_events/build_events.bundle.js new file mode 100644 index 0000000000..6c8986af3f --- /dev/null +++ b/frappe/public/js/frappe/build_events/build_events.bundle.js @@ -0,0 +1,48 @@ +import BuildError from "./BuildError.vue"; +import BuildSuccess from "./BuildSuccess.vue"; + +let $container = $("#build-events-overlay"); +let success = null; +let error = null; + +frappe.realtime.on("build_event", data => { + if (data.success) { + show_build_success(data); + } else if (data.error) { + show_build_error(data); + } +}); + +function show_build_success() { + if (error) { + error.hide(); + } + if (!success) { + let target = $('
') + .appendTo($container) + .get(0); + let vm = new Vue({ + el: target, + render: h => h(BuildSuccess) + }); + success = vm.$children[0]; + } + success.show(); +} + +function show_build_error(data) { + if (success) { + success.hide(); + } + if (!error) { + let target = $('
') + .appendTo($container) + .get(0); + let vm = new Vue({ + el: target, + render: h => h(BuildError) + }); + error = vm.$children[0]; + } + error.show(data); +} diff --git a/frappe/public/js/frappe/class.js b/frappe/public/js/frappe/class.js index 4f6dd0dc97..79ef2792ae 100644 --- a/frappe/public/js/frappe/class.js +++ b/frappe/public/js/frappe/class.js @@ -80,4 +80,4 @@ To subclass, use: // export global.Class = Class; - })(this); \ No newline at end of file + })(window); diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index c093a73689..46812f5fb6 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -24,12 +24,12 @@ $(document).ready(function() { frappe.start_app(); }); -frappe.Application = Class.extend({ - init: function() { +frappe.Application = class Application { + constructor() { this.startup(); - }, + } - startup: function() { + startup() { frappe.socketio.init(); frappe.model.init(); @@ -114,10 +114,8 @@ frappe.Application = Class.extend({ dialog.get_close_btn().toggle(false); }); - this.setup_user_group_listeners(); - // listen to build errors - this.setup_build_error_listener(); + this.setup_build_events(); if (frappe.sys_defaults.email_user_password) { var email_list = frappe.sys_defaults.email_user_password.split(','); @@ -162,7 +160,7 @@ frappe.Application = Class.extend({ }, 600000); // check every 10 minutes } } - }, + } set_route() { frappe.flags.setting_original_route = true; @@ -177,14 +175,14 @@ frappe.Application = Class.extend({ frappe.router.on('change', () => { $(".tooltip").hide(); }); - }, + } setup_frappe_vue() { Vue.prototype.__ = window.__; Vue.prototype.frappe = window.frappe; - }, + } - set_password: function(user) { + set_password(user) { var me=this; frappe.call({ method: 'frappe.core.doctype.user.user.get_email_awaiting', @@ -201,9 +199,9 @@ frappe.Application = Class.extend({ } } }); - }, + } - email_password_prompt: function(email_account,user,i) { + email_password_prompt(email_account,user,i) { var me = this; let d = new frappe.ui.Dialog({ title: __('Password missing in Email Account'), @@ -257,8 +255,8 @@ frappe.Application = Class.extend({ }); }); d.show(); - }, - load_bootinfo: function() { + } + load_bootinfo() { if(frappe.boot) { this.setup_workspaces(); frappe.model.sync(frappe.boot.docs); @@ -280,7 +278,7 @@ frappe.Application = Class.extend({ } else { this.set_as_guest(); } - }, + } setup_workspaces() { frappe.modules = {}; @@ -291,26 +289,26 @@ frappe.Application = Class.extend({ } if (!frappe.workspaces['home']) { // default workspace is settings for Frappe - frappe.workspaces['home'] = frappe.workspaces['build']; + frappe.workspaces['home'] = frappe.workspaces[Object.keys(frappe.workspaces)[0]]; } - }, + } - load_user_permissions: function() { + load_user_permissions() { frappe.defaults.update_user_permissions(); frappe.realtime.on('update_user_permissions', frappe.utils.debounce(() => { frappe.defaults.update_user_permissions(); }, 500)); - }, + } - check_metadata_cache_status: function() { + check_metadata_cache_status() { if(frappe.boot.metadata_version != localStorage.metadata_version) { frappe.assets.clear_local_storage(); frappe.assets.init_local_storage(); } - }, + } - set_globals: function() { + set_globals() { frappe.session.user = frappe.boot.user.name; frappe.session.logged_in_user = frappe.boot.user.name; frappe.session.user_email = frappe.boot.user.email; @@ -362,8 +360,8 @@ frappe.Application = Class.extend({ } } }); - }, - sync_pages: function() { + } + sync_pages() { // clear cached pages if timestamp is not found if(localStorage["page_info"]) { frappe.boot.allowed_pages = []; @@ -378,8 +376,8 @@ frappe.Application = Class.extend({ frappe.boot.allowed_pages = Object.keys(frappe.boot.page_info); } localStorage["page_info"] = JSON.stringify(frappe.boot.page_info); - }, - set_as_guest: function() { + } + set_as_guest() { frappe.session.user = 'Guest'; frappe.session.user_email = ''; frappe.session.user_fullname = 'Guest'; @@ -387,23 +385,23 @@ frappe.Application = Class.extend({ frappe.user_defaults = {}; frappe.user_roles = ['Guest']; frappe.sys_defaults = {}; - }, - make_page_container: function() { + } + make_page_container() { if ($("#body").length) { $(".splash").remove(); frappe.temp_container = $("