Merge branch 'develop' of https://github.com/frappe/frappe into print-format-builder-beta

.github/helper/install.sh

@@ -17,6 +17,7 @@ if [ "$TYPE" == "server" ]; then
 fi
 
 if [ "$DB" == "mariadb" ];then
+      sudo apt install mariadb-client-10.3
       mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
       mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
 
@@ -58,4 +59,4 @@ cd ../..
 bench start &
 bench --site test_site reinstall --yes
 if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
-bench build --app frappe
+CI=Yes bench build --app frappe

.github/helper/semgrep_rules/frappe_correctness.yml

@@ -131,3 +131,16 @@ rules:
       key `$X` is uselessly assigned twice. This could be a potential bug.
   languages: [python]
   severity: ERROR
+
+- id: frappe-using-db-sql
+  pattern-either:
+    - pattern: frappe.db.sql(...)
+    - pattern: frappe.db.sql_ddl(...)
+    - pattern: frappe.db.sql_list(...)
+  paths:
+    exclude:
+      - "test_*.py"
+  message: |
+    The PR contains a SQL query that may be re-written with frappe.qb (https://frappeframework.com/docs/user/en/api/query-builder) or the Database API (https://frappeframework.com/docs/user/en/api/database)
+  languages: [python]
+  severity: ERROR

.github/try-on-f-cloud-button.svg

@@ -0,0 +1,32 @@
+<svg width="201" height="60" viewBox="0 0 201 60" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g filter="url(#filter0_dd)">
+<rect x="4" y="2" width="193" height="52" rx="6" fill="#2490EF"/>
+<path d="M28 22.2891H32.8786V35.5H36.2088V22.2891H41.0874V19.5H28V22.2891Z" fill="white"/>
+<path d="M41.6982 35.5H45.0129V28.7109C45.0129 27.2344 46.0866 26.2188 47.5494 26.2188C48.0085 26.2188 48.6388 26.2969 48.95 26.3984V23.4453C48.6543 23.375 48.2419 23.3281 47.9074 23.3281C46.5691 23.3281 45.472 24.1094 45.0362 25.5938H44.9117V23.5H41.6982V35.5Z" fill="white"/>
+<path d="M52.8331 40C55.2996 40 56.6068 38.7344 57.2837 36.7969L61.9289 23.5156L58.4197 23.5L55.9221 32.3125H55.7976L53.3233 23.5H49.8374L54.1247 35.8437L53.9302 36.3516C53.4944 37.4766 52.6619 37.5312 51.4947 37.1719L50.7478 39.6562C51.2224 39.8594 51.9927 40 52.8331 40Z" fill="white"/>
+<path d="M73.6142 35.7344C77.2401 35.7344 79.4966 33.2422 79.4966 29.5469C79.4966 25.8281 77.2401 23.3438 73.6142 23.3438C69.9883 23.3438 67.7319 25.8281 67.7319 29.5469C67.7319 33.2422 69.9883 35.7344 73.6142 35.7344ZM73.6298 33.1562C71.9569 33.1562 71.101 31.6171 71.101 29.5233C71.101 27.4296 71.9569 25.8827 73.6298 25.8827C75.2715 25.8827 76.1274 27.4296 76.1274 29.5233C76.1274 31.6171 75.2715 33.1562 73.6298 33.1562Z" fill="white"/>
+<path d="M84.7253 28.5625C84.7331 27.0156 85.6512 26.1094 86.9895 26.1094C88.3201 26.1094 89.1215 26.9844 89.1137 28.4531V35.5H92.4284V27.8594C92.4284 25.0625 90.7945 23.3438 88.3046 23.3438C86.5306 23.3438 85.2466 24.2187 84.7097 25.6172H84.5697V23.5H81.4106V35.5H84.7253V28.5625Z" fill="white"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M102.429 19.5H113.429V22.3141H102.429V19.5ZM102.429 35.5V26.6794H112.699V29.4982H105.94V35.5H102.429Z" fill="white"/>
+<path d="M131.584 24.9625C131.09 21.5057 128.345 19.5 124.785 19.5C120.589 19.5 117.429 22.463 117.429 27.4924C117.429 32.5142 120.55 35.4848 124.785 35.4848C128.604 35.4848 131.137 33.0916 131.584 30.1211L128.651 30.1059C128.282 31.9293 126.745 32.9549 124.824 32.9549C122.22 32.9549 120.354 31.0632 120.354 27.4924C120.354 23.9824 122.204 22.0299 124.832 22.0299C126.784 22.0299 128.314 23.1011 128.651 24.9625H131.584Z" fill="white"/>
+<path d="M136.409 19.7124H133.571V35.2718H136.409V19.7124Z" fill="white"/>
+<path d="M144.031 35.5001C147.56 35.5001 149.803 33.0917 149.803 29.483C149.803 25.8667 147.56 23.4507 144.031 23.4507C140.502 23.4507 138.259 25.8667 138.259 29.483C138.259 33.0917 140.502 35.5001 144.031 35.5001ZM144.047 33.2969C142.094 33.2969 141.137 31.6103 141.137 29.4754C141.137 27.3406 142.094 25.6312 144.047 25.6312C145.968 25.6312 146.925 27.3406 146.925 29.4754C146.925 31.6103 145.968 33.2969 144.047 33.2969Z" fill="white"/>
+<path d="M159.338 30.3641C159.338 32.1419 158.028 33.0232 156.773 33.0232C155.409 33.0232 154.499 32.0887 154.499 30.6072V23.6025H151.66V31.0327C151.66 33.8361 153.307 35.4239 155.675 35.4239C157.479 35.4239 158.749 34.5046 159.298 33.1979H159.424V35.272H162.176V23.6025H159.338V30.3641Z" fill="white"/>
+<path d="M169.014 35.4769C171.084 35.4769 172.017 34.2841 172.464 33.4332H172.637V35.2718H175.429V19.7124H172.582V25.532H172.464C172.033 24.6887 171.147 23.4503 169.022 23.4503C166.238 23.4503 164.05 25.5624 164.05 29.4522C164.05 33.2965 166.175 35.4769 169.014 35.4769ZM169.806 33.2205C167.931 33.2205 166.943 31.6251 166.943 29.437C166.943 27.2642 167.916 25.7067 169.806 25.7067C171.633 25.7067 172.637 27.173 172.637 29.437C172.637 31.701 171.617 33.2205 169.806 33.2205Z" fill="white"/>
+</g>
+<defs>
+<filter id="filter0_dd" x="0" y="0" width="201" height="60" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feOffset/>
+<feGaussianBlur stdDeviation="0.25"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.5 0"/>
+<feBlend mode="normal" in2="BackgroundImageFix" result="effect1_dropShadow"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feOffset dy="2"/>
+<feGaussianBlur stdDeviation="2"/>
+<feColorMatrix type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.13 0"/>
+<feBlend mode="normal" in2="effect1_dropShadow" result="effect2_dropShadow"/>
+<feBlend mode="normal" in="SourceGraphic" in2="effect2_dropShadow" result="shape"/>
+</filter>
+</defs>
+</svg>

.github/workflows/docs-checker.yml

@@ -12,7 +12,7 @@ jobs:
       - name: 'Setup Environment'
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.7
 
       - name: 'Clone repo'
         uses: actions/checkout@v2

.github/workflows/patch-mariadb-tests.yml

@@ -9,7 +9,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     name: Patch Test
 
@@ -29,7 +29,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: '3.9'
 
       - name: Check if build should be run
         id: check-build
@@ -102,4 +102,25 @@ jobs:
           cd ~/frappe-bench/
           wget https://frappeframework.com/files/v10-frappe.sql.gz
           bench --site test_site --force restore ~/frappe-bench/v10-frappe.sql.gz
+
+          source env/bin/activate
+          cd apps/frappe/
+          git remote set-url upstream https://github.com/frappe/frappe.git
+          git fetch --all --tags
+
+          taglist=$(git tag --sort version:refname | grep -v "beta")
+          last_release=$(echo "$taglist" | tail -1 | cut -d . -f 1 | cut -c 2-)
+
+          for version in $(seq 12 "$last_release")
+          do
+              last_tag=$(echo "$taglist" | grep "v$version" | tail -1)
+              echo "Updating to $last_tag"
+              git checkout -q -f "$last_tag"
+              pip install -q -r requirements.txt
+              bench --site test_site migrate
+          done
+
+          echo "Updating to last commit"
+          git checkout -q -f "$GITHUB_SHA"
+          bench setup requirements --python
           bench --site test_site migrate

.github/workflows/publish-assets-develop.yml

@@ -18,7 +18,7 @@ jobs:
           node-version: 14
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.9'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/publish-assets-releases.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: '12.x'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.9'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/server-mariadb-tests.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -38,7 +38,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: '3.9'
 
       - name: Check if build should be run
         id: check-build
@@ -121,9 +121,11 @@ jobs:
           ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
       - name: Upload coverage data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: codecov/codecov-action@v2
         with:
           name: MariaDB
           fail_ci_if_error: true
           files: /home/runner/frappe-bench/sites/coverage.xml
           verbose: true
+          flags: server

.github/workflows/server-postgres-tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -41,7 +41,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: '3.9'
 
       - name: Check if build should be run
         id: check-build
@@ -124,9 +124,11 @@ jobs:
           ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
       - name: Upload coverage data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: codecov/codecov-action@v2
         with:
           name: Postgres
           fail_ci_if_error: true
           files: /home/runner/frappe-bench/sites/coverage.xml
           verbose: true
+          flags: server

.github/workflows/translation_linter.yml

@@ -1,22 +0,0 @@
-name: Frappe Linter
-on:
-  pull_request:
-    branches:
-      - develop
-      - version-12-hotfix
-      - version-11-hotfix
-jobs:
-  check_translation:
-    name: Translation Syntax Check
-    runs-on: ubuntu-18.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup python3
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.6
-    - name: Validating Translation Syntax
-      run: |
-        git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
-        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        python $GITHUB_WORKSPACE/.github/helper/translation.py $files

.github/workflows/ui-tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -37,7 +37,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: '3.9'
 
       - name: Check if build should be run
         id: check-build
@@ -122,12 +122,36 @@ jobs:
           DB: mariadb
           TYPE: ui
 
+      - name: Instrument Source Code
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        run: cd ~/frappe-bench/apps/frappe/ && npx nyc instrument -x 'frappe/public/dist/**' -x 'frappe/public/js/lib/**' -x '**/*.bundle.js' --compact=false --in-place frappe
+
+      - name: Build
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
+        run: cd ~/frappe-bench/ && bench build --apps frappe
+
       - name: Site Setup
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
 
       - name: UI Tests
         if: ${{ steps.check-build.outputs.build == 'strawberry' }}
-        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID
+        run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --with-coverage --headless --parallel --ci-build-id $GITHUB_RUN_ID
         env:
           CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb
+
+      - name: Check If Coverage Report Exists
+        id: check_coverage
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "/home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml"
+
+      - name: Upload Coverage Data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }}
+        uses: codecov/codecov-action@v2
+        with:
+          name: Cypress
+          fail_ci_if_error: true
+          directory: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/
+          verbose: true
+          flags: ui-tests

.gitignore

@@ -67,6 +67,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+.cypress-coverage
 
 # Translations
 *.mo

CODEOWNERS

@@ -15,5 +15,6 @@ core/							@surajshetty3416
 database          @gavindsouza
 model             @gavindsouza
 requirements.txt	@gavindsouza
+query_builder/	@gavindsouza
 commands/					@gavindsouza
 workspace         @shariquerik

README.md

@@ -27,7 +27,7 @@
 		<img src='https://www.codetriage.com/frappe/frappe/badges/users.svg'>
 	</a>
 	<a href="https://codecov.io/gh/frappe/frappe">
-	        <img src="https://codecov.io/gh/frappe/frappe/branch/develop/graph/badge.svg?token=XoTa679hIj"/>
+	        <img src="https://codecov.io/gh/frappe/frappe/branch/develop/graph/badge.svg?token=XoTa679hIj&flag=server"/>
 	</a>
 </div>
 
@@ -35,25 +35,36 @@
 
 Full-stack web application framework that uses Python and MariaDB on the server side and a tightly integrated client side library. Built for [ERPNext](https://erpnext.com)
 
-### Table of Contents
-* [Installation](https://frappeframework.com/docs/user/en/installation)
-* [Documentation](https://frappeframework.com/docs)
+<div align="center">
+	<a href="https://frappecloud.com/deploy?apps=frappe&source=frappe_readme">
+		<img src=".github/try-on-f-cloud-button.svg" height="40">
+	</a>
+</div>
+
+## Table of Contents
+* [Installation](#installation)
+* [Contributing](#contributing)
+* [Resources](#resources)
 * [License](#license)
 
-### Installation
+## Installation
 
 * [Install via Docker](https://github.com/frappe/frappe_docker)
 * [Install via Frappe Bench](https://github.com/frappe/bench)
+* [Offical Documentation](https://frappeframework.com/docs/user/en/installation)
+* [Managed Hosting on Frappe Cloud](https://frappecloud.com/deploy?apps=frappe&source=frappe_readme)
 
 ## Contributing
 
+1. [Code of Conduct](CODE_OF_CONDUCT.md)
 1. [Contribution Guidelines](https://github.com/frappe/erpnext/wiki/Contribution-Guidelines)
+1. [Security Policy](SECURITY.md)
 1. [Translations](https://translate.erpnext.com)
 
-### Website
+## Resources
 
-For details and documentation, see the website
-[https://frappeframework.com](https://frappeframework.com)
+1. [frappeframework.com](https://frappeframework.com) - Official documentation of the Frappe Framework.
+1. [frappe.school](https://frappe.school) - Pick from the various courses by the maintainers or from the community.
 
-### License
+## License
 This repository has been released under the [MIT License](LICENSE).

codecov.yml

@@ -1,9 +1,27 @@
 codecov:
   require_ci_to_pass: yes
+
+coverage:
   status:
+    patch: off
     project:
-      default:
+      default: false
+      server:
+        target: auto
         threshold: 0.5%
+        flags:
+          - server
+
 comment:
-  layout: "diff, flags, files"
+  layout: "diff, flags"
   require_changes: true
+
+flags:
+  server:
+    paths:
+      - ".*\\.py"
+    carryforward: true
+  ui-tests:
+    paths:
+      - ".*\\.js"
+    carryforward: true

cypress/fixtures/doctype_with_tab_break.js

@@ -0,0 +1,59 @@
+export default {
+	name: 'Form With Tab Break',
+	custom: 1,
+	actions: [],
+	doctype: 'DocType',
+	engine: 'InnoDB',
+	fields: [
+		{
+			fieldname: 'username',
+			fieldtype: 'Data',
+			label: 'Name',
+			options: 'Name'
+		},
+		{
+			fieldname: 'tab',
+			fieldtype: 'Tab Break',
+			label: 'Tab 2',
+		},
+		{
+			fieldname: 'Phone',
+			fieldtype: 'Data',
+			label: 'Phone',
+			options: 'Phone',
+			reqd: 1
+		},
+	],
+	links: [
+		{
+			"group": "Profile",
+			"link_doctype": "Contact",
+			"link_fieldname": "user"
+		},
+		{
+			"group": "Profile",
+			"link_doctype": "Chat Profile",
+			"link_fieldname": "user"
+		},
+	],
+	modified_by: 'Administrator',
+	module: 'Custom',
+	owner: 'Administrator',
+	permissions: [
+		{
+			create: 1,
+			delete: 1,
+			email: 1,
+			print: 1,
+			read: 1,
+			role: 'System Manager',
+			share: 1,
+			write: 1
+		}
+	],
+	quick_entry: 1,
+	autoname: "format: Test-{####}",
+	sort_field: 'modified',
+	sort_order: 'ASC',
+	track_changes: 1
+};

cypress/integration/api.js

@@ -31,8 +31,13 @@ context('API Resources', () => {
 	});
 
 	it('Removes the Comments', () => {
-		cy.get_list('Comment').then(body => body.data.forEach(comment => {
-			cy.remove_doc('Comment', comment.name);
-		}));
+		cy.get_list('Comment').then(body => {
+			let comment_names = [];
+			body.data.map(comment => comment_names.push(comment.name));
+			comment_names = [...new Set(comment_names)]; // remove duplicates
+			comment_names.forEach((comment_name) => {
+				cy.remove_doc('Comment', comment_name);
+			});
+		});
 	});
 });

cypress/integration/control_float.js

@@ -0,0 +1,93 @@
+context("Control Float", () => {
+	before(() => {
+		cy.login();
+		cy.visit("/app/website");
+	});
+
+	function get_dialog_with_float() {
+		return cy.dialog({
+			title: "Float Check",
+			fields: [
+				{
+					fieldname: "float_number",
+					fieldtype: "Float",
+					Label: "Float"
+				}
+			]
+		});
+	}
+
+	it("check value changes", () => {
+		get_dialog_with_float().as("dialog");
+
+		let data = get_data();
+		data.forEach(x => {
+			cy.window()
+				.its("frappe")
+				.then(frappe => {
+					frappe.boot.sysdefaults.number_format = x.number_format;
+				});
+			x.values.forEach(d => {
+				cy.get_field("float_number", "Float").clear();
+				cy.fill_field("float_number", d.input, "Float").blur();
+				cy.get_field("float_number", "Float").should(
+					"have.value",
+					d.blur_expected
+				);
+
+				cy.get_field("float_number", "Float").focus();
+				cy.get_field("float_number", "Float").blur();
+				cy.get_field("float_number", "Float").focus();
+				cy.get_field("float_number", "Float").should(
+					"have.value",
+					d.focus_expected
+				);
+			});
+		});
+	});
+
+	function get_data() {
+		return [
+			{
+				number_format: "#.###,##",
+				values: [
+					{
+						input: "364.87,334",
+						blur_expected: "36.487,334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "36487,334",
+						blur_expected: "36.487,334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "100",
+						blur_expected: "100,000",
+						focus_expected: "100"
+					}
+				]
+			},
+			{
+				number_format: "#,###.##",
+				values: [
+					{
+						input: "364,87.334",
+						blur_expected: "36,487.334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "36487.334",
+						blur_expected: "36,487.334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "100",
+						blur_expected: "100.000",
+						focus_expected: "100"
+					}
+				]
+			}
+		];
+	}
+});

cypress/integration/dashboard_links.js

@@ -9,17 +9,20 @@ context('Dashboard links', () => {
 		cy.clear_filters();
 
 		cy.visit('/app/user');
-		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click();
+		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true });
 
 		//To check if initially the dashboard contains only the "Contact" link and there is no counter
 		cy.get('[data-doctype="Contact"]').should('contain', 'Contact');
 
 		//Adding a new contact
-		cy.get('.btn[data-doctype="Contact"]').click();
+		cy.get('.document-link-badge[data-doctype="Contact"]').click();
+		cy.wait(300);
+		cy.findByRole('button', {name: 'Add Contact'}).should('be.visible');
+		cy.findByRole('button', {name: 'Add Contact'}).click();
 		cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type('Admin');
 		cy.findByRole('button', {name: 'Save'}).click();
 		cy.visit('/app/user');
-		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click();
+		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true });
 
 		//To check if the counter for contact doc is "1" after adding the contact
 		cy.get('[data-doctype="Contact"] > .count').should('contain', '1');
@@ -27,7 +30,7 @@ context('Dashboard links', () => {
 
 		//Deleting the newly created contact
 		cy.visit('/app/contact');
-		cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click();
+		cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click({ force: true });
 		cy.findByRole('button', {name: 'Actions'}).click();
 		cy.get('.actions-btn-group [data-label="Delete"]').click();
 		cy.findByRole('button', {name: 'Yes'}).click({delay: 700});
@@ -36,7 +39,7 @@ context('Dashboard links', () => {
 		//To check if the counter from the "Contact" doc link is removed
 		cy.wait(700);
 		cy.visit('/app/user');
-		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click();
+		cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true });
 		cy.get('[data-doctype="Contact"]').should('contain', 'Contact');
 	});
 
@@ -51,13 +54,12 @@ context('Dashboard links', () => {
 				cur_frm.dashboard.data.reports = [
 					{
 						'label': 'Reports',
-						'items': ['Permitted Documents For User']
+						'items': ['Website Analytics']
 					}
 				];
 				cur_frm.dashboard.render_report_links();
-				cy.get('[data-report="Permitted Documents For User"]').contains('Permitted Documents For User').click();
-				cy.findByText('Permitted Documents For User');
-				cy.findByPlaceholderText('User').should("have.value", "Administrator");
+				cy.get('[data-report="Website Analytics"]').contains('Website Analytics').click();
+				cy.findByText('Website Analytics');
 			});
 	});
 });

cypress/integration/datetime_field_form_validation.js

@@ -1,19 +1,19 @@
-context('Datetime Field Validation', () => {
-	before(() => {
-		cy.login();
-		cy.visit('/app/communication');
-		cy.window().its('frappe').then(frappe => {
-			frappe.call("frappe.tests.ui_test_helpers.create_communication_records");
-		});
-	});
+// TODO: Enable this again
+// currently this is flaky possibly because of different timezone in CI
 
-	// validating datetime field value when value is set from backend and get validated on form load.
-	it('datetime field form validation', () => {
-		cy.visit('/app/communication');
-		cy.get('a[title="Test Form Communication 1"]').invoke('attr', 'data-name')
-			.then((name) => {
-				cy.visit(`/app/communication/${name}`);
-				cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
-			});
-	});
-});
+// context('Datetime Field Validation', () => {
+// 	before(() => {
+// 		cy.login();
+// 		cy.visit('/app/communication');
+// 	});
+
+// 	it('datetime field form validation', () => {
+// 		// validating datetime field value when value is set from backend and get validated on form load.
+// 		cy.window().its('frappe').then(frappe => {
+// 			return frappe.xcall("frappe.tests.ui_test_helpers.create_communication_record");
+// 		}).then(doc => {
+// 			cy.visit(`/app/communication/${doc.name}`);
+// 			cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
+// 		});
+// 	});
+// });

cypress/integration/discussions.js

@@ -0,0 +1,79 @@
+context('Discussions', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app');
+		return cy.window().its('frappe').then(frappe => {
+			return frappe.call('frappe.tests.ui_test_helpers.create_data_for_discussions');
+		});
+	});
+
+	const reply_through_modal = () => {
+		cy.visit('/test-page-discussions');
+
+		// Open the modal
+		cy.get('.reply').click();
+		cy.wait(500);
+		cy.get('.discussion-modal').should('be.visible');
+
+		// Enter title
+		cy.get('.modal .topic-title').type('Discussion from tests')
+			.should('have.value', 'Discussion from tests');
+
+		// Enter comment
+		cy.get('.modal .comment-field')
+			.type('This is a discussion from the cypress ui tests.')
+			.should('have.value', 'This is a discussion from the cypress ui tests.');
+
+		// Submit
+		cy.get('.modal .submit-discussion').click();
+		cy.wait(2000);
+
+		// Check if discussion is added to page and content is visible
+		cy.get('.sidebar-parent:first .discussion-topic-title').should('have.text', 'Discussion from tests');
+		cy.get('.discussion-on-page:visible').should('have.class', 'show');
+		cy.get('.discussion-on-page:visible .reply-card .reply-text')
+			.should('have.text', 'This is a discussion from the cypress ui tests.\n');
+
+	};
+
+	const reply_through_comment_box = () => {
+		cy.get('.discussion-on-page:visible .comment-field')
+			.type('This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.')
+			.should('have.value', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.');
+
+		cy.get('.discussion-on-page:visible .submit-discussion').click();
+		cy.wait(3000);
+		cy.get('.discussion-on-page:visible').should('have.class', 'show');
+		cy.get('.discussion-on-page:visible').children(".reply-card").eq(1).children(".reply-text")
+			.should('have.text', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.\n');
+	};
+
+	const cancel_and_clear_comment_box = () => {
+		cy.get('.discussion-on-page:visible .comment-field')
+			.type('This is a discussion from the cypress ui tests.')
+			.should('have.value', 'This is a discussion from the cypress ui tests.');
+
+		cy.get('.discussion-on-page:visible .cancel-comment').click();
+		cy.get('.discussion-on-page:visible .comment-field').should('have.value', '');
+	};
+
+	const single_thread_discussion = () => {
+		cy.visit('/test-single-thread');
+		cy.get('.discussions-sidebar').should('have.length', 0);
+		cy.get('.reply').should('have.length', 0);
+
+		cy.get('.discussion-on-page .comment-field')
+			.type('This comment is being made on a single thread discussion.')
+			.should('have.value', 'This comment is being made on a single thread discussion.');
+
+		cy.get('.discussion-on-page .submit-discussion').click();
+		cy.wait(3000);
+		cy.get('.discussion-on-page').children(".reply-card").eq(-1).children(".reply-text")
+			.should('have.text', 'This comment is being made on a single thread discussion.\n');
+	};
+
+	it('reply through modal', reply_through_modal);
+	it('reply through comment box', reply_through_comment_box);
+	it('cancel and clear comment box', cancel_and_clear_comment_box);
+	it('single thread discussion', single_thread_discussion);
+});

cypress/integration/folder_navigation.js

@@ -71,7 +71,7 @@ context('Folder Navigation', () => {
 	it('Deleting Test Folder from the home', () => {
 	//Deleting the Test Folder added in the home directory
 		cy.visit('/app/file/view/home');
-		cy.get('.level-left > .list-subject > .list-row-checkbox').eq(0).click({force: true, delay: 500});
+		cy.get('.level-left > .list-subject > .file-select >.list-row-checkbox').eq(0).click({force: true, delay: 500});
 		cy.findByRole('button', {name: 'Actions'}).click();
 		cy.get('.actions-btn-group [data-label="Delete"]').click();
 		cy.findByRole('button', {name: 'Yes'}).click();

cypress/integration/form.js

@@ -8,7 +8,10 @@ context('Form', () => {
 	});
 	it('create a new form', () => {
 		cy.visit('/app/todo/new');
-		cy.fill_field('description', 'this is a test todo', 'Text Editor');
+		cy.get('[data-fieldname="description"] .ql-editor')
+			.first()
+			.click()
+			.type('this is a test todo');
 		cy.wait(300);
 		cy.get('.page-title').should('contain', 'Not Saved');
 		cy.intercept({

cypress/integration/form_tab_break.js

@@ -0,0 +1,31 @@
+import doctype_with_tab_break from '../fixtures/doctype_with_tab_break';
+const doctype_name = doctype_with_tab_break.name;
+context("Form Tab Break", () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+		return cy.insert_doc('DocType', doctype_with_tab_break, true);
+	});
+	it("Should switch tab and open correct tabs on validation error", () => {
+		cy.new_form(doctype_name);
+		// test tab switch
+		cy.findByRole("tab", {name: "Tab 2"}).click();
+		cy.findByText("Phone");
+		cy.findByRole("tab", {name: "Details"}).click();
+		cy.findByText("Name");
+
+		// form should switch to the tab with un-filled mandatory field
+		cy.fill_field("username", "Test");
+		cy.findByRole("button", {name: "Save"}).click();
+		cy.findByText("Missing Fields");
+		cy.hide_dialog();
+		cy.findByText("Phone");
+		cy.fill_field("phone", "12345678");
+		cy.findByRole("button", {name: "Save"}).click();
+
+		// After save, first tab should have dashboard
+		cy.get(".form-tabs > .nav-item").eq(0).click();
+		cy.findByText("Connections");
+
+	});
+});

cypress/integration/grid_configuration.js

@@ -0,0 +1,23 @@
+context('Grid Configuration', () => {
+	beforeEach(() => {
+		cy.login();
+		cy.visit('/app/doctype/User');
+	});
+	it('Set user wise grid settings', () => {
+		cy.wait(100);
+		cy.get('.frappe-control[data-fieldname="fields"]').as('table');
+		cy.get('@table').find('.icon-sm').click();
+		cy.wait(100);
+		cy.get('.frappe-control[data-fieldname="fields_html"]').as('modal');
+		cy.get('@modal').find('.add-new-fields').click();
+		cy.wait(100);
+		cy.get('[type="checkbox"][data-unit="read_only"]').check();
+		cy.findByRole('button', {name: 'Add'}).click();
+		cy.wait(100);
+		cy.get('[data-fieldname="options"]').invoke('attr', 'value', '1');
+		cy.get('.form-control.column-width[data-fieldname="options"]').trigger('change');
+		cy.findByRole('button', {name: 'Update'}).click();
+		cy.wait(200);
+		cy.get('[title="Read Only"').should('be.visible');
+	});
+});

cypress/integration/list_view.js

@@ -6,12 +6,29 @@ context('List View', () => {
 			return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow");
 		});
 	});
+
+	it('Keep checkbox checked after Bulk Update', () => {
+		cy.go_to_list('ToDo');
+		cy.get('.list-row-container .list-row-checkbox').click({ multiple: true, force: true });
+		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
+		cy.get('.dropdown-menu li:visible .dropdown-item .menu-item-label[data-label="Edit"]').click();
+
+		cy.get('.modal-body .form-control[data-fieldname="field"]').first().select('Due Date').wait(200);
+		cy.fill_field('value', '09-28-21', 'Date');
+
+		cy.get('.modal-footer .standard-actions .btn-primary').click();
+		cy.wait(500);
+
+		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
+		cy.get('.list-row-container .list-row-checkbox:checked').should('be.visible');
+	});
+
 	it('enables "Actions" button', () => {
-		const actions = ['Approve', 'Reject', 'Edit', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete'];
+		const actions = ['Approve', 'Reject', 'Edit', 'Export', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete'];
 		cy.go_to_list('ToDo');
 		cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ multiple: true, force: true });
 		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
-		cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 8).each((el, index) => {
+		cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 9).each((el, index) => {
 			cy.wrap(el).contains(actions[index]);
 		}).then((elements) => {
 			cy.intercept({
@@ -24,10 +41,11 @@ context('List View', () => {
 			}).as('real-time-update');
 			cy.wrap(elements).contains('Approve').click();
 			cy.wait(['@bulk-approval', '@real-time-update']);
-			cy.hide_dialog();
+			cy.wait(300);
+			cy.get_open_dialog().find('.btn-modal-close').click();
+			cy.reload();
 			cy.clear_filters();
 			cy.get('.list-row-container:visible').should('contain', 'Approved');
 		});
 	});
 });
-

cypress/integration/multi_select_dialog.js

@@ -0,0 +1,58 @@
+context('MultiSelectDialog', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app');
+	});
+
+	function open_multi_select_dialog() {
+		cy.window().its('frappe').then(frappe => {
+			new frappe.ui.form.MultiSelectDialog({
+				doctype: "Assignment Rule",
+				target: {},
+				setters: {
+					document_type: null,
+					priority: null
+				},
+				add_filters_group: 1,
+				allow_child_item_selection: 1,
+				child_fieldname: "assignment_days",
+				child_columns: ["day"]
+			});
+		});
+	}
+
+	it('multi select dialog api works', () => {
+		open_multi_select_dialog();
+		cy.get_open_dialog().should('contain', 'Select Assignment Rules');
+	});
+
+	it('checks for filters', () => {
+		['search_term', 'document_type', 'priority'].forEach(fieldname => {
+			cy.get_open_dialog().get(`.frappe-control[data-fieldname="${fieldname}"]`).should('exist');
+		});
+
+		// add_filters_group: 1 should add a filter group
+		cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should('exist');
+
+	});
+
+	it('checks for child item selection', () => {
+		cy.get_open_dialog()
+			.get(`.dt-row-header`).should('not.exist');
+
+		cy.get_open_dialog()
+			.get(`.frappe-control[data-fieldname="allow_child_item_selection"]`)
+			.should('exist')
+			.click();
+
+		cy.get_open_dialog()
+			.get(`.frappe-control[data-fieldname="child_selection_area"]`)
+			.should('exist');
+
+		cy.get_open_dialog()
+			.get(`.dt-row-header`).should('contain', 'Assignment Rule');
+
+		cy.get_open_dialog()
+			.get(`.dt-row-header`).should('contain', 'Day');
+	});
+});

cypress/integration/navigation.js

@@ -1,7 +1,6 @@
 context('Navigation', () => {
 	before(() => {
 		cy.login();
-		cy.visit('/app/website');
 	});
 	it('Navigate to route with hash in document name', () => {
 		cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true});
@@ -11,4 +10,16 @@ context('Navigation', () => {
 		cy.go('back');
 		cy.title().should('eq', 'Website');
 	});
+
+	it.only('Navigate to previous page after login', () => {
+		cy.visit('/app/todo');
+		cy.findByTitle('To Do').should('be.visible');
+		cy.request('/api/method/logout');
+		cy.reload();
+		cy.get('.btn-primary').contains('Login').click();
+		cy.location('pathname').should('eq', '/login');
+		cy.login();
+		cy.visit('/app');
+		cy.location('pathname').should('eq', '/app/todo');
+	});
 });

cypress/integration/relative_time_filters.js

@@ -1,44 +1,47 @@
-context('Relative Timeframe', () => {
-	before(() => {
-		cy.login();
-		cy.visit('/app/website');
-		cy.window().its('frappe').then(frappe => {
-			frappe.call("frappe.tests.ui_test_helpers.create_todo_records");
-		});
-	});
-	it('sets relative timespan filter for last week and filters list', () => {
-		cy.visit('/app/List/ToDo/List');
-		cy.clear_filters();
-		cy.get('.list-row:contains("this is fourth todo")').should('exist');
-		cy.add_filter();
-		cy.get('.fieldname-select-area').should('exist');
-		cy.get('.fieldname-select-area input').type("Due Date{enter}", { delay: 100 });
-		cy.get('select.condition.form-control').select("Timespan");
-		cy.get('.filter-field select.input-with-feedback.form-control').select("last week");
-		cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh');
-		cy.get('.filter-popover .apply-filters').click({ force: true });
-		cy.wait('@list_refresh');
-		cy.get('.list-row-container').its('length').should('eq', 1);
-		cy.get('.list-row-container').should('contain', 'this is second todo');
-		cy.intercept('POST', '/api/method/frappe.model.utils.user_settings.save')
-			.as('save_user_settings');
-		cy.clear_filters();
-		cy.wait('@save_user_settings');
-	});
-	it('sets relative timespan filter for next week and filters list', () => {
-		cy.visit('/app/List/ToDo/List');
-		cy.clear_filters();
-		cy.get('.list-row:contains("this is fourth todo")').should('exist');
-		cy.add_filter();
-		cy.get('.fieldname-select-area input').type("Due Date{enter}", { delay: 100 });
-		cy.get('select.condition.form-control').select("Timespan");
-		cy.get('.filter-field select.input-with-feedback.form-control').select("next week");
-		cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh');
-		cy.get('.filter-popover .apply-filters').click({ force: true });
-		cy.wait('@list_refresh');
-		cy.intercept('POST', '/api/method/frappe.model.utils.user_settings.save')
-			.as('save_user_settings');
-		cy.clear_filters();
-		cy.wait('@save_user_settings');
-	});
-});
+// TODO: Enable this again
+// currently this is flaky possibly because of different timezone in CI
+
+// context('Relative Timeframe', () => {
+// 	before(() => {
+// 		cy.login();
+// 		cy.visit('/app/website');
+// 		cy.window().its('frappe').then(frappe => {
+// 			frappe.call("frappe.tests.ui_test_helpers.create_todo_records");
+// 		});
+// 	});
+// 	it('sets relative timespan filter for last week and filters list', () => {
+// 		cy.visit('/app/List/ToDo/List');
+// 		cy.clear_filters();
+// 		cy.get('.list-row:contains("this is fourth todo")').should('exist');
+// 		cy.add_filter();
+// 		cy.get('.fieldname-select-area').should('exist');
+// 		cy.get('.fieldname-select-area input').type("Due Date{enter}", { delay: 100 });
+// 		cy.get('select.condition.form-control').select("Timespan");
+// 		cy.get('.filter-field select.input-with-feedback.form-control').select("last week");
+// 		cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh');
+// 		cy.get('.filter-popover .apply-filters').click({ force: true });
+// 		cy.wait('@list_refresh');
+// 		cy.get('.list-row-container').its('length').should('eq', 1);
+// 		cy.get('.list-row-container').should('contain', 'this is second todo');
+// 		cy.intercept('POST', '/api/method/frappe.model.utils.user_settings.save')
+// 			.as('save_user_settings');
+// 		cy.clear_filters();
+// 		cy.wait('@save_user_settings');
+// 	});
+// 	it('sets relative timespan filter for next week and filters list', () => {
+// 		cy.visit('/app/List/ToDo/List');
+// 		cy.clear_filters();
+// 		cy.get('.list-row:contains("this is fourth todo")').should('exist');
+// 		cy.add_filter();
+// 		cy.get('.fieldname-select-area input').type("Due Date{enter}", { delay: 100 });
+// 		cy.get('select.condition.form-control').select("Timespan");
+// 		cy.get('.filter-field select.input-with-feedback.form-control').select("next week");
+// 		cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh');
+// 		cy.get('.filter-popover .apply-filters').click({ force: true });
+// 		cy.wait('@list_refresh');
+// 		cy.intercept('POST', '/api/method/frappe.model.utils.user_settings.save')
+// 			.as('save_user_settings');
+// 		cy.clear_filters();
+// 		cy.wait('@save_user_settings');
+// 	});
+// });

cypress/integration/sidebar.js

@@ -6,12 +6,12 @@ context('Sidebar', () => {
 	});
 
 	it('Test for checking "Assigned To" counter value, adding filter and adding & removing an assignment', () => {
-		cy.click_sidebar_button(0);
+		cy.click_sidebar_button("Assigned To");
 
 		//To check if no filter is available in "Assigned To" dropdown
 		cy.get('.empty-state').should('contain', 'No filters found');
 
-		cy.click_sidebar_button(1);
+		cy.click_sidebar_button("Created By");
 
 		//To check if "Created By" dropdown contains filter
 		cy.get('.group-by-item > .dropdown-item').should('contain', 'Me');
@@ -22,7 +22,7 @@ context('Sidebar', () => {
 		cy.get_field('assign_to_me', 'Check').click();
 		cy.get('.modal-footer > .standard-actions > .btn-primary').click();
 		cy.visit('/app/doctype');
-		cy.click_sidebar_button(0);
+		cy.click_sidebar_button("Assigned To");
 
 		//To check if filter is added in "Assigned To" dropdown after assignment
 		cy.get('.group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item').should('contain', '1');
@@ -38,20 +38,19 @@ context('Sidebar', () => {
 		cy.get('.fieldname-select-area > .awesomplete > .form-control').should('have.value', 'Assigned To');
 		cy.get('.condition').should('have.value', 'like');
 		cy.get('.filter-field > .form-group > .input-with-feedback').should('have.value', '%Administrator%');
+		cy.click_filter_button();
 
 		//To remove the applied filter
-		cy.get('.filter-action-buttons > div > .btn-secondary').contains('Clear Filters').click();
-		cy.click_filter_button();
-		cy.get('.filter-selector > .btn').should('contain', 'Filter');
+		cy.clear_filters();
 
 		//To remove the assignment
 		cy.visit('/app/doctype');
 		cy.click_listview_row_item(0);
 		cy.get('.assignments > .avatar-group > .avatar > .avatar-frame').click();
 		cy.get('.remove-btn').click({force: true});
-		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-header > .modal-actions > .btn-modal-close').click();
+		cy.hide_dialog();
 		cy.visit('/app/doctype');
-		cy.click_sidebar_button(0);
+		cy.click_sidebar_button("Assigned To");
 		cy.get('.empty-state').should('contain', 'No filters found');
 	});
 });

cypress/integration/timeline.js

@@ -4,13 +4,14 @@ context('Timeline', () => {
 	before(() => {
 		cy.visit('/login');
 		cy.login();
-		cy.visit('/app/todo');
 	});
 
 	it('Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo', () => {
 		//Adding new ToDo
+		cy.visit('/app/todo');
 		cy.click_listview_primary_button('Add ToDo');
 		cy.findByRole('button', {name: 'Edit in full page'}).click();
+		cy.findByTitle('New ToDo').should('be.visible');
 		cy.get('[data-fieldname="description"] .ql-editor').eq(0).type('Test ToDo', {force: true});
 		cy.wait(200);
 		cy.findByRole('button', {name: 'Save'}).click();
@@ -28,28 +29,29 @@ context('Timeline', () => {
 		cy.get('.timeline-content').should('contain', 'Testing Timeline');
 
 		//Editing comment
-		cy.click_timeline_action_btn(0);
+		cy.click_timeline_action_btn("Edit");
 		cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(' 123');
-		cy.click_timeline_action_btn(0);
+		cy.click_timeline_action_btn("Save");
 
 		//To check if the edited comment text is visible in timeline content
 		cy.get('.timeline-content').should('contain', 'Testing Timeline 123');
 
 		//Discarding comment
-		cy.click_timeline_action_btn(0);
+		cy.click_timeline_action_btn("Edit");
 		cy.findByRole('button', {name: 'Dismiss'}).click();
 
 		//To check if after discarding the timeline content is same as previous
 		cy.get('.timeline-content').should('contain', 'Testing Timeline 123');
 
 		//Deleting the added comment
-		cy.get('.actions > .btn > .icon').first().click();
+		cy.get('.more-actions > .action-btn').click();
+		cy.get('.more-actions .dropdown-item').contains('Delete').click();
 		cy.findByRole('button', {name: 'Yes'}).click();
 		cy.click_modal_primary_button('Yes');
 
 		//Deleting the added ToDo
-		cy.get('.menu-btn-group button').eq(1).click();
-		cy.get('.menu-btn-group [data-label="Delete"]').click();
+		cy.get('[id="page-ToDo"] .menu-btn-group [data-original-title="Menu"]').click();
+		cy.get('[id="page-ToDo"] .menu-btn-group .dropdown-item').contains('Delete').click();
 		cy.findByRole('button', {name: 'Yes'}).click();
 	});
 
@@ -81,7 +83,7 @@ context('Timeline', () => {
 		cy.visit('/app/custom-submittable-doctype');
 		cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click();
 		cy.findByRole('button', {name: 'Actions'}).click();
-		cy.get('.actions-btn-group > .dropdown-menu > li > .grey-link').eq(7).click();
+		cy.get('.actions-btn-group > .dropdown-menu > li > .dropdown-item').contains("Delete").click();
 		cy.click_modal_primary_button('Yes', {force: true, delay: 700});
 
 		//Deleting the custom doctype

cypress/integration/timeline_email.js

@@ -5,14 +5,16 @@ context('Timeline Email', () => {
 		cy.visit('/app/todo');
 	});
 
-	it('Adding new ToDo, adding email and verifying timeline content for email attachment, deleting attachment and ToDo', () => {
-		//Adding new ToDo
+	it('Adding new ToDo', () => {
 		cy.click_listview_primary_button('Add ToDo');
 		cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500});
 		cy.fill_field("description", "Test ToDo",  "Text Editor");
 		cy.wait(500);
 		cy.get('.primary-action').contains('Save').click({force: true});
 		cy.wait(700);
+	});
+
+	it('Adding email and verifying timeline content for email attachment, deleting attachment and ToDo', () => {
 		cy.visit('/app/todo');
 		cy.get('.list-row > .level-left > .list-subject').eq(0).click();
 
@@ -41,11 +43,13 @@ context('Timeline Email', () => {
 		cy.get('#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .btn').click();
 		cy.get('#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .dropdown-menu > li > .grey-link').eq(9).click();
 		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').click();
+
 		cy.visit('/app/todo');
 		cy.get('.list-row > .level-left > .list-subject > .level-item.ellipsis > .ellipsis').eq(0).click();
 
 		//Removing the added attachment
 		cy.get('.attachment-row > .data-pill > .remove-btn > .icon').click();
+		cy.wait(500);
 		cy.get('.modal-footer:visible > .standard-actions > .btn-primary').contains('Yes').click();
 
 		//To check if the removed attachment is shown in the timeline content

cypress/plugins/index.js

@@ -11,7 +11,7 @@
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 
-module.exports = () => {
-	// `on` is used to hook into various events Cypress emits
-	// `config` is the resolved Cypress config
-};
+module.exports = (on, config) => {
+	require('@cypress/code-coverage/task')(on, config);
+	return config;
+};

cypress/support/commands.js

@@ -187,7 +187,7 @@ Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => {
 	if (fieldtype === 'Select') {
 		cy.get('@input').select(value);
 	} else {
-		cy.get('@input').type(value, {waitForAnimations: false, force: true});
+		cy.get('@input').type(value, {waitForAnimations: false, force: true, delay: 100});
 	}
 	return cy.get('@input');
 });
@@ -252,7 +252,8 @@ Cypress.Commands.add('new_form', doctype => {
 });
 
 Cypress.Commands.add('go_to_list', doctype => {
-	cy.visit(`/app/list/${doctype}/list`);
+	let dt_in_route = doctype.toLowerCase().replace(/ /g, '-');
+	cy.visit(`/app/${dt_in_route}`);
 });
 
 Cypress.Commands.add('clear_cache', () => {
@@ -316,7 +317,11 @@ Cypress.Commands.add('add_filter', () => {
 });
 
 Cypress.Commands.add('clear_filters', () => {
-	cy.get('.filter-section .filter-button').click();
+	cy.intercept({
+		method: 'POST',
+		url: 'api/method/frappe.model.utils.user_settings.save'
+	}).as('filter-saved');
+	cy.get('.filter-section .filter-button').click({force: true});
 	cy.wait(300);
 	cy.get('.filter-popover').should('exist');
 	cy.get('.filter-popover').find('.clear-filters').click();
@@ -324,16 +329,15 @@ Cypress.Commands.add('clear_filters', () => {
 	cy.window().its('cur_list').then(cur_list => {
 		cur_list && cur_list.filter_area && cur_list.filter_area.clear();
 	});
-
-	
+	cy.wait('@filter-saved');
 });
 
 Cypress.Commands.add('click_modal_primary_button', (btn_name) => {
 	cy.get('.modal-footer > .standard-actions > .btn-primary').contains(btn_name).trigger('click', {force: true});
 });
 
-Cypress.Commands.add('click_sidebar_button', (btn_no) => {
-	cy.get('.list-group-by-fields > .group-by-field > .btn').eq(btn_no).click();
+Cypress.Commands.add('click_sidebar_button', (btn_name) => {
+	cy.get('.list-group-by-fields .list-link > a').contains(btn_name).click({force: true});
 });
 
 Cypress.Commands.add('click_listview_row_item', (row_no) => {
@@ -348,6 +352,6 @@ Cypress.Commands.add('click_listview_primary_button', (btn_name) => {
 	cy.get('.primary-action').contains(btn_name).click({force: true});
 });
 
-Cypress.Commands.add('click_timeline_action_btn', (btn_no) => {
-	cy.get('.timeline-content > .timeline-message-box > .justify-between > .actions > .btn').eq(btn_no).first().click();
+Cypress.Commands.add('click_timeline_action_btn', (btn_name) => {
+	cy.get('.timeline-message-box .custom-actions > .btn').contains(btn_name).click();
 });

cypress/support/index.js

@@ -15,6 +15,7 @@
 
 // Import commands.js using ES2015 syntax:
 import './commands';
+import '@cypress/code-coverage/support';
 
 
 // Alternatively you can use CommonJS syntax:

esbuild/esbuild.js

@@ -44,6 +44,11 @@ let argv = yargs
 		type: "boolean",
 		description: "Run in watch mode and rebuild on file changes"
 	})
+	.option("live-reload", {
+		type: "boolean",
+		description: `Automatically reload web pages when assets are rebuilt.
+			Can only be used with the --watch flag.`
+	})
 	.option("production", {
 		type: "boolean",
 		description: "Run build in production mode"
@@ -104,6 +109,9 @@ async function execute() {
 		log_error("There were some problems during build");
 		log();
 		log(chalk.dim(e.stack));
+		if (process.env.CI) {
+			process.kill(process.pid);
+		}
 		return;
 	}
 
@@ -490,7 +498,8 @@ async function notify_redis({ error, success, changed_files }) {
 	if (success) {
 		payload = {
 			success: true,
-			changed_files
+			changed_files,
+			live_reload: argv["live-reload"]
 		};
 	}
 

frappe/__init__.py

@@ -235,12 +235,13 @@ def connect_replica():
 	from frappe.database import get_db
 	user = local.conf.db_name
 	password = local.conf.db_password
+	port = local.conf.replica_db_port
 
 	if local.conf.different_credentials_for_replica:
 		user = local.conf.replica_db_name
 		password = local.conf.replica_db_password
 
-	local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password)
+	local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port)
 
 	# swap db connections
 	local.primary_db = local.db
@@ -618,8 +619,6 @@ def read_only():
 
 			try:
 				retval = fn(*args, **get_newargs(fn, kwargs))
-			except:
-				raise
 			finally:
 				if local and hasattr(local, 'primary_db'):
 					local.db.close()
@@ -629,6 +628,29 @@ def read_only():
 		return wrapper_fn
 	return innfn
 
+def write_only():
+	# if replica connection exists, we have to replace it momentarily with the primary connection
+	def innfn(fn):
+		def wrapper_fn(*args, **kwargs):
+			primary_db = getattr(local, "primary_db", None)
+			replica_db = getattr(local, "replica_db", None)
+			in_read_only = getattr(local, "db", None) != primary_db
+
+			# switch to primary connection
+			if in_read_only and primary_db:
+				local.db = local.primary_db
+
+			try:
+				retval = fn(*args, **get_newargs(fn, kwargs))
+			finally:
+				# switch back to replica connection
+				if in_read_only and replica_db:
+					local.db = replica_db
+
+			return retval
+		return wrapper_fn
+	return innfn
+
 def only_for(roles, message=False):
 	"""Raise `frappe.PermissionError` if the user does not have any of the given **Roles**.
 
@@ -1458,7 +1480,10 @@ def get_value(*args, **kwargs):
 
 def as_json(obj, indent=1):
 	from frappe.utils.response import json_handler
-	return json.dumps(obj, indent=indent, sort_keys=True, default=json_handler, separators=(',', ': '))
+	try:
+		return json.dumps(obj, indent=indent, sort_keys=True, default=json_handler, separators=(',', ': '))
+	except TypeError:
+		return json.dumps(obj, indent=indent, default=json_handler, separators=(',', ': '))
 
 def are_emails_muted():
 	from frappe.utils import cint

frappe/build.py

@@ -1,10 +1,11 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import os
 import re
 import json
 import shutil
 import subprocess
+from subprocess import getoutput
 from io import StringIO
 from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
@@ -17,6 +18,8 @@ import psutil
 from urllib.parse import urlparse
 from simple_chalk import green
 from semantic_version import Version
+from requests import head
+from requests.exceptions import HTTPError
 
 
 timestamps = {}
@@ -24,6 +27,12 @@ app_paths = None
 sites_path = os.path.abspath(os.getcwd())
 
 
+class AssetsNotDownloadedError(Exception):
+	pass
+
+class AssetsDontExistError(HTTPError):
+	pass
+
 def download_file(url, prefix):
 	from requests import get
 
@@ -70,81 +79,94 @@ def build_missing_files():
 		bundle(build_mode, apps="frappe")
 
 
-def get_assets_link(frappe_head):
-	from subprocess import getoutput
-	from requests import head
-
+def get_assets_link(frappe_head) -> str:
 	tag = getoutput(
-			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			r" refs/tags/,,' -e 's/\^{}//'"
-			% frappe_head
-		)
+		r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+		r" refs/tags/,,' -e 's/\^{}//'"
+		% frappe_head
+	)
 
 	if tag:
 		# if tag exists, download assets from github release
-		url = "https://github.com/frappe/frappe/releases/download/{0}/assets.tar.gz".format(tag)
+		url = f"https://github.com/frappe/frappe/releases/download/{tag}/assets.tar.gz"
 	else:
-		url = "http://assets.frappeframework.com/{0}.tar.gz".format(frappe_head)
+		url = f"http://assets.frappeframework.com/{frappe_head}.tar.gz"
 
 	if not head(url):
-		raise ValueError("URL {0} doesn't exist".format(url))
+		reference = f"Release {tag}" if tag else f"Commit {frappe_head}"
+		raise AssetsDontExistError(f"Assets for {reference} don't exist")
 
 	return url
 
 
+def fetch_assets(url, frappe_head):
+	click.secho("Retrieving assets...", fg="yellow")
+
+	prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
+	assets_archive = download_file(url, prefix)
+
+	if not assets_archive:
+		raise AssetsNotDownloadedError(f"Assets could not be retrived from {url}")
+
+	print(f"\n{green('✔')} Downloaded Frappe assets from {url}")
+
+	return assets_archive
+
+
+def setup_assets(assets_archive):
+	import tarfile
+	directories_created = set()
+
+	click.secho("\nExtracting assets...\n", fg="yellow")
+	with tarfile.open(assets_archive) as tar:
+		for file in tar:
+			if not file.isdir():
+				dest = "." + file.name.replace("./frappe-bench/sites", "")
+				asset_directory = os.path.dirname(dest)
+				show = dest.replace("./assets/", "")
+
+				if asset_directory not in directories_created:
+					if not os.path.exists(asset_directory):
+						os.makedirs(asset_directory, exist_ok=True)
+					directories_created.add(asset_directory)
+
+				tar.makefile(file, dest)
+				print("{0} Restored {1}".format(green('✔'), show))
+
+	return directories_created
+
+
 def download_frappe_assets(verbose=True):
 	"""Downloads and sets up Frappe assets if they exist based on the current
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from subprocess import getoutput
-
-	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
 
-	if frappe_head:
+	if not frappe_head:
+		return False
+
+	try:
+		url = get_assets_link(frappe_head)
+		assets_archive = fetch_assets(url, frappe_head)
+		setup_assets(assets_archive)
+		build_missing_files()
+		return True
+
+	except AssetsDontExistError as e:
+		click.secho(str(e), fg="yellow")
+
+	except Exception as e:
+		# TODO: log traceback in bench.log
+		click.secho(str(e), fg="red")
+
+	finally:
 		try:
-			url = get_assets_link(frappe_head)
-			click.secho("Retrieving assets...", fg="yellow")
-			prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
-			assets_archive = download_file(url, prefix)
-			print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))
-
-			if assets_archive:
-				import tarfile
-				directories_created = set()
-
-				click.secho("\nExtracting assets...\n", fg="yellow")
-				with tarfile.open(assets_archive) as tar:
-					for file in tar:
-						if not file.isdir():
-							dest = "." + file.name.replace("./frappe-bench/sites", "")
-							asset_directory = os.path.dirname(dest)
-							show = dest.replace("./assets/", "")
-
-							if asset_directory not in directories_created:
-								if not os.path.exists(asset_directory):
-									os.makedirs(asset_directory, exist_ok=True)
-								directories_created.add(asset_directory)
-
-							tar.makefile(file, dest)
-							print("{0} Restored {1}".format(green('✔'), show))
-
-				build_missing_files()
-				return True
-			else:
-				raise
+			shutil.rmtree(os.path.dirname(assets_archive))
 		except Exception:
-			# TODO: log traceback in bench.log
-			click.secho("An Error occurred while downloading assets...", fg="red")
-			assets_setup = False
-		finally:
-			try:
-				shutil.rmtree(os.path.dirname(assets_archive))
-			except Exception:
-				pass
+			pass
 
-	return assets_setup
+	return False
 
 
 def symlink(target, link_name, overwrite=False):
@@ -224,7 +246,7 @@ def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, ver
 
 	check_node_executable()
 	frappe_app_path = frappe.get_app_path("frappe", "..")
-	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())
+	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env(), raise_err=True)
 
 
 def watch(apps=None):
@@ -235,6 +257,13 @@ def watch(apps=None):
 	if apps:
 		command += " --apps {apps}".format(apps=apps)
 
+	live_reload = frappe.utils.cint(
+		os.environ.get("LIVE_RELOAD", frappe.conf.live_reload)
+	)
+
+	if live_reload:
+		command += " --live-reload"
+
 	check_node_executable()
 	frappe_app_path = frappe.get_app_path("frappe", "..")
 	frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env())

frappe/commands/__init__.py

@@ -102,9 +102,24 @@ def get_commands():
 	from .site import commands as site_commands
 	from .translate import commands as translate_commands
 	from .utils import commands as utils_commands
-	from .redis import commands as redis_commands
+	from .redis_utils import commands as redis_commands
+
+	clickable_link = (
+		"\x1b]8;;https://frappeframework.com/docs\afrappeframework.com\x1b]8;;\a"
+	)
+	all_commands = (
+		scheduler_commands
+		+ site_commands
+		+ translate_commands
+		+ utils_commands
+		+ redis_commands
+	)
+
+	for command in all_commands:
+		if not command.help:
+			command.help = f"Refer to {clickable_link}"
+
+	return all_commands
 
-	all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
-	return list(set(all_commands))
 
 commands = get_commands()

frappe/commands/redis.py → frappe/commands/redis_utils.py

@@ -3,7 +3,7 @@ import os
 import click
 
 import frappe
-from frappe.utils.rq import RedisQueue
+from frappe.utils.redis_queue import RedisQueue
 from frappe.installer import update_site_config
 
 @click.command('create-rq-users')

frappe/commands/site.py

@@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
 		validate_database_sql
 	)
 
+	site = get_site(context)
+	frappe.init(site=site)
+
 	force = context.force or force
 	decompressed_file_name = extract_sql_from_archive(sql_file_path)
 
@@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
 	# check if valid SQL file
 	validate_database_sql(decompressed_file_name, _raise=not force)
 
-	site = get_site(context)
-	frappe.init(site=site)
-
 	# dont allow downgrading to older versions of frappe without force
 	if not force and is_downgrade(decompressed_file_name, verbose=True):
 		warn_message = (
@@ -474,7 +474,7 @@ def remove_from_installed_apps(context, app):
 
 @click.command('uninstall-app')
 @click.argument('app')
-@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False, multiple=True)
+@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False)
 @click.option('--dry-run', help='List all doctypes that will be deleted', is_flag=True, default=False)
 @click.option('--no-backup', help='Do not backup the site', is_flag=True, default=False)
 @click.option('--force', help='Force remove app from site', is_flag=True, default=False)
@@ -738,6 +738,131 @@ def build_search_index(context):
 	finally:
 		frappe.destroy()
 
+@click.command('trim-database')
+@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted')
+@click.option('--format', '-f', default='text', type=click.Choice(['json', 'text']), help='Output format')
+@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site')
+@pass_context
+def trim_database(context, dry_run, format, no_backup):
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	from frappe.utils.backups import scheduled_backup
+
+	ALL_DATA = {}
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+
+		TABLES_TO_DROP = []
+		STANDARD_TABLES = get_standard_tables()
+		information_schema = frappe.qb.Schema("information_schema")
+		table_name = frappe.qb.Field("table_name").as_("name")
+
+		queried_result = frappe.qb.from_(
+			information_schema.tables
+		).select(table_name).where(
+			information_schema.tables.table_schema == frappe.conf.db_name
+		).run()
+
+		database_tables = [x[0] for x in queried_result]
+		doctype_tables = frappe.get_all("DocType", pluck="name")
+
+		for x in database_tables:
+			doctype = x.lstrip("tab")
+			if not (doctype in doctype_tables or x.startswith("__") or x in STANDARD_TABLES):
+				TABLES_TO_DROP.append(x)
+
+		if not TABLES_TO_DROP:
+			if format == "text":
+				click.secho(f"No ghost tables found in {frappe.local.site}...Great!", fg="green")
+		else:
+			if not (no_backup or dry_run):
+				if format == "text":
+					print(f"Backing Up Tables: {', '.join(TABLES_TO_DROP)}")
+
+				odb = scheduled_backup(
+					ignore_conf=False,
+					include_doctypes=",".join(x.lstrip("tab") for x in TABLES_TO_DROP),
+					ignore_files=True,
+					force=True,
+				)
+				if format == "text":
+					odb.print_summary()
+					print("\nTrimming Database")
+
+			for table in TABLES_TO_DROP:
+				if format == "text":
+					print(f"* Dropping Table '{table}'...")
+				if not dry_run:
+					frappe.db.sql_ddl(f"drop table `{table}`")
+
+			ALL_DATA[frappe.local.site] = TABLES_TO_DROP
+		frappe.destroy()
+
+	if format == "json":
+		import json
+		print(json.dumps(ALL_DATA, indent=1))
+
+
+def get_standard_tables():
+	import re
+
+	tables = []
+	sql_file = os.path.join(
+		"..", "apps", "frappe", "frappe", "database", frappe.conf.db_type, f'framework_{frappe.conf.db_type}.sql'
+	)
+	content = open(sql_file).read().splitlines()
+
+	for line in content:
+		table_found = re.search(r"""CREATE TABLE ("|`)(.*)?("|`) \(""", line)
+		if table_found:
+			tables.append(table_found.group(2))
+
+	return tables
+
+@click.command('trim-tables')
+@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted')
+@click.option('--format', '-f', default='table', type=click.Choice(['json', 'table']), help='Output format')
+@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site')
+@pass_context
+def trim_tables(context, dry_run, format, no_backup):
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	from frappe.model.meta import trim_tables
+	from frappe.utils.backups import scheduled_backup
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+
+		if not (no_backup or dry_run):
+			click.secho(f"Taking backup for {frappe.local.site}", fg="green")
+			odb = scheduled_backup(ignore_files=False, force=True)
+			odb.print_summary()
+
+		try:
+			trimmed_data = trim_tables(dry_run=dry_run, quiet=format == 'json')
+
+			if format == 'table' and not dry_run:
+				click.secho(f"The following data have been removed from {frappe.local.site}", fg='green')
+
+			handle_data(trimmed_data, format=format)
+		finally:
+			frappe.destroy()
+
+def handle_data(data: dict, format='json'):
+	if format == 'json':
+		import json
+		print(json.dumps({frappe.local.site: data}, indent=1, sort_keys=True))
+	else:
+		from frappe.utils.commands import render_table
+		data = [["DocType", "Fields"]] + [[table, ", ".join(columns)] for table, columns in data.items()]
+		render_table(data)
+
+
 commands = [
 	add_system_manager,
 	backup,
@@ -766,5 +891,7 @@ commands = [
 	add_to_hosts,
 	start_ngrok,
 	build_search_index,
-	partial_restore
+	partial_restore,
+	trim_tables,
+	trim_database,
 ]

frappe/commands/utils.py

@@ -12,10 +12,9 @@ from frappe.exceptions import SiteNotSpecifiedError
 from frappe.utils import update_progress_bar, cint
 from frappe.coverage import CodeCoverage
 
-DATA_IMPORT_DEPRECATION = click.style(
+DATA_IMPORT_DEPRECATION = (
 	"[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n"
-	"Use `data-import` command instead to import data via 'Data Import'.",
-	fg="yellow"
+	"Use `data-import` command instead to import data via 'Data Import'."
 )
 
 
@@ -364,7 +363,7 @@ def import_doc(context, path, force=False):
 @click.option('--no-email', default=True, is_flag=True, help='Send email if applicable')
 @pass_context
 def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
-	click.secho(DATA_IMPORT_DEPRECATION)
+	click.secho(DATA_IMPORT_DEPRECATION, fg="yellow")
 	sys.exit(1)
 
 
@@ -408,20 +407,47 @@ def bulk_rename(context, doctype, path):
 	frappe.destroy()
 
 
+@click.command('db-console')
+@pass_context
+def database(context):
+	"""
+		Enter into the Database console for given site.
+	"""
+	site = get_site(context)
+	if not site:
+		raise SiteNotSpecifiedError
+	frappe.init(site=site)
+	if not frappe.conf.db_type or frappe.conf.db_type == "mariadb":
+		_mariadb()
+	elif frappe.conf.db_type == "postgres":
+		_psql()
+
+
 @click.command('mariadb')
 @pass_context
 def mariadb(context):
 	"""
 		Enter into mariadb console for a given site.
 	"""
-	import os
-
 	site  = get_site(context)
 	if not site:
 		raise SiteNotSpecifiedError
 	frappe.init(site=site)
+	_mariadb()
+
+
+@click.command('postgres')
+@pass_context
+def postgres(context):
+	"""
+		Enter into postgres console for a given site.
+	"""
+	site  = get_site(context)
+	frappe.init(site=site)
+	_psql()
+
 
-	# This is assuming you're within the bench instance.
+def _mariadb():
 	mysql = find_executable('mysql')
 	os.execv(mysql, [
 		mysql,
@@ -434,15 +460,7 @@ def mariadb(context):
 		"-A"])
 
 
-@click.command('postgres')
-@pass_context
-def postgres(context):
-	"""
-		Enter into postgres console for a given site.
-	"""
-	site  = get_site(context)
-	frappe.init(site=site)
-	# This is assuming you're within the bench instance.
+def _psql():
 	psql = find_executable('psql')
 	subprocess.run([ psql, '-d', frappe.conf.db_name])
 
@@ -485,6 +503,12 @@ frappe.db.connect()
 	])
 
 
+def _console_cleanup():
+	# Execute rollback_observers on console close
+	frappe.db.rollback()
+	frappe.destroy()
+
+
 @click.command('console')
 @click.option(
 	'--autoreload',
@@ -500,6 +524,9 @@ def console(context, autoreload=False):
 	frappe.local.lang = frappe.db.get_default("lang")
 
 	from IPython.terminal.embed import InteractiveShellEmbed
+	from atexit import register
+
+	register(_console_cleanup)
 
 	terminal = InteractiveShellEmbed()
 	if autoreload:
@@ -525,6 +552,74 @@ def console(context, autoreload=False):
 	terminal()
 
 
+@click.command('transform-database', help="Change tables' internal settings changing engine and row formats")
+@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'")
+@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)")
+@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)")
+@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred")
+@pass_context
+def transform_database(context, table, engine, row_format, failfast):
+	"Transform site database through given parameters"
+	site = get_site(context)
+	check_table = []
+	add_line = False
+	skipped = 0
+	frappe.init(site=site)
+
+	if frappe.conf.db_type and frappe.conf.db_type != "mariadb":
+		click.secho("This command only has support for MariaDB databases at this point", fg="yellow")
+		sys.exit(1)
+
+	if not (engine or row_format):
+		click.secho("Values for `--engine` or `--row_format` must be set")
+		sys.exit(1)
+
+	frappe.connect()
+
+	if table == "all":
+		information_schema = frappe.qb.Schema("information_schema")
+		queried_tables = frappe.qb.from_(
+			information_schema.tables
+		).select("table_name").where(
+			(information_schema.tables.row_format != row_format)
+			& (information_schema.tables.table_schema == frappe.conf.db_name)
+		).run()
+		tables = [x[0] for x in queried_tables]
+	else:
+		tables = [x.strip() for x in table.split(",")]
+
+	total = len(tables)
+
+	for current, table in enumerate(tables):
+		values_to_set = ""
+		if engine:
+			values_to_set += f" ENGINE={engine}"
+		if row_format:
+			values_to_set += f" ROW_FORMAT={row_format}"
+
+		try:
+			frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}")
+			update_progress_bar("Updating table schema", current - skipped, total)
+			add_line = True
+
+		except Exception as e:
+			check_table.append([table, e.args])
+			skipped += 1
+
+			if failfast:
+				break
+
+	if add_line:
+		print()
+
+	for errored_table in check_table:
+		table, err = errored_table
+		err_msg = f"{table}: ERROR {err[0]}: {err[1]}"
+		click.secho(err_msg, fg="yellow")
+
+	frappe.destroy()
+
+
 @click.command('run-tests')
 @click.option('--app', help="For App")
 @click.option('--doctype', help="For DocType")
@@ -592,9 +687,10 @@ def run_parallel_tests(context, app, build_number, total_builds, with_coverage=F
 @click.argument('app')
 @click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
 @click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
+@click.option('--with-coverage', is_flag=True, help="Generate coverage report")
 @click.option('--ci-build-id')
 @pass_context
-def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
+def run_ui_tests(context, app, headless=False, parallel=True, with_coverage=False, ci_build_id=None):
 	"Run UI tests"
 	site = get_site(context)
 	app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@@ -604,6 +700,7 @@ def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	# override baseUrl using env variable
 	site_env = f'CYPRESS_baseUrl={site_url}'
 	password_env = f'CYPRESS_adminPassword={admin_password}' if admin_password else ''
+	coverage_env = f'CYPRESS_coverage={str(with_coverage).lower()}'
 
 	os.chdir(app_base_path)
 
@@ -611,22 +708,23 @@ def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
 	cypress_path = f"{node_bin}/cypress"
 	plugin_path = f"{node_bin}/../cypress-file-upload"
 	testing_library_path = f"{node_bin}/../@testing-library"
+	coverage_plugin_path = f"{node_bin}/../@cypress/code-coverage"
 
 	# check if cypress in path...if not, install it.
 	if not (
 		os.path.exists(cypress_path)
 		and os.path.exists(plugin_path)
 		and os.path.exists(testing_library_path)
+		and os.path.exists(coverage_plugin_path)
 		and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6
 	):
 		# install cypress
 		click.secho("Installing Cypress...", fg="yellow")
-		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 --no-lockfile")
+		frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile")
 
 	# run for headless mode
 	run_or_open = 'run --browser firefox --record' if headless else 'open'
-	command = '{site_env} {password_env} {cypress} {run_or_open}'
-	formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
+	formatted_command = f'{site_env} {password_env} {coverage_env} {cypress_path} {run_or_open}'
 
 	if parallel:
 		formatted_command += ' --parallel'
@@ -811,6 +909,8 @@ commands = [
 	build,
 	clear_cache,
 	clear_website_cache,
+	database,
+	transform_database,
 	jupyter,
 	console,
 	destroy_all_sessions,

frappe/contacts/address_and_contact.py

@@ -178,4 +178,4 @@ def set_link_title(doc):
 	for link in doc.links:
 		if not link.link_title:
 			linked_doc = frappe.get_doc(link.link_doctype, link.link_name)
-			link.link_title = linked_doc.get("title_field") or linked_doc.get("name")
+			link.link_title = linked_doc.get_title() or link.link_name

frappe/contacts/doctype/address/address.py

@@ -65,7 +65,7 @@ class Address(Document):
 
 	def has_link(self, doctype, name):
 		for link in self.links:
-			if link.link_doctype==doctype and link.link_name== name:
+			if link.link_doctype == doctype and link.link_name == name:
 				return True
 
 	def has_common_link(self, doc):

frappe/contacts/doctype/contact/contact.py

@@ -47,14 +47,14 @@ class Contact(Document):
 	def get_link_for(self, link_doctype):
 		'''Return the link name, if exists for the given link DocType'''
 		for link in self.links:
-			if link.link_doctype==link_doctype:
+			if link.link_doctype == link_doctype:
 				return link.link_name
 
 		return None
 
 	def has_link(self, doctype, name):
 		for link in self.links:
-			if link.link_doctype==doctype and link.link_name== name:
+			if link.link_doctype == doctype and link.link_name == name:
 				return True
 
 	def has_common_link(self, doc):

frappe/core/doctype/access_log/access_log.py

@@ -1,6 +1,7 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 import frappe
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
 from frappe.model.document import Document
 
 
@@ -9,25 +10,41 @@ class AccessLog(Document):
 
 
 @frappe.whitelist()
-def make_access_log(doctype=None, document=None, method=None, file_type=None,
-		report_name=None, filters=None, page=None, columns=None):
+@frappe.write_only()
+@retry(
+	stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError)
+)
+def make_access_log(
+	doctype=None,
+	document=None,
+	method=None,
+	file_type=None,
+	report_name=None,
+	filters=None,
+	page=None,
+	columns=None,
+):
 
 	user = frappe.session.user
+	in_request = frappe.request and frappe.request.method == "GET"
 
-	doc = frappe.get_doc({
-		'doctype': 'Access Log',
-		'user': user,
-		'export_from': doctype,
-		'reference_document': document,
-		'file_type': file_type,
-		'report_name': report_name,
-		'page': page,
-		'method': method,
-		'filters': frappe.utils.cstr(filters) if filters else None,
-		'columns': columns
-	})
+	doc = frappe.get_doc(
+		{
+			"doctype": "Access Log",
+			"user": user,
+			"export_from": doctype,
+			"reference_document": document,
+			"file_type": file_type,
+			"report_name": report_name,
+			"page": page,
+			"method": method,
+			"filters": frappe.utils.cstr(filters) if filters else None,
+			"columns": columns,
+		}
+	)
 	doc.insert(ignore_permissions=True)
 
 	# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
-	if frappe.request and frappe.request.method == 'GET':
+	# dont commit in test mode
+	if not frappe.flags.in_test or in_request:
 		frappe.db.commit()

frappe/core/doctype/communication/communication.py

@@ -255,7 +255,7 @@ class Communication(Document, CommunicationEmailMixin):
 	def set_delivery_status(self, commit=False):
 		'''Look into the status of Email Queue linked to this Communication and set the Delivery Status of this Communication'''
 		delivery_status = None
-		status_counts = Counter(frappe.db.sql_list('''select status from `tabEmail Queue` where communication=%s''', self.name))
+		status_counts = Counter(frappe.get_all("Email Queue", pluck="status", filters={"communication": self.name}))
 		if self.sent_or_received == "Received":
 			return
 

frappe/core/doctype/communication/mixins.py

@@ -217,17 +217,7 @@ class CommunicationEmailMixin:
 		if not emails:
 			return []
 
-		disabled_users = frappe.db.sql_list("""
-			SELECT
-				email
-			FROM
-				`tabUser`
-			where
-				email in %(emails)s
-				and
-				thread_notify=0
-		""", {'emails': tuple(emails)})
-		return disabled_users
+		return frappe.get_all("User", pluck="email", filters={"email": ["in", emails], "thread_notify": 0})
 
 	@staticmethod
 	def filter_disabled_users(emails):
@@ -236,17 +226,7 @@ class CommunicationEmailMixin:
 		if not emails:
 			return []
 
-		disabled_users = frappe.db.sql_list("""
-			SELECT
-				email
-			FROM
-				`tabUser`
-			where
-				email in %(emails)s
-				and
-				enabled=0
-		""", {'emails': tuple(emails)})
-		return disabled_users
+		return frappe.get_all("User", pluck="email", filters={"email": ["in", emails], "enabled": 0})
 
 	def sendmail_input_dict(self, print_html=None, print_format=None,
 			send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):

frappe/core/doctype/data_export/exporter.py

@@ -261,6 +261,7 @@ class DataExporter:
 			self.writer.writerow([self.data_keys.data_separator])
 
 	def add_data(self):
+		from frappe.query_builder import DocType
 		if self.template and not self.with_data:
 			return
 
@@ -305,9 +306,15 @@ class DataExporter:
 			if self.all_doctypes:
 				# add child tables
 				for c in self.child_doctypes:
-					for ci, child in enumerate(frappe.db.sql("""select * from `tab{0}`
-						where parent=%s and parentfield=%s order by idx""".format(c['doctype']),
-						(doc.name, c['parentfield']), as_dict=1)):
+					child_doctype_table = DocType(c["doctype"])
+					data_row = (
+						frappe.qb.from_(child_doctype_table)
+						.select("*")
+						.where(child_doctype_table.parent == doc.name)
+						.where(child_doctype_table.parentfield == c["parentfield"])
+						.orderby(child_doctype_table.idx)
+					)
+					for ci, child in enumerate(data_row.run()):
 						self.add_data_row(rows, c['doctype'], c['parentfield'], child, ci)
 
 			for row in rows:

frappe/core/doctype/doctype/doctype.py

@@ -23,6 +23,7 @@ from frappe.modules.import_file import get_file_path
 from frappe.model.meta import Meta
 from frappe.desk.utils import validate_route_conflict
 from frappe.website.utils import clear_cache
+from frappe.query_builder.functions import Concat
 
 class InvalidFieldNameError(frappe.ValidationError): pass
 class UniqueFieldnameError(frappe.ValidationError): pass
@@ -274,6 +275,8 @@ class DocType(Document):
 							d.fieldname = d.fieldname + '_section'
 						elif d.fieldtype=='Column Break':
 							d.fieldname = d.fieldname + '_column'
+						elif d.fieldtype=='Tab Break':
+							d.fieldname = d.fieldname + '_tab'
 					else:
 						d.fieldname = d.fieldtype.lower().replace(" ","_") + "_" + str(d.idx)
 				else:
@@ -463,7 +466,7 @@ class DocType(Document):
 			return
 
 		# check if atleast 1 record exists
-		if not (frappe.db.table_exists(self.name) and frappe.db.sql("select name from `tab{}` limit 1".format(self.name))):
+		if not (frappe.db.table_exists(self.name) and frappe.get_all(self.name, fields=["name"], limit=1, as_list=True)):
 			return
 
 		existing_property_setter = frappe.db.get_value("Property Setter", {"doc_type": self.name,
@@ -569,17 +572,17 @@ class DocType(Document):
 	def make_amendable(self):
 		"""If is_submittable is set, add amended_from docfields."""
 		if self.is_submittable:
-			if not frappe.db.sql("""select name from tabDocField
-				where fieldname = 'amended_from' and parent = %s""", self.name):
-					self.append("fields", {
-						"label": "Amended From",
-						"fieldtype": "Link",
-						"fieldname": "amended_from",
-						"options": self.name,
-						"read_only": 1,
-						"print_hide": 1,
-						"no_copy": 1
-					})
+			docfield_exists = frappe.get_all("DocField", filters={"fieldname": "amended_from", "parent": self.name}, pluck="name", limit=1)
+			if not docfield_exists:
+				self.append("fields", {
+					"label": "Amended From",
+					"fieldtype": "Link",
+					"fieldname": "amended_from",
+					"options": self.name,
+					"read_only": 1,
+					"print_hide": 1,
+					"no_copy": 1
+				})
 
 	def make_repeatable(self):
 		"""If allow_auto_repeat is set, add auto_repeat custom field."""
@@ -704,12 +707,13 @@ def validate_series(dt, autoname=None, name=None):
 		and (not autoname.startswith('format:')):
 
 		prefix = autoname.split('.')[0]
-		used_in = frappe.db.sql("""
-			SELECT `name`
-			FROM `tabDocType`
-			WHERE `autoname` LIKE CONCAT(%s, '.%%')
-			AND `name`!=%s
-		""", (prefix, name))
+		doctype = frappe.qb.DocType("DocType")
+		used_in = (frappe.qb
+					.from_(doctype)
+					.select(doctype.name)
+					.where(doctype.autoname.like(Concat(prefix,".%")))
+					.where(doctype.name != name)
+					).run()
 		if used_in:
 			frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0]))
 

frappe/core/doctype/document_naming_rule/document_naming_rule.json

@@ -41,6 +41,7 @@
    "fieldname": "counter",
    "fieldtype": "Int",
    "label": "Counter",
+   "no_copy": 1,
    "read_only": 1
   },
   {
@@ -79,7 +80,7 @@
  ],
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-11-04 14:38:14.836056",
+ "modified": "2021-09-13 20:07:47.617615",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Document Naming Rule",

frappe/core/doctype/feedback/test_feedback.py

@@ -5,6 +5,13 @@ import frappe
 import unittest
 
 class TestFeedback(unittest.TestCase):
+	def tearDown(self):
+		frappe.form_dict.reference_doctype = None
+		frappe.form_dict.reference_name = None
+		frappe.form_dict.rating = None
+		frappe.form_dict.feedback = None
+		frappe.local.request_ip = None
+
 	def test_feedback_creation_updation(self):
 		from frappe.website.doctype.blog_post.test_blog_post import make_test_blog
 		test_blog = make_test_blog()
@@ -12,7 +19,14 @@ class TestFeedback(unittest.TestCase):
 		frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"})
 
 		from frappe.templates.includes.feedback.feedback import add_feedback, update_feedback
-		feedback = add_feedback('Blog Post', test_blog.name, 5, 'New feedback')
+
+		frappe.form_dict.reference_doctype = 'Blog Post'
+		frappe.form_dict.reference_name = test_blog.name
+		frappe.form_dict.rating = 5
+		frappe.form_dict.feedback = 'New feedback'
+		frappe.local.request_ip = '127.0.0.1'
+
+		feedback = add_feedback()
 
 		self.assertEqual(feedback.feedback, 'New feedback')
 		self.assertEqual(feedback.rating, 5)

frappe/core/doctype/file/file.py

@@ -813,7 +813,7 @@ def extract_images_from_doc(doc, fieldname):
 		doc.set(fieldname, content)
 
 
-def extract_images_from_html(doc, content):
+def extract_images_from_html(doc, content, is_private=False):
 	frappe.flags.has_dataurl = False
 
 	def _save_file(match):
@@ -846,7 +846,8 @@ def extract_images_from_html(doc, content):
 			"attached_to_doctype": doctype,
 			"attached_to_name": name,
 			"content": content,
-			"decode": False
+			"decode": False,
+			"is_private": is_private
 		})
 		_file.save(ignore_permissions=True)
 		file_url = _file.file_url

frappe/core/doctype/file/test_file.py

@@ -204,10 +204,14 @@ class TestFile(unittest.TestCase):
 
 
 	def delete_test_data(self):
-		for f in frappe.db.sql('''select name, file_name from tabFile where
-			is_home_folder = 0 and is_attachments_folder = 0 order by creation desc'''):
-			frappe.delete_doc("File", f[0])
-
+		test_file_data = frappe.db.get_all(
+			"File",
+			pluck="name",
+			filters={"is_home_folder": 0, "is_attachments_folder": 0},
+			order_by="creation desc",
+		)
+		for f in test_file_data:
+			frappe.delete_doc("File", f)
 
 	def upload_file(self):
 		_file = frappe.get_doc({

frappe/core/doctype/language/language.json

@@ -7,6 +7,7 @@
  "document_type": "Setup",
  "engine": "InnoDB",
  "field_order": [
+  "enabled",
   "language_code",
   "language_name",
   "flag",
@@ -39,15 +40,22 @@
    "fieldtype": "Link",
    "label": "Based On",
    "options": "Language"
+  },
+  {
+   "default": "1",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "label": "Enabled"
   }
  ],
  "icon": "fa fa-globe",
  "in_create": 1,
  "links": [],
- "modified": "2020-04-16 22:11:33.066852",
+ "modified": "2021-10-18 14:02:06.818219",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Language",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {

frappe/core/doctype/log_settings/log_settings.py

@@ -38,7 +38,7 @@ def has_unseen_error_log(user):
 			'message': _("You have unseen {0}").format('<a href="/app/List/Error%20Log/List"> Error Logs </a>')
 		}
 
-	if frappe.db.sql_list("select name from `tabError Log` where seen = 0 limit 1"):
+	if frappe.get_all("Error Log", filters={"seen": 0}, limit=1):
 		log_settings = frappe.get_cached_doc('Log Settings')
 
 		if log_settings.users_to_notify:

frappe/core/doctype/navbar_settings/navbar_settings.py

@@ -22,7 +22,6 @@ class NavbarSettings(Document):
 		if not frappe.flags.in_patch and (len(before_save_items) > len(after_save_items)):
 			frappe.throw(_("Please hide the standard navbar items instead of deleting them"))
 
-@frappe.whitelist(allow_guest=True)
 def get_app_logo():
 	app_logo = frappe.db.get_single_value('Navbar Settings', 'app_logo', cache=True)
 	if not app_logo:

frappe/core/doctype/package_release/package_release.py

@@ -6,16 +6,27 @@ from frappe.model.document import Document
 from frappe.modules.export_file import export_doc
 import os
 import subprocess
+from frappe.query_builder.functions import Max
+
 
 class PackageRelease(Document):
 	def set_version(self):
 		# set the next patch release by default
+		doctype = frappe.qb.DocType("Package Release")
 		if not self.major:
-			self.major = frappe.db.max('Package Release', 'major', dict(package=self.package))
+			self.major = frappe.qb.from_(doctype) \
+				.where(doctype.package == self.package) \
+				.select(Max(doctype.minor)).run()[0][0] or 0
+
 		if not self.minor:
-			self.minor = frappe.db.max('Package Release', 'minor', dict(package=self.package))
+			self.minor = frappe.qb.from_(doctype) \
+				.where(doctype.package == self.package) \
+				.select(Max("minor")).run()[0][0] or 0
 		if not self.patch:
-			self.patch = frappe.db.max('Package Release', 'patch', dict(package=self.package)) + 1
+			value = frappe.qb.from_(doctype) \
+				.where(doctype.package == self.package) \
+				.select(Max("patch")).run()[0][0] or 0
+			self.patch = value + 1
 
 	def autoname(self):
 		self.set_version()

frappe/core/doctype/server_script/server_script.py

@@ -94,7 +94,7 @@ class ServerScript(Document):
 		Args:
 			doc (Document): Executes script with for a certain document's events
 		"""
-		safe_exec(self.script, _locals={"doc": doc})
+		safe_exec(self.script, _locals={"doc": doc}, restrict_commit_rollback=True)
 
 	def execute_scheduled_method(self):
 		"""Specific to Scheduled Jobs via Server Scripts

frappe/core/doctype/server_script/test_server_script.py

@@ -59,6 +59,26 @@ conditions = '1 = 1'
 		reference_doctype = 'Note',
 		script = '''
 frappe.method_that_doesnt_exist("do some magic")
+'''
+	),
+	dict(
+		name='test_todo_commit',
+		script_type = 'DocType Event',
+		doctype_event = 'Before Save',
+		reference_doctype = 'ToDo',
+		disabled = 1,
+		script = '''
+frappe.db.commit()
+'''
+	),
+	dict(
+		name='test_cache_methods',
+		script_type = 'DocType Event',
+		doctype_event = 'Before Save',
+		reference_doctype = 'ToDo',
+		disabled = 1,
+		script = '''
+frappe.cache().set_value('test_key', doc.name)
 '''
 	)
 ]
@@ -119,3 +139,24 @@ class TestServerScript(unittest.TestCase):
 
 		self.assertTrue("invalid python code" in str(se.exception).lower(),
 				msg="Python code validation not working")
+
+	def test_commit_in_doctype_event(self):
+		server_script = frappe.get_doc('Server Script', 'test_todo_commit')
+		server_script.disabled = 0
+		server_script.save()
+
+		self.assertRaises(AttributeError, frappe.get_doc(dict(doctype='ToDo', description='test me')).insert)
+
+		server_script.disabled = 1
+		server_script.save()
+
+	def test_cache_methods_in_server_script(self):
+		server_script = frappe.get_doc('Server Script', 'test_cache_methods')
+		server_script.disabled = 0
+		server_script.save()
+
+		todo = frappe.get_doc(dict(doctype='ToDo', description='test me')).insert()
+		self.assertEqual(todo.name, frappe.cache().get_value('test_key'))
+
+		server_script.disabled = 1
+		server_script.save()

frappe/core/doctype/sms_settings/sms_settings.json

@@ -1,238 +1,80 @@
 {
- "allow_copy": 1, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2013-01-10 16:34:24", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "editable_grid": 0, 
+ "actions": [],
+ "allow_copy": 1,
+ "creation": "2013-01-10 16:34:24",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "sms_gateway_url",
+  "message_parameter",
+  "receiver_parameter",
+  "static_parameters_section",
+  "parameters",
+  "use_post"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Eg. smsgateway.com/api/send_sms.cgi", 
-   "fieldname": "sms_gateway_url", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "SMS Gateway URL", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "description": "Eg. smsgateway.com/api/send_sms.cgi",
+   "fieldname": "sms_gateway_url",
+   "fieldtype": "Small Text",
+   "in_list_view": 1,
+   "label": "SMS Gateway URL",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Enter url parameter for message", 
-   "fieldname": "message_parameter", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Message Parameter", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "description": "Enter url parameter for message",
+   "fieldname": "message_parameter",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Message Parameter",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Enter url parameter for receiver nos", 
-   "fieldname": "receiver_parameter", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Receiver Parameter", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "description": "Enter url parameter for receiver nos",
+   "fieldname": "receiver_parameter",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Receiver Parameter",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "static_parameters_section", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0, 
+   "fieldname": "static_parameters_section",
+   "fieldtype": "Column Break",
    "width": "50%"
-  }, 
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Enter static url parameters here (Eg. sender=ERPNext, username=ERPNext, password=1234 etc.)", 
-   "fieldname": "parameters", 
-   "fieldtype": "Table", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Static Parameters", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "SMS Parameter", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "description": "Enter static url parameters here (Eg. sender=ERPNext, username=ERPNext, password=1234 etc.)",
+   "fieldname": "parameters",
+   "fieldtype": "Table",
+   "label": "Static Parameters",
+   "options": "SMS Parameter"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "use_post", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Use POST", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "default": "0",
+   "fieldname": "use_post",
+   "fieldtype": "Check",
+   "label": "Use POST"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-cog", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2021-03-02 18:06:00.868688", 
- "modified_by": "Administrator", 
- "module": "Core", 
- "name": "SMS Settings", 
- "owner": "Administrator", 
+ ],
+ "icon": "fa fa-cog",
+ "idx": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2021-09-21 19:45:26.809793",
+ "modified_by": "Administrator",
+ "module": "Core",
+ "name": "SMS Settings",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 0, 
-   "email": 0, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 0, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "track_changes": 1, 
- "track_seen": 0
-}
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "track_changes": 1
+}

frappe/core/doctype/transaction_log/transaction_log.py

@@ -14,10 +14,9 @@ class TransactionLog(Document):
 		self.row_index = index
 		self.timestamp = now_datetime()
 		if index != 1:
-			prev_hash = frappe.db.sql(
-				"SELECT `chaining_hash` FROM `tabTransaction Log` WHERE `row_index` = '{0}'".format(index - 1))
+			prev_hash = frappe.get_all("Transaction Log", filters={"row_index":str(index-1)}, pluck="chaining_hash", limit=1)
 			if prev_hash:
-				self.previous_hash = prev_hash[0][0]
+				self.previous_hash = prev_hash[0]
 			else:
 				self.previous_hash = "Indexing broken"
 		else:

frappe/core/doctype/user/user.json

@@ -202,7 +202,8 @@
    "fieldname": "role_profile_name",
    "fieldtype": "Link",
    "label": "Role Profile",
-   "options": "Role Profile"
+   "options": "Role Profile",
+   "permlevel": 1
   },
   {
    "fieldname": "roles_html",
@@ -670,7 +671,7 @@
   }
  ],
  "max_attachments": 5,
- "modified": "2021-02-02 16:11:06.037543",
+ "modified": "2021-10-18 16:56:05.578379",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "User",

frappe/core/doctype/user/user.py

@@ -788,7 +788,7 @@ def sign_up(email, full_name, redirect_to):
 			return 2, _("Please ask your administrator to verify your sign-up")
 
 @frappe.whitelist(allow_guest=True)
-@rate_limit(key='user', limit=get_password_reset_limit, seconds = 24*60*60, methods=['POST'])
+@rate_limit(limit=get_password_reset_limit, seconds = 24*60*60, methods=['POST'])
 def reset_password(user):
 	if user=="Administrator":
 		return 'not allowed'

frappe/core/doctype/user_permission/user_permission.py

@@ -54,7 +54,7 @@ class UserPermission(Document):
 			ref_link = frappe.get_desk_link(self.doctype, overlap_exists[0].name)
 			frappe.throw(_("{0} has already assigned default value for {1}.").format(ref_link, self.allow))
 
-@frappe.whitelist(allow_guest=True)
+@frappe.whitelist()
 def get_user_permissions(user=None):
 	'''Get all users permissions for the user as a dict of doctype'''
 	# if this is called from client-side,

frappe/core/doctype/user_type/user_type.py

@@ -36,8 +36,11 @@ class UserType(Document):
 		if not self.user_doctypes:
 			return
 
-		modules = frappe.get_all('DocType', fields=['distinct module as module'],
-			filters={'name': ('in', [d.document_type for d in self.user_doctypes])})
+		modules = frappe.get_all("DocType",
+			fields=["module"],
+			filters={"name": ("in", [d.document_type for d in self.user_doctypes])},
+			distinct=True,
+		)
 
 		self.set('user_type_modules', [])
 		for row in modules:

frappe/core/doctype/version/version.css

@@ -1,21 +0,0 @@
-.version-info {
-	overflow: auto;
-}
-
-.version-info pre {
-	border: 0px;
-	margin: 0px;
-	background-color: inherit;
-}
-
-.version-info .table {
-	background-color: inherit;
-}
-
-.version-info .success {
-	background-color: #dff0d8 !important;
-}
-
-.version-info .danger {
-	background-color: #f2dede !important;
-}

frappe/core/doctype/version/version.py

@@ -1,8 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-# License: MIT. See LICENSE
-
 import frappe, json
 
 from frappe.model.document import Document

frappe/core/page/background_jobs/background_jobs.py

@@ -67,7 +67,8 @@ def get_info(show_failed=False) -> List[Dict]:
 			fail_registry = queue.failed_job_registry
 			for job_id in fail_registry.get_job_ids():
 				job = queue.fetch_job(job_id)
-				add_job(job, queue.name)
+				if job:
+					add_job(job, queue.name)
 
 	return jobs
 

frappe/core/page/permission_manager/permission_manager.js

@@ -325,15 +325,15 @@ frappe.PermissionEngine = class PermissionEngine {
 			.attr("data-doctype", d.parent)
 			.attr("data-role", d.role)
 			.attr("data-permlevel", d.permlevel)
-			.click(function () {
+			.on("click", () => {
 				return frappe.call({
 					module: "frappe.core",
 					page: "permission_manager",
 					method: "remove",
 					args: {
-						doctype: $(this).attr("data-doctype"),
-						role: $(this).attr("data-role"),
-						permlevel: $(this).attr("data-permlevel")
+						doctype: d.parent,
+						role: d.role,
+						permlevel: d.permlevel
 					},
 					callback: (r) => {
 						if (r.exc) {

frappe/custom/doctype/custom_field/custom_field.json

@@ -1,460 +1,458 @@
 {
- "actions": [],
- "allow_import": 1,
- "creation": "2013-01-10 16:34:01",
- "description": "Adds a custom field to a DocType",
- "doctype": "DocType",
- "document_type": "Setup",
- "engine": "InnoDB",
- "field_order": [
-  "dt",
-  "module",
-  "label",
-  "label_help",
-  "fieldname",
-  "insert_after",
-  "length",
-  "column_break_6",
-  "fieldtype",
-  "precision",
-  "hide_seconds",
-  "hide_days",
-  "options",
-  "fetch_from",
-  "fetch_if_empty",
-  "options_help",
-  "section_break_11",
-  "collapsible",
-  "collapsible_depends_on",
-  "default",
-  "depends_on",
-  "mandatory_depends_on",
-  "read_only_depends_on",
-  "properties",
-  "non_negative",
-  "reqd",
-  "unique",
-  "read_only",
-  "ignore_user_permissions",
-  "hidden",
-  "print_hide",
-  "print_hide_if_no_value",
-  "print_width",
-  "no_copy",
-  "allow_on_submit",
-  "in_list_view",
-  "in_standard_filter",
-  "in_global_search",
-  "in_preview",
-  "bold",
-  "report_hide",
-  "search_index",
-  "allow_in_quick_entry",
-  "ignore_xss_filter",
-  "translatable",
-  "hide_border",
-  "description",
-  "permlevel",
-  "width",
-  "columns"
- ],
- "fields": [
-  {
-   "bold": 1,
-   "fieldname": "dt",
-   "fieldtype": "Link",
-   "in_filter": 1,
-   "in_list_view": 1,
-   "label": "Document",
-   "oldfieldname": "dt",
-   "oldfieldtype": "Link",
-   "options": "DocType",
-   "reqd": 1,
-   "search_index": 1
-  },
-  {
-   "bold": 1,
-   "fieldname": "label",
-   "fieldtype": "Data",
-   "in_filter": 1,
-   "label": "Label",
-   "no_copy": 1,
-   "oldfieldname": "label",
-   "oldfieldtype": "Data"
-  },
-  {
-   "fieldname": "label_help",
-   "fieldtype": "HTML",
-   "label": "Label Help",
-   "oldfieldtype": "HTML"
-  },
-  {
-   "fieldname": "fieldname",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Fieldname",
-   "no_copy": 1,
-   "oldfieldname": "fieldname",
-   "oldfieldtype": "Data",
-   "read_only": 1
-  },
-  {
-   "description": "Select the label after which you want to insert new field.",
-   "fieldname": "insert_after",
-   "fieldtype": "Select",
-   "label": "Insert After",
-   "no_copy": 1,
-   "oldfieldname": "insert_after",
-   "oldfieldtype": "Select"
-  },
-  {
-   "fieldname": "column_break_6",
-   "fieldtype": "Column Break"
-  },
-  {
-   "bold": 1,
-   "default": "Data",
-   "fieldname": "fieldtype",
-   "fieldtype": "Select",
-   "in_filter": 1,
-   "in_list_view": 1,
-   "label": "Field Type",
-   "oldfieldname": "fieldtype",
-   "oldfieldtype": "Select",
-   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature",
-   "reqd": 1
-  },
-  {
-   "depends_on": "eval:in_list([\"Float\", \"Currency\", \"Percent\"], doc.fieldtype)",
-   "description": "Set non-standard precision for a Float or Currency field",
-   "fieldname": "precision",
-   "fieldtype": "Select",
-   "label": "Precision",
-   "options": "\n0\n1\n2\n3\n4\n5\n6\n7\n8\n9"
-  },
-  {
-   "fieldname": "options",
-   "fieldtype": "Small Text",
-   "in_list_view": 1,
-   "label": "Options",
-   "oldfieldname": "options",
-   "oldfieldtype": "Text"
-  },
-  {
-   "fieldname": "fetch_from",
-   "fieldtype": "Small Text",
-   "label": "Fetch From"
-  },
-  {
-   "default": "0",
-   "description": "If checked, this field will be not overwritten based on Fetch From if a value already exists.",
-   "fieldname": "fetch_if_empty",
-   "fieldtype": "Check",
-   "label": "Fetch If Empty"
-  },
-  {
-   "fieldname": "options_help",
-   "fieldtype": "HTML",
-   "label": "Options Help",
-   "oldfieldtype": "HTML"
-  },
-  {
-   "fieldname": "section_break_11",
-   "fieldtype": "Section Break"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:doc.fieldtype==\"Section Break\"",
-   "fieldname": "collapsible",
-   "fieldtype": "Check",
-   "label": "Collapsible"
-  },
-  {
-   "depends_on": "eval:doc.fieldtype==\"Section Break\"",
-   "fieldname": "collapsible_depends_on",
-   "fieldtype": "Code",
-   "label": "Collapsible Depends On"
-  },
-  {
-   "fieldname": "default",
-   "fieldtype": "Text",
-   "label": "Default Value",
-   "oldfieldname": "default",
-   "oldfieldtype": "Text"
-  },
-  {
-   "fieldname": "depends_on",
-   "fieldtype": "Code",
-   "label": "Depends On",
-   "length": 255
-  },
-  {
-   "fieldname": "description",
-   "fieldtype": "Text",
-   "label": "Field Description",
-   "oldfieldname": "description",
-   "oldfieldtype": "Text",
-   "print_width": "300px",
-   "width": "300px"
-  },
-  {
-   "default": "0",
-   "fieldname": "permlevel",
-   "fieldtype": "Int",
-   "label": "Permission Level",
-   "oldfieldname": "permlevel",
-   "oldfieldtype": "Int"
-  },
-  {
-   "fieldname": "width",
-   "fieldtype": "Data",
-   "label": "Width",
-   "oldfieldname": "width",
-   "oldfieldtype": "Data"
-  },
-  {
-   "description": "Number of columns for a field in a List View or a Grid (Total Columns should be less than 11)",
-   "fieldname": "columns",
-   "fieldtype": "Int",
-   "label": "Columns"
-  },
-  {
-   "fieldname": "properties",
-   "fieldtype": "Column Break",
-   "oldfieldtype": "Column Break",
-   "print_width": "50%",
-   "width": "50%"
-  },
-  {
-   "default": "0",
-   "fieldname": "reqd",
-   "fieldtype": "Check",
-   "in_list_view": 1,
-   "label": "Is Mandatory Field",
-   "oldfieldname": "reqd",
-   "oldfieldtype": "Check"
-  },
-  {
-   "default": "0",
-   "fieldname": "unique",
-   "fieldtype": "Check",
-   "label": "Unique"
-  },
-  {
-   "default": "0",
-   "fieldname": "read_only",
-   "fieldtype": "Check",
-   "label": "Read Only"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:doc.fieldtype===\"Link\"",
-   "fieldname": "ignore_user_permissions",
-   "fieldtype": "Check",
-   "label": "Ignore User Permissions"
-  },
-  {
-   "default": "0",
-   "fieldname": "hidden",
-   "fieldtype": "Check",
-   "label": "Hidden"
-  },
-  {
-   "default": "0",
-   "fieldname": "print_hide",
-   "fieldtype": "Check",
-   "label": "Print Hide",
-   "oldfieldname": "print_hide",
-   "oldfieldtype": "Check"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:[\"Int\", \"Float\", \"Currency\", \"Percent\"].indexOf(doc.fieldtype)!==-1",
-   "fieldname": "print_hide_if_no_value",
-   "fieldtype": "Check",
-   "label": "Print Hide If No Value"
-  },
-  {
-   "fieldname": "print_width",
-   "fieldtype": "Data",
-   "hidden": 1,
-   "label": "Print Width",
-   "no_copy": 1,
-   "print_hide": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "no_copy",
-   "fieldtype": "Check",
-   "label": "No Copy",
-   "oldfieldname": "no_copy",
-   "oldfieldtype": "Check"
-  },
-  {
-   "default": "0",
-   "fieldname": "allow_on_submit",
-   "fieldtype": "Check",
-   "label": "Allow on Submit",
-   "oldfieldname": "allow_on_submit",
-   "oldfieldtype": "Check"
-  },
-  {
-   "default": "0",
-   "fieldname": "in_list_view",
-   "fieldtype": "Check",
-   "label": "In List View"
-  },
-  {
-   "default": "0",
-   "fieldname": "in_standard_filter",
-   "fieldtype": "Check",
-   "label": "In Standard Filter"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:([\"Data\", \"Select\", \"Table\", \"Text\", \"Text Editor\", \"Link\", \"Small Text\", \"Long Text\", \"Read Only\", \"Heading\", \"Dynamic Link\"].indexOf(doc.fieldtype) !== -1)",
-   "fieldname": "in_global_search",
-   "fieldtype": "Check",
-   "label": "In Global Search"
-  },
-  {
-   "default": "0",
-   "fieldname": "bold",
-   "fieldtype": "Check",
-   "label": "Bold"
-  },
-  {
-   "default": "0",
-   "fieldname": "report_hide",
-   "fieldtype": "Check",
-   "label": "Report Hide",
-   "oldfieldname": "report_hide",
-   "oldfieldtype": "Check"
-  },
-  {
-   "default": "0",
-   "fieldname": "search_index",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "label": "Index",
-   "no_copy": 1,
-   "print_hide": 1
-  },
-  {
-   "default": "0",
-   "description": "Don't HTML Encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field",
-   "fieldname": "ignore_xss_filter",
-   "fieldtype": "Check",
-   "label": "Ignore XSS Filter"
-  },
-  {
-   "default": "1",
-   "depends_on": "eval:['Data', 'Select', 'Text', 'Small Text', 'Text Editor'].includes(doc.fieldtype)",
-   "fieldname": "translatable",
-   "fieldtype": "Check",
-   "label": "Translatable"
-  },
-  {
-   "depends_on": "eval:in_list(['Data', 'Link', 'Dynamic Link', 'Password', 'Select', 'Read Only', 'Attach', 'Attach Image', 'Int'], doc.fieldtype)",
-   "fieldname": "length",
-   "fieldtype": "Int",
-   "label": "Length"
-  },
-  {
-   "fieldname": "mandatory_depends_on",
-   "fieldtype": "Code",
-   "label": "Mandatory Depends On",
-   "length": 255
-  },
-  {
-   "fieldname": "read_only_depends_on",
-   "fieldtype": "Code",
-   "label": "Read Only Depends On",
-   "length": 255
-  },
-  {
-   "default": "0",
-   "fieldname": "allow_in_quick_entry",
-   "fieldtype": "Check",
-   "label": "Allow in Quick Entry"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:!in_list(['Table', 'Table MultiSelect'], doc.fieldtype);",
-   "fieldname": "in_preview",
-   "fieldtype": "Check",
-   "label": "In Preview"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:doc.fieldtype=='Duration'",
-   "fieldname": "hide_seconds",
-   "fieldtype": "Check",
-   "label": "Hide Seconds"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:doc.fieldtype=='Duration'",
-   "fieldname": "hide_days",
-   "fieldtype": "Check",
-   "label": "Hide Days"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:doc.fieldtype=='Section Break'",
-   "fieldname": "hide_border",
-   "fieldtype": "Check",
-   "label": "Hide Border"
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:in_list([\"Int\", \"Float\", \"Currency\"], doc.fieldtype)",
-   "fieldname": "non_negative",
-   "fieldtype": "Check",
-   "label": "Non Negative"
-  },
-  {
-   "fieldname": "module",
-   "fieldtype": "Link",
-   "label": "Module (for export)",
-   "options": "Module Def"
-  }
- ],
- "icon": "fa fa-glass",
- "idx": 1,
- "index_web_pages_for_search": 1,
- "links": [],
- "modified": "2021-09-04 12:45:22.810120",
- "modified_by": "Administrator",
- "module": "Custom",
- "name": "Custom Field",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "Administrator",
-   "share": 1,
-   "write": 1
-  },
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "search_fields": "dt,label,fieldtype,options",
- "sort_field": "modified",
- "sort_order": "ASC",
- "track_changes": 1
+    "actions": [],
+    "allow_import": 1,
+    "creation": "2013-01-10 16:34:01",
+    "description": "Adds a custom field to a DocType",
+    "doctype": "DocType",
+    "document_type": "Setup",
+    "engine": "InnoDB",
+    "field_order": [
+        "dt",
+        "module",
+        "label",
+        "label_help",
+        "fieldname",
+        "insert_after",
+        "length",
+        "column_break_6",
+        "fieldtype",
+        "precision",
+        "hide_seconds",
+        "hide_days",
+        "options",
+        "fetch_from",
+        "fetch_if_empty",
+        "options_help",
+        "section_break_11",
+        "collapsible",
+        "collapsible_depends_on",
+        "default",
+        "depends_on",
+        "mandatory_depends_on",
+        "read_only_depends_on",
+        "properties",
+        "non_negative",
+        "reqd",
+        "unique",
+        "read_only",
+        "ignore_user_permissions",
+        "hidden",
+        "print_hide",
+        "print_hide_if_no_value",
+        "print_width",
+        "no_copy",
+        "allow_on_submit",
+        "in_list_view",
+        "in_standard_filter",
+        "in_global_search",
+        "in_preview",
+        "bold",
+        "report_hide",
+        "search_index",
+        "allow_in_quick_entry",
+        "ignore_xss_filter",
+        "translatable",
+        "hide_border",
+        "description",
+        "permlevel",
+        "width",
+        "columns"
+    ],
+    "fields": [{
+            "bold": 1,
+            "fieldname": "dt",
+            "fieldtype": "Link",
+            "in_filter": 1,
+            "in_list_view": 1,
+            "label": "Document",
+            "oldfieldname": "dt",
+            "oldfieldtype": "Link",
+            "options": "DocType",
+            "reqd": 1,
+            "search_index": 1
+        },
+        {
+            "bold": 1,
+            "fieldname": "label",
+            "fieldtype": "Data",
+            "in_filter": 1,
+            "label": "Label",
+            "no_copy": 1,
+            "oldfieldname": "label",
+            "oldfieldtype": "Data"
+        },
+        {
+            "fieldname": "label_help",
+            "fieldtype": "HTML",
+            "label": "Label Help",
+            "oldfieldtype": "HTML"
+        },
+        {
+            "fieldname": "fieldname",
+            "fieldtype": "Data",
+            "in_list_view": 1,
+            "label": "Fieldname",
+            "no_copy": 1,
+            "oldfieldname": "fieldname",
+            "oldfieldtype": "Data",
+            "read_only": 1
+        },
+        {
+            "description": "Select the label after which you want to insert new field.",
+            "fieldname": "insert_after",
+            "fieldtype": "Select",
+            "label": "Insert After",
+            "no_copy": 1,
+            "oldfieldname": "insert_after",
+            "oldfieldtype": "Select"
+        },
+        {
+            "fieldname": "column_break_6",
+            "fieldtype": "Column Break"
+        },
+        {
+            "bold": 1,
+            "default": "Data",
+            "fieldname": "fieldtype",
+            "fieldtype": "Select",
+            "in_filter": 1,
+            "in_list_view": 1,
+            "label": "Field Type",
+            "oldfieldname": "fieldtype",
+            "oldfieldtype": "Select",
+            "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature\nTab Break",
+            "reqd": 1
+        },
+        {
+            "depends_on": "eval:in_list([\"Float\", \"Currency\", \"Percent\"], doc.fieldtype)",
+            "description": "Set non-standard precision for a Float or Currency field",
+            "fieldname": "precision",
+            "fieldtype": "Select",
+            "label": "Precision",
+            "options": "\n0\n1\n2\n3\n4\n5\n6\n7\n8\n9"
+        },
+        {
+            "fieldname": "options",
+            "fieldtype": "Small Text",
+            "in_list_view": 1,
+            "label": "Options",
+            "oldfieldname": "options",
+            "oldfieldtype": "Text"
+        },
+        {
+            "fieldname": "fetch_from",
+            "fieldtype": "Small Text",
+            "label": "Fetch From"
+        },
+        {
+            "default": "0",
+            "description": "If checked, this field will be not overwritten based on Fetch From if a value already exists.",
+            "fieldname": "fetch_if_empty",
+            "fieldtype": "Check",
+            "label": "Fetch If Empty"
+        },
+        {
+            "fieldname": "options_help",
+            "fieldtype": "HTML",
+            "label": "Options Help",
+            "oldfieldtype": "HTML"
+        },
+        {
+            "fieldname": "section_break_11",
+            "fieldtype": "Section Break"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:doc.fieldtype==\"Section Break\"",
+            "fieldname": "collapsible",
+            "fieldtype": "Check",
+            "label": "Collapsible"
+        },
+        {
+            "depends_on": "eval:doc.fieldtype==\"Section Break\"",
+            "fieldname": "collapsible_depends_on",
+            "fieldtype": "Code",
+            "label": "Collapsible Depends On"
+        },
+        {
+            "fieldname": "default",
+            "fieldtype": "Text",
+            "label": "Default Value",
+            "oldfieldname": "default",
+            "oldfieldtype": "Text"
+        },
+        {
+            "fieldname": "depends_on",
+            "fieldtype": "Code",
+            "label": "Depends On",
+            "length": 255
+        },
+        {
+            "fieldname": "description",
+            "fieldtype": "Text",
+            "label": "Field Description",
+            "oldfieldname": "description",
+            "oldfieldtype": "Text",
+            "print_width": "300px",
+            "width": "300px"
+        },
+        {
+            "default": "0",
+            "fieldname": "permlevel",
+            "fieldtype": "Int",
+            "label": "Permission Level",
+            "oldfieldname": "permlevel",
+            "oldfieldtype": "Int"
+        },
+        {
+            "fieldname": "width",
+            "fieldtype": "Data",
+            "label": "Width",
+            "oldfieldname": "width",
+            "oldfieldtype": "Data"
+        },
+        {
+            "description": "Number of columns for a field in a List View or a Grid (Total Columns should be less than 11)",
+            "fieldname": "columns",
+            "fieldtype": "Int",
+            "label": "Columns"
+        },
+        {
+            "fieldname": "properties",
+            "fieldtype": "Column Break",
+            "oldfieldtype": "Column Break",
+            "print_width": "50%",
+            "width": "50%"
+        },
+        {
+            "default": "0",
+            "fieldname": "reqd",
+            "fieldtype": "Check",
+            "in_list_view": 1,
+            "label": "Is Mandatory Field",
+            "oldfieldname": "reqd",
+            "oldfieldtype": "Check"
+        },
+        {
+            "default": "0",
+            "fieldname": "unique",
+            "fieldtype": "Check",
+            "label": "Unique"
+        },
+        {
+            "default": "0",
+            "fieldname": "read_only",
+            "fieldtype": "Check",
+            "label": "Read Only"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:doc.fieldtype===\"Link\"",
+            "fieldname": "ignore_user_permissions",
+            "fieldtype": "Check",
+            "label": "Ignore User Permissions"
+        },
+        {
+            "default": "0",
+            "fieldname": "hidden",
+            "fieldtype": "Check",
+            "label": "Hidden"
+        },
+        {
+            "default": "0",
+            "fieldname": "print_hide",
+            "fieldtype": "Check",
+            "label": "Print Hide",
+            "oldfieldname": "print_hide",
+            "oldfieldtype": "Check"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:[\"Int\", \"Float\", \"Currency\", \"Percent\"].indexOf(doc.fieldtype)!==-1",
+            "fieldname": "print_hide_if_no_value",
+            "fieldtype": "Check",
+            "label": "Print Hide If No Value"
+        },
+        {
+            "fieldname": "print_width",
+            "fieldtype": "Data",
+            "hidden": 1,
+            "label": "Print Width",
+            "no_copy": 1,
+            "print_hide": 1
+        },
+        {
+            "default": "0",
+            "fieldname": "no_copy",
+            "fieldtype": "Check",
+            "label": "No Copy",
+            "oldfieldname": "no_copy",
+            "oldfieldtype": "Check"
+        },
+        {
+            "default": "0",
+            "fieldname": "allow_on_submit",
+            "fieldtype": "Check",
+            "label": "Allow on Submit",
+            "oldfieldname": "allow_on_submit",
+            "oldfieldtype": "Check"
+        },
+        {
+            "default": "0",
+            "fieldname": "in_list_view",
+            "fieldtype": "Check",
+            "label": "In List View"
+        },
+        {
+            "default": "0",
+            "fieldname": "in_standard_filter",
+            "fieldtype": "Check",
+            "label": "In Standard Filter"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:([\"Data\", \"Select\", \"Table\", \"Text\", \"Text Editor\", \"Link\", \"Small Text\", \"Long Text\", \"Read Only\", \"Heading\", \"Dynamic Link\"].indexOf(doc.fieldtype) !== -1)",
+            "fieldname": "in_global_search",
+            "fieldtype": "Check",
+            "label": "In Global Search"
+        },
+        {
+            "default": "0",
+            "fieldname": "bold",
+            "fieldtype": "Check",
+            "label": "Bold"
+        },
+        {
+            "default": "0",
+            "fieldname": "report_hide",
+            "fieldtype": "Check",
+            "label": "Report Hide",
+            "oldfieldname": "report_hide",
+            "oldfieldtype": "Check"
+        },
+        {
+            "default": "0",
+            "fieldname": "search_index",
+            "fieldtype": "Check",
+            "hidden": 1,
+            "label": "Index",
+            "no_copy": 1,
+            "print_hide": 1
+        },
+        {
+            "default": "0",
+            "description": "Don't HTML Encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field",
+            "fieldname": "ignore_xss_filter",
+            "fieldtype": "Check",
+            "label": "Ignore XSS Filter"
+        },
+        {
+            "default": "1",
+            "depends_on": "eval:['Data', 'Select', 'Text', 'Small Text', 'Text Editor'].includes(doc.fieldtype)",
+            "fieldname": "translatable",
+            "fieldtype": "Check",
+            "label": "Translatable"
+        },
+        {
+            "depends_on": "eval:in_list(['Data', 'Link', 'Dynamic Link', 'Password', 'Select', 'Read Only', 'Attach', 'Attach Image', 'Int'], doc.fieldtype)",
+            "fieldname": "length",
+            "fieldtype": "Int",
+            "label": "Length"
+        },
+        {
+            "fieldname": "mandatory_depends_on",
+            "fieldtype": "Code",
+            "label": "Mandatory Depends On",
+            "length": 255
+        },
+        {
+            "fieldname": "read_only_depends_on",
+            "fieldtype": "Code",
+            "label": "Read Only Depends On",
+            "length": 255
+        },
+        {
+            "default": "0",
+            "fieldname": "allow_in_quick_entry",
+            "fieldtype": "Check",
+            "label": "Allow in Quick Entry"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:!in_list(['Table', 'Table MultiSelect'], doc.fieldtype);",
+            "fieldname": "in_preview",
+            "fieldtype": "Check",
+            "label": "In Preview"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:doc.fieldtype=='Duration'",
+            "fieldname": "hide_seconds",
+            "fieldtype": "Check",
+            "label": "Hide Seconds"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:doc.fieldtype=='Duration'",
+            "fieldname": "hide_days",
+            "fieldtype": "Check",
+            "label": "Hide Days"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:doc.fieldtype=='Section Break'",
+            "fieldname": "hide_border",
+            "fieldtype": "Check",
+            "label": "Hide Border"
+        },
+        {
+            "default": "0",
+            "depends_on": "eval:in_list([\"Int\", \"Float\", \"Currency\"], doc.fieldtype)",
+            "fieldname": "non_negative",
+            "fieldtype": "Check",
+            "label": "Non Negative"
+        },
+        {
+            "fieldname": "module",
+            "fieldtype": "Link",
+            "label": "Module (for export)",
+            "options": "Module Def"
+        }
+    ],
+    "icon": "fa fa-glass",
+    "idx": 1,
+    "index_web_pages_for_search": 1,
+    "links": [],
+    "modified": "2021-09-04 12:45:23.810120",
+    "modified_by": "Administrator",
+    "module": "Custom",
+    "name": "Custom Field",
+    "owner": "Administrator",
+    "permissions": [{
+            "create": 1,
+            "delete": 1,
+            "email": 1,
+            "print": 1,
+            "read": 1,
+            "report": 1,
+            "role": "Administrator",
+            "share": 1,
+            "write": 1
+        },
+        {
+            "create": 1,
+            "delete": 1,
+            "email": 1,
+            "print": 1,
+            "read": 1,
+            "report": 1,
+            "role": "System Manager",
+            "share": 1,
+            "write": 1
+        }
+    ],
+    "search_fields": "dt,label,fieldtype,options",
+    "sort_field": "modified",
+    "sort_order": "ASC",
+    "track_changes": 1
 }

frappe/custom/doctype/custom_field/custom_field.py

@@ -18,7 +18,7 @@ class CustomField(Document):
 		if not self.fieldname:
 			label = self.label
 			if not label:
-				if self.fieldtype in ["Section Break", "Column Break"]:
+				if self.fieldtype in ["Section Break", "Column Break", "Tab Break"]:
 					label = self.fieldtype + "_" + str(self.idx)
 				else:
 					frappe.throw(_("Label is mandatory"))

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -82,7 +82,7 @@
    "label": "Type",
    "oldfieldname": "fieldtype",
    "oldfieldtype": "Select",
-   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime",
+   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nTab Break",
    "reqd": 1,
    "search_index": 1
   },
@@ -428,7 +428,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-07-10 21:57:24.479749",
+ "modified": "2021-07-11 21:57:24.479749",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",

frappe/custom/doctype/property_setter/property_setter.py

@@ -34,7 +34,7 @@ class PropertySetter(Document):
 			fields=['fieldname', 'label', 'fieldtype'],
 			filters={
 				'parent': dt,
-				'fieldtype': ['not in', ('Section Break', 'Column Break', 'HTML', 'Read Only', 'Fold') + frappe.model.table_fields],
+				'fieldtype': ['not in', ('Section Break', 'Column Break', 'Tab Break', 'HTML', 'Read Only', 'Fold') + frappe.model.table_fields],
 				'fieldname': ['!=', '']
 			},
 			order_by='label asc',

frappe/data/sample_site_config.json

@@ -1,45 +0,0 @@
-{
- "db_name": "testdb",
- "db_password": "password",
- "mute_emails": true,
- 
-  "limits": {
-  	"emails": 1500,
-	"space": 0.157,
-  	"expiry": "2016-07-25",
-  	"users": 1
-  },
-
- "developer_mode": 1,
- "auto_cache_clear": true,
- "disable_website_cache": true,
- "max_file_size": 1000000,
-
- "mail_server": "localhost",
- "mail_login": null,
- "mail_password": null,
- "mail_port": 25,
- "use_ssl": 0,
- "auto_email_id": "hello@example.com",
-
- "google_analytics_id": "google_analytics_id",
- "google_analytics_anonymize_ip": 1,
-
- "google_login": {
-	 "client_id": "google_client_id",
-	 "client_secret": "google_client_secret"
- },
- "github_login": {
-	 "client_id": "github_client_id",
-	 "client_secret": "github_client_secret"
- },
- "facebook_login": {
-	 "client_id": "facebook_client_id",
-	 "client_secret": "facebook_client_secret"
- },
-
- "celery_broker": "redis://localhost",
- "celery_result_backend": null,
- "scheduler_interval": 300,
- "celery_queue_per_site": true
-}

frappe/database/database.py

@@ -14,8 +14,13 @@ import frappe.model.meta
 
 from frappe import _
 from time import time
-from frappe.utils import now, getdate, cast, get_datetime, get_table_name
+from frappe.utils import now, getdate, cast, get_datetime
 from frappe.model.utils.link_count import flush_local_link_count
+from frappe.query_builder.functions import Count
+from frappe.query_builder.functions import Min, Max, Avg, Sum
+from frappe.query_builder.utils import Column
+from .query import Query
+from pypika.terms import PseudoColumn
 
 
 class Database(object):
@@ -55,6 +60,7 @@ class Database(object):
 
 		self.password = password or frappe.conf.db_password
 		self.value_cache = {}
+		self.query = Query()
 
 	def setup_type_map(self):
 		pass
@@ -77,7 +83,7 @@ class Database(object):
 		pass
 
 	def sql(self, query, values=(), as_dict = 0, as_list = 0, formatted = 0,
-		debug=0, ignore_ddl=0, as_utf8=0, auto_commit=0, update=None, explain=False):
+		debug=0, ignore_ddl=0, as_utf8=0, auto_commit=0, update=None, explain=False, run=True):
 		"""Execute a SQL query and fetch all rows.
 
 		:param query: SQL query.
@@ -90,7 +96,7 @@ class Database(object):
 		:param as_utf8: Encode values as UTF 8.
 		:param auto_commit: Commit after executing the query.
 		:param update: Update this dict to all rows (if returned `as_dict`).
-
+		:param run: Returns query without executing it if False.
 		Examples:
 
 			# return customer names as dicts
@@ -105,6 +111,9 @@ class Database(object):
 
 		"""
 		query = str(query)
+		if not run:
+			return query
+
 		if re.search(r'ifnull\(', query, flags=re.IGNORECASE):
 			# replaces ifnull in query with coalesce
 			query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE)
@@ -310,59 +319,6 @@ class Database(object):
 			nres.append(nr)
 		return nres
 
-	def build_conditions(self, filters):
-		"""Convert filters sent as dict, lists to SQL conditions. filter's key
-		is passed by map function, build conditions like:
-
-		* ifnull(`fieldname`, default_value) = %(fieldname)s
-		* `fieldname` [=, !=, >, >=, <, <=] %(fieldname)s
-		"""
-		conditions = []
-		values = {}
-		def _build_condition(key):
-			"""
-				filter's key is passed by map function
-				build conditions like:
-					* ifnull(`fieldname`, default_value) = %(fieldname)s
-					* `fieldname` [=, !=, >, >=, <, <=] %(fieldname)s
-			"""
-			_operator = "="
-			_rhs = " %(" + key + ")s"
-			value = filters.get(key)
-			values[key] = value
-			if isinstance(value, (list, tuple)):
-				# value is a tuple like ("!=", 0)
-				_operator = value[0]
-				values[key] = value[1]
-				if isinstance(value[1], (tuple, list)):
-					# value is a list in tuple ("in", ("A", "B"))
-					_rhs = " ({0})".format(", ".join(self.escape(v) for v in value[1]))
-					del values[key]
-
-			if _operator not in ["=", "!=", ">", ">=", "<", "<=", "like", "in", "not in", "not like"]:
-				_operator = "="
-
-			if "[" in key:
-				split_key = key.split("[")
-				condition = "coalesce(`" + split_key[0] + "`, " + split_key[1][:-1] + ") " \
-					+ _operator + _rhs
-			else:
-				condition = "`" + key + "` " + _operator + _rhs
-
-			conditions.append(condition)
-
-		if isinstance(filters, int):
-			# docname is a number, convert to string
-			filters = str(filters)
-
-		if isinstance(filters, str):
-			filters = { "name": filters }
-
-		for f in filters:
-			_build_condition(f)
-
-		return " and ".join(conditions), values
-
 	def get(self, doctype, filters=None, as_dict=True, cache=False):
 		"""Returns `get_value` with fieldname='*'"""
 		return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache)
@@ -424,9 +380,8 @@ class Database(object):
 			(doctype, filters, fieldname) in self.value_cache:
 			return self.value_cache[(doctype, filters, fieldname)]
 
-		if not order_by: order_by = 'modified desc'
-
 		if isinstance(filters, list):
+			order_by = order_by or "modified_desc"
 			out = self._get_value_for_many_names(doctype, filters, fieldname, debug=debug)
 
 		else:
@@ -439,6 +394,7 @@ class Database(object):
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:
+					order_by = order_by or "modified"
 					out = self._get_values_from_table(fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update)
 				except Exception as e:
 					if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)):
@@ -567,32 +523,23 @@ class Database(object):
 		return self.get_single_value(*args, **kwargs)
 
 	def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, update=None, for_update=False):
-		fl = []
+		field_objects = []
+
+		for field in fields:
+			if "(" in field or " as " in field:
+				field_objects.append(PseudoColumn(field))
+			else:
+				field_objects.append(field)
+
+		criterion = self.query.build_conditions(table=doctype, filters=filters, orderby=order_by, for_update=for_update)
+
 		if isinstance(fields, (list, tuple)):
-			for f in fields:
-				if "(" in f or " as " in f: # function
-					fl.append(f)
-				else:
-					fl.append("`" + f + "`")
-			fl = ", ".join(fl)
+			query = criterion.select(*field_objects)
 		else:
-			fl = fields
 			if fields=="*":
+				query = criterion.select(fields)
 				as_dict = True
-
-		conditions, values = self.build_conditions(filters)
-
-		order_by = ("order by " + order_by) if order_by else ""
-
-		r = self.sql("select {fields} from `tab{doctype}` {where} {conditions} {order_by} {for_update}"
-			.format(
-				for_update = 'for update' if for_update else '',
-				fields = fl,
-				doctype = doctype,
-				where = "where" if conditions else "",
-				conditions = conditions,
-				order_by = order_by),
-			values, as_dict=as_dict, debug=debug, update=update)
+		r = self.sql(query, as_dict=as_dict, debug=debug, update=update)
 
 		return r
 
@@ -819,50 +766,34 @@ class Database(object):
 			except Exception:
 				return None
 
+	def min(self, dt, fieldname, filters=None, **kwargs):
+		return self.query.build_conditions(dt, filters=filters).select(Min(Column(fieldname))).run(**kwargs)[0][0] or 0
+
+	def max(self, dt, fieldname, filters=None, **kwargs):
+		return self.query.build_conditions(dt, filters=filters).select(Max(Column(fieldname))).run(**kwargs)[0][0] or 0
+
+	def avg(self, dt, fieldname, filters=None, **kwargs):
+		return self.query.build_conditions(dt, filters=filters).select(Avg(Column(fieldname))).run(**kwargs)[0][0] or 0
+
+	def sum(self, dt, fieldname, filters=None, **kwargs):
+		return self.query.build_conditions(dt, filters=filters).select(Sum(Column(fieldname))).run(**kwargs)[0][0] or 0
+
 	def count(self, dt, filters=None, debug=False, cache=False):
 		"""Returns `COUNT(*)` for given DocType and filters."""
 		if cache and not filters:
 			cache_count = frappe.cache().get_value('doctype:count:{}'.format(dt))
 			if cache_count is not None:
 				return cache_count
+		query = self.query.build_conditions(table=dt, filters=filters).select(Count("*"))
 		if filters:
-			conditions, filters = self.build_conditions(filters)
-			count = self.sql("""select count(*)
-				from `tab%s` where %s""" % (dt, conditions), filters, debug=debug)[0][0]
+			count = self.sql(query, debug=debug)[0][0]
 			return count
 		else:
-			count = self.sql("""select count(*)
-				from `tab%s`""" % (dt,))[0][0]
-
+			count = self.sql(query, debug=debug)[0][0]
 			if cache:
 				frappe.cache().set_value('doctype:count:{}'.format(dt), count, expires_in_sec = 86400)
-
 			return count
 
-	def sum(self, dt, fieldname, filters=None):
-		return self._get_aggregation('SUM', dt, fieldname, filters)
-
-	def avg(self, dt, fieldname, filters=None):
-		return self._get_aggregation('AVG', dt, fieldname, filters)
-
-	def min(self, dt, fieldname, filters=None):
-		return self._get_aggregation('MIN', dt, fieldname, filters)
-
-	def max(self, dt, fieldname, filters=None):
-		return self._get_aggregation('MAX', dt, fieldname, filters)
-
-	def _get_aggregation(self, function, dt, fieldname, filters=None):
-		if not self.has_column(dt, fieldname):
-			frappe.throw(frappe._('Invalid column'), self.InvalidColumnName)
-
-		query = f'SELECT {function}({fieldname}) AS value FROM `tab{dt}`'
-		values = ()
-		if filters:
-			conditions, values = self.build_conditions(filters)
-			query = f"{query} WHERE {conditions}"
-
-		return self.sql(query, values)[0][0] or 0
-
 	@staticmethod
 	def format_date(date):
 		return getdate(date).strftime("%Y-%m-%d")
@@ -919,13 +850,13 @@ class Database(object):
 			WHERE table_name = 'tab{0}' AND column_name = '{1}' '''.format(doctype, column))[0][0]
 
 	def has_index(self, table_name, index_name):
-		pass
+		raise NotImplementedError
 
 	def add_index(self, doctype, fields, index_name=None):
-		pass
+		raise NotImplementedError
 
 	def add_unique(self, doctype, fields, constraint_name=None):
-		pass
+		raise NotImplementedError
 
 	@staticmethod
 	def get_index_name(fields):
@@ -951,7 +882,7 @@ class Database(object):
 	def escape(s, percent=True):
 		"""Excape quotes and percent in given string."""
 		# implemented in specific class
-		pass
+		raise NotImplementedError
 
 	@staticmethod
 	def is_column_missing(e):
@@ -984,16 +915,9 @@ class Database(object):
 		"""
 		values = ()
 		filters = filters or kwargs.get("conditions")
-		table = get_table_name(doctype)
-		query = f"DELETE FROM `{table}`"
-
+		query = self.query.build_conditions(table=doctype, filters=filters).delete()
 		if "debug" not in kwargs:
 			kwargs["debug"] = debug
-
-		if filters:
-			conditions, values = self.build_conditions(filters)
-			query = f"{query} WHERE {conditions}"
-
 		return self.sql(query, values, **kwargs)
 
 	def truncate(self, doctype: str):

frappe/database/mariadb/database.py

@@ -22,11 +22,11 @@ class MariaDBDatabase(Database):
 	def setup_type_map(self):
 		self.db_type = 'mariadb'
 		self.type_map = {
-			'Currency':		('decimal', '18,6'),
+			'Currency':		('decimal', '21,9'),
 			'Int':			('int', '11'),
 			'Long Int':		('bigint', '20'),
-			'Float':		('decimal', '18,6'),
-			'Percent':		('decimal', '18,6'),
+			'Float':		('decimal', '21,9'),
+			'Percent':		('decimal', '21,9'),
 			'Check':		('int', '1'),
 			'Small Text':	('text', ''),
 			'Long Text':	('longtext', ''),
@@ -51,7 +51,7 @@ class MariaDBDatabase(Database):
 			'Color':		('varchar', self.VARCHAR_LEN),
 			'Barcode':		('longtext', ''),
 			'Geolocation':	('longtext', ''),
-			'Duration':		('decimal', '18,6'),
+			'Duration':		('decimal', '21,9'),
 			'Icon':			('varchar', self.VARCHAR_LEN)
 		}
 
@@ -135,8 +135,8 @@ class MariaDBDatabase(Database):
 		table_name = get_table_name(doctype)
 		return self.sql(f"DESC `{table_name}`")
 
-	def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]:
-		table_name = get_table_name(table)
+	def change_column_type(self, doctype: str, column: str, type: str) -> Union[List, Tuple]:
+		table_name = get_table_name(doctype)
 		return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL")
 
 	# exception types
@@ -195,7 +195,7 @@ class MariaDBDatabase(Database):
 				`password` TEXT NOT NULL,
 				`encrypted` INT(1) NOT NULL DEFAULT 0,
 				PRIMARY KEY (`doctype`, `name`, `fieldname`)
-			) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
+			) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
 
 	def create_global_search_table(self):
 		if not '__global_search' in self.get_tables():
@@ -256,11 +256,11 @@ class MariaDBDatabase(Database):
 				index_name=index_name
 			))
 
-	def add_index(self, doctype, fields, index_name=None):
+	def add_index(self, doctype: str, fields: List, index_name: str = None):
 		"""Creates an index with given fields if not already created.
 		Index name will be `fieldname1_fieldname2_index`"""
 		index_name = index_name or self.get_index_name(fields)
-		table_name = 'tab' + doctype
+		table_name = get_table_name(doctype)
 		if not self.has_index(table_name, index_name):
 			self.commit()
 			self.sql("""ALTER TABLE `%s`

frappe/database/mariadb/framework_mariadb.sql

@@ -72,7 +72,7 @@ CREATE TABLE `tabDocField` (
   KEY `label` (`label`),
   KEY `fieldtype` (`fieldtype`),
   KEY `fieldname` (`fieldname`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
 --
@@ -109,7 +109,7 @@ CREATE TABLE `tabDocPerm` (
   `email` int(1) NOT NULL DEFAULT 1,
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabDocType Action`
@@ -133,7 +133,7 @@ CREATE TABLE `tabDocType Action` (
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`),
   KEY `modified` (`modified`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;
 
 --
 -- Table structure for table `tabDocType Action`
@@ -156,7 +156,7 @@ CREATE TABLE `tabDocType Link` (
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`),
   KEY `modified` (`modified`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=COMPRESSED;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;
 
 --
 -- Table structure for table `tabDocType`
@@ -226,9 +226,10 @@ CREATE TABLE `tabDocType` (
   `email_append_to` int(1) NOT NULL DEFAULT 0,
   `subject_field` varchar(255) DEFAULT NULL,
   `sender_field` varchar(255) DEFAULT NULL,
+  `migration_hash` varchar(255) DEFAULT NULL,
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabSeries`
@@ -239,7 +240,7 @@ CREATE TABLE `tabSeries` (
   `name` varchar(100),
   `current` int(10) NOT NULL DEFAULT 0,
   PRIMARY KEY(`name`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
 --
@@ -256,7 +257,7 @@ CREATE TABLE `tabSessions` (
   `device` varchar(255) DEFAULT 'desktop',
   `status` varchar(20) DEFAULT NULL,
   KEY `sid` (`sid`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 
 --
@@ -269,7 +270,7 @@ CREATE TABLE `tabSingles` (
   `field` varchar(255) DEFAULT NULL,
   `value` text,
   KEY `singles_doctype_field_index` (`doctype`, `field`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `__Auth`
@@ -283,7 +284,7 @@ CREATE TABLE `__Auth` (
 	`password` TEXT NOT NULL,
 	`encrypted` INT(1) NOT NULL DEFAULT 0,
 	PRIMARY KEY (`doctype`, `name`, `fieldname`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabFile`
@@ -311,7 +312,7 @@ CREATE TABLE `tabFile` (
   KEY `parent` (`parent`),
   KEY `attached_to_name` (`attached_to_name`),
   KEY `attached_to_doctype` (`attached_to_doctype`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --
 -- Table structure for table `tabDefaultValue`
@@ -334,4 +335,4 @@ CREATE TABLE `tabDefaultValue` (
   PRIMARY KEY (`name`),
   KEY `parent` (`parent`),
   KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`)
-) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

frappe/database/mariadb/schema.py

@@ -4,18 +4,22 @@ from frappe.database.schema import DBTable
 
 class MariaDBTable(DBTable):
 	def create(self):
-		add_text = ''
+		additional_definitions = ""
+		engine = self.meta.get("engine") or "InnoDB"
+		varchar_len = frappe.db.VARCHAR_LEN
 
 		# columns
 		column_defs = self.get_column_definitions()
-		if column_defs: add_text += ',\n'.join(column_defs) + ',\n'
+		if column_defs:
+			additional_definitions += ',\n'.join(column_defs) + ',\n'
 
 		# index
 		index_defs = self.get_index_definitions()
-		if index_defs: add_text += ',\n'.join(index_defs) + ',\n'
+		if index_defs:
+			additional_definitions += ',\n'.join(index_defs) + ',\n'
 
 		# create table
-		frappe.db.sql("""create table `%s` (
+		query = f"""create table `{self.table_name}` (
 			name varchar({varchar_len}) not null primary key,
 			creation datetime(6),
 			modified datetime(6),
@@ -26,13 +30,15 @@ class MariaDBTable(DBTable):
 			parentfield varchar({varchar_len}),
 			parenttype varchar({varchar_len}),
 			idx int(8) not null default '0',
-			%sindex parent(parent),
+			{additional_definitions}
+			index parent(parent),
 			index modified(modified))
 			ENGINE={engine}
-			ROW_FORMAT=COMPRESSED
+			ROW_FORMAT=DYNAMIC
 			CHARACTER SET=utf8mb4
-			COLLATE=utf8mb4_unicode_ci""".format(varchar_len=frappe.db.VARCHAR_LEN,
-				engine=self.meta.get("engine") or 'InnoDB') % (self.table_name, add_text))
+			COLLATE=utf8mb4_unicode_ci"""
+
+		frappe.db.sql(query)
 
 	def alter(self):
 		for col in self.columns.values():

frappe/database/mariadb/setup_db.py

@@ -34,25 +34,23 @@ def setup_database(force, source_sql, verbose, no_mariadb_socket=False):
 	db_name = frappe.local.conf.db_name
 	root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password)
 	dbman = DbManager(root_conn)
+	dbman_kwargs = {}
+	if no_mariadb_socket:
+		dbman_kwargs["host"] = "%"
+
 	if force or (db_name not in dbman.get_database_list()):
-		dbman.delete_user(db_name)
-		if no_mariadb_socket:
-			dbman.delete_user(db_name, host="%")
+		dbman.delete_user(db_name, **dbman_kwargs)
 		dbman.drop_database(db_name)
 	else:
 		raise Exception("Database %s already exists" % (db_name,))
 
-	dbman.create_user(db_name, frappe.conf.db_password)
-	if no_mariadb_socket:
-		dbman.create_user(db_name, frappe.conf.db_password, host="%")
+	dbman.create_user(db_name, frappe.conf.db_password, **dbman_kwargs)
 	if verbose: print("Created user %s" % db_name)
 
 	dbman.create_database(db_name)
 	if verbose: print("Created database %s" % db_name)
 
-	dbman.grant_all_privileges(db_name, db_name)
-	if no_mariadb_socket:
-		dbman.grant_all_privileges(db_name, db_name, host="%")
+	dbman.grant_all_privileges(db_name, db_name, **dbman_kwargs)
 	dbman.flush_privileges()
 	if verbose: print("Granted privileges to user %s and database %s" % (db_name, db_name))
 

frappe/database/postgres/database.py

@@ -4,6 +4,7 @@ from typing import List, Tuple, Union
 import psycopg2
 import psycopg2.extensions
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION
 
 import frappe
 from frappe.database.database import Database
@@ -31,11 +32,11 @@ class PostgresDatabase(Database):
 	def setup_type_map(self):
 		self.db_type = 'postgres'
 		self.type_map = {
-			'Currency':		('decimal', '18,6'),
+			'Currency':		('decimal', '21,9'),
 			'Int':			('bigint', None),
 			'Long Int':		('bigint', None),
-			'Float':		('decimal', '18,6'),
-			'Percent':		('decimal', '18,6'),
+			'Float':		('decimal', '21,9'),
+			'Percent':		('decimal', '21,9'),
 			'Check':		('smallint', None),
 			'Small Text':	('text', ''),
 			'Long Text':	('text', ''),
@@ -60,7 +61,7 @@ class PostgresDatabase(Database):
 			'Color':		('varchar', self.VARCHAR_LEN),
 			'Barcode':		('text', ''),
 			'Geolocation':	('text', ''),
-			'Duration':		('decimal', '18,6'),
+			'Duration':		('decimal', '21,9'),
 			'Icon':			('varchar', self.VARCHAR_LEN)
 		}
 
@@ -171,7 +172,7 @@ class PostgresDatabase(Database):
 
 	@staticmethod
 	def is_data_too_long(e):
-		return e.pgcode == '22001'
+		return e.pgcode == STRING_DATA_RIGHT_TRUNCATION
 
 	def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]:
 		old_name = get_table_name(old_name)
@@ -182,8 +183,8 @@ class PostgresDatabase(Database):
 		table_name = get_table_name(doctype)
 		return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'")
 
-	def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]:
-		table_name = get_table_name(table)
+	def change_column_type(self, doctype: str, column: str, type: str) -> Union[List, Tuple]:
+		table_name = get_table_name(doctype)
 		return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}')
 
 	def create_auth_table(self):
@@ -258,14 +259,14 @@ class PostgresDatabase(Database):
 		return self.sql("""SELECT 1 FROM pg_indexes WHERE tablename='{table_name}'
 			and indexname='{index_name}' limit 1""".format(table_name=table_name, index_name=index_name))
 
-	def add_index(self, doctype, fields, index_name=None):
+	def add_index(self, doctype: str, fields: List, index_name: str = None):
 		"""Creates an index with given fields if not already created.
 		Index name will be `fieldname1_fieldname2_index`"""
+		table_name = get_table_name(doctype)
 		index_name = index_name or self.get_index_name(fields)
-		table_name = 'tab' + doctype
+		fields_str = '", "'.join(re.sub(r"\(.*\)", "", field) for field in fields)
 
-		self.commit()
-		self.sql("""CREATE INDEX IF NOT EXISTS "{}" ON `{}`("{}")""".format(index_name, table_name, '", "'.join(fields)))
+		self.sql_ddl(f'CREATE INDEX IF NOT EXISTS "{index_name}" ON `{table_name}` ("{fields_str}")')
 
 	def add_unique(self, doctype, fields, constraint_name=None):
 		if isinstance(fields, str):

frappe/database/postgres/framework_postgres.sql

@@ -231,6 +231,7 @@ CREATE TABLE "tabDocType" (
   "email_append_to" smallint NOT NULL DEFAULT 0,
   "subject_field" varchar(255) DEFAULT NULL,
   "sender_field" varchar(255) DEFAULT NULL,
+  "migration_hash" varchar(255) DEFAULT NULL,
   PRIMARY KEY ("name")
 ) ;
 

frappe/database/query.py

@@ -0,0 +1,267 @@
+import operator
+from typing import Any, Dict, List, Tuple, Union
+
+import frappe
+from frappe.query_builder import Criterion, Order, Field
+
+
+def like(key: str, value: str) -> frappe.qb:
+	"""Wrapper method for `LIKE`
+
+	Args:
+		key (str): field
+		value (str): criterion
+
+	Returns:
+		frappe.qb: `frappe.qb object with `LIKE`
+	"""
+	return Field(key).like(value)
+
+
+def func_in(key: str, value: Union[List, Tuple]) -> frappe.qb:
+	"""Wrapper method for `IN`
+
+	Args:
+		key (str): field
+		value (Union[int, str]): criterion
+
+	Returns:
+		frappe.qb: `frappe.qb object with `IN`
+	"""
+	return Field(key).isin(value)
+
+
+def not_like(key: str, value: str) -> frappe.qb:
+	"""Wrapper method for `NOT LIKE`
+
+	Args:
+		key (str): field
+		value (str): criterion
+
+	Returns:
+		frappe.qb: `frappe.qb object with `NOT LIKE`
+	"""
+	return Field(key).not_like(value)
+
+
+def func_not_in(key: str, value: Union[List, Tuple]):
+	"""Wrapper method for `NOT IN`
+
+	Args:
+		key (str): field
+		value (Union[int, str]): criterion
+
+	Returns:
+		frappe.qb: `frappe.qb object with `NOT IN`
+	"""
+	return Field(key).notin(value)
+
+
+def func_regex(key: str, value: str) -> frappe.qb:
+	"""Wrapper method for `REGEX`
+
+	Args:
+		key (str): field
+		value (str): criterion
+
+	Returns:
+		frappe.qb: `frappe.qb object with `REGEX`
+	"""
+	return Field(key).regex(value)
+
+
+def func_between(key: str, value: Union[List, Tuple]) -> frappe.qb:
+	"""Wrapper method for `BETWEEN`
+
+	Args:
+		key (str): field
+		value (Union[int, str]): criterion
+
+	Returns:
+		frappe.qb: `frappe.qb object with `BETWEEN`
+	"""
+	return Field(key)[slice(*value)]
+
+def make_function(key: Any, value: Union[int, str]):
+	"""returns fucntion query
+
+	Args:
+		key (Any): field
+		value (Union[int, str]): criterion
+
+	Returns:
+		frappe.qb: frappe.qb object
+	"""
+	return OPERATOR_MAP[value[0]](key, value[1])
+
+
+def change_orderby(order: str):
+	"""Convert orderby to standart Order object
+
+	Args:
+		order (str): Field, order
+
+	Returns:
+		tuple: field, order
+	"""
+	order = order.split()
+	if order[1].lower() == "asc":
+		orderby, order = order[0], Order.asc
+		return orderby, order
+	orderby, order = order[0], Order.desc
+	return orderby, order
+
+
+OPERATOR_MAP = {
+					"+": operator.add,
+					"=": operator.eq,
+					"-": operator.sub,
+					"!=": operator.ne,
+					"<": operator.lt,
+					">": operator.gt,
+					"<=": operator.le,
+					">=": operator.ge,
+					"in": func_in,
+					"not in": func_not_in,
+					"like": like,
+					"not like": not_like,
+					"regex": func_regex,
+					"between": func_between
+				}
+
+
+class Query:
+	def get_condition(self, table: str, **kwargs) -> frappe.qb:
+		"""Get initial table object
+
+		Args:
+			table (str): DocType
+
+		Returns:
+			frappe.qb: DocType with initial condition
+		"""
+		if kwargs.get("update"):
+			return frappe.qb.update(table)
+		if kwargs.get("into"):
+			return frappe.qb.into(table)
+		return frappe.qb.from_(table)
+
+	def criterion_query(self, table: str, criterion: Criterion, **kwargs) -> frappe.qb:
+		"""Generate filters from Criterion objects
+
+		Args:
+			table (str): DocType
+			criterion (Criterion): Filters
+
+		Returns:
+			frappe.qb: condition object
+		"""
+		condition = self.get_condition(table, **kwargs)
+		return condition.where(criterion)
+
+	def add_conditions(self, conditions: frappe.qb, **kwargs):
+		"""Adding additional conditions
+
+		Args:
+			conditions (frappe.qb): built conditions
+
+		Returns:
+			conditions (frappe.qb): frappe.qb object
+		"""
+		if kwargs.get("orderby"):
+			orderby = kwargs.get("orderby")
+			order = kwargs.get("order") if kwargs.get("order") else Order.desc
+			if isinstance(orderby, str) and len(orderby.split()) > 1:
+				orderby, order = change_orderby(orderby)
+			conditions = conditions.orderby(orderby, order=order)
+
+		if kwargs.get("limit"):
+			conditions = conditions.limit(kwargs.get("limit"))
+
+		if kwargs.get("distinct"):
+			conditions = conditions.distinct()
+
+		if kwargs.get("for_update"):
+			conditions = conditions.for_update()
+
+		return conditions
+
+	def misc_query(self, table: str, filters: Union[List, Tuple] = None, **kwargs):
+		"""Build conditions using the given Lists or Tuple filters
+
+		Args:
+			table (str): DocType
+			filters (Union[List, Tuple], optional): Filters. Defaults to None.
+		"""
+		conditions = self.get_condition(table, **kwargs)
+		if not filters:
+			return conditions
+		if isinstance(filters, list):
+			for f in filters:
+				if not isinstance(f, (list, tuple)):
+					_operator = OPERATOR_MAP[filters[1]]
+					if not isinstance(filters[0], str):
+						conditions = make_function(filters[0], filters[2])
+						break
+					conditions = conditions.where(_operator(Field(filters[0]), filters[2]))
+					break
+				else:
+					_operator = OPERATOR_MAP[f[1]]
+					conditions = conditions.where(_operator(Field(f[0]), f[2]))
+
+		conditions = self.add_conditions(conditions, **kwargs)
+		return conditions
+
+	def dict_query(self, table: str, filters: Dict[str, Union[str, int]] = None, **kwargs) -> frappe.qb:
+		"""Build conditions using the given dictionary filters
+
+		Args:
+			table (str): DocType
+			filters (Dict[str, Union[str, int]], optional): Filters. Defaults to None.
+
+		Returns:
+			frappe.qb: conditions object
+		"""
+		conditions = self.get_condition(table, **kwargs)
+		if not filters:
+			return conditions
+
+		for key in filters:
+			value = filters.get(key)
+			_operator = OPERATOR_MAP["="]
+
+			if not isinstance(key, str):
+				conditions = conditions.where(make_function(key, value))
+				continue
+			if isinstance(value, (list, tuple)):
+				if isinstance(value[1], (list, tuple)) or value[0] in list(OPERATOR_MAP.keys())[-4:]:
+					_operator = OPERATOR_MAP[value[0]]
+					conditions = conditions.where(_operator(key, value[1]))
+				else:
+					_operator = OPERATOR_MAP[value[0]]
+					conditions = conditions.where(_operator(Field(key), value[1]))
+			else:
+				conditions = conditions.where(_operator(Field(key), value))
+		conditions = self.add_conditions(conditions, **kwargs)
+		return conditions
+
+	def build_conditions(self, table: str, filters: Union[Dict[str, Union[str, int]], str, int] = None, **kwargs) -> frappe.qb:
+		"""Build conditions for sql query
+
+		Args:
+			filters (Union[Dict[str, Union[str, int]], str, int]): conditions in Dict
+			table (str): DocType
+
+		Returns:
+			frappe.qb: frappe.qb conditions object
+		"""
+		if isinstance(filters, Criterion):
+			return self.criterion_query(table, filters, **kwargs)
+
+		if isinstance(filters, int) or isinstance(filters, str):
+			filters = {"name": str(filters)}
+
+		if isinstance(filters, (list, tuple)):
+			return self.misc_query(table, filters, **kwargs)
+
+		return self.dict_query(filters=filters, table=table, **kwargs)

frappe/database/schema.py

@@ -303,6 +303,8 @@ def get_definition(fieldtype, precision=None, length=None):
 	size = d[1] if d[1] else None
 
 	if size:
+		# This check needs to exist for backward compatibility.
+		# Till V13, default size used for float, currency and percent are (18, 6).
 		if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6:
 			size = '21,9'
 

frappe/desk/doctype/note/note.json

@@ -1,322 +1,106 @@
 {
-    "allow_copy": 0, 
-    "allow_guest_to_view": 0, 
-    "allow_import": 0, 
-    "allow_rename": 1, 
-    "beta": 0, 
-    "creation": "2013-05-24 13:41:00", 
-    "custom": 0, 
-    "description": "", 
-    "docstatus": 0, 
-    "doctype": "DocType", 
-    "document_type": "Document", 
-    "editable_grid": 0, 
-    "fields": [
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 0, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "fieldname": "title", 
-      "fieldtype": "Data", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 1, 
-      "in_list_view": 1, 
-      "in_standard_filter": 0, 
-      "label": "Title", 
-      "length": 0, 
-      "no_copy": 1, 
-      "permlevel": 0, 
-      "print_hide": 1, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 1, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 1, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "description": "", 
-      "fieldname": "public", 
-      "fieldtype": "Check", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 0, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Public", 
-      "length": 0, 
-      "no_copy": 0, 
-      "permlevel": 0, 
-      "print_hide": 1, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 1, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "depends_on": "public", 
-      "fieldname": "notify_on_login", 
-      "fieldtype": "Check", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 0, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Notify users with a popup when they log in", 
-      "length": 0, 
-      "no_copy": 0, 
-      "permlevel": 0, 
-      "precision": "", 
-      "print_hide": 0, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 1, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "default": "0", 
-      "depends_on": "notify_on_login", 
-      "description": "If enabled, users will be notified every time they login. If not enabled, users will only be notified once.", 
-      "fieldname": "notify_on_every_login", 
-      "fieldtype": "Check", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 0, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Notify Users On Every Login", 
-      "length": 0, 
-      "no_copy": 0, 
-      "permlevel": 0, 
-      "precision": "", 
-      "print_hide": 0, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 0, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "depends_on": "eval:doc.notify_on_login && doc.public", 
-      "fieldname": "expire_notification_on", 
-      "fieldtype": "Date", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 0, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Expire Notification On", 
-      "length": 0, 
-      "no_copy": 0, 
-      "permlevel": 0, 
-      "precision": "", 
-      "print_hide": 0, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 1, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 1, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "description": "Help: To link to another record in the system, use \"#Form/Note/[Note Name]\" as the Link URL. (don't use \"http://\")", 
-      "fieldname": "content", 
-      "fieldtype": "Text Editor", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 1, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Content", 
-      "length": 0, 
-      "no_copy": 0, 
-      "permlevel": 0, 
-      "print_hide": 0, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 0, 
-      "collapsible": 1, 
-      "columns": 0, 
-      "fieldname": "seen_by_section", 
-      "fieldtype": "Section Break", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 0, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Seen By", 
-      "length": 0, 
-      "no_copy": 0, 
-      "permlevel": 0, 
-      "precision": "", 
-      "print_hide": 0, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }, 
-     {
-      "allow_bulk_edit": 0, 
-      "allow_in_quick_entry": 0, 
-      "allow_on_submit": 0, 
-      "bold": 0, 
-      "collapsible": 0, 
-      "columns": 0, 
-      "fieldname": "seen_by", 
-      "fieldtype": "Table", 
-      "hidden": 0, 
-      "ignore_user_permissions": 0, 
-      "ignore_xss_filter": 0, 
-      "in_filter": 0, 
-      "in_global_search": 0, 
-      "in_list_view": 0, 
-      "in_standard_filter": 0, 
-      "label": "Seen By Table", 
-      "length": 0, 
-      "no_copy": 0, 
-      "options": "Note Seen By", 
-      "permlevel": 0, 
-      "precision": "", 
-      "print_hide": 0, 
-      "print_hide_if_no_value": 0, 
-      "read_only": 0, 
-      "remember_last_selected_value": 0, 
-      "report_hide": 0, 
-      "reqd": 0, 
-      "search_index": 0, 
-      "set_only_once": 0, 
-      "translatable": 0, 
-      "unique": 0
-     }
-    ], 
-    "has_web_view": 0, 
-    "hide_heading": 0, 
-    "hide_toolbar": 0, 
-    "icon": "fa fa-file-text", 
-    "idx": 1, 
-    "image_view": 0, 
-    "in_create": 0, 
-    "is_submittable": 0, 
-    "issingle": 0, 
-    "istable": 0, 
-    "max_attachments": 0, 
-    "modified": "2018-09-21 15:15:44.909636", 
-    "modified_by": "Administrator", 
-    "module": "Desk", 
-    "name": "Note", 
-    "owner": "Administrator", 
-    "permissions": [
-     {
-      "amend": 0, 
-      "cancel": 0, 
-      "create": 1, 
-      "delete": 1, 
-      "email": 1, 
-      "export": 0, 
-      "if_owner": 0, 
-      "import": 0, 
-      "permlevel": 0, 
-      "print": 1, 
-      "read": 1, 
-      "report": 0, 
-      "role": "All", 
-      "set_user_permissions": 0, 
-      "share": 1, 
-      "submit": 0, 
-      "write": 1
-     }
-    ], 
-    "quick_entry": 1, 
-    "read_only": 0, 
-    "read_only_onload": 1, 
-    "show_name_in_global_search": 0, 
-    "sort_order": "ASC", 
-    "track_changes": 1, 
-    "track_seen": 0, 
-    "track_views": 0
-   }
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2013-05-24 13:41:00",
+ "doctype": "DocType",
+ "document_type": "Document",
+ "engine": "InnoDB",
+ "field_order": [
+  "title",
+  "public",
+  "notify_on_login",
+  "notify_on_every_login",
+  "expire_notification_on",
+  "content",
+  "seen_by_section",
+  "seen_by"
+ ],
+ "fields": [
+  {
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "in_global_search": 1,
+   "in_list_view": 1,
+   "label": "Title",
+   "no_copy": 1,
+   "print_hide": 1,
+   "reqd": 1
+  },
+  {
+   "bold": 1,
+   "default": "0",
+   "fieldname": "public",
+   "fieldtype": "Check",
+   "label": "Public",
+   "print_hide": 1
+  },
+  {
+   "bold": 1,
+   "default": "0",
+   "depends_on": "public",
+   "fieldname": "notify_on_login",
+   "fieldtype": "Check",
+   "label": "Notify users with a popup when they log in"
+  },
+  {
+   "bold": 1,
+   "default": "0",
+   "depends_on": "notify_on_login",
+   "description": "If enabled, users will be notified every time they login. If not enabled, users will only be notified once.",
+   "fieldname": "notify_on_every_login",
+   "fieldtype": "Check",
+   "label": "Notify Users On Every Login"
+  },
+  {
+   "depends_on": "eval:doc.notify_on_login && doc.public",
+   "fieldname": "expire_notification_on",
+   "fieldtype": "Date",
+   "label": "Expire Notification On",
+   "search_index": 1
+  },
+  {
+   "bold": 1,
+   "description": "Help: To link to another record in the system, use \"/app/note/[Note Name]\" as the Link URL. (don't use \"http://\")",
+   "fieldname": "content",
+   "fieldtype": "Text Editor",
+   "in_global_search": 1,
+   "label": "Content"
+  },
+  {
+   "collapsible": 1,
+   "fieldname": "seen_by_section",
+   "fieldtype": "Section Break",
+   "label": "Seen By"
+  },
+  {
+   "fieldname": "seen_by",
+   "fieldtype": "Table",
+   "label": "Seen By Table",
+   "options": "Note Seen By"
+  }
+ ],
+ "icon": "fa fa-file-text",
+ "idx": 1,
+ "links": [],
+ "modified": "2021-09-18 10:57:51.352643",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Note",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "All",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "ASC",
+ "track_changes": 1
+}

frappe/desk/doctype/system_console/system_console.js

@@ -10,15 +10,95 @@ frappe.ui.form.on('System Console', {
 			description: __('Execute Console script'),
 			ignore_inputs: true,
 		});
+		frm.set_value("type", "Python");
 	},
 
 	refresh: function(frm) {
 		frm.disable_save();
 		frm.page.set_primary_action(__("Execute"), $btn => {
-			$btn.text(__('Executing...'));
-			return frm.execute_action("Execute").then(() => {
-				$btn.text(__('Execute'));
-			});
+			$btn.text(__("Executing..."));
+			return frm
+				.execute_action("Execute")
+				.then(() => frm.trigger("render_sql_output"))
+				.finally(() => $btn.text(__("Execute")));
+		});
+	},
+
+	type: function(frm) {
+		if (frm.doc.type == "Python") {
+			frm.set_value("output", "");
+			if (frm.sql_output) {
+				frm.sql_output.destroy();
+				frm.get_field("sql_output").html("");
+			}
+		}
+	},
+
+	render_sql_output: function(frm) {
+		if (frm.doc.type !== "SQL") return;
+		if (frm.sql_output) {
+			frm.sql_output.destroy();
+			frm.get_field("sql_output").html("");
+		}
+
+		if (frm.doc.output.startsWith("Traceback")) {
+			return;
+		}
+
+		let result = JSON.parse(frm.doc.output);
+		frm.set_value("output", `${result.length} ${result.length == 1 ? 'row' : 'rows'}`);
+
+		if (result.length) {
+			let columns = Object.keys(result[0]);
+			frm.sql_output = new DataTable(
+				frm.get_field("sql_output").$wrapper.get(0),
+				{
+					columns,
+					data: result
+				}
+			);
+		}
+	},
+
+	show_processlist: function(frm) {
+		if (frm.doc.show_processlist) {
+			// keep refreshing every 5 seconds
+			frm.events.refresh_processlist(frm);
+			frm.processlist_interval = setInterval(() => frm.events.refresh_processlist(frm), 5000);
+		} else {
+			if (frm.processlist_interval) {
+
+				// end it
+				clearInterval(frm.processlist_interval);
+				frm.get_field("processlist").html('');
+			}
+		}
+	},
+
+	refresh_processlist: function(frm) {
+		let timestamp = new Date();
+		frappe.call('frappe.desk.doctype.system_console.system_console.show_processlist').then(r => {
+			let rows = '';
+			for (let row of r.message) {
+				rows += `<tr>
+					<td>${row.Id}</td>
+					<td>${row.Time}</td>
+					<td>${row.State}</td>
+					<td>${row.Info}</td>
+					<td>${row.Progress}</td>
+				</tr>`
+			}
+			frm.get_field('processlist').html(`
+				<p class='text-muted'>Requested on: ${timestamp}</p>
+				<table class='table-bordered' style='width: 100%'>
+				<thead><tr>
+					<th width='10%'>Id</ht>
+					<th width='10%'>Time</ht>
+					<th width='10%'>State</ht>
+					<th width='60%'>Info</ht>
+					<th width='10%'>Progress</ht>
+				</tr></thead>
+				<tbody>${rows}</thead>`);
 		});
 	}
 });

frappe/desk/doctype/system_console/system_console.json

@@ -17,9 +17,15 @@
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
+  "execute_section",
+  "type",
   "console",
   "commit",
-  "output"
+  "output",
+  "sql_output",
+  "database_processes_section",
+  "show_processlist",
+  "processlist"
  ],
  "fields": [
   {
@@ -40,13 +46,47 @@
    "fieldname": "commit",
    "fieldtype": "Check",
    "label": "Commit"
+  },
+  {
+   "fieldname": "execute_section",
+   "fieldtype": "Section Break",
+   "label": "Execute"
+  },
+  {
+   "fieldname": "database_processes_section",
+   "fieldtype": "Section Break",
+   "label": "Database Processes"
+  },
+  {
+   "default": "0",
+   "fieldname": "show_processlist",
+   "fieldtype": "Check",
+   "label": "Show Processlist"
+  },
+  {
+   "fieldname": "processlist",
+   "fieldtype": "HTML",
+   "label": "processlist"
+  },
+  {
+   "default": "Python",
+   "fieldname": "type",
+   "fieldtype": "Select",
+   "label": "Type",
+   "options": "Python\nSQL"
+  },
+  {
+   "depends_on": "eval:doc.type == 'SQL'",
+   "fieldname": "sql_output",
+   "fieldtype": "HTML",
+   "label": "SQL Output"
   }
  ],
  "hide_toolbar": 1,
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-08-21 14:44:35.296877",
+ "modified": "2021-09-15 17:17:44.844767",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "System Console",
@@ -65,4 +105,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
+}

frappe/desk/doctype/system_console/system_console.py

@@ -5,7 +5,7 @@
 import json
 
 import frappe
-from frappe.utils.safe_exec import safe_exec
+from frappe.utils.safe_exec import safe_exec, read_sql
 from frappe.model.document import Document
 
 class SystemConsole(Document):
@@ -13,8 +13,11 @@ class SystemConsole(Document):
 		frappe.only_for('System Manager')
 		try:
 			frappe.debug_log = []
-			safe_exec(self.console)
-			self.output = '\n'.join(frappe.debug_log)
+			if self.type == 'Python':
+				safe_exec(self.console)
+				self.output = '\n'.join(frappe.debug_log)
+			elif self.type == 'SQL':
+				self.output = frappe.as_json(read_sql(self.console, as_dict=1))
 		except: # noqa: E722
 			self.output = frappe.get_traceback()
 
@@ -33,4 +36,9 @@ class SystemConsole(Document):
 def execute_code(doc):
 	console = frappe.get_doc(json.loads(doc))
 	console.run()
-	return console.as_dict()
+	return console.as_dict()
+
+@frappe.whitelist()
+def show_processlist():
+	frappe.only_for('System Manager')
+	return frappe.db.sql('show full processlist', as_dict=1)

frappe/desk/doctype/tag/tag.py

@@ -128,46 +128,35 @@ def delete_tags_for_document(doc):
 	})
 
 def update_tags(doc, tags):
-	"""
-		Adds tags for documents
-		:param doc: Document to be added to global tags
-	"""
+	"""Adds tags for documents
 
+	:param doc: Document to be added to global tags
+	"""
 	new_tags = {tag.strip() for tag in tags.split(",") if tag}
-
-	for tag in new_tags:
-		if not frappe.db.exists("Tag Link", {"parenttype": doc.doctype, "parent": doc.name, "tag": tag}):
-			frappe.get_doc({
-				"doctype": "Tag Link",
-				"document_type": doc.doctype,
-				"document_name": doc.name,
-				"parenttype": doc.doctype,
-				"parent": doc.name,
-				"title": doc.get_title() or '',
-				"tag": tag
-			}).insert(ignore_permissions=True)
-
 	existing_tags = [tag.tag for tag in frappe.get_list("Tag Link", filters={
 			"document_type": doc.doctype,
 			"document_name": doc.name
 		}, fields=["tag"])]
 
-	deleted_tags = get_deleted_tags(new_tags, existing_tags)
-
-	if deleted_tags:
-		for tag in deleted_tags:
-			delete_tag_for_document(doc.doctype, doc.name, tag)
-
-def get_deleted_tags(new_tags, existing_tags):
-
-	return list(set(existing_tags) - set(new_tags))
-
-def delete_tag_for_document(dt, dn, tag):
-	frappe.db.delete("Tag Link", {
-		"document_type": dt,
-		"document_name": dn,
-		"tag": tag
-	})
+	added_tags = set(new_tags) - set(existing_tags)
+	for tag in added_tags:
+		frappe.get_doc({
+			"doctype": "Tag Link",
+			"document_type": doc.doctype,
+			"document_name": doc.name,
+			"parenttype": doc.doctype,
+			"parent": doc.name,
+			"title": doc.get_title() or '',
+			"tag": tag
+		}).insert(ignore_permissions=True)
+
+	deleted_tags = list(set(existing_tags) - set(new_tags))
+	for tag in deleted_tags:
+		frappe.db.delete("Tag Link", {
+			"document_type": doc.doctype,
+			"document_name": doc.name,
+			"tag": tag
+		})
 
 @frappe.whitelist()
 def get_documents_for_tag(tag):

frappe/desk/doctype/tag_link/tag_link.json

@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "creation": "2019-09-24 13:25:36.435685",
  "doctype": "DocType",
  "editable_grid": 1,
@@ -44,7 +45,8 @@
    "read_only": 1
   }
  ],
- "modified": "2019-10-03 16:42:35.932409",
+ "links": [],
+ "modified": "2021-09-20 16:53:37.217998",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Tag Link",
@@ -61,6 +63,17 @@
    "role": "System Manager",
    "share": 1,
    "write": 1
+  },
+  {
+   "create": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "All",
+   "share": 1,
+   "write": 1
   }
  ],
  "read_only": 1,

frappe/desk/doctype/workspace/workspace.json

@@ -165,8 +165,6 @@
    "default": "0",
    "fieldname": "is_standard",
    "fieldtype": "Check",
-   "in_list_view": 1,
-   "in_standard_filter": 1,
    "label": "Is Standard",
    "search_index": 1
   },
@@ -181,7 +179,6 @@
    "depends_on": "eval:doc.extends_another_page == 1 || doc.for_user",
    "fieldname": "extends",
    "fieldtype": "Link",
-   "in_standard_filter": 1,
    "label": "Extends",
    "options": "Workspace",
    "search_index": 1
@@ -228,6 +225,8 @@
    "default": "0",
    "fieldname": "public",
    "fieldtype": "Check",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
    "label": "Public"
   },
   {
@@ -265,11 +264,13 @@
    "label": "Roles"
   }
  ],
+ "in_create": 1,
  "links": [],
- "modified": "2021-08-30 18:47:18.227154",
+ "modified": "2021-09-16 12:01:06.450621",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {

frappe/desk/doctype/workspace/workspace.py

@@ -208,17 +208,17 @@ def save_page(title, icon, parent, public, sb_public_items, sb_private_items, de
 	if loads(deleted_pages):
 		return delete_pages(loads(deleted_pages))
 
-	return {"name": title, "public": public}
+	return {"name": title, "public": public, "label": doc.label}
 
 def delete_pages(deleted_pages):
 	for page in deleted_pages:
 		if page.get("public") and "Workspace Manager" not in frappe.get_roles():
-			return {"name": page.get("title"), "public": 1}
+			return {"name": page.get("title"), "public": 1, "label": page.get("label")}
 
 		if frappe.db.exists("Workspace", page.get("name")):
 			frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True)
 
-	return {"name": "Home", "public": 1}
+	return {"name": "Home", "public": 1, "label": "Home"}
 
 def sort_pages(sb_public_items, sb_private_items):
 	wspace_public_pages = get_page_list(['name', 'title'], {'public': 1})

frappe/desk/form/load.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+from typing import Dict, List, Union
 import frappe, json
 import frappe.utils
 import frappe.share
@@ -12,7 +13,7 @@ from frappe.desk.form.document_follow import is_document_followed
 from frappe import _
 from urllib.parse import quote
 
-@frappe.whitelist(allow_guest=True)
+@frappe.whitelist()
 def getdoc(doctype, name, user=None):
 	"""
 	Loads a doclist for a given document. This method is called directly from the client.
@@ -51,7 +52,7 @@ def getdoc(doctype, name, user=None):
 
 	frappe.response.docs.append(doc)
 
-@frappe.whitelist(allow_guest=True)
+@frappe.whitelist()
 def getdoctype(doctype, with_parent=False, cached_timestamp=None):
 	"""load doctype"""
 
@@ -105,9 +106,10 @@ def get_docinfo(doc=None, doctype=None, name=None):
 		"assignment_logs": get_comments(doc.doctype, doc.name, 'assignment'),
 		"permissions": get_doc_permissions(doc),
 		"shared": frappe.share.get_users(doc.doctype, doc.name),
-		"info_logs": get_comments(doc.doctype, doc.name, 'Info'),
+		"info_logs": get_comments(doc.doctype, doc.name, comment_type=['Info', 'Edit', 'Label']),
 		"share_logs": get_comments(doc.doctype, doc.name, 'share'),
 		"like_logs": get_comments(doc.doctype, doc.name, 'Like'),
+		"workflow_logs": get_comments(doc.doctype, doc.name, comment_type="Workflow"),
 		"views": get_view_logs(doc.doctype, doc.name),
 		"energy_point_logs": get_point_logs(doc.doctype, doc.name),
 		"additional_timeline_content": get_additional_timeline_content(doc.doctype, doc.name),
@@ -138,10 +140,11 @@ def get_communications(doctype, name, start=0, limit=20):
 	return _get_communications(doctype, name, start, limit)
 
 
-def get_comments(doctype, name, comment_type='Comment'):
-	comment_types = [comment_type]
+def get_comments(doctype: str, name: str, comment_type : Union[str, List[str]] = "Comment") -> List[frappe._dict]:
+	if isinstance(comment_type, list):
+		comment_types = comment_type
 
-	if comment_type == 'share':
+	elif comment_type == 'share':
 		comment_types = ['Shared', 'Unshared']
 
 	elif comment_type == 'assignment':
@@ -150,15 +153,21 @@ def get_comments(doctype, name, comment_type='Comment'):
 	elif comment_type == 'attachment':
 		comment_types = ['Attachment', 'Attachment Removed']
 
-	comments = frappe.get_all('Comment', fields = ['name', 'creation', 'content', 'owner', 'comment_type'], filters=dict(
-		reference_doctype = doctype,
-		reference_name = name,
-		comment_type = ['in', comment_types]
-	))
+	else:
+		comment_types = [comment_type]
+
+	comments = frappe.get_all("Comment",
+		fields=["name", "creation", "content", "owner", "comment_type"],
+		filters={
+			"reference_doctype": doctype,
+			"reference_name": name,
+			"comment_type": ['in', comment_types],
+		}
+	)
 
 	# convert to markdown (legacy ?)
-	if comment_type == 'Comment':
-		for c in comments:
+	for c in comments:
+		if c.comment_type == "Comment":
 			c.content = frappe.utils.markdown(c.content)
 
 	return comments

frappe/desk/form/utils.py

@@ -66,7 +66,8 @@ def add_comment(reference_doctype, reference_name, content, comment_email, comme
 		comment_type='Comment',
 		comment_by=comment_by
 	))
-	doc.content = extract_images_from_html(doc, content)
+	reference_doc = frappe.get_doc(reference_doctype, reference_name)
+	doc.content = extract_images_from_html(reference_doc, content, is_private=True)
 	doc.insert(ignore_permissions=True)
 
 	follow_document(doc.reference_doctype, doc.reference_name, frappe.session.user)

frappe/desk/listview.py

@@ -2,7 +2,7 @@
 # License: MIT. See LICENSE
 import frappe
 
-@frappe.whitelist(allow_guest=True)
+@frappe.whitelist()
 def get_list_settings(doctype):
 	try:
 		return frappe.get_cached_doc("List View Settings", doctype)

frappe/desk/page/leaderboard/leaderboard.js

@@ -141,7 +141,7 @@ class Leaderboard {
 	}
 
 	create_date_range_field() {
-		let timespan_field = $(this.parent).find(`.frappe-control[data-original-title=${__('Timespan')}]`);
+		let timespan_field = $(this.parent).find(`.frappe-control[data-original-title="${__('Timespan')}"]`);
 		this.date_range_field = $(`<div class="from-date-field"></div>`).insertAfter(timespan_field).hide();
 
 		let date_field = frappe.ui.form.make_control({

frappe/desk/reportview.py

@@ -14,7 +14,7 @@ from frappe.utils import cstr, format_duration
 from frappe.model.base_document import get_controller
 
 
-@frappe.whitelist(allow_guest=True)
+@frappe.whitelist()
 @frappe.read_only()
 def get():
 	args = get_form_params()
@@ -121,12 +121,14 @@ def validate_filters(data, filters):
 
 def setup_group_by(data):
 	'''Add columns for aggregated values e.g. count(name)'''
-	if data.group_by:
+	if data.group_by and data.aggregate_function:
 		if data.aggregate_function.lower() not in ('count', 'sum', 'avg'):
 			frappe.throw(_('Invalid aggregate function'))
 
 		if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field):
 			data.fields.append('{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column'.format(**data))
+			if data.aggregate_on_field:
+				data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`")
 		else:
 			raise_invalid_field(data.aggregate_on_field)
 

frappe/desk/treeview.py

@@ -69,13 +69,11 @@ def make_tree_args(**kwarg):
 
 	doctype = kwarg['doctype']
 	parent_field = 'parent_' + doctype.lower().replace(' ', '_')
-	name_field = kwarg.get('name_field', doctype.lower().replace(' ', '_') + '_name')
 
 	if kwarg['is_root'] == 'false': kwarg['is_root'] = False
 	if kwarg['is_root'] == 'true': kwarg['is_root'] = True
 
 	kwarg.update({
-		name_field: kwarg[name_field],
 		parent_field: kwarg.get("parent") or kwarg.get(parent_field)
 	})