Merge branch 'develop' into set-property-after-alert-fix

.git-blame-ignore-revs

@@ -10,3 +10,6 @@
 
 # Replace use of Class.extend with native JS class
 fe20515c23a3ac41f1092bf0eaf0a0a452ec2e85
+
+# Updating license headers
+34460265554242a8d05fb09f049033b1117e1a2b

.github/helper/install.sh

@@ -17,6 +17,7 @@ if [ "$TYPE" == "server" ]; then
 fi
 
 if [ "$DB" == "mariadb" ];then
+      sudo apt install mariadb-client-10.3
       mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'";
       mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'";
 
@@ -58,4 +59,4 @@ cd ../..
 bench start &
 bench --site test_site reinstall --yes
 if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi
-bench build --app frappe
+bench build --app frappe

.github/workflows/docs-checker.yml

@@ -12,7 +12,7 @@ jobs:
       - name: 'Setup Environment'
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.7
 
       - name: 'Clone repo'
         uses: actions/checkout@v2

.github/workflows/patch-mariadb-tests.yml

@@ -9,7 +9,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     name: Patch Test
 

.github/workflows/publish-assets-develop.yml

@@ -18,7 +18,7 @@ jobs:
           node-version: 14
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.7'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/publish-assets-releases.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: '12.x'
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.6'
+          python-version: '3.7'
       - name: Set up bench and build assets
         run: |
           npm install -g yarn

.github/workflows/server-mariadb-tests.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -121,9 +121,10 @@ jobs:
           ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
       - name: Upload coverage data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: codecov/codecov-action@v2
         with:
           name: MariaDB
           fail_ci_if_error: true
           files: /home/runner/frappe-bench/sites/coverage.xml
-          verbose: true
+          verbose: true

.github/workflows/server-postgres-tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
@@ -124,6 +124,7 @@ jobs:
           ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
 
       - name: Upload coverage data
+        if: ${{ steps.check-build.outputs.build == 'strawberry' }}
         uses: codecov/codecov-action@v2
         with:
           name: Postgres

.github/workflows/translation_linter.yml

@@ -1,22 +0,0 @@
-name: Frappe Linter
-on:
-  pull_request:
-    branches:
-      - develop
-      - version-12-hotfix
-      - version-11-hotfix
-jobs:
-  check_translation:
-    name: Translation Syntax Check
-    runs-on: ubuntu-18.04
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup python3
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.6
-    - name: Validating Translation Syntax
-      run: |
-        git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
-        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        python $GITHUB_WORKSPACE/.github/helper/translation.py $files

.github/workflows/ui-tests.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false

LICENSE

@@ -1,6 +1,6 @@
 The MIT License
 
-Copyright (c) 2016-2018 Frappe Technologies Pvt. Ltd. <developers@frappe.io>
+Copyright (c) 2016-2021 Frappe Technologies Pvt. Ltd. <developers@frappe.io>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

codecov.yml

@@ -1,9 +1,13 @@
 codecov:
   require_ci_to_pass: yes
+
+coverage:
   status:
     project:
       default:
+        target: auto
         threshold: 0.5%
+
 comment:
-  layout: "diff, flags, files"
+  layout: "diff"
   require_changes: true

cypress/fixtures/doctype_with_tab_break.js

@@ -0,0 +1,59 @@
+export default {
+	name: 'Form With Tab Break',
+	custom: 1,
+	actions: [],
+	doctype: 'DocType',
+	engine: 'InnoDB',
+	fields: [
+		{
+			fieldname: 'username',
+			fieldtype: 'Data',
+			label: 'Name',
+			options: 'Name'
+		},
+		{
+			fieldname: 'tab',
+			fieldtype: 'Tab Break',
+			label: 'Tab 2',
+		},
+		{
+			fieldname: 'Phone',
+			fieldtype: 'Data',
+			label: 'Phone',
+			options: 'Phone',
+			reqd: 1
+		},
+	],
+	links: [
+		{
+			"group": "Profile",
+			"link_doctype": "Contact",
+			"link_fieldname": "user"
+		},
+		{
+			"group": "Profile",
+			"link_doctype": "Chat Profile",
+			"link_fieldname": "user"
+		},
+	],
+	modified_by: 'Administrator',
+	module: 'Custom',
+	owner: 'Administrator',
+	permissions: [
+		{
+			create: 1,
+			delete: 1,
+			email: 1,
+			print: 1,
+			read: 1,
+			role: 'System Manager',
+			share: 1,
+			write: 1
+		}
+	],
+	quick_entry: 1,
+	autoname: "format: Test-{####}",
+	sort_field: 'modified',
+	sort_order: 'ASC',
+	track_changes: 1
+};

cypress/integration/api.js

@@ -31,8 +31,13 @@ context('API Resources', () => {
 	});
 
 	it('Removes the Comments', () => {
-		cy.get_list('Comment').then(body => body.data.forEach(comment => {
-			cy.remove_doc('Comment', comment.name);
-		}));
+		cy.get_list('Comment').then(body => {
+			let comment_names = [];
+			body.data.map(comment => comment_names.push(comment.name));
+			comment_names = [...new Set(comment_names)]; // remove duplicates
+			comment_names.forEach((comment_name) => {
+				cy.remove_doc('Comment', comment_name);
+			});
+		});
 	});
 });

cypress/integration/control_float.js

@@ -0,0 +1,93 @@
+context("Control Float", () => {
+	before(() => {
+		cy.login();
+		cy.visit("/app/website");
+	});
+
+	function get_dialog_with_float() {
+		return cy.dialog({
+			title: "Float Check",
+			fields: [
+				{
+					fieldname: "float_number",
+					fieldtype: "Float",
+					Label: "Float"
+				}
+			]
+		});
+	}
+
+	it("check value changes", () => {
+		get_dialog_with_float().as("dialog");
+
+		let data = get_data();
+		data.forEach(x => {
+			cy.window()
+				.its("frappe")
+				.then(frappe => {
+					frappe.boot.sysdefaults.number_format = x.number_format;
+				});
+			x.values.forEach(d => {
+				cy.get_field("float_number", "Float").clear();
+				cy.fill_field("float_number", d.input, "Float").blur();
+				cy.get_field("float_number", "Float").should(
+					"have.value",
+					d.blur_expected
+				);
+
+				cy.get_field("float_number", "Float").focus();
+				cy.get_field("float_number", "Float").blur();
+				cy.get_field("float_number", "Float").focus();
+				cy.get_field("float_number", "Float").should(
+					"have.value",
+					d.focus_expected
+				);
+			});
+		});
+	});
+
+	function get_data() {
+		return [
+			{
+				number_format: "#.###,##",
+				values: [
+					{
+						input: "364.87,334",
+						blur_expected: "36.487,334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "36487,334",
+						blur_expected: "36.487,334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "100",
+						blur_expected: "100,000",
+						focus_expected: "100"
+					}
+				]
+			},
+			{
+				number_format: "#,###.##",
+				values: [
+					{
+						input: "364,87.334",
+						blur_expected: "36,487.334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "36487.334",
+						blur_expected: "36,487.334",
+						focus_expected: "36487.334"
+					},
+					{
+						input: "100",
+						blur_expected: "100.000",
+						focus_expected: "100"
+					}
+				]
+			}
+		];
+	}
+});

cypress/integration/datetime_field_form_validation.js

@@ -1,19 +1,19 @@
-context('Datetime Field Validation', () => {
-	before(() => {
-		cy.login();
-		cy.visit('/app/communication');
-		cy.window().its('frappe').then(frappe => {
-			frappe.call("frappe.tests.ui_test_helpers.create_communication_records");
-		});
-	});
+// TODO: Enable this again
+// currently this is flaky possibly because of different timezone in CI
 
-	// validating datetime field value when value is set from backend and get validated on form load.
-	it('datetime field form validation', () => {
-		cy.visit('/app/communication');
-		cy.get('a[title="Test Form Communication 1"]').invoke('attr', 'data-name')
-			.then((name) => {
-				cy.visit(`/app/communication/${name}`);
-				cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
-			});
-	});
-});
+// context('Datetime Field Validation', () => {
+// 	before(() => {
+// 		cy.login();
+// 		cy.visit('/app/communication');
+// 	});
+
+// 	it('datetime field form validation', () => {
+// 		// validating datetime field value when value is set from backend and get validated on form load.
+// 		cy.window().its('frappe').then(frappe => {
+// 			return frappe.xcall("frappe.tests.ui_test_helpers.create_communication_record");
+// 		}).then(doc => {
+// 			cy.visit(`/app/communication/${doc.name}`);
+// 			cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red');
+// 		});
+// 	});
+// });

cypress/integration/form_tab_break.js

@@ -0,0 +1,31 @@
+import doctype_with_tab_break from '../fixtures/doctype_with_tab_break';
+const doctype_name = doctype_with_tab_break.name;
+context("Form Tab Break", () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+		return cy.insert_doc('DocType', doctype_with_tab_break, true);
+	});
+	it("Should switch tab and open correct tabs on validation error", () => {
+		cy.new_form(doctype_name);
+		// test tab switch
+		cy.findByRole("tab", {name: "Tab 2"}).click();
+		cy.findByText("Phone");
+		cy.findByRole("tab", {name: "Details"}).click();
+		cy.findByText("Name");
+
+		// form should switch to the tab with un-filled mandatory field
+		cy.fill_field("username", "Test");
+		cy.findByRole("button", {name: "Save"}).click();
+		cy.findByText("Missing Fields");
+		cy.hide_dialog();
+		cy.findByText("Phone");
+		cy.fill_field("phone", "12345678");
+		cy.findByRole("button", {name: "Save"}).click();
+
+		// After save, first tab should have dashboard
+		cy.get(".form-tabs > .nav-item").eq(0).click();
+		cy.findByText("Connections");
+
+	});
+});

cypress/integration/list_view.js

@@ -6,12 +6,34 @@ context('List View', () => {
 			return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow");
 		});
 	});
+
+	it('Keep checkbox checked after Bulk Update', () => {
+		cy.go_to_list('ToDo');
+		cy.get('.list-row-container .list-row-checkbox').click({ multiple: true, force: true });
+		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
+		cy.get('.dropdown-menu li:visible .dropdown-item .menu-item-label[data-label="Edit"]').click();
+
+		cy.get('.modal-body .form-control[data-fieldname="field"]').first().select('Due Date').wait(200);
+		cy.get('.modal-body .frappe-control[data-fieldname="value"] input:visible').first().focus();
+		cy.get('.datepickers-container .datepicker.active').should('be.visible');
+
+		cy.get('.datepickers-container .datepicker.active .datepicker--cell-day.-current-').click({force: true});
+		cy.get('.modal-body .frappe-control[data-fieldname="value"] input:visible').first().focus();
+		cy.get('.datepickers-container .datepicker.active .datepicker--cell-day.-current-').click({force: true});
+
+		cy.get('.modal-footer .standard-actions .btn-primary').click();
+		cy.wait(500);
+
+		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
+		cy.get('.list-row-container .list-row-checkbox:checked').should('be.visible');
+	});
+
 	it('enables "Actions" button', () => {
-		const actions = ['Approve', 'Reject', 'Edit', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete'];
+		const actions = ['Approve', 'Reject', 'Edit', 'Export', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete'];
 		cy.go_to_list('ToDo');
 		cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ multiple: true, force: true });
 		cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click();
-		cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 8).each((el, index) => {
+		cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 9).each((el, index) => {
 			cy.wrap(el).contains(actions[index]);
 		}).then((elements) => {
 			cy.intercept({
@@ -30,4 +52,3 @@ context('List View', () => {
 		});
 	});
 });
-

cypress/integration/multi_select_dialog.js

@@ -0,0 +1,58 @@
+context('MultiSelectDialog', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app');
+	});
+
+	function open_multi_select_dialog() {
+		cy.window().its('frappe').then(frappe => {
+			new frappe.ui.form.MultiSelectDialog({
+				doctype: "Assignment Rule",
+				target: {},
+				setters: {
+					document_type: null,
+					priority: null
+				},
+				add_filters_group: 1,
+				allow_child_item_selection: 1,
+				child_fieldname: "assignment_days",
+				child_columns: ["day"]
+			});
+		});
+	}
+
+	it('multi select dialog api works', () => {
+		open_multi_select_dialog();
+		cy.get_open_dialog().should('contain', 'Select Assignment Rules');
+	});
+
+	it('checks for filters', () => {
+		['search_term', 'document_type', 'priority'].forEach(fieldname => {
+			cy.get_open_dialog().get(`.frappe-control[data-fieldname="${fieldname}"]`).should('exist');
+		});
+
+		// add_filters_group: 1 should add a filter group
+		cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should('exist');
+
+	});
+
+	it('checks for child item selection', () => {
+		cy.get_open_dialog()
+			.get(`.dt-row-header`).should('not.exist');
+
+		cy.get_open_dialog()
+			.get(`.frappe-control[data-fieldname="allow_child_item_selection"]`)
+			.should('exist')
+			.click();
+
+		cy.get_open_dialog()
+			.get(`.frappe-control[data-fieldname="child_selection_area"]`)
+			.should('exist');
+
+		cy.get_open_dialog()
+			.get(`.dt-row-header`).should('contain', 'Assignment Rule');
+
+		cy.get_open_dialog()
+			.get(`.dt-row-header`).should('contain', 'Day');
+	});
+});

cypress/integration/navigation.js

@@ -1,7 +1,6 @@
 context('Navigation', () => {
 	before(() => {
 		cy.login();
-		cy.visit('/app/website');
 	});
 	it('Navigate to route with hash in document name', () => {
 		cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true});
@@ -11,4 +10,15 @@ context('Navigation', () => {
 		cy.go('back');
 		cy.title().should('eq', 'Website');
 	});
+
+	it.only('Navigate to previous page after login', () => {
+		cy.visit('/app/todo');
+		cy.request('/api/method/logout');
+		cy.reload();
+		cy.get('.btn-primary').contains('Login').click();
+		cy.location('pathname').should('eq', '/login');
+		cy.login();
+		cy.visit('/app');
+		cy.location('pathname').should('eq', '/app/todo');
+	});
 });

cypress/integration/sidebar.js

@@ -6,12 +6,12 @@ context('Sidebar', () => {
 	});
 
 	it('Test for checking "Assigned To" counter value, adding filter and adding & removing an assignment', () => {
-		cy.click_sidebar_button(0);
+		cy.click_sidebar_button("Assigned To");
 
 		//To check if no filter is available in "Assigned To" dropdown
 		cy.get('.empty-state').should('contain', 'No filters found');
 
-		cy.click_sidebar_button(1);
+		cy.click_sidebar_button("Created By");
 
 		//To check if "Created By" dropdown contains filter
 		cy.get('.group-by-item > .dropdown-item').should('contain', 'Me');
@@ -22,7 +22,7 @@ context('Sidebar', () => {
 		cy.get_field('assign_to_me', 'Check').click();
 		cy.get('.modal-footer > .standard-actions > .btn-primary').click();
 		cy.visit('/app/doctype');
-		cy.click_sidebar_button(0);
+		cy.click_sidebar_button("Assigned To");
 
 		//To check if filter is added in "Assigned To" dropdown after assignment
 		cy.get('.group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item').should('contain', '1');
@@ -38,20 +38,19 @@ context('Sidebar', () => {
 		cy.get('.fieldname-select-area > .awesomplete > .form-control').should('have.value', 'Assigned To');
 		cy.get('.condition').should('have.value', 'like');
 		cy.get('.filter-field > .form-group > .input-with-feedback').should('have.value', '%Administrator%');
+		cy.click_filter_button();
 
 		//To remove the applied filter
-		cy.get('.filter-action-buttons > div > .btn-secondary').contains('Clear Filters').click();
-		cy.click_filter_button();
-		cy.get('.filter-selector > .btn').should('contain', 'Filter');
+		cy.clear_filters();
 
 		//To remove the assignment
 		cy.visit('/app/doctype');
 		cy.click_listview_row_item(0);
 		cy.get('.assignments > .avatar-group > .avatar > .avatar-frame').click();
 		cy.get('.remove-btn').click({force: true});
-		cy.get('.modal.show > .modal-dialog > .modal-content > .modal-header > .modal-actions > .btn-modal-close').click();
+		cy.hide_dialog();
 		cy.visit('/app/doctype');
-		cy.click_sidebar_button(0);
+		cy.click_sidebar_button("Assigned To");
 		cy.get('.empty-state').should('contain', 'No filters found');
 	});
 });

cypress/integration/timeline.js

@@ -4,11 +4,11 @@ context('Timeline', () => {
 	before(() => {
 		cy.visit('/login');
 		cy.login();
-		cy.visit('/app/todo');
 	});
 
 	it('Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo', () => {
 		//Adding new ToDo
+		cy.visit('/app/todo');
 		cy.click_listview_primary_button('Add ToDo');
 		cy.findByRole('button', {name: 'Edit in full page'}).click();
 		cy.get('[data-fieldname="description"] .ql-editor').eq(0).type('Test ToDo', {force: true});
@@ -28,15 +28,15 @@ context('Timeline', () => {
 		cy.get('.timeline-content').should('contain', 'Testing Timeline');
 
 		//Editing comment
-		cy.click_timeline_action_btn(0);
+		cy.click_timeline_action_btn("Edit");
 		cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(' 123');
-		cy.click_timeline_action_btn(0);
+		cy.click_timeline_action_btn("Save");
 
 		//To check if the edited comment text is visible in timeline content
 		cy.get('.timeline-content').should('contain', 'Testing Timeline 123');
 
 		//Discarding comment
-		cy.click_timeline_action_btn(0);
+		cy.click_timeline_action_btn("Edit");
 		cy.findByRole('button', {name: 'Dismiss'}).click();
 
 		//To check if after discarding the timeline content is same as previous
@@ -81,7 +81,7 @@ context('Timeline', () => {
 		cy.visit('/app/custom-submittable-doctype');
 		cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click();
 		cy.findByRole('button', {name: 'Actions'}).click();
-		cy.get('.actions-btn-group > .dropdown-menu > li > .grey-link').eq(7).click();
+		cy.get('.actions-btn-group > .dropdown-menu > li > .dropdown-item').contains("Delete").click();
 		cy.click_modal_primary_button('Yes', {force: true, delay: 700});
 
 		//Deleting the custom doctype

cypress/support/commands.js

@@ -187,7 +187,7 @@ Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => {
 	if (fieldtype === 'Select') {
 		cy.get('@input').select(value);
 	} else {
-		cy.get('@input').type(value, {waitForAnimations: false, force: true});
+		cy.get('@input').type(value, {waitForAnimations: false, force: true, delay: 100});
 	}
 	return cy.get('@input');
 });
@@ -252,7 +252,8 @@ Cypress.Commands.add('new_form', doctype => {
 });
 
 Cypress.Commands.add('go_to_list', doctype => {
-	cy.visit(`/app/list/${doctype}/list`);
+	let dt_in_route = doctype.toLowerCase().replace(/ /g, '-');
+	cy.visit(`/app/${dt_in_route}`);
 });
 
 Cypress.Commands.add('clear_cache', () => {
@@ -316,7 +317,11 @@ Cypress.Commands.add('add_filter', () => {
 });
 
 Cypress.Commands.add('clear_filters', () => {
-	cy.get('.filter-section .filter-button').click();
+	cy.intercept({
+		method: 'POST',
+		url: 'api/method/frappe.model.utils.user_settings.save'
+	}).as('filter-saved');
+	cy.get('.filter-section .filter-button').click({force: true});
 	cy.wait(300);
 	cy.get('.filter-popover').should('exist');
 	cy.get('.filter-popover').find('.clear-filters').click();
@@ -324,16 +329,15 @@ Cypress.Commands.add('clear_filters', () => {
 	cy.window().its('cur_list').then(cur_list => {
 		cur_list && cur_list.filter_area && cur_list.filter_area.clear();
 	});
-
-	
+	cy.wait('@filter-saved');
 });
 
 Cypress.Commands.add('click_modal_primary_button', (btn_name) => {
 	cy.get('.modal-footer > .standard-actions > .btn-primary').contains(btn_name).trigger('click', {force: true});
 });
 
-Cypress.Commands.add('click_sidebar_button', (btn_no) => {
-	cy.get('.list-group-by-fields > .group-by-field > .btn').eq(btn_no).click();
+Cypress.Commands.add('click_sidebar_button', (btn_name) => {
+	cy.get('.list-group-by-fields .list-link > a').contains(btn_name).click({force: true});
 });
 
 Cypress.Commands.add('click_listview_row_item', (row_no) => {
@@ -348,6 +352,6 @@ Cypress.Commands.add('click_listview_primary_button', (btn_name) => {
 	cy.get('.primary-action').contains(btn_name).click({force: true});
 });
 
-Cypress.Commands.add('click_timeline_action_btn', (btn_no) => {
-	cy.get('.timeline-content > .timeline-message-box > .justify-between > .actions > .btn').eq(btn_no).first().click();
+Cypress.Commands.add('click_timeline_action_btn', (btn_name) => {
+	cy.get('.timeline-content > .timeline-message-box > .justify-between > .actions > .btn').contains(btn_name).click();
 });

frappe/__init__.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 """
 Frappe - Low Code Open Source Framework in Python and JS
 
@@ -235,12 +235,13 @@ def connect_replica():
 	from frappe.database import get_db
 	user = local.conf.db_name
 	password = local.conf.db_password
+	port = local.conf.replica_db_port
 
 	if local.conf.different_credentials_for_replica:
 		user = local.conf.replica_db_name
 		password = local.conf.replica_db_password
 
-	local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password)
+	local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port)
 
 	# swap db connections
 	local.primary_db = local.db
@@ -618,8 +619,6 @@ def read_only():
 
 			try:
 				retval = fn(*args, **get_newargs(fn, kwargs))
-			except:
-				raise
 			finally:
 				if local and hasattr(local, 'primary_db'):
 					local.db.close()
@@ -629,6 +628,29 @@ def read_only():
 		return wrapper_fn
 	return innfn
 
+def write_only():
+	# if replica connection exists, we have to replace it momentarily with the primary connection
+	def innfn(fn):
+		def wrapper_fn(*args, **kwargs):
+			primary_db = getattr(local, "primary_db", None)
+			replica_db = getattr(local, "replica_db", None)
+			in_read_only = getattr(local, "db", None) != primary_db
+
+			# switch to primary connection
+			if in_read_only and primary_db:
+				local.db = local.primary_db
+
+			try:
+				retval = fn(*args, **get_newargs(fn, kwargs))
+			finally:
+				# switch back to replica connection
+				if in_read_only and replica_db:
+					local.db = replica_db
+
+			return retval
+		return wrapper_fn
+	return innfn
+
 def only_for(roles, message=False):
 	"""Raise `frappe.PermissionError` if the user does not have any of the given **Roles**.
 

frappe/api.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 import base64
 import binascii
 import json

frappe/app.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import os
 import logging

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import unittest
 from frappe.utils import random_string

frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 # import frappe
 from frappe.model.document import Document

frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 # import frappe
 from frappe.model.document import Document

frappe/automation/doctype/auto_repeat/auto_repeat.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe import _

frappe/automation/doctype/auto_repeat/test_auto_repeat.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import unittest
 
 import frappe

frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 # import frappe
 from frappe.model.document import Document

frappe/automation/doctype/milestone/milestone.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/automation/doctype/milestone/test_milestone.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 #import frappe
 import unittest
 

frappe/automation/doctype/milestone_tracker/milestone_tracker.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import frappe.cache_manager
 import unittest

frappe/boot.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 """
 bootstrap client session
 """

frappe/build.py

@@ -1,10 +1,11 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# License: MIT. See LICENSE
 import os
 import re
 import json
 import shutil
 import subprocess
+from subprocess import getoutput
 from io import StringIO
 from tempfile import mkdtemp, mktemp
 from distutils.spawn import find_executable
@@ -17,6 +18,8 @@ import psutil
 from urllib.parse import urlparse
 from simple_chalk import green
 from semantic_version import Version
+from requests import head
+from requests.exceptions import HTTPError
 
 
 timestamps = {}
@@ -24,6 +27,12 @@ app_paths = None
 sites_path = os.path.abspath(os.getcwd())
 
 
+class AssetsNotDownloadedError(Exception):
+	pass
+
+class AssetsDontExistError(HTTPError):
+	pass
+
 def download_file(url, prefix):
 	from requests import get
 
@@ -70,81 +79,94 @@ def build_missing_files():
 		bundle(build_mode, apps="frappe")
 
 
-def get_assets_link(frappe_head):
-	from subprocess import getoutput
-	from requests import head
-
+def get_assets_link(frappe_head) -> str:
 	tag = getoutput(
-			r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
-			r" refs/tags/,,' -e 's/\^{}//'"
-			% frappe_head
-		)
+		r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
+		r" refs/tags/,,' -e 's/\^{}//'"
+		% frappe_head
+	)
 
 	if tag:
 		# if tag exists, download assets from github release
-		url = "https://github.com/frappe/frappe/releases/download/{0}/assets.tar.gz".format(tag)
+		url = f"https://github.com/frappe/frappe/releases/download/{tag}/assets.tar.gz"
 	else:
-		url = "http://assets.frappeframework.com/{0}.tar.gz".format(frappe_head)
+		url = f"http://assets.frappeframework.com/{frappe_head}.tar.gz"
 
 	if not head(url):
-		raise ValueError("URL {0} doesn't exist".format(url))
+		reference = f"Release {tag}" if tag else f"Commit {frappe_head}"
+		raise AssetsDontExistError(f"Assets for {reference} don't exist")
 
 	return url
 
 
+def fetch_assets(url, frappe_head):
+	click.secho("Retrieving assets...", fg="yellow")
+
+	prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
+	assets_archive = download_file(url, prefix)
+
+	if not assets_archive:
+		raise AssetsNotDownloadedError(f"Assets could not be retrived from {url}")
+
+	print(f"\n{green('✔')} Downloaded Frappe assets from {url}")
+
+	return assets_archive
+
+
+def setup_assets(assets_archive):
+	import tarfile
+	directories_created = set()
+
+	click.secho("\nExtracting assets...\n", fg="yellow")
+	with tarfile.open(assets_archive) as tar:
+		for file in tar:
+			if not file.isdir():
+				dest = "." + file.name.replace("./frappe-bench/sites", "")
+				asset_directory = os.path.dirname(dest)
+				show = dest.replace("./assets/", "")
+
+				if asset_directory not in directories_created:
+					if not os.path.exists(asset_directory):
+						os.makedirs(asset_directory, exist_ok=True)
+					directories_created.add(asset_directory)
+
+				tar.makefile(file, dest)
+				print("{0} Restored {1}".format(green('✔'), show))
+
+	return directories_created
+
+
 def download_frappe_assets(verbose=True):
 	"""Downloads and sets up Frappe assets if they exist based on the current
 	commit HEAD.
 	Returns True if correctly setup else returns False.
 	"""
-	from subprocess import getoutput
-
-	assets_setup = False
 	frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
 
-	if frappe_head:
+	if not frappe_head:
+		return False
+
+	try:
+		url = get_assets_link(frappe_head)
+		assets_archive = fetch_assets(url, frappe_head)
+		setup_assets(assets_archive)
+		build_missing_files()
+		return True
+
+	except AssetsDontExistError as e:
+		click.secho(str(e), fg="yellow")
+
+	except Exception as e:
+		# TODO: log traceback in bench.log
+		click.secho(str(e), fg="red")
+
+	finally:
 		try:
-			url = get_assets_link(frappe_head)
-			click.secho("Retrieving assets...", fg="yellow")
-			prefix = mkdtemp(prefix="frappe-assets-", suffix=frappe_head)
-			assets_archive = download_file(url, prefix)
-			print("\n{0} Downloaded Frappe assets from {1}".format(green('✔'), url))
-
-			if assets_archive:
-				import tarfile
-				directories_created = set()
-
-				click.secho("\nExtracting assets...\n", fg="yellow")
-				with tarfile.open(assets_archive) as tar:
-					for file in tar:
-						if not file.isdir():
-							dest = "." + file.name.replace("./frappe-bench/sites", "")
-							asset_directory = os.path.dirname(dest)
-							show = dest.replace("./assets/", "")
-
-							if asset_directory not in directories_created:
-								if not os.path.exists(asset_directory):
-									os.makedirs(asset_directory, exist_ok=True)
-								directories_created.add(asset_directory)
-
-							tar.makefile(file, dest)
-							print("{0} Restored {1}".format(green('✔'), show))
-
-				build_missing_files()
-				return True
-			else:
-				raise
+			shutil.rmtree(os.path.dirname(assets_archive))
 		except Exception:
-			# TODO: log traceback in bench.log
-			click.secho("An Error occurred while downloading assets...", fg="red")
-			assets_setup = False
-		finally:
-			try:
-				shutil.rmtree(os.path.dirname(assets_archive))
-			except Exception:
-				pass
-
-	return assets_setup
+			pass
+
+	return False
 
 
 def symlink(target, link_name, overwrite=False):

frappe/cache_manager.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import frappe, json
 from frappe.model.document import Document

frappe/chat/doctype/chat_token/chat_token.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/client.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 import frappe
 from frappe import _
 import frappe.model

frappe/commands/__init__.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Web Notes Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import sys
 import click
@@ -102,9 +102,24 @@ def get_commands():
 	from .site import commands as site_commands
 	from .translate import commands as translate_commands
 	from .utils import commands as utils_commands
-	from .redis import commands as redis_commands
+	from .redis_utils import commands as redis_commands
+
+	clickable_link = (
+		"\x1b]8;;https://frappeframework.com/docs\afrappeframework.com\x1b]8;;\a"
+	)
+	all_commands = (
+		scheduler_commands
+		+ site_commands
+		+ translate_commands
+		+ utils_commands
+		+ redis_commands
+	)
+
+	for command in all_commands:
+		if not command.help:
+			command.help = f"Refer to {clickable_link}"
+
+	return all_commands
 
-	all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
-	return list(set(all_commands))
 
 commands = get_commands()

frappe/commands/redis.py → frappe/commands/redis_utils.py

@@ -3,7 +3,7 @@ import os
 import click
 
 import frappe
-from frappe.utils.rq import RedisQueue
+from frappe.utils.redis_queue import RedisQueue
 from frappe.installer import update_site_config
 
 @click.command('create-rq-users')

frappe/commands/site.py

@@ -67,6 +67,9 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
 		validate_database_sql
 	)
 
+	site = get_site(context)
+	frappe.init(site=site)
+
 	force = context.force or force
 	decompressed_file_name = extract_sql_from_archive(sql_file_path)
 
@@ -85,9 +88,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas
 	# check if valid SQL file
 	validate_database_sql(decompressed_file_name, _raise=not force)
 
-	site = get_site(context)
-	frappe.init(site=site)
-
 	# dont allow downgrading to older versions of frappe without force
 	if not force and is_downgrade(decompressed_file_name, verbose=True):
 		warn_message = (
@@ -474,7 +474,7 @@ def remove_from_installed_apps(context, app):
 
 @click.command('uninstall-app')
 @click.argument('app')
-@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False, multiple=True)
+@click.option('--yes', '-y', help='To bypass confirmation prompt for uninstalling the app', is_flag=True, default=False)
 @click.option('--dry-run', help='List all doctypes that will be deleted', is_flag=True, default=False)
 @click.option('--no-backup', help='Do not backup the site', is_flag=True, default=False)
 @click.option('--force', help='Force remove app from site', is_flag=True, default=False)
@@ -738,6 +738,131 @@ def build_search_index(context):
 	finally:
 		frappe.destroy()
 
+@click.command('trim-database')
+@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted')
+@click.option('--format', '-f', default='text', type=click.Choice(['json', 'text']), help='Output format')
+@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site')
+@pass_context
+def trim_database(context, dry_run, format, no_backup):
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	from frappe.utils.backups import scheduled_backup
+
+	ALL_DATA = {}
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+
+		TABLES_TO_DROP = []
+		STANDARD_TABLES = get_standard_tables()
+		information_schema = frappe.qb.Schema("information_schema")
+		table_name = frappe.qb.Field("table_name").as_("name")
+
+		queried_result = frappe.qb.from_(
+			information_schema.tables
+		).select(table_name).where(
+			information_schema.tables.table_schema == frappe.conf.db_name
+		).run()
+
+		database_tables = [x[0] for x in queried_result]
+		doctype_tables = frappe.get_all("DocType", pluck="name")
+
+		for x in database_tables:
+			doctype = x.lstrip("tab")
+			if not (doctype in doctype_tables or x.startswith("__") or x in STANDARD_TABLES):
+				TABLES_TO_DROP.append(x)
+
+		if not TABLES_TO_DROP:
+			if format == "text":
+				click.secho(f"No ghost tables found in {frappe.local.site}...Great!", fg="green")
+		else:
+			if not (no_backup or dry_run):
+				if format == "text":
+					print(f"Backing Up Tables: {', '.join(TABLES_TO_DROP)}")
+
+				odb = scheduled_backup(
+					ignore_conf=False,
+					include_doctypes=",".join(x.lstrip("tab") for x in TABLES_TO_DROP),
+					ignore_files=True,
+					force=True,
+				)
+				if format == "text":
+					odb.print_summary()
+					print("\nTrimming Database")
+
+			for table in TABLES_TO_DROP:
+				if format == "text":
+					print(f"* Dropping Table '{table}'...")
+				if not dry_run:
+					frappe.db.sql_ddl(f"drop table `{table}`")
+
+			ALL_DATA[frappe.local.site] = TABLES_TO_DROP
+		frappe.destroy()
+
+	if format == "json":
+		import json
+		print(json.dumps(ALL_DATA, indent=1))
+
+
+def get_standard_tables():
+	import re
+
+	tables = []
+	sql_file = os.path.join(
+		"..", "apps", "frappe", "frappe", "database", frappe.conf.db_type, f'framework_{frappe.conf.db_type}.sql'
+	)
+	content = open(sql_file).read().splitlines()
+
+	for line in content:
+		table_found = re.search(r"""CREATE TABLE ("|`)(.*)?("|`) \(""", line)
+		if table_found:
+			tables.append(table_found.group(2))
+
+	return tables
+
+@click.command('trim-tables')
+@click.option('--dry-run', is_flag=True, default=False, help='Show what would be deleted')
+@click.option('--format', '-f', default='table', type=click.Choice(['json', 'table']), help='Output format')
+@click.option('--no-backup', is_flag=True, default=False, help='Do not backup the site')
+@pass_context
+def trim_tables(context, dry_run, format, no_backup):
+	if not context.sites:
+		raise SiteNotSpecifiedError
+
+	from frappe.model.meta import trim_tables
+	from frappe.utils.backups import scheduled_backup
+
+	for site in context.sites:
+		frappe.init(site=site)
+		frappe.connect()
+
+		if not (no_backup or dry_run):
+			click.secho(f"Taking backup for {frappe.local.site}", fg="green")
+			odb = scheduled_backup(ignore_files=False, force=True)
+			odb.print_summary()
+
+		try:
+			trimmed_data = trim_tables(dry_run=dry_run, quiet=format == 'json')
+
+			if format == 'table' and not dry_run:
+				click.secho(f"The following data have been removed from {frappe.local.site}", fg='green')
+
+			handle_data(trimmed_data, format=format)
+		finally:
+			frappe.destroy()
+
+def handle_data(data: dict, format='json'):
+	if format == 'json':
+		import json
+		print(json.dumps({frappe.local.site: data}, indent=1, sort_keys=True))
+	else:
+		from frappe.utils.commands import render_table
+		data = [["DocType", "Fields"]] + [[table, ", ".join(columns)] for table, columns in data.items()]
+		render_table(data)
+
+
 commands = [
 	add_system_manager,
 	backup,
@@ -766,5 +891,7 @@ commands = [
 	add_to_hosts,
 	start_ngrok,
 	build_search_index,
-	partial_restore
+	partial_restore,
+	trim_tables,
+	trim_database,
 ]

frappe/commands/utils.py

@@ -408,20 +408,47 @@ def bulk_rename(context, doctype, path):
 	frappe.destroy()
 
 
+@click.command('db-console')
+@pass_context
+def database(context):
+	"""
+		Enter into the Database console for given site.
+	"""
+	site = get_site(context)
+	if not site:
+		raise SiteNotSpecifiedError
+	frappe.init(site=site)
+	if not frappe.conf.db_type or frappe.conf.db_type == "mariadb":
+		_mariadb()
+	elif frappe.conf.db_type == "postgres":
+		_psql()
+
+
 @click.command('mariadb')
 @pass_context
 def mariadb(context):
 	"""
 		Enter into mariadb console for a given site.
 	"""
-	import os
-
 	site  = get_site(context)
 	if not site:
 		raise SiteNotSpecifiedError
 	frappe.init(site=site)
+	_mariadb()
+
+
+@click.command('postgres')
+@pass_context
+def postgres(context):
+	"""
+		Enter into postgres console for a given site.
+	"""
+	site  = get_site(context)
+	frappe.init(site=site)
+	_psql()
 
-	# This is assuming you're within the bench instance.
+
+def _mariadb():
 	mysql = find_executable('mysql')
 	os.execv(mysql, [
 		mysql,
@@ -434,15 +461,7 @@ def mariadb(context):
 		"-A"])
 
 
-@click.command('postgres')
-@pass_context
-def postgres(context):
-	"""
-		Enter into postgres console for a given site.
-	"""
-	site  = get_site(context)
-	frappe.init(site=site)
-	# This is assuming you're within the bench instance.
+def _psql():
 	psql = find_executable('psql')
 	subprocess.run([ psql, '-d', frappe.conf.db_name])
 
@@ -525,6 +544,74 @@ def console(context, autoreload=False):
 	terminal()
 
 
+@click.command('transform-database', help="Change tables' internal settings changing engine and row formats")
+@click.option('--table', required=True, help="Comma separated name of tables to convert. To convert all tables, pass 'all'")
+@click.option('--engine', default=None, type=click.Choice(["InnoDB", "MyISAM"]), help="Choice of storage engine for said table(s)")
+@click.option('--row_format', default=None, type=click.Choice(["DYNAMIC", "COMPACT", "REDUNDANT", "COMPRESSED"]), help="Set ROW_FORMAT parameter for said table(s)")
+@click.option('--failfast', is_flag=True, default=False, help="Exit on first failure occurred")
+@pass_context
+def transform_database(context, table, engine, row_format, failfast):
+	"Transform site database through given parameters"
+	site = get_site(context)
+	check_table = []
+	add_line = False
+	skipped = 0
+	frappe.init(site=site)
+
+	if frappe.conf.db_type and frappe.conf.db_type != "mariadb":
+		click.secho("This command only has support for MariaDB databases at this point", fg="yellow")
+		sys.exit(1)
+
+	if not (engine or row_format):
+		click.secho("Values for `--engine` or `--row_format` must be set")
+		sys.exit(1)
+
+	frappe.connect()
+
+	if table == "all":
+		information_schema = frappe.qb.Schema("information_schema")
+		queried_tables = frappe.qb.from_(
+			information_schema.tables
+		).select("table_name").where(
+			(information_schema.tables.row_format != row_format)
+			& (information_schema.tables.table_schema == frappe.conf.db_name)
+		).run()
+		tables = [x[0] for x in queried_tables]
+	else:
+		tables = [x.strip() for x in table.split(",")]
+
+	total = len(tables)
+
+	for current, table in enumerate(tables):
+		values_to_set = ""
+		if engine:
+			values_to_set += f" ENGINE={engine}"
+		if row_format:
+			values_to_set += f" ROW_FORMAT={row_format}"
+
+		try:
+			frappe.db.sql(f"ALTER TABLE `{table}`{values_to_set}")
+			update_progress_bar("Updating table schema", current - skipped, total)
+			add_line = True
+
+		except Exception as e:
+			check_table.append([table, e.args])
+			skipped += 1
+
+			if failfast:
+				break
+
+	if add_line:
+		print()
+
+	for errored_table in check_table:
+		table, err = errored_table
+		err_msg = f"{table}: ERROR {err[0]}: {err[1]}"
+		click.secho(err_msg, fg="yellow")
+
+	frappe.destroy()
+
+
 @click.command('run-tests')
 @click.option('--app', help="For App")
 @click.option('--doctype', help="For DocType")
@@ -811,6 +898,8 @@ commands = [
 	build,
 	clear_cache,
 	clear_website_cache,
+	database,
+	transform_database,
 	jupyter,
 	console,
 	destroy_all_sessions,

frappe/contacts/address_and_contact.py

@@ -1,5 +1,5 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: GNU General Public License v3. See license.txt
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# License: MIT. See LICENSE
 
 import frappe
 

frappe/contacts/doctype/address/address.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 
@@ -65,7 +65,7 @@ class Address(Document):
 
 	def has_link(self, doctype, name):
 		for link in self.links:
-			if link.link_doctype==doctype and link.link_name== name:
+			if link.link_doctype == doctype and link.link_name == name:
 				return True
 
 	def has_common_link(self, doc):

frappe/contacts/doctype/address/test_address.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe, unittest
 from frappe.contacts.doctype.address.address import get_address_display
 

frappe/contacts/doctype/address_template/address_template.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/contacts/doctype/address_template/test_address_template.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe, unittest
 
 class TestAddressTemplate(unittest.TestCase):

frappe/contacts/doctype/contact/contact.py

@@ -1,5 +1,5 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: GNU General Public License v3. See license.txt
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# License: MIT. See LICENSE
 import frappe
 from frappe.utils import cstr, has_gravatar
 from frappe import _
@@ -47,14 +47,14 @@ class Contact(Document):
 	def get_link_for(self, link_doctype):
 		'''Return the link name, if exists for the given link DocType'''
 		for link in self.links:
-			if link.link_doctype==link_doctype:
+			if link.link_doctype == link_doctype:
 				return link.link_name
 
 		return None
 
 	def has_link(self, doctype, name):
 		for link in self.links:
-			if link.link_doctype==doctype and link.link_name== name:
+			if link.link_doctype == doctype and link.link_name == name:
 				return True
 
 	def has_common_link(self, doc):

frappe/contacts/doctype/contact/test_contact.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import unittest
 

frappe/contacts/doctype/contact_email/contact_email.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 # import frappe
 from frappe.model.document import Document

frappe/contacts/doctype/contact_phone/contact_phone.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 # import frappe
 from frappe.model.document import Document

frappe/contacts/doctype/gender/gender.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 from frappe.model.document import Document
 

frappe/contacts/doctype/gender/test_gender.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import unittest
 
 class TestGender(unittest.TestCase):

frappe/contacts/doctype/salutation/salutation.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 from frappe.model.document import Document
 

frappe/contacts/doctype/salutation/test_salutation.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import unittest
 
 class TestSalutation(unittest.TestCase):

frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 import frappe
 from frappe import _
 

frappe/core/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE

frappe/core/doctype/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 

frappe/core/doctype/access_log/access_log.py

@@ -1,6 +1,7 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# Copyright (c) 2021, Frappe Technologies and contributors
+# License: MIT. See LICENSE
 import frappe
+from tenacity import retry, retry_if_exception_type, stop_after_attempt
 from frappe.model.document import Document
 
 
@@ -9,25 +10,41 @@ class AccessLog(Document):
 
 
 @frappe.whitelist()
-def make_access_log(doctype=None, document=None, method=None, file_type=None,
-		report_name=None, filters=None, page=None, columns=None):
+@frappe.write_only()
+@retry(
+	stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError)
+)
+def make_access_log(
+	doctype=None,
+	document=None,
+	method=None,
+	file_type=None,
+	report_name=None,
+	filters=None,
+	page=None,
+	columns=None,
+):
 
 	user = frappe.session.user
+	in_request = frappe.request and frappe.request.method == "GET"
 
-	doc = frappe.get_doc({
-		'doctype': 'Access Log',
-		'user': user,
-		'export_from': doctype,
-		'reference_document': document,
-		'file_type': file_type,
-		'report_name': report_name,
-		'page': page,
-		'method': method,
-		'filters': frappe.utils.cstr(filters) if filters else None,
-		'columns': columns
-	})
+	doc = frappe.get_doc(
+		{
+			"doctype": "Access Log",
+			"user": user,
+			"export_from": doctype,
+			"reference_document": document,
+			"file_type": file_type,
+			"report_name": report_name,
+			"page": page,
+			"method": method,
+			"filters": frappe.utils.cstr(filters) if filters else None,
+			"columns": columns,
+		}
+	)
 	doc.insert(ignore_permissions=True)
 
 	# `frappe.db.commit` added because insert doesnt `commit` when called in GET requests like `printview`
-	if frappe.request and frappe.request.method == 'GET':
+	# dont commit in test mode
+	if not frappe.flags.in_test or in_request:
 		frappe.db.commit()

frappe/core/doctype/access_log/test_access_log.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 
 # imports - standard imports
 import unittest

frappe/core/doctype/activity_log/activity_log.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 from frappe import _
 from frappe.utils import get_fullname, now

frappe/core/doctype/activity_log/feed.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 import frappe.permissions

frappe/core/doctype/activity_log/test_activity_log.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import unittest
 import time

frappe/core/doctype/block_module/block_module.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/comment/comment.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 import frappe
 from frappe import _
 import json

frappe/core/doctype/comment/test_comment.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe, json
 import unittest
 

frappe/core/doctype/communication/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 

frappe/core/doctype/communication/communication.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 from collections import Counter
 import frappe

frappe/core/doctype/communication/email.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 import json

frappe/core/doctype/communication/test_communication.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import unittest
 from urllib.parse import quote
 

frappe/core/doctype/communication_link/communication_link.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/custom_docperm/custom_docperm.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/custom_docperm/test_custom_docperm.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import unittest
 

frappe/core/doctype/custom_role/custom_role.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/custom_role/test_custom_role.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import unittest
 

frappe/core/doctype/data_export/data_export.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 from frappe.model.document import Document
 

frappe/core/doctype/data_export/exporter.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe import _

frappe/core/doctype/data_import/data_import.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import os
 

frappe/core/doctype/data_import/exporter.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import typing
 

frappe/core/doctype/data_import/importer.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import os
 import io

frappe/core/doctype/data_import/test_data_import.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 # import frappe
 import unittest
 

frappe/core/doctype/data_import/test_exporter.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import unittest
 import frappe
 from frappe.core.doctype.data_import.exporter import Exporter

frappe/core/doctype/data_import/test_importer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import unittest
 import frappe
 from frappe.core.doctype.data_import.importer import Importer

frappe/core/doctype/defaultvalue/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 

frappe/core/doctype/defaultvalue/defaultvalue.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 

frappe/core/doctype/deleted_document/deleted_document.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 import json

frappe/core/doctype/deleted_document/test_deleted_document.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 import frappe
 import unittest
 

frappe/core/doctype/docfield/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 

frappe/core/doctype/docfield/docfield.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/docperm/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 

frappe/core/doctype/docperm/docperm.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 

frappe/core/doctype/docshare/docshare.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
-# For license information, please see license.txt
+# License: MIT. See LICENSE
 
 import frappe
 from frappe.model.document import Document

frappe/core/doctype/docshare/test_docshare.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# See license.txt
+# License: MIT. See LICENSE
 
 import frappe
 import frappe.share

frappe/core/doctype/doctype/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# License: MIT. See LICENSE
 

frappe/core/doctype/doctype/doctype.js

@@ -61,9 +61,73 @@ frappe.ui.form.on('DocType', {
 			__('In Grid View') : __('In List View');
 
 		frm.events.autoname(frm);
+		frm.events.set_naming_rule_description(frm);
+	},
+
+	naming_rule: function(frm) {
+		// set the "autoname" property based on naming_rule
+		if (frm.doc.naming_rule && !frm.__from_autoname) {
+
+			// flag to avoid recursion
+			frm.__from_naming_rule = true;
+
+			if (frm.doc.naming_rule=='Set by user') {
+				frm.set_value('autoname', 'Prompt');
+			} else if (frm.doc.naming_rule=='By fieldname') {
+				frm.set_value('autoname', 'field:');
+			} else if (frm.doc.naming_rule=='By "Naming Series" field') {
+				frm.set_value('autoname', 'naming_series:');
+			} else if (frm.doc.naming_rule=='Expression') {
+				frm.set_value('autoname', 'format:');
+			} else if (frm.doc.naming_rule=='Expression (old style)') {
+				// pass
+			} else if (frm.doc.naming_rule=='Random') {
+				frm.set_value('autoname', 'hash');
+			}
+			setTimeout(() =>frm.__from_naming_rule = false, 500);
+
+			frm.events.set_naming_rule_description(frm);
+		}
+
+	},
+
+	set_naming_rule_description(frm) {
+		let naming_rule_description = {
+			'Set by user': '',
+			'By fieldname': 'Format: <code>field:[fieldname]</code>. Valid fieldname must exist',
+			'By "Naming Series" field': 'Format: <code>naming_series:[fieldname]</code>. Fieldname called <code>naming_series</code> must exist',
+			'Expression': 'Format: <code>format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####}</code> - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.',
+			'Expression (old style)': 'Format: <code>EXAMPLE-.#####</code> Series by prefix (separated by a dot)',
+			'Random': '',
+			'By script': ''
+		};
+
+		if (frm.doc.naming_rule) {
+			frm.get_field('autoname').set_description(naming_rule_description[frm.doc.naming_rule]);
+		}
 	},
 
 	autoname: function(frm) {
+		// set naming_rule based on autoname (for old doctypes where its not been set)
+		if (frm.doc.autoname && !frm.doc.naming_rule && !frm.__from_naming_rule) {
+			// flag to avoid recursion
+			frm.__from_autoname = true;
+			if (frm.doc.autoname.toLowerCase() === 'prompt') {
+				frm.set_value('naming_rule', 'Set by user');
+			} else if (frm.doc.autoname.startsWith('field:')) {
+				frm.set_value('naming_rule', 'By fieldname');
+			} else if (frm.doc.autoname.startsWith('naming_series:')) {
+				frm.set_value('naming_rule', 'By "Naming Series" field');
+			} else if (frm.doc.autoname.startsWith('format:')) {
+				frm.set_value('naming_rule', 'Expression');
+			} else if (frm.doc.autoname.toLowerCase() === 'hash') {
+				frm.set_value('naming_rule', 'Random');
+			} else {
+				frm.set_value('naming_rule', 'Expression (old style)');
+			}
+			setTimeout(() => frm.__from_autoname = false, 500);
+		}
+
 		frm.set_df_property('fields', 'reqd', frm.doc.autoname !== 'Prompt');
 	}
 });